Merge pull request #3476 from DataDog/bm-log_correlation #8031
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: System Tests | |
| on: | |
| push: | |
| branches: | |
| - "**" | |
| workflow_dispatch: {} | |
| schedule: | |
| - cron: '00 04 * * 2-6' | |
| env: | |
| REGISTRY: ghcr.io | |
| jobs: | |
| build-harness: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - agent | |
| runs-on: ubuntu-latest | |
| name: Build (${{ matrix.image }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| - name: Pull released image | |
| run: | | |
| docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest | |
| docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest system_tests/${{ matrix.image }}:latest | |
| - name: Build image | |
| run: ./build.sh --images ${{ matrix.image }} | |
| - name: List images | |
| run: | | |
| docker image list | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Tag image for CI run | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Push image for CI run | |
| run: | | |
| docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Tag image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest | |
| - name: Push image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | | |
| docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}:latest | |
| build-apps: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - weblog | |
| library: | |
| - ruby | |
| app: | |
| - rack | |
| # - sinatra14 # DEV-2.0: Uncomment after https://github.com/DataDog/system-tests/pull/1882 is merged | |
| - sinatra20 | |
| - sinatra21 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| runs-on: ubuntu-latest | |
| name: Build (${{ matrix.app }}) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| - name: Checkout dd-trace-rb | |
| uses: actions/checkout@v4 | |
| with: | |
| path: 'binaries/dd-trace-rb' | |
| - name: Pull released image | |
| run: | | |
| docker pull ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest | |
| docker tag ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest system_tests/${{ matrix.image }}:latest | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - name: Build | |
| run: ./build.sh --library ${{ matrix.library }} --weblog-variant ${{ matrix.app }} --images ${{ matrix.image }} | |
| - name: Tag image for CI run | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Push image for CI run | |
| run: | | |
| docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| - name: Tag image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | |
| docker tag system_tests/${{ matrix.image }}:latest ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest | |
| - name: Push image for release | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: | | |
| docker push ghcr.io/datadog/dd-trace-rb/system-tests/${{ matrix.image }}-${{ matrix.app }}:latest | |
| test: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| library: | |
| - ruby | |
| app: | |
| - rack | |
| # - sinatra14 # DEV-2.0: Uncomment after https://github.com/DataDog/system-tests/pull/1882 is merged | |
| - sinatra20 | |
| - sinatra21 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| scenario: | |
| - DEFAULT | |
| - APPSEC_DISABLED | |
| - APPSEC_BLOCKING_FULL_DENYLIST | |
| - APPSEC_REQUEST_BLOCKING | |
| include: | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES_NOCACHE | |
| - library: ruby | |
| app: rack | |
| scenario: REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD_NOCACHE | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CUSTOM_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_MISSING_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CORRUPTED_RULES | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_LOW_WAF_TIMEOUT | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_CUSTOM_OBFUSCATION | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_RATE_LIMITER | |
| - library: ruby | |
| app: rails70 | |
| scenario: APPSEC_AUTO_EVENTS_EXTENDED | |
| - library: ruby | |
| app: rails70 | |
| scenario: APPSEC_API_SECURITY | |
| - library: ruby | |
| app: rack | |
| scenario: APPSEC_RULES_MONITORING_WITH_ERRORS | |
| - library: ruby | |
| app: rack | |
| scenario: SAMPLING | |
| - library: ruby | |
| app: rack | |
| scenario: PROFILING | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_APP_STARTED_PRODUCTS_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_DEPENDENCY_LOADED_TEST_FOR_DEPENDENCY_COLLECTION_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_LOG_GENERATION_DISABLED | |
| - library: ruby | |
| app: rack | |
| scenario: TELEMETRY_METRIC_GENERATION_DISABLED | |
| runs-on: ubuntu-latest | |
| needs: | |
| - build-harness | |
| - build-apps | |
| name: Test (${{ matrix.app }}, ${{ matrix.scenario }}) | |
| steps: | |
| - name: Setup python 3.9 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| - name: Pull agent image | |
| run: | | |
| docker pull ghcr.io/datadog/dd-trace-rb/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ghcr.io/datadog/dd-trace-rb/system-tests/agent:gha${{ github.run_id }}-g${{ github.sha }} system_tests/agent:latest | |
| - name: Pull app image | |
| run: | | |
| docker pull ghcr.io/datadog/dd-trace-rb/system-tests/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} | |
| docker tag ghcr.io/datadog/dd-trace-rb/system-tests/weblog-${{ matrix.app }}:gha${{ github.run_id }}-g${{ github.sha }} system_tests/weblog:latest | |
| - name: List images | |
| run: | | |
| docker image list | |
| - name: Run scenario | |
| run: ./build.sh --images runner && ./run.sh ${{ matrix.scenario }} | |
| env: | |
| DD_API_KEY: ${{ secrets.DD_APPSEC_SYSTEM_TESTS_API_KEY }} | |
| - name: Archive logs (per scenario) | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| name: system-tests-${{ matrix.library }}-${{ matrix.app }}-${{ matrix.scenario }}-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| path: logs* | |
| - name: Archive logs (aggregated) | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| path: logs* | |
| all_test_successful: | |
| name: System tests successful | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| steps: | |
| - name: System tests successful | |
| run: echo "Done" | |
| aggregate: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| library: | |
| - ruby | |
| app: | |
| - rack | |
| # - sinatra14 # DEV-2.0: Uncomment after https://github.com/DataDog/system-tests/pull/1882 is merged | |
| - sinatra20 | |
| - sinatra21 | |
| - rails50 | |
| - rails51 | |
| - rails52 | |
| - rails60 | |
| - rails61 | |
| - rails70 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| if: ${{ always() }} | |
| name: Aggregate (${{ matrix.app }}) | |
| steps: | |
| - name: Setup python 3.9 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.9' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: 'DataDog/system-tests' | |
| - name: Retrieve logs | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: system-tests-${{ matrix.library }}-${{ matrix.app }}-logs-gha${{ github.run_id }}-g${{ github.sha }} | |
| path: . | |
| - name: Print fancy log report | |
| run: | | |
| find logs* | |
| python utils/scripts/markdown_logs.py >> $GITHUB_STEP_SUMMARY | |
| cleanup: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| image: | |
| - runner | |
| - agent | |
| - weblog-rack | |
| # - weblog-sinatra14 # DEV-2.0: Uncomment after https://github.com/DataDog/system-tests/pull/1882 is merged | |
| - weblog-sinatra20 | |
| - weblog-sinatra21 | |
| - weblog-rails50 | |
| - weblog-rails51 | |
| - weblog-rails52 | |
| - weblog-rails60 | |
| - weblog-rails61 | |
| - weblog-rails70 | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| if: ${{ always() }} | |
| name: Cleanup (${{ matrix.image }}) | |
| steps: | |
| - name: Log in to the Container registry | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ${{ env.REGISTRY }} -u ${{ github.actor }} --password-stdin | |
| - uses: actions/delete-package-versions@v4 | |
| with: | |
| package-version-ids: 'gha${{ github.run_id }}-g${{ github.sha }}' | |
| package-name: 'system-tests/${{ matrix.image }}' | |
| package-type: 'container' | |
| continue-on-error: true |