From 3d9647825f69a9ce13fdde4d916f373053104b98 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 17 Jul 2023 09:52:21 -0500 Subject: [PATCH] add env vars for datadog ci visibility (#8097) * add env vars for datadog ci visibility * modify pytest command for tracing * fix posargs * move env vars to job that needs them * add test repeater to DD * swap flags # Conflicts: # .github/workflows/test-repeater.yml # dev-requirements.txt --- .github/workflows/main.yml | 7 +- .github/workflows/test-repeater.yml | 155 ++++++++++++++++++++++++++++ dev-requirements.txt | 1 + tox.ini | 2 + 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/test-repeater.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6adbf6a21fd..4b7035e1d44 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -132,6 +132,11 @@ jobs: DBT_TEST_USER_1: dbt_test_user_1 DBT_TEST_USER_2: dbt_test_user_2 DBT_TEST_USER_3: dbt_test_user_3 + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} steps: - name: Check out the repository @@ -162,7 +167,7 @@ jobs: tox --version - name: Run tests - run: tox + run: tox -- --ddtrace - name: Get current date if: always() diff --git a/.github/workflows/test-repeater.yml b/.github/workflows/test-repeater.yml new file mode 100644 index 00000000000..e414792140c --- /dev/null +++ b/.github/workflows/test-repeater.yml @@ -0,0 +1,155 @@ +# **what?** +# This workflow will test all test(s) at the input path given number of times to determine if it's flaky or not. You can test with any supported OS/Python combination. +# This is batched in 10 to allow more test iterations faster. + +# **why?** +# Testing if a test is flaky and if a previously flaky test has been fixed. This allows easy testing on supported python versions and OS combinations. + +# **when?** +# This is triggered manually from dbt-core. + +name: Flaky Tester + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to check out' + type: string + required: true + default: 'main' + test_path: + description: 'Path to single test to run (ex: tests/functional/retry/test_retry.py::TestRetry::test_fail_fast)' + type: string + required: true + default: 'tests/functional/...' + python_version: + description: 'Version of Python to Test Against' + type: choice + options: + - '3.8' + - '3.9' + - '3.10' + - '3.11' + os: + description: 'OS to run test in' + type: choice + options: + - 'ubuntu-latest' + - 'macos-latest' + - 'windows-latest' + num_runs_per_batch: + description: 'Max number of times to run the test per batch. We always run 10 batches.' + type: number + required: true + default: '50' + +permissions: read-all + +defaults: + run: + shell: bash + +jobs: + debug: + runs-on: ubuntu-latest + steps: + - name: "[DEBUG] Output Inputs" + run: | + echo "Branch: ${{ inputs.branch }}" + echo "test_path: ${{ inputs.test_path }}" + echo "python_version: ${{ inputs.python_version }}" + echo "os: ${{ inputs.os }}" + echo "num_runs_per_batch: ${{ inputs.num_runs_per_batch }}" + + pytest: + runs-on: ${{ inputs.os }} + strategy: + # run all batches, even if one fails. This informs how flaky the test may be. + fail-fast: false + # using a matrix to speed up the jobs since the matrix will run in parallel when runners are available + matrix: + batch: ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10"] + env: + PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv" + DBT_TEST_USER_1: dbt_test_user_1 + DBT_TEST_USER_2: dbt_test_user_2 + DBT_TEST_USER_3: dbt_test_user_3 + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: ${{ secrets.DATADOG_API_KEY }} + DD_SITE: datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} + + steps: + - name: "Checkout code" + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + + - name: "Setup Python" + uses: actions/setup-python@v4 + with: + python-version: "${{ inputs.python_version }}" + + - name: "Setup Dev Environment" + run: make dev + + - name: "Set up postgres (linux)" + if: inputs.os == 'ubuntu-latest' + run: make setup-db + + # mac and windows don't use make due to limitations with docker with those runners in GitHub + - name: "Set up postgres (macos)" + if: inputs.os == 'macos-latest' + uses: ./.github/actions/setup-postgres-macos + + - name: "Set up postgres (windows)" + if: inputs.os == 'windows-latest' + uses: ./.github/actions/setup-postgres-windows + + - name: "Test Command" + id: command + run: | + test_command="python -m pytest ${{ inputs.test_path }}" + echo "test_command=$test_command" >> $GITHUB_OUTPUT + + - name: "Run test ${{ inputs.num_runs_per_batch }} times" + id: pytest + run: | + set +e + for ((i=1; i<=${{ inputs.num_runs_per_batch }}; i++)) + do + echo "Running pytest iteration $i..." + python -m pytest --ddtrace ${{ inputs.test_path }} + exit_code=$? + + if [[ $exit_code -eq 0 ]]; then + success=$((success + 1)) + echo "Iteration $i: Success" + else + failure=$((failure + 1)) + echo "Iteration $i: Failure" + fi + + echo + echo "===========================" + echo "Successful runs: $success" + echo "Failed runs: $failure" + echo "===========================" + echo + done + + echo "failure=$failure" >> $GITHUB_OUTPUT + + - name: "Success and Failure Summary: ${{ inputs.os }}/Python ${{ inputs.python_version }}" + run: | + echo "Batch: ${{ matrix.batch }}" + echo "Successful runs: ${{ steps.pytest.outputs.success }}" + echo "Failed runs: ${{ steps.pytest.outputs.failure }}" + + - name: "Error for Failures" + if: ${{ steps.pytest.outputs.failure }} + run: | + echo "Batch ${{ matrix.batch }} failed ${{ steps.pytest.outputs.failure }} of ${{ inputs.num_runs_per_batch }} tests" + exit 1 diff --git a/dev-requirements.txt b/dev-requirements.txt index e13aa4628ea..251b9a0ed43 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,6 @@ black==22.6.0 bumpversion +ddtrace flake8 flaky freezegun==0.3.12 diff --git a/tox.ini b/tox.ini index 8461b9c8e4f..c4cf6f750eb 100644 --- a/tox.ini +++ b/tox.ini @@ -22,6 +22,8 @@ passenv = DBT_* POSTGRES_TEST_* PYTEST_ADDOPTS + DD_SERVICE + DD_ENV commands = {envpython} -m pytest --cov=core -m profile_postgres {posargs} test/integration {envpython} -m pytest --cov=core {posargs} tests/functional