integration.yml

# **what?**
# This workflow runs all integration tests for supported OS
# and python versions and core adapters. If triggered by PR,
# the workflow will only run tests for adapters related
# to code changes. Use the `test all` and `test ${adapter}`
# label to run all or additional tests. Use `ok to test`
# label to mark PRs from forked repositories that are safe
# to run integration tests for. Requires secrets to run
# against different warehouses.

# **why?**
# This checks the functionality of dbt from a user's perspective
# and attempts to catch functional regressions.

# **when?**
# This workflow will run on every push to a protected branch
# and when manually triggered. It will also run for all PRs, including
# PRs from forks. The workflow will be skipped until there is a label
# to mark the PR as safe to run.

name: Adapter Integration Tests

run-name: "${{ (contains(github.event_name, 'workflow_') && inputs.name) || github.event_name }}: ${{ (contains(github.event_name, 'workflow_') &&  inputs.adapter_branch) || github.ref_name }} by @${{ github.actor }}"

on:
  # pushes to release branches
  push:
    branches:
      - "main"
      - "develop"
      - "*.latest"
      - "releases/*"
  # all PRs, important to note that `pull_request_target` workflows
  # will run in the context of the target branch of a PR
  pull_request_target:
  # manual trigger
  workflow_dispatch:
    inputs:
      name:
        description: "Name to associate with run (example: 'dbt-adapters-242')"
        required: false
        type: string
        default: "Adapter Integration Tests"
      adapter_branch:
        description: "The branch of this adapter repository to use"
        type: string
        required: false
        default: "main"
      dbt_adapters_branch:
        description: "The branch of dbt-adapters to use"
        type: string
        required: false
        default: "main"
      dbt_core_branch:
        description: "The branch of dbt-core to use"
        type: string
        required: false
        default: "main"
      dbt_common_branch:
        description: "The branch of dbt-common to use"
        type: string
        required: false
        default: "main"

# explicitly turn off permissions for `GITHUB_TOKEN`
permissions: read-all

# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}
  cancel-in-progress: true

# sets default shell to bash, for all operating systems
defaults:
  run:
    shell: bash

jobs:
  test:
    name: redshift / python ${{ matrix.python-version }} / ${{ matrix.os }}

    # run if not a PR from a forked repository or has a label to mark as safe to test
    if: >-
      github.event_name != 'pull_request_target' ||
      github.event.pull_request.head.repo.full_name == github.repository ||
      contains(github.event.pull_request.labels.*.name, 'ok to test')
    runs-on: ${{ matrix.os }}

    strategy:
      fail-fast: false
      max-parallel: 3
      matrix:
        os: [ubuntu-22.04, macos-12, windows-2022]
        python-version: ["3.8"]
        include:
          - os: ubuntu-22.04
            python-version: "3.9"
          - os: ubuntu-22.04
            python-version: "3.10"
          - os: ubuntu-22.04
            python-version: "3.11"
          - os: ubuntu-22.04
            python-version: "3.12"

    env:
      TOXENV: integration-redshift
      PYTEST_ADDOPTS: "-v --color=yes -n4 --csv integration_results.csv"
      DBT_INVOCATION_ENV: github-actions
      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
      DD_SITE: datadoghq.com
      DD_ENV: ci
      DD_SERVICE: ${{ github.event.repository.name }}

    steps:
      - name: Check out the repository (push)
        if: github.event_name == 'push'
        uses: actions/checkout@v4
        with:
          persist-credentials: false

      - name: Check out the repository (workflow_dispatch)
        if: github.event_name == 'workflow_dispatch'
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          ref: ${{ inputs.adapter_branch }}

      # explicitly checkout the branch for the PR,
      # this is necessary for the `pull_request_target` event
      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Update Adapters and Core branches (update dev_requirements.txt)
        if: github.event_name == 'workflow_dispatch'
        run: |
          ./.github/scripts/update_dev_dependency_branches.sh \
            ${{ inputs.dbt_adapters_branch }} \
            ${{ inputs.dbt_core_branch }} \
            ${{ inputs.dbt_common_branch }}
          cat dev-requirements.txt

      - name: Install python dependencies
        run: |
          python -m pip install  --user --upgrade pip
          python -m pip install tox
          python -m pip --version
          tox --version

      - name: Create AWS IAM profiles
        run: |
          aws configure --profile $AWS_USER_PROFILE set aws_access_key_id $AWS_USER_ACCESS_KEY_ID
          aws configure --profile $AWS_USER_PROFILE set aws_secret_access_key $AWS_USER_SECRET_ACCESS_KEY
          aws configure --profile $AWS_USER_PROFILE set region $AWS_REGION
          aws configure --profile $AWS_USER_PROFILE set output json

          aws configure --profile $AWS_SOURCE_PROFILE set aws_access_key_id $AWS_ROLE_ACCESS_KEY_ID
          aws configure --profile $AWS_SOURCE_PROFILE set aws_secret_access_key $AWS_ROLE_SECRET_ACCESS_KEY
          aws configure --profile $AWS_SOURCE_PROFILE set region $AWS_REGION
          aws configure --profile $AWS_SOURCE_PROFILE set output json

          aws configure --profile $AWS_ROLE_PROFILE set source_profile $AWS_SOURCE_PROFILE
          aws configure --profile $AWS_ROLE_PROFILE set role_arn $AWS_ROLE_ARN
          aws configure --profile $AWS_ROLE_PROFILE set region $AWS_REGION
          aws configure --profile $AWS_ROLE_PROFILE set output json
        env:
          AWS_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
          AWS_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
          AWS_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
          AWS_SOURCE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}-user
          AWS_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
          AWS_ROLE_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_ROLE_ACCESS_KEY_ID }}
          AWS_ROLE_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_SECRET_ACCESS_KEY }}
          AWS_ROLE_ARN: ${{ secrets.REDSHIFT_TEST_IAM_ROLE_ARN }}
          AWS_REGION: ${{ vars.REDSHIFT_TEST_REGION }}

      - name: Run tox (redshift)
        env:
          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
          REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
          REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
          REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
          REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
          REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
          REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
          DBT_TEST_USER_1: dbt_test_user_1
          DBT_TEST_USER_2: dbt_test_user_2
          DBT_TEST_USER_3: dbt_test_user_3
        run: tox -- -m "not flaky" --ddtrace

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: logs
          path: ./logs

      - name: Get current date
        if: always()
        id: date
        run: |
          echo "date=$(date +'%Y-%m-%dT%H_%M_%S')" >> $GITHUB_OUTPUT #no colons allowed for artifacts

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: integration_results_${{ matrix.python-version }}_${{ matrix.os }}_redshift-${{ steps.date.outputs.date }}.csv
          path: integration_results.csv

  test-flaky:
    name: redshift / python ${{ matrix.python-version }} / ubuntu-22.04 - flaky

    # run this after the norm integration tests to avoid collisions
    needs: test

    # run if not a PR from a forked repository or has a label to mark as safe to test
    if: >-
      github.event_name != 'pull_request_target' ||
      github.event.pull_request.head.repo.full_name == github.repository ||
      contains(github.event.pull_request.labels.*.name, 'ok to test')
    runs-on: ubuntu-22.04

    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]

    env:
      TOXENV: integration-redshift
      PYTEST_ADDOPTS: "-v --color=yes -n1 --csv integration_results.csv"
      DBT_INVOCATION_ENV: github-actions
      DD_CIVISIBILITY_AGENTLESS_ENABLED: true
      DD_INSTRUMENTATION_TELEMETRY_ENABLED: false
      DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
      DD_SITE: datadoghq.com
      DD_ENV: ci
      DD_SERVICE: ${{ github.event.repository.name }}

    steps:
      - name: Check out the repository (push)
        if: github.event_name == 'push'
        uses: actions/checkout@v3
        with:
          persist-credentials: false

      - name: Check out the repository (workflow_dispatch)
        if: github.event_name == 'workflow_dispatch'
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          ref: ${{ inputs.adapter_branch }}

      # explicitly checkout the branch for the PR,
      # this is necessary for the `pull_request_target` event
      - name: Check out the repository (PR)
        if: github.event_name == 'pull_request_target'
        uses: actions/checkout@v3
        with:
          persist-credentials: false
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install python dependencies
        run: |
          python -m pip install  --user --upgrade pip
          python -m pip install tox
          python -m pip --version
          tox --version

      - name: Update Adapters and Core branches (update dev_requirements.txt)
        if: github.event_name == 'workflow_dispatch'
        run: |
          ./.github/scripts/update_dev_dependency_branches.sh \
            ${{ inputs.dbt_adapters_branch }} \
            ${{ inputs.dbt_core_branch }} \
            ${{ inputs.dbt_common_branch }}
          cat dev-requirements.txt


      - name: Run tox (redshift)
        env:
          REDSHIFT_TEST_DBNAME: ${{ secrets.REDSHIFT_TEST_DBNAME }}
          REDSHIFT_TEST_PASS: ${{ secrets.REDSHIFT_TEST_PASS }}
          REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
          REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
          REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
          REDSHIFT_TEST_REGION: ${{ vars.REDSHIFT_TEST_REGION }}
          REDSHIFT_TEST_CLUSTER_ID: ${{ vars.REDSHIFT_TEST_CLUSTER_ID }}
          REDSHIFT_TEST_IAM_USER_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_USER_PROFILE }}
          REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID: ${{ vars.REDSHIFT_TEST_IAM_USER_ACCESS_KEY_ID }}
          REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY: ${{ secrets.REDSHIFT_TEST_IAM_USER_SECRET_ACCESS_KEY }}
          REDSHIFT_TEST_IAM_ROLE_PROFILE: ${{ vars.REDSHIFT_TEST_IAM_ROLE_PROFILE }}
          DBT_TEST_USER_1: dbt_test_user_1
          DBT_TEST_USER_2: dbt_test_user_2
          DBT_TEST_USER_3: dbt_test_user_3
        run: tox -- -m flaky --ddtrace

  require-label-comment:
    runs-on: ubuntu-22.04

    needs: test

    permissions:
      pull-requests: write

    steps:
      - name: Needs permission PR comment
        if: >-
          needs.test.result == 'skipped' &&
          github.event_name == 'pull_request_target' &&
          github.event.pull_request.head.repo.full_name != github.repository
        uses: unsplash/comment-on-pr@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          msg: |
            "You do not have permissions to run integration tests, @dbt-labs/core "\
            "needs to label this PR with `ok to test` in order to run integration tests!"
          check_for_duplicate_msg: true

  post-failure:
    runs-on: ubuntu-22.04
    needs: test
    if: ${{ failure() }}

    steps:
      - name: Posting scheduled run failures
        uses: ravsamhq/notify-slack-action@v2
        if: ${{ github.event_name == 'schedule' }}
        with:
          notification_title: 'Redshift nightly integration test failed'
          status: ${{ job.status }}
    env:
      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }}