Performance Refactor for privacy-experience
#6882
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Backend Code Checks | |
| on: | |
| pull_request: | |
| # This only ignores paths if there are _no_ commits on the branch that touch other files | |
| paths-ignore: | |
| - "**.md" | |
| - "clients/**" | |
| push: | |
| branches: | |
| - "main" | |
| env: | |
| IMAGE: ethyca/fides:local | |
| DEFAULT_PYTHON_VERSION: "3.10.12" | |
| jobs: | |
| ############### | |
| ## Prechecks ## | |
| ############### | |
| Collect-Tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set Up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
| cache: "pip" | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Run Static Check | |
| run: nox -s collect_tests | |
| Build: | |
| needs: Collect-Tests | |
| strategy: | |
| matrix: | |
| # NOTE: These are the currently supported/tested Python Versions | |
| python_version: ["3.8.17", "3.9.17", "3.10.12"] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build container | |
| uses: docker/build-push-action@v4 | |
| with: | |
| builder: ${{ steps.buildx.outputs.name }} | |
| context: . | |
| build-args: PYTHON_VERSION=${{ matrix.python_version }} | |
| target: prod | |
| outputs: type=docker,dest=/tmp/python-${{ matrix.python_version }}.tar | |
| push: false | |
| tags: ${{ env.IMAGE }} | |
| - name: Upload container | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: python-${{ matrix.python_version }} | |
| path: /tmp/python-${{ matrix.python_version }}.tar | |
| retention-days: 1 | |
| ################### | |
| ## Static Checks ## | |
| ################### | |
| Static-Checks: | |
| strategy: | |
| matrix: | |
| session_name: | |
| [ | |
| "isort", | |
| "black", | |
| "mypy", | |
| "pylint", | |
| "xenon", | |
| "check_install", | |
| '"pytest(nox)"', | |
| ] | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set Up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
| cache: "pip" | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Install Dev Requirements | |
| run: pip install -r dev-requirements.txt | |
| - name: Run Static Check | |
| run: nox -s ${{ matrix.session_name }} | |
| ################## | |
| ## Performance ## | |
| ################## | |
| Performance-Checks: | |
| needs: Check-Container-Startup | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| steps: | |
| - name: Download container | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-${{ env.DEFAULT_PYTHON_VERSION }} | |
| path: /tmp/ | |
| - name: Load image | |
| run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set Up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
| cache: "pip" | |
| - name: Install Rust/Cargo | |
| run: curl -y https://sh.rustup.rs -sSf | sh | |
| - name: Install Drill | |
| run: cargo install drill | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Run Performance Tests | |
| run: nox -s performance_tests | |
| ################# | |
| ## Misc Checks ## | |
| ################# | |
| Check-Container-Startup: | |
| needs: Build | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Download container | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-${{ env.DEFAULT_PYTHON_VERSION }} | |
| path: /tmp/ | |
| - name: Load image | |
| run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set Up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
| cache: "pip" | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Run test suite | |
| run: nox -s check_container_startup | |
| Misc-Tests: | |
| needs: Check-Container-Startup | |
| strategy: | |
| matrix: | |
| test_selection: | |
| - "check_fides_annotations" | |
| - "fides_db_scan" | |
| - "docs_check" | |
| - "minimal_config_startup" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| continue-on-error: true | |
| steps: | |
| - name: Download container | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-${{ env.DEFAULT_PYTHON_VERSION }} | |
| path: /tmp/ | |
| - name: Load image | |
| run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set Up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
| cache: "pip" | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Run test suite | |
| run: nox -s "${{ matrix.test_selection }}" | |
| ################ | |
| ## Safe Tests ## | |
| ################ | |
| Safe-Tests: | |
| needs: Check-Container-Startup | |
| strategy: | |
| matrix: | |
| python_version: ["3.8.17", "3.9.17", "3.10.12"] | |
| test_selection: | |
| - "ctl-not-external" | |
| - "ops-unit" | |
| - "ops-integration" | |
| - "lib" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| continue-on-error: true | |
| steps: | |
| - name: Download container | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-${{ matrix.python_version }} | |
| path: /tmp/ | |
| - name: Load image | |
| run: docker load --input /tmp/python-${{ matrix.python_version }}.tar | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set Up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
| cache: "pip" | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Run test suite | |
| run: nox -s "pytest(${{ matrix.test_selection }})" | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: true | |
| ################## | |
| ## Unsafe Tests ## | |
| ################## | |
| # NOTE: Matrixes aren't used here due to the danger of race conditions for external resources | |
| Pytest-Ctl-External: | |
| needs: Check-Container-Startup | |
| strategy: | |
| max-parallel: 1 # This prevents collisions in shared external resources | |
| matrix: | |
| python_version: ["3.8.17", "3.9.17", "3.10.12"] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| # In PRs run with the "unsafe" label, or run on a "push" event to main | |
| if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' | |
| steps: | |
| - name: Download container | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-${{ matrix.python_version }} | |
| path: /tmp/ | |
| - name: Load image | |
| run: docker load --input /tmp/python-${{ matrix.python_version }}.tar | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Run external test suite | |
| run: nox -s "pytest(ctl-external)" | |
| env: | |
| SNOWFLAKE_FIDESCTL_PASSWORD: ${{ secrets.SNOWFLAKE_FIDESCTL_PASSWORD }} | |
| REDSHIFT_FIDESCTL_PASSWORD: ${{ secrets.REDSHIFT_FIDESCTL_PASSWORD }} | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CTL_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CTL_SECRET_ACCESS_KEY }} | |
| OKTA_CLIENT_TOKEN: ${{ secrets.OKTA_FIDESCTL_CLIENT_TOKEN }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| BIGQUERY_CONFIG: ${{ secrets.BIGQUERY_CONFIG }} | |
| DYNAMODB_REGION: ${{ secrets.DYNAMODB_REGION }} | |
| DYNAMODB_ACCESS_KEY_ID: ${{ secrets.DYNAMODB_ACCESS_KEY_ID }} | |
| DYNAMODB_ACCESS_KEY: ${{ secrets.DYNAMODB_ACCESS_KEY }} | |
| External-Datastores: | |
| needs: Check-Container-Startup | |
| strategy: | |
| max-parallel: 1 # This prevents collisions in shared external resources | |
| matrix: | |
| python_version: ["3.8.17", "3.9.17", "3.10.12"] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| # In PRs run with the "unsafe" label, or run on a "push" event to main | |
| if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' | |
| steps: | |
| - name: Download container | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-${{ matrix.python_version }} | |
| path: /tmp/ | |
| - name: Load image | |
| run: docker load --input /tmp/python-${{ matrix.python_version }}.tar | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Integration Tests (External) | |
| env: | |
| BIGQUERY_KEYFILE_CREDS: ${{ secrets.BIGQUERY_KEYFILE_CREDS }} | |
| BIGQUERY_DATASET: fidesopstest | |
| DYNAMODB_REGION: ${{ secrets.DYNAMODB_REGION }} | |
| DYNAMODB_ACCESS_KEY_ID: ${{ secrets.DYNAMODB_ACCESS_KEY_ID }} | |
| DYNAMODB_ACCESS_KEY: ${{ secrets.DYNAMODB_ACCESS_KEY }} | |
| REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }} | |
| REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }} | |
| REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }} | |
| REDSHIFT_TEST_PASSWORD: ${{ secrets.REDSHIFT_TEST_PASSWORD }} | |
| REDSHIFT_TEST_DATABASE: ${{ secrets.REDSHIFT_TEST_DATABASE }} | |
| REDSHIFT_TEST_DB_SCHEMA: ${{ secrets.REDSHIFT_TEST_DB_SCHEMA }} | |
| SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER }} | |
| SNOWFLAKE_TEST_USER_LOGIN_NAME: ${{ secrets.SNOWFLAKE_TEST_USER_LOGIN_NAME }} | |
| SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }} | |
| SNOWFLAKE_TEST_WAREHOUSE_NAME: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE_NAME }} | |
| SNOWFLAKE_TEST_DATABASE_NAME: ${{ secrets.SNOWFLAKE_TEST_DATABASE_NAME }} | |
| SNOWFLAKE_TEST_SCHEMA_NAME: ${{ secrets.SNOWFLAKE_TEST_SCHEMA_NAME }} | |
| run: nox -s "pytest(ops-external-datastores)" | |
| External-SaaS-Connectors: | |
| needs: Check-Container-Startup | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| # In PRs run with the "unsafe" label, or run on a "push" event to main | |
| if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push' | |
| permissions: | |
| contents: read | |
| id-token: write | |
| strategy: | |
| max-parallel: 1 # This prevents collisions in shared external resources | |
| matrix: | |
| python_version: ["3.8.17", "3.9.17", "3.10.12"] | |
| steps: | |
| - name: Download container | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-${{ matrix.python_version }} | |
| path: /tmp/ | |
| - name: Load image | |
| run: docker load --input /tmp/python-${{ matrix.python_version }}.tar | |
| - name: Install Nox | |
| run: pip install nox>=2022 | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Get Vault Token | |
| uses: hashicorp/vault-action@v2.5.0 | |
| with: | |
| url: ${{ secrets.VAULT_ADDR }} | |
| namespace: ${{ secrets.VAULT_NAMESPACE }} | |
| method: jwt | |
| role: ${{ secrets.VAULT_ROLE }} | |
| exportToken: True | |
| - name: SaaS Connector Tests | |
| env: | |
| VAULT_ADDR: ${{ secrets.VAULT_ADDR }} | |
| VAULT_NAMESPACE: ${{ secrets.VAULT_NAMESPACE }} | |
| run: nox -s "pytest(ops-saas)" |