From ce3dd39ee8213cf32503f61cbdeb31223b662078 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 9 Nov 2023 20:04:44 -0800 Subject: [PATCH] Adding support for configuring enabled actions (#4374) Co-authored-by: Adam Sachs --- CHANGELOG.md | 4 +- .../e2e/system-integrations-plus.cy.ts | 81 ++++++++ .../cypress/e2e/system-integrations.cy.ts | 23 +++ .../fixtures/connectors/connection_types.json | 3 +- .../forms/ConnectorParameters.tsx | 26 ++- .../forms/ConnectorParametersForm.tsx | 85 +++++++- .../system_portal_config/types.ts | 1 + .../admin-ui/src/features/plus/plus.slice.ts | 42 ++++ .../models/ConnectionConfigurationResponse.ts | 2 + .../api/models/ConnectionSystemTypeMap.ts | 2 + .../v1/endpoints/privacy_request_endpoints.py | 16 +- src/fides/api/common_exceptions.py | 4 + src/fides/api/models/manual_webhook.py | 33 ++-- .../connection_config.py | 6 +- .../connection_secrets_mysql.py | 2 +- .../connection_secrets_postgres.py | 2 +- .../connection_secrets_redshift.py | 2 +- .../connection_type_system_map.py | 4 +- .../saas_config_template_values.py | 23 ++- .../api/schemas/saas/connector_template.py | 4 +- src/fides/api/schemas/saas/saas_config.py | 30 +++ .../saas/connector_registry_service.py | 42 ++-- .../privacy_request/request_runner_service.py | 4 +- src/fides/api/task/graph_task.py | 49 ++++- src/fides/api/util/connection_type.py | 57 ++---- .../test_connection_config_endpoints.py | 92 ++++++--- .../test_connection_template_endpoints.py | 69 ++++++- .../test_policy_webhook_endpoints.py | 1 + tests/ops/api/v1/endpoints/test_system.py | 77 +++++++- tests/ops/graph/test_graph.py | 3 + .../saas/connector_runner.py | 8 +- .../integration_tests/test_enabled_actions.py | 187 ++++++++++++++++++ tests/ops/models/test_manual_webhook.py | 27 +++ .../test_connector_registry_service.py | 3 - .../test_connector_template_loaders.py | 6 +- tests/ops/task/test_graph_task.py | 91 +++++++++ tests/ops/util/test_connection_type.py | 32 ++- 37 files changed, 991 insertions(+), 152 deletions(-) create mode 100644 clients/admin-ui/cypress/e2e/system-integrations-plus.cy.ts create mode 100644 tests/ops/integration_tests/test_enabled_actions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 692b26cd55..60b0653f2d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,10 +28,8 @@ The types of changes are: - Erasure support for Qualtrics [#4371](https://github.com/ethyca/fides/pull/4371) - Erasure support for Ada Chatbot [#4382](https://github.com/ethyca/fides/pull/4382) - Erasure support for Typeform [#4366](https://github.com/ethyca/fides/pull/4366) - -## Added - - Added notice that a system is GVL when adding/editing from system form [#4327](https://github.com/ethyca/fides/pull/4327) +- Added the ability to select the request types to enable per integration (for plus users) [#4374](https://github.com/ethyca/fides/pull/4374) ### Changed - Add filtering and pagination to bulk vendor add table [#4351](https://github.com/ethyca/fides/pull/4351) diff --git a/clients/admin-ui/cypress/e2e/system-integrations-plus.cy.ts b/clients/admin-ui/cypress/e2e/system-integrations-plus.cy.ts new file mode 100644 index 0000000000..9d55ad8738 --- /dev/null +++ b/clients/admin-ui/cypress/e2e/system-integrations-plus.cy.ts @@ -0,0 +1,81 @@ +import { stubPlus, stubSystemCrud } from "cypress/support/stubs"; + +import { SYSTEM_ROUTE } from "~/features/common/nav/v2/routes"; + +describe("System integrations", () => { + beforeEach(() => { + cy.login(); + cy.intercept("GET", "/api/v1/system", { + fixture: "systems/systems.json", + }).as("getSystems"); + cy.intercept("GET", "/api/v1/connection_type*", { + fixture: "connectors/connection_types.json", + }).as("getConnectionTypes"); + cy.intercept("GET", "/api/v1/connection_type/postgres/secret", { + fixture: "connectors/postgres_secret.json", + }).as("getPostgresConnectorSecret"); + stubPlus(true); + stubSystemCrud(); + cy.visit(SYSTEM_ROUTE); + }); + + it("should render the integration configuration panel when navigating to integrations tab", () => { + cy.getByTestId("system-fidesctl_system").within(() => { + cy.getByTestId("more-btn").click(); + cy.getByTestId("edit-btn").click(); + }); + cy.getByTestId("tab-Integrations").click(); + cy.getByTestId("tab-panel-Integrations").should("exist"); + }); + + describe("Integration search", () => { + beforeEach(() => { + cy.getByTestId("system-fidesctl_system").within(() => { + cy.getByTestId("more-btn").click(); + cy.getByTestId("edit-btn").click(); + }); + cy.getByTestId("tab-Integrations").click(); + cy.getByTestId("select-dropdown-btn").click(); + }); + + it("should display Shopify when searching with upper case letters", () => { + cy.getByTestId("input-search-integrations").type("Sho"); + cy.getByTestId("select-dropdown-list") + .find('[role="menuitem"] p') + .should("contain.text", "Shopify"); + }); + + it("should display Shopify when searching with lower case letters", () => { + cy.getByTestId("input-search-integrations").type("sho"); + cy.getByTestId("select-dropdown-list") + .find('[role="menuitem"] p') + .should("contain.text", "Shopify"); + }); + }); + + describe("Integration form contents", () => { + beforeEach(() => { + cy.getByTestId("system-fidesctl_system").within(() => { + cy.getByTestId("more-btn").click(); + cy.getByTestId("edit-btn").click(); + }); + cy.getByTestId("tab-Integrations").click(); + cy.getByTestId("select-dropdown-btn").click(); + + cy.getByTestId("input-search-integrations").type("PostgreSQL"); + cy.getByTestId("select-dropdown-list") + .contains('[role="menuitem"]', "PostgreSQL") + .click(); + }); + + // Verify Postgres shows access and erasure by default + it("should display Request types (enabled-actions) field", () => { + cy.getByTestId("enabled-actions").should("exist"); + cy.getByTestId("enabled-actions").within(() => { + cy.contains("access"); + cy.contains("erasure"); + cy.contains("consent").should("not.exist"); + }); + }); + }); +}); diff --git a/clients/admin-ui/cypress/e2e/system-integrations.cy.ts b/clients/admin-ui/cypress/e2e/system-integrations.cy.ts index 8b0535d744..e4f523a878 100644 --- a/clients/admin-ui/cypress/e2e/system-integrations.cy.ts +++ b/clients/admin-ui/cypress/e2e/system-integrations.cy.ts @@ -11,6 +11,9 @@ describe("System integrations", () => { cy.intercept("GET", "/api/v1/connection_type*", { fixture: "connectors/connection_types.json", }).as("getConnectionTypes"); + cy.intercept("GET", "/api/v1/connection_type/postgres/secret", { + fixture: "connectors/postgres_secret.json", + }).as("getPostgresConnectorSecret"); stubPlus(false); stubSystemCrud(); cy.visit(SYSTEM_ROUTE); @@ -49,4 +52,24 @@ describe("System integrations", () => { .should("contain.text", "Shopify"); }); }); + + describe("Integration form contents", () => { + beforeEach(() => { + cy.getByTestId("system-fidesctl_system").within(() => { + cy.getByTestId("more-btn").click(); + cy.getByTestId("edit-btn").click(); + }); + cy.getByTestId("tab-Integrations").click(); + cy.getByTestId("select-dropdown-btn").click(); + + cy.getByTestId("input-search-integrations").type("PostgreSQL"); + cy.getByTestId("select-dropdown-list") + .contains('[role="menuitem"]', "PostgreSQL") + .click(); + }); + + it("should not Request types (enabled-actions) field", () => { + cy.getByTestId("enabled-actions").should("not.exist"); + }); + }); }); diff --git a/clients/admin-ui/cypress/fixtures/connectors/connection_types.json b/clients/admin-ui/cypress/fixtures/connectors/connection_types.json index e3426a058c..01c55ea9e9 100644 --- a/clients/admin-ui/cypress/fixtures/connectors/connection_types.json +++ b/clients/admin-ui/cypress/fixtures/connectors/connection_types.json @@ -34,7 +34,8 @@ "identifier": "postgres", "type": "database", "human_readable": "PostgreSQL", - "encoded_icon": null + "encoded_icon": null, + "supported_actions": ["access", "erasure"] }, { "identifier": "redshift", diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx index 700ebd6460..f61734eacf 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParameters.tsx @@ -18,11 +18,16 @@ import { useMemo, useState } from "react"; import { useAppDispatch, useAppSelector } from "~/app/hooks"; import DocsLink from "~/features/common/DocsLink"; +import { useFeatures } from "~/features/common/features"; import RightArrow from "~/features/common/Icon/RightArrow"; import { DEFAULT_TOAST_PARAMS } from "~/features/common/toast"; import { useGetConnectionTypeSecretSchemaQuery } from "~/features/connection-type"; import TestConnectionMessage from "~/features/datastore-connections/system_portal_config/TestConnectionMessage"; import TestData from "~/features/datastore-connections/TestData"; +import { + useCreatePlusSaasConnectionConfigMutation, + usePatchPlusSystemConnectionConfigsMutation, +} from "~/features/plus/plus.slice"; import { ConnectionConfigSecretsRequest, selectActiveSystem, @@ -33,6 +38,7 @@ import { } from "~/features/system/system.slice"; import { AccessLevel, + ActionType, BulkPutConnectionConfiguration, ConnectionConfigurationResponse, ConnectionSystemTypeMap, @@ -80,6 +86,9 @@ const createSaasConnector = async ( instance_key: generateIntegrationKey(systemFidesKey, connectionOption), saas_connector_type: connectionOption.identifier, secrets: {}, + ...(values.enabled_actions + ? { enabled_actions: values.enabled_actions } + : {}), }; const params: CreateSaasConnectionConfig = { @@ -111,6 +120,7 @@ export const patchConnectionConfig = async ( ? connectionConfig.key : generateIntegrationKey(systemFidesKey, connectionOption); + // the enabled_actions are conditionally added if plus is enabled const params1: Omit = { access: AccessLevel.WRITE, @@ -120,6 +130,9 @@ export const patchConnectionConfig = async ( description: values.description, disabled: false, key, + ...(values.enabled_actions + ? { enabled_actions: values.enabled_actions as ActionType[] } + : {}), }; const payload = await patchFunc({ systemFidesKey, @@ -211,15 +224,20 @@ export const useConnectorForm = ({ }); const [createSassConnectionConfig] = useCreateSassConnectionConfigMutation(); + const [createPlusSaasConnectionConfig] = + useCreatePlusSaasConnectionConfigMutation(); const [getAuthorizationUrl] = useLazyGetAuthorizationUrlQuery(); const [updateSystemConnectionSecrets] = usePatchSystemConnectionSecretsMutation(); const [patchDatastoreConnection] = usePatchSystemConnectionConfigsMutation(); + const [patchPlusDatastoreConnection] = + usePatchPlusSystemConnectionConfigsMutation(); const [deleteDatastoreConnection, deleteDatastoreConnectionResult] = useDeleteSystemConnectionConfigMutation(); const { data: allDatasetConfigs } = useGetConnectionConfigDatasetConfigsQuery( connectionConfig?.key || "" ); + const { plus: isPlusEnabled } = useFeatures(); const originalSecrets = useMemo( () => (connectionConfig ? { ...connectionConfig.secrets } : {}), @@ -255,7 +273,9 @@ export const useConnectorForm = ({ secretsSchema!, connectionOption, systemFidesKey, - createSassConnectionConfig + isPlusEnabled + ? createPlusSaasConnectionConfig + : createSassConnectionConfig ); // eslint-disable-next-line no-param-reassign connectionConfig = response.connection; @@ -265,7 +285,9 @@ export const useConnectorForm = ({ connectionOption, systemFidesKey, connectionConfig!, - patchDatastoreConnection + isPlusEnabled + ? patchPlusDatastoreConnection + : patchDatastoreConnection ); if ( !connectionConfig && diff --git a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx index 056b275bbf..6f91c09991 100644 --- a/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx +++ b/clients/admin-ui/src/features/datastore-connections/system_portal_config/forms/ConnectorParametersForm.tsx @@ -1,4 +1,5 @@ import { + Box, Button, ButtonGroup, CircleHelpIcon, @@ -16,7 +17,7 @@ import { Tooltip, VStack, } from "@fidesui/react"; -import { Option } from "common/form/inputs"; +import { Option, SelectInput } from "common/form/inputs"; import { ConnectionTypeSecretSchemaProperty, ConnectionTypeSecretSchemaReponse, @@ -28,6 +29,7 @@ import _ from "lodash"; import React from "react"; import { DatastoreConnectionStatus } from "src/features/datastore-connections/types"; +import { useFeatures } from "~/features/common/features"; import DisableConnectionModal from "~/features/datastore-connections/DisableConnectionModal"; import DatasetConfigField from "~/features/datastore-connections/system_portal_config/forms/fields/DatasetConfigField/DatasetConfigField"; import { @@ -95,6 +97,7 @@ const ConnectorParametersForm: React.FC = ({ }) => { const [trigger, { isLoading, isFetching }] = useLazyGetDatastoreConnectionStatusQuery(); + const { plus: isPlusEnabled } = useFeatures(); const validateField = (label: string, value: string, type?: string) => { let error; @@ -228,6 +231,9 @@ const ConnectorParametersForm: React.FC = ({ connectionConfig.connection_type === ConnectionType.SAAS ? (connectionConfig.saas_config?.fides_key as string) : connectionConfig.key; + initialValues.enabled_actions = ( + connectionConfig.enabled_actions || [] + ).map((action) => action.toString()); // @ts-ignore initialValues.secrets = { ...connectionConfig.secrets }; @@ -248,6 +254,13 @@ const ConnectorParametersForm: React.FC = ({ return initialValues; } + + if (_.isEmpty(initialValues.enabled_actions)) { + initialValues.enabled_actions = connectionOption.supported_actions.map( + (action) => action.toString() + ); + } + return fillInDefaults(initialValues, secretsSchema); }; @@ -351,7 +364,7 @@ const ConnectorParametersForm: React.FC = ({ {({ field }: { field: FieldInputProps }) => ( - {getFormLabel("instance_key", "Integration Identifier")} + {getFormLabel("instance_key", "Integration identifier")} = ({ )} {/* Dynamic connector secret fields */} - {connectionOption.type !== SystemType.MANUAL && secretsSchema ? Object.entries(secretsSchema.properties).map( ([key, item]) => { @@ -398,6 +410,73 @@ const ConnectorParametersForm: React.FC = ({ } ) : null} + {isPlusEnabled && + connectionOption.supported_actions.length > 1 && ( + { + let error; + if (!value || value.length === 0) { + error = "At least one request type must be selected"; + } + return error; + }} + > + {({ + field, + form, + }: { + field: FieldInputProps; + form: any; + }) => ( + + {/* Known as enabled_actions throughout the front-end and back-end but it's displayed to the user as "Request types" */} + {getFormLabel("enabled_actions", "Request types")} + + + ({ + label: action, + value: action, + }) + )} + fieldName={field.name} + size="sm" + isMulti + /> + + + {props.errors.enabled_actions} + + + + + + + + + )} + + )} {SystemType.DATABASE === connectionOption.type && !isCreatingConnectionConfig ? ( ["Custom Assets"], }), + patchPlusSystemConnectionConfigs: build.mutation< + BulkPutConnectionConfiguration, + { + systemFidesKey: string; + connectionConfigs: (Omit< + ConnectionConfigurationResponse, + "created_at" + > & { + enabled_actions?: string[]; + })[]; + } + >({ + query: ({ systemFidesKey, connectionConfigs }) => ({ + url: `/plus/system/${systemFidesKey}/connection`, + method: "PATCH", + body: connectionConfigs, + }), + invalidatesTags: ["Datamap", "System", "Datastore Connection"], + }), + createPlusSaasConnectionConfig: build.mutation< + CreateSaasConnectionConfigResponse, + CreateSaasConnectionConfig + >({ + query: (params) => { + const url = `/plus/system/${params.systemFidesKey}${CONNECTION_ROUTE}/instantiate/${params.connectionConfig.saas_connector_type}`; + + return { + url, + method: "POST", + body: { ...params.connectionConfig }, + }; + }, + // Creating a connection config also creates a dataset behind the scenes + invalidatesTags: () => ["Datastore Connection", "Datasets", "System"], + }), }), }); @@ -360,6 +400,8 @@ export const { usePostSystemVendorsMutation, useGetSystemHistoryQuery, useUpdateCustomAssetMutation, + usePatchPlusSystemConnectionConfigsMutation, + useCreatePlusSaasConnectionConfigMutation, } = plusApi; export const selectHealth: (state: RootState) => HealthCheck | undefined = diff --git a/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts b/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts index b1143f050f..9060c70d33 100644 --- a/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts +++ b/clients/admin-ui/src/types/api/models/ConnectionConfigurationResponse.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { AccessLevel } from "./AccessLevel"; +import type { ActionType } from "./ActionType"; import type { ConnectionType } from "./ConnectionType"; import type { SaaSConfigBase } from "./SaaSConfigBase"; @@ -23,4 +24,5 @@ export type ConnectionConfigurationResponse = { saas_config?: SaaSConfigBase; secrets?: any; authorized?: boolean; + enabled_actions?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/ConnectionSystemTypeMap.ts b/clients/admin-ui/src/types/api/models/ConnectionSystemTypeMap.ts index 59f76dfe08..2216b965e6 100644 --- a/clients/admin-ui/src/types/api/models/ConnectionSystemTypeMap.ts +++ b/clients/admin-ui/src/types/api/models/ConnectionSystemTypeMap.ts @@ -2,6 +2,7 @@ /* tslint:disable */ /* eslint-disable */ +import type { ActionType } from "./ActionType"; import type { ConnectionType } from "./ConnectionType"; import type { SystemType } from "./SystemType"; @@ -15,4 +16,5 @@ export type ConnectionSystemTypeMap = { encoded_icon?: string; authorization_required?: boolean; user_guide?: string; + supported_actions: Array; }; diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index 341864ccfc..686f5479dd 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -1655,19 +1655,21 @@ def resume_privacy_request_from_requires_input( detail=f"Cannot resume privacy request from 'requires_input': privacy request '{privacy_request.id}' status = {privacy_request.status.value}.", # type: ignore ) + action_type = None + if privacy_request.policy.get_rules_for_action(ActionType.access): + action_type = ActionType.access + elif privacy_request.policy.get_rules_for_action(ActionType.erasure): + action_type = ActionType.erasure + access_manual_webhooks: List[AccessManualWebhook] = AccessManualWebhook.get_enabled( - db + db, action_type ) try: for manual_webhook in access_manual_webhooks: # check the access or erasure cache based on the privacy request's action type - if privacy_request.policy.get_rules_for_action( - action_type=ActionType.access - ): + if action_type == ActionType.access: privacy_request.get_manual_webhook_access_input_strict(manual_webhook) - if privacy_request.policy.get_rules_for_action( - action_type=ActionType.erasure - ): + if action_type == ActionType.erasure: privacy_request.get_manual_webhook_erasure_input_strict(manual_webhook) except ( NoCachedManualWebhookEntry, diff --git a/src/fides/api/common_exceptions.py b/src/fides/api/common_exceptions.py index 952f165ea2..d1d26b311d 100644 --- a/src/fides/api/common_exceptions.py +++ b/src/fides/api/common_exceptions.py @@ -131,6 +131,10 @@ class CollectionDisabled(BaseException): """Collection is attached to disabled ConnectionConfig""" +class ActionDisabled(BaseException): + """Collection is attached to a ConnectionConfig that has not enabled the given action""" + + class NotSupportedForCollection(BaseException): """The given action is not supported for this type of collection""" diff --git a/src/fides/api/models/manual_webhook.py b/src/fides/api/models/manual_webhook.py index 76108a1e86..a38ced8b83 100644 --- a/src/fides/api/models/manual_webhook.py +++ b/src/fides/api/models/manual_webhook.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List, Optional from pydantic import BaseConfig, create_model -from sqlalchemy import Column, ForeignKey, String, text +from sqlalchemy import Column, ForeignKey, String, or_, text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableList from sqlalchemy.orm import Session, relationship @@ -9,6 +9,7 @@ from fides.api.db.base_class import Base from fides.api.models.connectionconfig import ConnectionConfig from fides.api.schemas.base_class import FidesSchema +from fides.api.schemas.policy import ActionType class AccessManualWebhook(Base): @@ -103,15 +104,25 @@ def empty_fields_dict(self) -> Dict[str, None]: } @classmethod - def get_enabled(cls, db: Session) -> List["AccessManualWebhook"]: + def get_enabled( + cls, db: Session, action_type: Optional[ActionType] = None + ) -> List["AccessManualWebhook"]: """Get all enabled access manual webhooks with fields""" - return ( - db.query(cls) - .filter( - AccessManualWebhook.connection_config_id == ConnectionConfig.id, - ConnectionConfig.disabled.is_(False), - AccessManualWebhook.fields != text("'null'"), - AccessManualWebhook.fields != "[]", - ) - .all() + + query = db.query(cls).filter( + AccessManualWebhook.connection_config_id == ConnectionConfig.id, + ConnectionConfig.disabled.is_(False), + AccessManualWebhook.fields != text("'null'"), + AccessManualWebhook.fields != "[]", ) + + # Add action_type filter only if action_type is provided + if action_type is not None: + query = query.filter( + or_( + ConnectionConfig.enabled_actions.contains([action_type]), + ConnectionConfig.enabled_actions.is_(None), + ) + ) + + return query.all() diff --git a/src/fides/api/schemas/connection_configuration/connection_config.py b/src/fides/api/schemas/connection_configuration/connection_config.py index 6bbba89105..ab8638e7ed 100644 --- a/src/fides/api/schemas/connection_configuration/connection_config.py +++ b/src/fides/api/schemas/connection_configuration/connection_config.py @@ -28,14 +28,13 @@ class CreateConnectionConfiguration(BaseModel): access: AccessLevel disabled: Optional[bool] = False description: Optional[str] - enabled_actions: Optional[List[ActionType]] = None class Config: """Restrict adding other fields through this schema and set orm_mode to support mapping to ConnectionConfig""" orm_mode = True use_enum_values = True - extra = Extra.forbid + extra = Extra.ignore class CreateConnectionConfigurationWithSecrets(CreateConnectionConfiguration): @@ -46,7 +45,7 @@ class CreateConnectionConfigurationWithSecrets(CreateConnectionConfiguration): class Config: orm_mode = True - extra = Extra.forbid + extra = Extra.ignore def mask_sensitive_fields( @@ -110,6 +109,7 @@ class ConnectionConfigurationResponse(BaseModel): saas_config: Optional[SaaSConfigBase] secrets: Optional[Dict[str, Any]] authorized: Optional[bool] = False + enabled_actions: Optional[List[ActionType]] @root_validator() def mask_sensitive_values(cls, values: Dict[str, Any]) -> Dict[str, Any]: diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_mysql.py b/src/fides/api/schemas/connection_configuration/connection_secrets_mysql.py index 71e9e9ab1e..44a0c8c010 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_mysql.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_mysql.py @@ -37,7 +37,7 @@ class MySQLSchema(ConnectionConfigSecretsSchema): ) ssh_required: bool = Field( False, - title="SSH Required", + title="SSH required", description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your MySQL server is behind a firewall and requires SSH tunneling for remote connections.", ) diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_postgres.py b/src/fides/api/schemas/connection_configuration/connection_secrets_postgres.py index f80f2a8310..0966c79981 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_postgres.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_postgres.py @@ -41,7 +41,7 @@ class PostgreSQLSchema(ConnectionConfigSecretsSchema): ) ssh_required: bool = Field( False, - title="SSH Required", + title="SSH required", description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your PostgreSQL server is behind a firewall and requires SSH tunneling for remote connections.", ) diff --git a/src/fides/api/schemas/connection_configuration/connection_secrets_redshift.py b/src/fides/api/schemas/connection_configuration/connection_secrets_redshift.py index d4c9008db0..ccff0d3941 100644 --- a/src/fides/api/schemas/connection_configuration/connection_secrets_redshift.py +++ b/src/fides/api/schemas/connection_configuration/connection_secrets_redshift.py @@ -40,7 +40,7 @@ class RedshiftSchema(ConnectionConfigSecretsSchema): ) ssh_required: bool = Field( False, - title="SSH Required", + title="SSH required", description="Indicates whether an SSH tunnel is required for the connection. Enable this option if your Redshift database is behind a firewall and requires SSH tunneling for remote connections.", ) diff --git a/src/fides/api/schemas/connection_configuration/connection_type_system_map.py b/src/fides/api/schemas/connection_configuration/connection_type_system_map.py index d31fb4c615..a94afefc9e 100644 --- a/src/fides/api/schemas/connection_configuration/connection_type_system_map.py +++ b/src/fides/api/schemas/connection_configuration/connection_type_system_map.py @@ -1,9 +1,10 @@ -from typing import Optional, Union +from typing import List, Optional, Union from pydantic import BaseModel from fides.api.models.connectionconfig import ConnectionType from fides.api.schemas.connection_configuration.enums.system_type import SystemType +from fides.api.schemas.policy import ActionType class ConnectionSystemTypeMap(BaseModel): @@ -17,6 +18,7 @@ class ConnectionSystemTypeMap(BaseModel): encoded_icon: Optional[str] authorization_required: Optional[bool] = False user_guide: Optional[str] + supported_actions: List[ActionType] class Config: """Use enum values and set orm mode""" diff --git a/src/fides/api/schemas/connection_configuration/saas_config_template_values.py b/src/fides/api/schemas/connection_configuration/saas_config_template_values.py index 1ba0fad11f..f8485a5d1d 100644 --- a/src/fides/api/schemas/connection_configuration/saas_config_template_values.py +++ b/src/fides/api/schemas/connection_configuration/saas_config_template_values.py @@ -1,8 +1,9 @@ -from typing import Optional +from typing import Any, Dict, Optional from fideslang.validation import FidesKey -from pydantic import BaseModel +from pydantic import BaseModel, Extra +from fides.api.models.connectionconfig import AccessLevel, ConnectionType from fides.api.schemas.connection_configuration import connection_secrets_schemas @@ -14,3 +15,21 @@ class SaasConnectionTemplateValues(BaseModel): description: Optional[str] # For ConnectionConfig secrets: connection_secrets_schemas # For ConnectionConfig instance_key: FidesKey # For DatasetConfig.fides_key + + class Config: + extra = Extra.ignore + + def generate_config_data_from_template( + self, config_from_template: Dict + ) -> Dict[str, Any]: + """Generate a config data object (dict) based on the template values""" + data = { + "key": self.key if self.key else self.instance_key, + "description": self.description, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, # does this need to be passed as a method arg instead? + "saas_config": config_from_template, + } + if self.name: + data["name"] = self.name + return data diff --git a/src/fides/api/schemas/saas/connector_template.py b/src/fides/api/schemas/saas/connector_template.py index 585507d9b7..6cec2450aa 100644 --- a/src/fides/api/schemas/saas/connector_template.py +++ b/src/fides/api/schemas/saas/connector_template.py @@ -1,8 +1,9 @@ -from typing import Optional +from typing import List, Optional from fideslang.models import Dataset from pydantic import BaseModel, validator +from fides.api.schemas.policy import ActionType from fides.api.schemas.saas.saas_config import SaaSConfig from fides.api.util.saas_util import load_config_from_string, load_dataset_from_string @@ -19,6 +20,7 @@ class ConnectorTemplate(BaseModel): human_readable: str authorization_required: bool user_guide: Optional[str] + supported_actions: List[ActionType] @validator("config") def validate_config(cls, config: str) -> str: diff --git a/src/fides/api/schemas/saas/saas_config.py b/src/fides/api/schemas/saas/saas_config.py index 577f0ce9d1..4a2a41a40d 100644 --- a/src/fides/api/schemas/saas/saas_config.py +++ b/src/fides/api/schemas/saas/saas_config.py @@ -15,6 +15,7 @@ ) from fides.api.schemas.base_class import FidesSchema from fides.api.schemas.limiter.rate_limit_config import RateLimitConfig +from fides.api.schemas.policy import ActionType from fides.api.schemas.saas.shared_schemas import HTTPMethod @@ -478,6 +479,35 @@ def resolve_param_reference( reference = FidesDatasetReference.parse_obj(secrets[reference]) return reference + @property + def supported_actions(self) -> List[ActionType]: + """Returns a list containing the privacy actions supported by the SaaS config.""" + + supported_actions = [] + + # check for access + if any( + requests.read + for requests in [endpoint.requests for endpoint in self.endpoints] + ): + supported_actions.append(ActionType.access) + + # check for erasure + if ( + any( + request.update or request.delete + for request in [endpoint.requests for endpoint in self.endpoints] + ) + or self.data_protection_request + ): + supported_actions.append(ActionType.erasure) + + # check for consent + if self.consent_requests: + supported_actions.append(ActionType.consent) + + return supported_actions + class SaaSConfigValidationDetails(FidesSchema): """ diff --git a/src/fides/api/service/connectors/saas/connector_registry_service.py b/src/fides/api/service/connectors/saas/connector_registry_service.py index 8879a6d461..7799287aa4 100644 --- a/src/fides/api/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/service/connectors/saas/connector_registry_service.py @@ -13,11 +13,7 @@ from fides.api.api.deps import get_api_session from fides.api.common_exceptions import ValidationError from fides.api.cryptography.cryptographic_util import str_to_b64_str -from fides.api.models.connectionconfig import ( - AccessLevel, - ConnectionConfig, - ConnectionType, -) +from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.custom_connector_template import CustomConnectorTemplate from fides.api.models.datasetconfig import DatasetConfig from fides.api.schemas.connection_configuration.saas_config_template_values import ( @@ -70,15 +66,15 @@ def _load_connector_templates(self) -> None: logger.info("Loading connectors templates from the data/saas directory") for file in os.listdir("data/saas/config"): if file.endswith(".yml"): - config_file = os.path.join("data/saas/config", file) - config_dict = load_config(config_file) - connector_type = config_dict["type"] - human_readable = config_dict["name"] - user_guide = config_dict.get("user_guide") - authentication = config_dict["client_config"].get("authentication") + config_path = os.path.join("data/saas/config", file) + config = SaaSConfig(**load_config(config_path)) + + connector_type = config.type + + authentication = config.client_config.authentication authorization_required = ( authentication is not None - and authentication.get("strategy") + and authentication.strategy == OAuth2AuthorizationCodeAuthenticationStrategy.name ) @@ -95,14 +91,15 @@ def _load_connector_templates(self) -> None: FileConnectorTemplateLoader.get_connector_templates()[ connector_type ] = ConnectorTemplate( - config=load_yaml_as_string(config_file), + config=load_yaml_as_string(config_path), dataset=load_yaml_as_string( f"data/saas/dataset/{connector_type}_dataset.yml" ), icon=icon, - human_readable=human_readable, + human_readable=config.name, authorization_required=authorization_required, - user_guide=user_guide, + user_guide=config.user_guide, + supported_actions=config.supported_actions, ) except Exception: logger.exception("Unable to load {} connector", connector_type) @@ -175,6 +172,7 @@ def _register_template( human_readable=template.name, authorization_required=authorization_required, user_guide=config.user_guide, + supported_actions=config.supported_actions, ) # register the template in the loader's template dictionary @@ -327,17 +325,9 @@ def create_connection_config_from_template_no_save( template.config, "", template_values.instance_key ) - data = { - "key": template_values.key - if template_values.key - else template_values.instance_key, - "description": template_values.description, - "connection_type": ConnectionType.saas, - "access": AccessLevel.write, - "saas_config": config_from_template, - } - if template_values.name: - data["name"] = template_values.name + data = template_values.generate_config_data_from_template( + config_from_template=config_from_template + ) if system_id: data["system_id"] = system_id diff --git a/src/fides/api/service/privacy_request/request_runner_service.py b/src/fides/api/service/privacy_request/request_runner_service.py index 95d58905f8..d1154ba3e7 100644 --- a/src/fides/api/service/privacy_request/request_runner_service.py +++ b/src/fides/api/service/privacy_request/request_runner_service.py @@ -109,7 +109,7 @@ def get_manual_webhook_access_inputs( return ManualWebhookResults(manual_data=manual_inputs, proceed=True) try: - for manual_webhook in AccessManualWebhook.get_enabled(db): + for manual_webhook in AccessManualWebhook.get_enabled(db, ActionType.access): manual_inputs[manual_webhook.connection_config.key] = [ privacy_request.get_manual_webhook_access_input_strict(manual_webhook) ] @@ -135,7 +135,7 @@ def get_manual_webhook_erasure_inputs( # Don't fetch manual inputs unless this policy has an access rule return ManualWebhookResults(manual_data=manual_inputs, proceed=True) try: - for manual_webhook in AccessManualWebhook().get_enabled(db): + for manual_webhook in AccessManualWebhook().get_enabled(db, ActionType.erasure): manual_inputs[manual_webhook.connection_config.key] = [ privacy_request.get_manual_webhook_erasure_input_strict(manual_webhook) ] diff --git a/src/fides/api/task/graph_task.py b/src/fides/api/task/graph_task.py index 000652e53d..14eea60222 100644 --- a/src/fides/api/task/graph_task.py +++ b/src/fides/api/task/graph_task.py @@ -1,3 +1,4 @@ +# pylint: disable=too-many-lines import copy import traceback from abc import ABC @@ -13,6 +14,7 @@ from sqlalchemy.orm import Session from fides.api.common_exceptions import ( + ActionDisabled, CollectionDisabled, NotSupportedForCollection, PrivacyRequestErasureEmailSendRequired, @@ -89,6 +91,7 @@ def result(*args: Any, **kwargs: Any) -> Any: for attempt in range(CONFIG.execution.task_retry_count + 1): try: self.skip_if_disabled() + self.skip_if_action_disabled(action_type) # Create ExecutionLog with status in_processing or retrying if attempt: self.log_retry(action_type) @@ -115,6 +118,7 @@ def result(*args: Any, **kwargs: Any) -> Any: return 0 except ( CollectionDisabled, + ActionDisabled, NotSupportedForCollection, ) as exc: traceback.print_exc() @@ -533,6 +537,23 @@ def skip_if_disabled(self) -> None: f"ConnectionConfig {connection_config.key} is disabled.", ) + def skip_if_action_disabled(self, action_type: ActionType) -> None: + """Skip execution for the given collection if it is attached to a ConnectionConfig that does not have the given action_type enabled.""" + + # the access action is never disabled since it provides data that is needed for erasure requests + if action_type == ActionType.access: + return + + connection_config: ConnectionConfig = self.connector.configuration + if ( + connection_config.enabled_actions is not None + and action_type not in connection_config.enabled_actions + ): + raise ActionDisabled( + f"Skipping collection {self.traversal_node.node.address}. " + f"The {action_type} action is disabled for connection config with key '{connection_config.key}'.", + ) + @retry(action_type=ActionType.access, default_return=[]) def access_request(self, *inputs: List[Row]) -> List[Row]: """Run an access request on a single node.""" @@ -762,7 +783,33 @@ def termination_fn( privacy_request.cache_data_use_map(_format_data_use_map_for_caching(env)) v = delayed(get(dsk, TERMINATOR_ADDRESS, num_workers=1)) - return v.compute() + access_results = v.compute() + filtered_access_results = filter_by_enabled_actions( + access_results, connection_configs + ) + return filtered_access_results + + +def filter_by_enabled_actions( + access_results: Dict[str, Any], connection_configs: List[ConnectionConfig] +) -> Dict[str, Any]: + """Removes any access results that are associated with a connection config that doesn't have the access action enabled.""" + + # create a map between the dataset and its connection config's enabled actions + dataset_enabled_actions = {} + for config in connection_configs: + for dataset in config.datasets: + dataset_enabled_actions[dataset.fides_key] = config.enabled_actions + + # use the enabled actions map to filter out the access results + filtered_access_results = {} + for key, value in access_results.items(): + dataset_name = key.split(":")[0] + enabled_action = dataset_enabled_actions.get(dataset_name) + if enabled_action is None or ActionType.access in enabled_action: + filtered_access_results[key] = value + + return filtered_access_results def get_cached_data_for_erasures( diff --git a/src/fides/api/util/connection_type.py b/src/fides/api/util/connection_type.py index 9e193726ee..32027a6ac2 100644 --- a/src/fides/api/util/connection_type.py +++ b/src/fides/api/util/connection_type.py @@ -2,8 +2,6 @@ from typing import Any, Set -import yaml - from fides.api.common_exceptions import NoSuchConnectionTypeSecretSchemaError from fides.api.models.connectionconfig import ConnectionType from fides.api.schemas.connection_configuration import ( @@ -94,38 +92,9 @@ def saas_request_type_filter(connection_type: str) -> bool: if template is None: # shouldn't happen, but we can be safe return False - saas_config = SaaSConfig(**yaml.safe_load(template.config).get("saas_config")) - has_access = bool( - next( - ( - request.read - for request in [ - endpoint.requests for endpoint in saas_config.endpoints - ] - ), - None, - ) - ) - has_erasure = ( - bool( - next( - ( - request.update or request.delete - for request in [ - endpoint.requests for endpoint in saas_config.endpoints - ] - ), - None, - ) - ) - or saas_config.data_protection_request - ) - has_consent = saas_config.consent_requests - - return bool( - (ActionType.consent in action_types and has_consent) - or (ActionType.access in action_types and has_access) - or (ActionType.erasure in action_types and has_erasure) + # Check if the necessary actions are supported + return any( + action_type in template.supported_actions for action_type in action_types ) connection_system_types: list[ConnectionSystemTypeMap] = [] @@ -157,6 +126,7 @@ def saas_request_type_filter(connection_type: str) -> bool: identifier=item, type=SystemType.database, human_readable=ConnectionType(item).human_readable, + supported_actions=[ActionType.access, ActionType.erasure], ) for item in database_types ] @@ -181,12 +151,13 @@ def saas_request_type_filter(connection_type: str) -> bool: encoded_icon=connector_template.icon, authorization_required=connector_template.authorization_required, user_guide=connector_template.user_guide, + supported_actions=connector_template.supported_actions, ) ) - if ( - system_type == SystemType.manual or system_type is None - ) and ActionType.access in action_types: + if (system_type == SystemType.manual or system_type is None) and ( + ActionType.access in action_types or ActionType.erasure in action_types + ): manual_types: list[str] = sorted( [ manual_type.value @@ -201,6 +172,7 @@ def saas_request_type_filter(connection_type: str) -> bool: identifier=item, type=SystemType.manual, human_readable=ConnectionType(item).human_readable, + supported_actions=[ActionType.access, ActionType.erasure], ) for item in manual_types ] @@ -229,11 +201,16 @@ def saas_request_type_filter(connection_type: str) -> bool: connection_system_types.extend( [ ConnectionSystemTypeMap( - identifier=item, + identifier=email_type, type=SystemType.email, - human_readable=ConnectionType(item).human_readable, + human_readable=ConnectionType(email_type).human_readable, + supported_actions=[ + ActionType.consent + if ConnectionType(email_type) in CONSENT_EMAIL_CONNECTOR_TYPES + else ActionType.erasure + ], ) - for item in email_types + for email_type in email_types ] ) return connection_system_types diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 636dc0acef..110a08f65d 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -248,43 +248,50 @@ def test_patch_connection_saas_with_secrets_new( assert config.secrets == payload[0]["secrets"] @pytest.mark.parametrize( - "payload", + "payload, expected", [ - [ - { - "name": "My Main Postgres DB", - "key": "postgres_key", - "connection_type": "postgres", - "access": "write", - "secrets": { - "username": "test", - "password": "test", - "dbname": "test", - "db_schema": "test", + ( + [ + { + "name": "My Main Postgres DB", + "key": "postgres_key", + "connection_type": "postgres", + "access": "write", + "secrets": { + "username": "test", + "password": "test", + "dbname": "test", + "db_schema": "test", + }, + "port": 5432, }, - "port": 5432, - }, - ], - [ - { - "instance_key": "mailchimp_instance_1", - "secrets": { - "bad": "test_mailchimp", + ], + 200, + ), + ( + [ + { + "instance_key": "mailchimp_instance_1", + "secrets": { + "bad": "test_mailchimp", + }, + "name": "My Mailchimp Test", + "description": "Mailchimp ConnectionConfig description", + "saas_connector_type": "mailchimp", + "key": "mailchimp_1", }, - "name": "My Mailchimp Test", - "description": "Mailchimp ConnectionConfig description", - "saas_connector_type": "mailchimp", - "key": "mailchimp_1", - }, - ], + ], + 422, + ), ], ) def test_patch_connection_invalid_secrets( - self, payload, url, api_client, generate_auth_header + self, payload, expected, url, api_client, generate_auth_header ): + # extra fields are ignored but missing fields are still invalid auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) response = api_client.patch(url, headers=auth_header, json=payload) - assert response.status_code == 422 + assert response.status_code == expected def test_patch_connections_bulk_create( self, api_client: TestClient, db: Session, generate_auth_header, url, payload @@ -639,7 +646,6 @@ def test_patch_connections_failed_response( "access": "write", "disabled": False, "description": None, - "enabled_actions": None, } assert response_body["failed"][1]["data"] == { "name": "My Mongo DB", @@ -648,7 +654,6 @@ def test_patch_connections_failed_response( "access": "read", "disabled": False, "description": None, - "enabled_actions": None, } @mock.patch("fides.api.main.prepare_and_log_request") @@ -738,6 +743,31 @@ def test_patch_connections_no_name(self, api_client, generate_auth_header, url): assert response.json()["succeeded"][0]["name"] is None assert response.json()["succeeded"][1]["name"] is None + def test_patch_connections_ignore_enabled_actions( + self, db, api_client: TestClient, generate_auth_header, url + ) -> None: + payload = [ + { + "name": "My Connection", + "key": "my_connection", + "connection_type": "postgres", + "access": "write", + "enabled_actions": ["access"], + } + ] + + auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) + response = api_client.patch(url, headers=auth_header, json=payload) + + assert 200 == response.status_code + response_body = response.json() + assert response_body["succeeded"][0]["enabled_actions"] is None + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "my_connection") + ).first() + assert connection_config.enabled_actions is None + class TestGetConnections: @pytest.fixture(scope="function") @@ -782,6 +812,7 @@ def test_get_connection_configs( "disabled", "description", "authorized", + "enabled_actions", } assert connection["key"] == "my_postgres_db_1" @@ -1194,6 +1225,7 @@ def test_get_connection_config( "saas_config", "secrets", "authorized", + "enabled_actions", } assert response_body["key"] == "my_postgres_db_1" diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index bc908f50b2..2c30cfb43f 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -67,6 +67,7 @@ def test_get_connection_types( "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access.value, ActionType.erasure.value], } in data first_saas_type = ConnectorRegistry.connector_types().pop() first_saas_template = ConnectorRegistry.get_connector_template(first_saas_type) @@ -77,6 +78,9 @@ def test_get_connection_types( "encoded_icon": first_saas_template.icon, "authorization_required": first_saas_template.authorization_required, "user_guide": first_saas_template.user_guide, + "supported_actions": [ + action.value for action in first_saas_template.supported_actions + ], } in data assert "saas" not in [item["identifier"] for item in data] @@ -156,6 +160,9 @@ def test_search_connection_types( "encoded_icon": saas_template[1].icon, "authorization_required": saas_template[1].authorization_required, "user_guide": saas_template[1].user_guide, + "supported_actions": [ + action.value for action in saas_template[1].supported_actions + ], } for saas_template in expected_saas_templates ] @@ -184,6 +191,9 @@ def test_search_connection_types( "encoded_icon": saas_template[1].icon, "authorization_required": saas_template[1].authorization_required, "user_guide": saas_template[1].user_guide, + "supported_actions": [ + action.value for action in saas_template[1].supported_actions + ], } for saas_template in expected_saas_templates ] @@ -202,6 +212,7 @@ def test_search_connection_types( "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access.value, ActionType.erasure.value], } in data assert { "identifier": ConnectionType.redshift.value, @@ -210,6 +221,7 @@ def test_search_connection_types( "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access.value, ActionType.erasure.value], } in data for expected_data in expected_saas_data: assert expected_data in data, f"{expected_data} not in" @@ -236,6 +248,9 @@ def test_search_connection_types_case_insensitive( "encoded_icon": saas_template[1].icon, "authorization_required": saas_template[1].authorization_required, "user_guide": saas_template[1].user_guide, + "supported_actions": [ + action.value for action in saas_template[1].supported_actions + ], } for saas_template in expected_saas_types ] @@ -253,6 +268,7 @@ def test_search_connection_types_case_insensitive( "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access.value, ActionType.erasure.value], } in data for expected_data in expected_saas_data: @@ -275,6 +291,9 @@ def test_search_connection_types_case_insensitive( "encoded_icon": saas_template[1].icon, "authorization_required": saas_template[1].authorization_required, "user_guide": saas_template[1].user_guide, + "supported_actions": [ + action.value for action in saas_template[1].supported_actions + ], } for saas_template in expected_saas_types ] @@ -291,6 +310,7 @@ def test_search_connection_types_case_insensitive( "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access, ActionType.erasure], } in data assert { "identifier": ConnectionType.redshift.value, @@ -299,6 +319,7 @@ def test_search_connection_types_case_insensitive( "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access, ActionType.erasure], } in data for expected_data in expected_saas_data: @@ -363,6 +384,10 @@ def test_search_manual_system_type(self, api_client, generate_auth_header, url): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ + ActionType.access.value, + ActionType.erasure.value, + ], } ] @@ -381,6 +406,7 @@ def test_search_email_type(self, api_client, generate_auth_header, url): "type": "email", "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.erasure.value], }, { "encoded_icon": None, @@ -389,6 +415,7 @@ def test_search_email_type(self, api_client, generate_auth_header, url): "type": "email", "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.consent.value], }, { "encoded_icon": None, @@ -397,6 +424,7 @@ def test_search_email_type(self, api_client, generate_auth_header, url): "type": "email", "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.erasure.value], }, { "encoded_icon": None, @@ -405,6 +433,7 @@ def test_search_email_type(self, api_client, generate_auth_header, url): "type": "email", "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.consent.value], }, ] @@ -462,6 +491,10 @@ def connection_type_objects(self): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ + ActionType.access.value, + ActionType.erasure.value, + ], }, ConnectionType.manual_webhook.value: { "identifier": ConnectionType.manual_webhook.value, @@ -470,6 +503,10 @@ def connection_type_objects(self): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ + ActionType.access.value, + ActionType.erasure.value, + ], }, GOOGLE_ANALYTICS: { "identifier": GOOGLE_ANALYTICS, @@ -478,6 +515,10 @@ def connection_type_objects(self): "encoded_icon": google_analytics_template.icon, "authorization_required": True, "user_guide": google_analytics_template.user_guide, + "supported_actions": [ + action.value + for action in google_analytics_template.supported_actions + ], }, MAILCHIMP_TRANSACTIONAL: { "identifier": MAILCHIMP_TRANSACTIONAL, @@ -486,6 +527,10 @@ def connection_type_objects(self): "encoded_icon": mailchimp_transactional_template.icon, "authorization_required": False, "user_guide": mailchimp_transactional_template.user_guide, + "supported_actions": [ + action.value + for action in mailchimp_transactional_template.supported_actions + ], }, SEGMENT: { "identifier": SEGMENT, @@ -494,6 +539,9 @@ def connection_type_objects(self): "encoded_icon": segment_template.icon, "authorization_required": False, "user_guide": segment_template.user_guide, + "supported_actions": [ + action.value for action in segment_template.supported_actions + ], }, STRIPE: { "identifier": STRIPE, @@ -502,6 +550,9 @@ def connection_type_objects(self): "encoded_icon": stripe_template.icon, "authorization_required": False, "user_guide": stripe_template.user_guide, + "supported_actions": [ + action.value for action in stripe_template.supported_actions + ], }, ZENDESK: { "identifier": ZENDESK, @@ -510,6 +561,9 @@ def connection_type_objects(self): "encoded_icon": zendesk_template.icon, "authorization_required": False, "user_guide": zendesk_template.user_guide, + "supported_actions": [ + action.value for action in zendesk_template.supported_actions + ], }, DOORDASH: { "identifier": DOORDASH, @@ -518,6 +572,9 @@ def connection_type_objects(self): "encoded_icon": doordash_template.icon, "authorization_required": False, "user_guide": doordash_template.user_guide, + "supported_actions": [ + action.value for action in doordash_template.supported_actions + ], }, ConnectionType.sovrn.value: { "identifier": ConnectionType.sovrn.value, @@ -526,6 +583,7 @@ def connection_type_objects(self): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.consent.value], }, ConnectionType.attentive.value: { "identifier": ConnectionType.attentive.value, @@ -534,6 +592,7 @@ def connection_type_objects(self): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.erasure.value], }, } @@ -594,11 +653,11 @@ def connection_type_objects(self): STRIPE, ZENDESK, ConnectionType.attentive.value, + ConnectionType.manual_webhook.value, ], [ GOOGLE_ANALYTICS, MAILCHIMP_TRANSACTIONAL, - ConnectionType.manual_webhook.value, # manual webhook is not erasure DOORDASH, # doordash does not have erasures ConnectionType.sovrn.value, ], @@ -631,9 +690,9 @@ def connection_type_objects(self): STRIPE, ZENDESK, ConnectionType.attentive.value, + ConnectionType.manual_webhook.value, ], [ - ConnectionType.manual_webhook.value, # manual webhook is not erasure DOORDASH, # doordash does not have erasures ], ), @@ -1005,7 +1064,7 @@ def test_get_connection_secret_schema_mysql( "type": "string", }, "ssh_required": { - "title": "SSH Required", + "title": "SSH required", "description": "Indicates whether an SSH tunnel is required for the connection. Enable this option if your MySQL server is behind a firewall and requires SSH tunneling for remote connections.", "default": False, "type": "boolean", @@ -1059,7 +1118,7 @@ def test_get_connection_secret_schema_postgres( "type": "string", }, "ssh_required": { - "title": "SSH Required", + "title": "SSH required", "description": "Indicates whether an SSH tunnel is required for the connection. Enable this option if your PostgreSQL server is behind a firewall and requires SSH tunneling for remote connections.", "default": False, "type": "boolean", @@ -1113,7 +1172,7 @@ def test_get_connection_secret_schema_redshift( "type": "string", }, "ssh_required": { - "title": "SSH Required", + "title": "SSH required", "description": "Indicates whether an SSH tunnel is required for the connection. Enable this option if your Redshift database is behind a firewall and requires SSH tunneling for remote connections.", "default": False, "type": "boolean", diff --git a/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py b/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py index 1b37a31c21..7150594927 100644 --- a/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_policy_webhook_endpoints.py @@ -38,6 +38,7 @@ def embedded_http_connection_config(connection_config: ConnectionConfig) -> Dict "saas_config": None, "secrets": None, "authorized": False, + "enabled_actions": None, } diff --git a/tests/ops/api/v1/endpoints/test_system.py b/tests/ops/api/v1/endpoints/test_system.py index 1ac3a7f5f3..1519d92738 100644 --- a/tests/ops/api/v1/endpoints/test_system.py +++ b/tests/ops/api/v1/endpoints/test_system.py @@ -23,6 +23,7 @@ from fides.api.models.manual_webhook import AccessManualWebhook from fides.api.models.privacy_request import PrivacyRequestStatus from fides.api.models.sql_models import Dataset, System +from fides.api.schemas.policy import ActionType from fides.common.api.scope_registry import ( CONNECTION_CREATE_OR_UPDATE, CONNECTION_DELETE, @@ -65,7 +66,6 @@ def payload(): "username": "test", "password": "test", }, - "enabled_actions": ["access", "erasure"], } ] @@ -295,6 +295,7 @@ def test_get_connection_configs( "disabled", "description", "authorized", + "enabled_actions", } connection_keys = [connection["key"] for connection in connections] assert response_body["items"][0]["key"] in connection_keys @@ -966,3 +967,77 @@ def test_instantiate_connection_from_template( dataset_config.delete(db) connection_config.delete(db) dataset_config.ctl_dataset.delete(db=db) + + def test_instantiate_connection_from_template_ignore_enabled_actions( + self, db, generate_auth_header, api_client, base_url + ): + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + assert connection_config is None + + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + assert dataset_config is None + + auth_header = generate_auth_header(scopes=[SAAS_CONNECTION_INSTANTIATE]) + request_body = { + "instance_key": "secondary_mailchimp_instance", + "secrets": { + "domain": "test_mailchimp_domain", + "username": "test_mailchimp_username", + "api_key": "test_mailchimp_api_key", + }, + "name": "Mailchimp Connector", + "description": "Mailchimp ConnectionConfig description", + "key": "mailchimp_connection_config", + "enabled_actions": [ActionType.access.value], + } + resp = api_client.post( + base_url.format(saas_connector_type="mailchimp"), + headers=auth_header, + json=request_body, + ) + + assert resp.status_code == 200 + assert set(resp.json().keys()) == {"connection", "dataset"} + connection_data = resp.json()["connection"] + assert connection_data["key"] == "mailchimp_connection_config" + assert connection_data["name"] == "Mailchimp Connector" + assert connection_data["secrets"]["api_key"] == "**********" + assert connection_data["enabled_actions"] is None + + dataset_data = resp.json()["dataset"] + assert dataset_data["fides_key"] == "secondary_mailchimp_instance" + + connection_config = ConnectionConfig.filter( + db=db, conditions=(ConnectionConfig.key == "mailchimp_connection_config") + ).first() + dataset_config = DatasetConfig.filter( + db=db, + conditions=(DatasetConfig.fides_key == "secondary_mailchimp_instance"), + ).first() + + assert connection_config is not None + assert dataset_config is not None + assert connection_config.name == "Mailchimp Connector" + assert connection_config.description == "Mailchimp ConnectionConfig description" + + assert connection_config.access == AccessLevel.write + assert connection_config.connection_type == ConnectionType.saas + assert connection_config.saas_config is not None + assert connection_config.disabled is False + assert connection_config.disabled_at is None + assert connection_config.last_test_timestamp is None + assert connection_config.last_test_succeeded is None + assert connection_config.system_id is not None + assert connection_config.enabled_actions is None + + assert dataset_config.connection_config_id == connection_config.id + assert dataset_config.ctl_dataset_id is not None + + dataset_config.delete(db) + connection_config.delete(db) + dataset_config.ctl_dataset.delete(db=db) diff --git a/tests/ops/graph/test_graph.py b/tests/ops/graph/test_graph.py index a83d53627c..ed9cca2b10 100644 --- a/tests/ops/graph/test_graph.py +++ b/tests/ops/graph/test_graph.py @@ -95,6 +95,9 @@ def log_retry(self, _: ActionType): def skip_if_disabled(self) -> bool: return False + def skip_if_action_disabled(self, action_type: ActionType): + return False + @retry(action_type=ActionType.access, default_return=[]) def test_function(self): self.call_count += 1 diff --git a/tests/ops/integration_tests/saas/connector_runner.py b/tests/ops/integration_tests/saas/connector_runner.py index a0cd60a0f8..6b378907fc 100644 --- a/tests/ops/integration_tests/saas/connector_runner.py +++ b/tests/ops/integration_tests/saas/connector_runner.py @@ -66,7 +66,7 @@ def __init__( # create and save the connection config and dataset config to the database self.connection_config = _connection_config(db, self.config, secrets) - self.dataset_config = _dataset_config(db, self.connection_config, self.dataset) + self.dataset_config = dataset_config(db, self.connection_config, self.dataset) def test_connection(self): """Connection test using the connectors test_request""" @@ -257,11 +257,13 @@ def _external_connection_config(db: Session, fides_key) -> ConnectionConfig: return connection_config -def _dataset_config( +def dataset_config( db: Session, connection_config: ConnectionConfig, dataset: Dict[str, Any], ) -> DatasetConfig: + """Helper function to persist a dataset config and link it to a connection config.""" + fides_key = dataset["fides_key"] connection_config.name = fides_key connection_config.key = fides_key @@ -331,7 +333,7 @@ def _process_external_references( db, f"{connector_type}_external_dataset" ) graph_list.append( - _dataset_config( + dataset_config( db, external_connection_config, _external_dataset( diff --git a/tests/ops/integration_tests/test_enabled_actions.py b/tests/ops/integration_tests/test_enabled_actions.py new file mode 100644 index 0000000000..c249ba012c --- /dev/null +++ b/tests/ops/integration_tests/test_enabled_actions.py @@ -0,0 +1,187 @@ +import pytest +from fideslang.models import Dataset + +from fides.api.graph.graph import DatasetGraph +from fides.api.models.connectionconfig import ActionType +from fides.api.models.datasetconfig import convert_dataset_to_graph +from fides.api.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fides.api.task import graph_task +from fides.api.task.graph_task import get_cached_data_for_erasures +from tests.ops.integration_tests.saas.connector_runner import dataset_config +from tests.ops.service.privacy_request.test_request_runner_service import ( + get_privacy_request_results, +) + + +@pytest.mark.integration +class TestEnabledActions: + @pytest.fixture(scope="function") + def dataset_graph( + self, + db, + example_datasets, + integration_postgres_config, + ) -> DatasetGraph: + dataset_postgres = Dataset(**example_datasets[0]) + dataset_config(db, integration_postgres_config, dataset_postgres.dict()) + graph = convert_dataset_to_graph( + dataset_postgres, integration_postgres_config.key + ) + return DatasetGraph(graph) + + @pytest.mark.asyncio + async def test_access_disabled( + self, + db, + policy, + integration_postgres_config, + dataset_graph, + ) -> None: + """Disable the access request for one connection config and verify the access results""" + + # disable the access action type for Postgres + integration_postgres_config.enabled_actions = [ActionType.erasure] + integration_postgres_config.save(db) + privacy_request = PrivacyRequest(id="test_disable_postgres_access") + + access_results = await graph_task.run_access_request( + privacy_request, + policy, + dataset_graph, + [integration_postgres_config], + {"email": "customer-1@example.com"}, + db, + ) + + assert access_results == {} + + @pytest.mark.asyncio + async def test_erasure_disabled( + self, + db, + policy, + erasure_policy, + integration_postgres_config, + dataset_graph, + ) -> None: + """Disable the erasure request for one connection config and verify the erasure results""" + + # disable the erasure action type for Postgres + integration_postgres_config.enabled_actions = [ActionType.access] + integration_postgres_config.save(db) + privacy_request = PrivacyRequest(id="test_disable_postgres_erasure") + + access_results = await graph_task.run_access_request( + privacy_request, + policy, + dataset_graph, + [integration_postgres_config], + {"email": "customer-1@example.com"}, + db, + ) + + # the access results should contain results from postgres + postgres_dataset = integration_postgres_config.datasets[0].fides_key + assert {key.split(":")[0] for key in access_results} == { + postgres_dataset, + } + + erasure_results = await graph_task.run_erasure( + privacy_request, + erasure_policy, + dataset_graph, + [integration_postgres_config], + {"email": "customer-1@example.com"}, + get_cached_data_for_erasures(privacy_request.id), + db, + ) + + # the erasure results should be empty + assert erasure_results == {} + + @pytest.mark.asyncio + async def test_access_disabled_for_manual_webhook_integrations( + self, + db, + policy, + integration_postgres_config, + integration_manual_webhook_config, + access_manual_webhook, + run_privacy_request_task, + ) -> None: + pr = get_privacy_request_results( + db, + policy, + run_privacy_request_task, + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "customer-1@example.com"}, + }, + ) + db.refresh(pr) + + # verify the request is paused if the manual webhook has the access action enabled + assert pr.status == PrivacyRequestStatus.requires_input + + integration_manual_webhook_config.enabled_actions = [ActionType.erasure] + integration_manual_webhook_config.save(db) + + pr = get_privacy_request_results( + db, + policy, + run_privacy_request_task, + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "customer-1@example.com"}, + }, + ) + db.refresh(pr) + + # verify the request completes if the manual webhook has the access action disabled + assert pr.status == PrivacyRequestStatus.complete + + @pytest.mark.asyncio + async def test_erasure_disabled_for_manual_webhook_integrations( + self, + db, + policy, + erasure_policy, + integration_postgres_config, + integration_manual_webhook_config, + access_manual_webhook, + run_privacy_request_task, + ) -> None: + pr = get_privacy_request_results( + db, + erasure_policy, + run_privacy_request_task, + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "customer-1@example.com"}, + }, + ) + db.refresh(pr) + + # verify the request is paused if the manual webhook has the erasure action enabled + assert pr.status == PrivacyRequestStatus.requires_input + + integration_manual_webhook_config.enabled_actions = [ActionType.access] + integration_manual_webhook_config.save(db) + + pr = get_privacy_request_results( + db, + erasure_policy, + run_privacy_request_task, + { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": "customer-1@example.com"}, + }, + ) + db.refresh(pr) + + # verify the request completes if the manual webhook has the erasure action disabled + assert pr.status == PrivacyRequestStatus.complete diff --git a/tests/ops/models/test_manual_webhook.py b/tests/ops/models/test_manual_webhook.py index d2361f8e9c..6b2f94b306 100644 --- a/tests/ops/models/test_manual_webhook.py +++ b/tests/ops/models/test_manual_webhook.py @@ -1,6 +1,7 @@ import pytest from fides.api.models.manual_webhook import AccessManualWebhook +from fides.api.schemas.policy import ActionType class TestManualWebhook: @@ -21,6 +22,32 @@ def test_get_enabled_no_enabled_webhooks(self, db): def test_get_enabled_webhooks(self, db, access_manual_webhook): assert AccessManualWebhook.get_enabled(db) == [access_manual_webhook] + def test_get_enabled_access_webhooks( + self, db, integration_manual_webhook_config, access_manual_webhook + ): + assert AccessManualWebhook.get_enabled(db, ActionType.access) == [ + access_manual_webhook + ] + + integration_manual_webhook_config.enabled_actions = [ActionType.erasure] + integration_manual_webhook_config.save(db) + + # don't return webhook if the access action is disabled + assert AccessManualWebhook.get_enabled(db, ActionType.access) == [] + + def test_get_enabled_erasure_webhooks( + self, db, integration_manual_webhook_config, access_manual_webhook + ): + assert AccessManualWebhook.get_enabled(db, ActionType.erasure) == [ + access_manual_webhook + ] + + integration_manual_webhook_config.enabled_actions = [ActionType.access] + integration_manual_webhook_config.save(db) + + # don't return webhook if the erasure action is disabled + assert AccessManualWebhook.get_enabled(db, ActionType.erasure) == [] + @pytest.mark.usefixtures("access_manual_webhook") def test_get_enabled_webhooks_connection_config_disabled( self, db, integration_manual_webhook_config diff --git a/tests/ops/service/connectors/test_connector_registry_service.py b/tests/ops/service/connectors/test_connector_registry_service.py index 5818163f0b..bf55406805 100644 --- a/tests/ops/service/connectors/test_connector_registry_service.py +++ b/tests/ops/service/connectors/test_connector_registry_service.py @@ -2,11 +2,9 @@ from unittest import mock from unittest.mock import Mock -import yaml from fideslang.models import DatasetCollection from fides.api.models.datasetconfig import DatasetConfig -from fides.api.schemas.saas.connector_template import ConnectorTemplate from fides.api.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, update_saas_configs, @@ -19,7 +17,6 @@ replace_config_placeholders, replace_dataset_placeholders, ) -from fides.config.helpers import load_file NEW_CONFIG_DESCRIPTION = "new test config description" NEW_DATASET_DESCRIPTION = "new test dataset description" diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index 43f087a600..e38b15ce89 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -1,4 +1,3 @@ -import os from io import BytesIO from unittest import mock from unittest.mock import MagicMock @@ -8,6 +7,7 @@ from fides.api.common_exceptions import NoSuchSaaSRequestOverrideException from fides.api.models.custom_connector_template import CustomConnectorTemplate +from fides.api.schemas.policy import ActionType from fides.api.schemas.saas.connector_template import ConnectorTemplate from fides.api.service.authentication.authentication_strategy import ( AuthenticationStrategy, @@ -27,7 +27,6 @@ load_yaml_as_string, replace_version, ) -from fides.config import CONFIG from tests.ops.test_helpers.saas_test_utils import create_zip_file @@ -189,6 +188,7 @@ def test_custom_connector_template_loader( human_readable="Planet Express", authorization_required=False, user_guide=None, + supported_actions=[ActionType.access], ) } @@ -382,6 +382,7 @@ def test_custom_connector_replacement_replaceable_with_update_not_available( human_readable="Planet Express", authorization_required=False, user_guide=None, + supported_actions=[ActionType.access], ) } mock_delete.assert_not_called() @@ -427,6 +428,7 @@ def test_custom_connector_replacement_not_replaceable( human_readable="Zendesk", authorization_required=False, user_guide="https://docs.ethyca.com/user-guides/integrations/saas-integrations/zendesk", + supported_actions=[ActionType.access, ActionType.erasure], ) } mock_delete.assert_not_called() diff --git a/tests/ops/task/test_graph_task.py b/tests/ops/task/test_graph_task.py index a4e14b5789..0aada504de 100644 --- a/tests/ops/task/test_graph_task.py +++ b/tests/ops/task/test_graph_task.py @@ -35,6 +35,7 @@ _format_data_use_map_for_caching, build_affected_field_logs, collect_queries, + filter_by_enabled_actions, start_function, update_erasure_mapping_from_cache, ) @@ -1007,3 +1008,93 @@ def test_errored_consent_task_for_connector_no_relevant_preferences( .order_by(ExecutionLog.created_at.desc()) ) assert logs.first().status == ExecutionLogStatus.error + + +class TestFilterByEnabledActions: + def test_filter_by_enabled_actions_enabled_actions_none(self): + access_results = { + "dataset1:collection": "data", + "dataset2:collection": "data", + } + connection_configs = [ + ConnectionConfig( + datasets=[ + DatasetConfig(fides_key="dataset1"), + ], + enabled_actions=None, + ), + ConnectionConfig( + datasets=[DatasetConfig(fides_key="dataset2")], + enabled_actions=None, + ), + ] + + assert filter_by_enabled_actions(access_results, connection_configs) == { + "dataset1:collection": "data", + "dataset2:collection": "data", + } + + def test_filter_by_enabled_actions_access_enabled(self): + access_results = { + "dataset1:collection": "data", + "dataset2:collection": "data", + } + connection_configs = [ + ConnectionConfig( + datasets=[ + DatasetConfig(fides_key="dataset1"), + ], + enabled_actions=[ActionType.access], + ), + ConnectionConfig( + datasets=[DatasetConfig(fides_key="dataset2")], + enabled_actions=[ActionType.access], + ), + ] + + assert filter_by_enabled_actions(access_results, connection_configs) == { + "dataset1:collection": "data", + "dataset2:collection": "data", + } + + def test_filter_by_enabled_actions_only_erasure(self): + access_results = { + "dataset1:collection": "data", + "dataset2:collection": "data", + } + connection_configs = [ + ConnectionConfig( + datasets=[ + DatasetConfig(fides_key="dataset1"), + ], + enabled_actions=[ActionType.erasure], + ), + ConnectionConfig( + datasets=[DatasetConfig(fides_key="dataset2")], + enabled_actions=[ActionType.erasure], + ), + ] + + assert filter_by_enabled_actions(access_results, connection_configs) == {} + + def test_filter_by_enabled_actions_mixed_actions(self): + access_results = { + "dataset1:collection": "data", + "dataset2:collection": "data", + } + connection_configs = [ + ConnectionConfig( + datasets=[ + DatasetConfig(fides_key="dataset1"), + ], + enabled_actions=[ActionType.access], + ), + ConnectionConfig( + datasets=[DatasetConfig(fides_key="dataset2")], + enabled_actions=[ActionType.erasure], + ), + ] + + assert filter_by_enabled_actions(access_results, connection_configs) == { + "dataset1:collection": "data", + } diff --git a/tests/ops/util/test_connection_type.py b/tests/ops/util/test_connection_type.py index e61dfb2f06..cd1d1776c8 100644 --- a/tests/ops/util/test_connection_type.py +++ b/tests/ops/util/test_connection_type.py @@ -22,6 +22,7 @@ def test_get_connection_types(): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access.value, ActionType.erasure.value], } in data first_saas_type = ConnectorRegistry.connector_types().pop() first_saas_template = ConnectorRegistry.get_connector_template(first_saas_type) @@ -32,6 +33,9 @@ def test_get_connection_types(): "encoded_icon": first_saas_template.icon, "authorization_required": first_saas_template.authorization_required, "user_guide": first_saas_template.user_guide, + "supported_actions": [ + action.value for action in first_saas_template.supported_actions + ], } in data assert "saas" not in [item.identifier for item in data] @@ -46,6 +50,7 @@ def test_get_connection_types(): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.consent.value], } in data @@ -78,6 +83,7 @@ def connection_type_objects(): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access.value, ActionType.erasure.value], }, ConnectionType.manual_webhook.value: { "identifier": ConnectionType.manual_webhook.value, @@ -86,6 +92,7 @@ def connection_type_objects(): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.access.value, ActionType.erasure.value], }, GOOGLE_ANALYTICS: { "identifier": GOOGLE_ANALYTICS, @@ -94,6 +101,9 @@ def connection_type_objects(): "encoded_icon": google_analytics_template.icon, "authorization_required": google_analytics_template.authorization_required, "user_guide": google_analytics_template.user_guide, + "supported_actions": [ + action.value for action in google_analytics_template.supported_actions + ], }, MAILCHIMP_TRANSACTIONAL: { "identifier": MAILCHIMP_TRANSACTIONAL, @@ -102,6 +112,10 @@ def connection_type_objects(): "encoded_icon": mailchimp_transactional_template.icon, "authorization_required": mailchimp_transactional_template.authorization_required, "user_guide": mailchimp_transactional_template.user_guide, + "supported_actions": [ + action.value + for action in mailchimp_transactional_template.supported_actions + ], }, SEGMENT: { "identifier": SEGMENT, @@ -110,6 +124,9 @@ def connection_type_objects(): "encoded_icon": segment_template.icon, "authorization_required": segment_template.authorization_required, "user_guide": segment_template.user_guide, + "supported_actions": [ + action.value for action in segment_template.supported_actions + ], }, STRIPE: { "identifier": STRIPE, @@ -118,6 +135,9 @@ def connection_type_objects(): "encoded_icon": stripe_template.icon, "authorization_required": stripe_template.authorization_required, "user_guide": stripe_template.user_guide, + "supported_actions": [ + action.value for action in stripe_template.supported_actions + ], }, ZENDESK: { "identifier": ZENDESK, @@ -126,6 +146,9 @@ def connection_type_objects(): "encoded_icon": zendesk_template.icon, "authorization_required": zendesk_template.authorization_required, "user_guide": zendesk_template.user_guide, + "supported_actions": [ + action.value for action in zendesk_template.supported_actions + ], }, DOORDASH: { "identifier": DOORDASH, @@ -134,6 +157,9 @@ def connection_type_objects(): "encoded_icon": doordash_template.icon, "authorization_required": doordash_template.authorization_required, "user_guide": doordash_template.user_guide, + "supported_actions": [ + action.value for action in doordash_template.supported_actions + ], }, ConnectionType.sovrn.value: { "identifier": ConnectionType.sovrn.value, @@ -142,6 +168,7 @@ def connection_type_objects(): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.consent.value], }, ConnectionType.attentive.value: { "identifier": ConnectionType.attentive.value, @@ -150,6 +177,7 @@ def connection_type_objects(): "encoded_icon": None, "authorization_required": False, "user_guide": None, + "supported_actions": [ActionType.erasure.value], }, } @@ -195,11 +223,11 @@ def connection_type_objects(): STRIPE, ZENDESK, ConnectionType.attentive.value, + ConnectionType.manual_webhook.value, ], [ GOOGLE_ANALYTICS, MAILCHIMP_TRANSACTIONAL, - ConnectionType.manual_webhook.value, # manual webhook is not erasure DOORDASH, # doordash does not have erasures ConnectionType.sovrn.value, ], @@ -232,9 +260,9 @@ def connection_type_objects(): STRIPE, ZENDESK, ConnectionType.attentive.value, + ConnectionType.manual_webhook.value, ], [ - ConnectionType.manual_webhook.value, # manual webhook is not erasure DOORDASH, # doordash does not have erasures ], ),