From acf64b165e6ea9f0fed6a1228d85274da525eca3 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Fri, 25 Oct 2024 10:58:38 -0300 Subject: [PATCH 01/19] fix test connection config endpoints tests --- .../test_connection_config_endpoints.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index c04f67139a..44d62b3bf7 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -34,7 +34,7 @@ page_size = Params().size -@pytest.mark.skip(reason="move to plus in progress") + class TestPatchConnections: @pytest.fixture(scope="function") def url(self) -> str: @@ -771,7 +771,7 @@ def test_patch_connections_ignore_enabled_actions( assert connection_config.enabled_actions is None -@pytest.mark.skip(reason="move to plus in progress") + class TestGetConnections: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy) -> str: @@ -1177,7 +1177,7 @@ def test_orphaned_connections_filter( config.delete(db) -@pytest.mark.skip(reason="move to plus in progress") + class TestGetConnection: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy, connection_config) -> str: @@ -1238,7 +1238,7 @@ def test_get_connection_config( assert response_body["last_test_timestamp"] is None -@pytest.mark.skip(reason="move to plus in progress") + class TestDeleteConnection: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy, connection_config) -> str: @@ -1337,7 +1337,7 @@ def test_delete_manual_webhook_connection_config( privacy_request_requires_input.status == PrivacyRequestStatus.in_processing ) - @pytest.mark.skip(reason="move to plus in progress") + def test_delete_saas_connection_config( self, url, @@ -1346,14 +1346,14 @@ def test_delete_saas_connection_config( generate_auth_header, ) -> None: secrets = { - "domain": "test_sendgrid_domain", - "api_key": "test_sendgrid_api_key", + "domain": "test_hubspot_domain", + "private_app_token": "test_hubspot_api_key", } connection_config, dataset_config = instantiate_connector( db, - "sendgrid", - "secondary_sendgrid_instance", - "Sendgrid ConnectionConfig description", + "hubspot", + "secondary_hubspot_instance", + "hubspot ConnectionConfig description", secrets, ) dataset = dataset_config.ctl_dataset @@ -1371,7 +1371,7 @@ def test_delete_saas_connection_config( assert db.query(Dataset).filter_by(id=dataset.id).first() is None -@pytest.mark.skip(reason="move to plus in progress") + class TestPutConnectionConfigSecrets: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy, connection_config) -> str: @@ -2202,7 +2202,7 @@ def test_put_saas_example_connection_config_secrets_missing_saas_config( ) -@pytest.mark.skip(reason="move to plus in progress") + class TestPatchConnectionConfigSecrets: def test_patch_connection_config_secrets_not_authenticated( self, api_client: TestClient, generate_auth_header, connection_config From e696b9fe4da17cfaeabe04961415658e7f093b0e Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 10:50:28 -0300 Subject: [PATCH 02/19] fix test_connection_template_endpoints --- .../test_connection_template_endpoints.py | 149 +++++------------- 1 file changed, 43 insertions(+), 106 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index cadd2685d0..46b41b3c38 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -28,7 +28,7 @@ ) -@pytest.mark.skip(reason="move to plus in progress") + class TestGetConnections: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy) -> str: @@ -104,17 +104,18 @@ def test_get_connection_types_size_param( len(data) == len(ConnectionType) + len(ConnectorRegistry.connector_types()) - 4 ) # there are 4 connection types that are not returned by the endpoint - # this value is > 50, so we've efectively tested our "default" size is - # > than the default of 50 (it's 100!) + # this value is > 20, so we've efectively tested our "default" size is + # > than the default of 20 (it's 100!) # ensure specifying size works as expected auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) - resp = api_client.get(url + "size=50", headers=auth_header) + resp = api_client.get(url + "size=20", headers=auth_header) data = resp.json()["items"] + assert resp.status_code == 200 assert ( - len(data) == 50 - ) # should be 50 items in response since we explicitly set size=50 + len(data) == 20 + ) # should be 20 items in response since we explicitly set size=20 # ensure specifying size and page works as expected auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) @@ -463,15 +464,11 @@ def test_search_email_type(self, api_client, generate_auth_header, url): ] -DOORDASH = "doordash" -GOOGLE_ANALYTICS = "google_analytics" -MAILCHIMP_TRANSACTIONAL = "mailchimp_transactional" -SEGMENT = "segment" +HUBSPOT = "hubspot" +MAILCHIMP = "mailchimp" STRIPE = "stripe" -ZENDESK = "zendesk" -@pytest.mark.skip(reason="move to plus in progress") class TestGetConnectionsActionTypeParams: """ Class specifically for testing the "action type" query params for the get connection types endpoint. @@ -498,16 +495,9 @@ def url_with_params(self) -> str: @pytest.fixture def connection_type_objects(self): - google_analytics_template = ConnectorRegistry.get_connector_template( - GOOGLE_ANALYTICS - ) - mailchimp_transactional_template = ConnectorRegistry.get_connector_template( - MAILCHIMP_TRANSACTIONAL - ) - stripe_template = ConnectorRegistry.get_connector_template("stripe") - zendesk_template = ConnectorRegistry.get_connector_template("zendesk") - doordash_template = ConnectorRegistry.get_connector_template(DOORDASH) - segment_template = ConnectorRegistry.get_connector_template(SEGMENT) + hubspot_template = ConnectorRegistry.get_connector_template(HUBSPOT) + mailchimp_template = ConnectorRegistry.get_connector_template(MAILCHIMP) + stripe_template = ConnectorRegistry.get_connector_template(STRIPE) return { ConnectionType.postgres.value: { @@ -534,39 +524,28 @@ def connection_type_objects(self): ActionType.erasure.value, ], }, - GOOGLE_ANALYTICS: { - "identifier": GOOGLE_ANALYTICS, + HUBSPOT: { + "identifier": HUBSPOT, "type": SystemType.saas.value, - "human_readable": google_analytics_template.human_readable, - "encoded_icon": google_analytics_template.icon, - "authorization_required": True, - "user_guide": google_analytics_template.user_guide, - "supported_actions": [ - action.value - for action in google_analytics_template.supported_actions - ], - }, - MAILCHIMP_TRANSACTIONAL: { - "identifier": MAILCHIMP_TRANSACTIONAL, - "type": SystemType.saas.value, - "human_readable": mailchimp_transactional_template.human_readable, - "encoded_icon": mailchimp_transactional_template.icon, + "human_readable": hubspot_template.human_readable, + "encoded_icon": hubspot_template.icon, "authorization_required": False, - "user_guide": mailchimp_transactional_template.user_guide, + "user_guide": hubspot_template.user_guide, "supported_actions": [ action.value - for action in mailchimp_transactional_template.supported_actions + for action in hubspot_template.supported_actions ], }, - SEGMENT: { - "identifier": SEGMENT, + MAILCHIMP: { + "identifier": MAILCHIMP, "type": SystemType.saas.value, - "human_readable": segment_template.human_readable, - "encoded_icon": segment_template.icon, + "human_readable": mailchimp_template.human_readable, + "encoded_icon": mailchimp_template.icon, "authorization_required": False, - "user_guide": segment_template.user_guide, + "user_guide": mailchimp_template.user_guide, "supported_actions": [ - action.value for action in segment_template.supported_actions + action.value + for action in mailchimp_template.supported_actions ], }, STRIPE: { @@ -580,28 +559,6 @@ def connection_type_objects(self): action.value for action in stripe_template.supported_actions ], }, - ZENDESK: { - "identifier": ZENDESK, - "type": SystemType.saas.value, - "human_readable": zendesk_template.human_readable, - "encoded_icon": zendesk_template.icon, - "authorization_required": False, - "user_guide": zendesk_template.user_guide, - "supported_actions": [ - action.value for action in zendesk_template.supported_actions - ], - }, - DOORDASH: { - "identifier": DOORDASH, - "type": SystemType.saas.value, - "human_readable": doordash_template.human_readable, - "encoded_icon": doordash_template.icon, - "authorization_required": False, - "user_guide": doordash_template.user_guide, - "supported_actions": [ - action.value for action in doordash_template.supported_actions - ], - }, ConnectionType.sovrn.value: { "identifier": ConnectionType.sovrn.value, "type": SystemType.email.value, @@ -630,27 +587,23 @@ def connection_type_objects(self): [ ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, + HUBSPOT, STRIPE, - ZENDESK, - SEGMENT, + MAILCHIMP, ConnectionType.attentive_email.value, - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value, ], [], ), ( [ActionType.consent], - [GOOGLE_ANALYTICS, MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value], + [ConnectionType.sovrn.value], [ ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, + HUBSPOT, STRIPE, - ZENDESK, - SEGMENT, + MAILCHIMP, ConnectionType.attentive_email.value, ], ), @@ -659,14 +612,11 @@ def connection_type_objects(self): [ ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, - SEGMENT, + HUBSPOT, + MAILCHIMP, STRIPE, - ZENDESK, ], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value, ConnectionType.attentive_email.value, ], @@ -675,31 +625,25 @@ def connection_type_objects(self): [ActionType.erasure], [ ConnectionType.postgres.value, - SEGMENT, # segment has DPR so it is an erasure + HUBSPOT, STRIPE, - ZENDESK, + MAILCHIMP, ConnectionType.attentive_email.value, ConnectionType.manual_webhook.value, ], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, - DOORDASH, # doordash does not have erasures ConnectionType.sovrn.value, ], ), ( [ActionType.consent, ActionType.access], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value, ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, - SEGMENT, + HUBSPOT, STRIPE, - ZENDESK, + MAILCHIMP, ], [ ConnectionType.attentive_email.value, @@ -708,40 +652,33 @@ def connection_type_objects(self): ( [ActionType.consent, ActionType.erasure], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, + HUBSPOT, + STRIPE, + MAILCHIMP, ConnectionType.sovrn.value, ConnectionType.postgres.value, - SEGMENT, # segment has DPR so it is an erasure - STRIPE, - ZENDESK, ConnectionType.attentive_email.value, ConnectionType.manual_webhook.value, ], - [ - DOORDASH, # doordash does not have erasures - ], + [], ), ( [ActionType.access, ActionType.erasure], [ ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, - SEGMENT, + HUBSPOT, STRIPE, - ZENDESK, + MAILCHIMP, ConnectionType.attentive_email.value, ], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value, ], ), ], ) - @pytest.mark.skip(reason="move to plus in progress") + def test_get_connection_types_action_type_filter( self, action_types, From a308a3464be3cec76ee4e2fe423469a9f103f747 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 10:52:36 -0300 Subject: [PATCH 03/19] run static checks --- .../api/v1/endpoints/test_connection_config_endpoints.py | 7 ------- .../v1/endpoints/test_connection_template_endpoints.py | 8 ++------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 44d62b3bf7..c7c5c93f13 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -34,7 +34,6 @@ page_size = Params().size - class TestPatchConnections: @pytest.fixture(scope="function") def url(self) -> str: @@ -771,7 +770,6 @@ def test_patch_connections_ignore_enabled_actions( assert connection_config.enabled_actions is None - class TestGetConnections: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy) -> str: @@ -1177,7 +1175,6 @@ def test_orphaned_connections_filter( config.delete(db) - class TestGetConnection: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy, connection_config) -> str: @@ -1238,7 +1235,6 @@ def test_get_connection_config( assert response_body["last_test_timestamp"] is None - class TestDeleteConnection: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy, connection_config) -> str: @@ -1337,7 +1333,6 @@ def test_delete_manual_webhook_connection_config( privacy_request_requires_input.status == PrivacyRequestStatus.in_processing ) - def test_delete_saas_connection_config( self, url, @@ -1371,7 +1366,6 @@ def test_delete_saas_connection_config( assert db.query(Dataset).filter_by(id=dataset.id).first() is None - class TestPutConnectionConfigSecrets: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy, connection_config) -> str: @@ -2202,7 +2196,6 @@ def test_put_saas_example_connection_config_secrets_missing_saas_config( ) - class TestPatchConnectionConfigSecrets: def test_patch_connection_config_secrets_not_authenticated( self, api_client: TestClient, generate_auth_header, connection_config diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 46b41b3c38..7b81e59e64 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -28,7 +28,6 @@ ) - class TestGetConnections: @pytest.fixture(scope="function") def url(self, oauth_client: ClientDetail, policy) -> str: @@ -532,8 +531,7 @@ def connection_type_objects(self): "authorization_required": False, "user_guide": hubspot_template.user_guide, "supported_actions": [ - action.value - for action in hubspot_template.supported_actions + action.value for action in hubspot_template.supported_actions ], }, MAILCHIMP: { @@ -544,8 +542,7 @@ def connection_type_objects(self): "authorization_required": False, "user_guide": mailchimp_template.user_guide, "supported_actions": [ - action.value - for action in mailchimp_template.supported_actions + action.value for action in mailchimp_template.supported_actions ], }, STRIPE: { @@ -678,7 +675,6 @@ def connection_type_objects(self): ), ], ) - def test_get_connection_types_action_type_filter( self, action_types, From 72a1e8b4a487ccc58f386e50e237a5c1ebf903ab Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 11:06:50 -0300 Subject: [PATCH 04/19] updated fixture and fix test_dataset_endpoints --- .../saas/connection_template_fixtures.py | 14 ++++---- .../v1/endpoints/test_dataset_endpoints.py | 32 +++++++++---------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/fixtures/saas/connection_template_fixtures.py b/tests/fixtures/saas/connection_template_fixtures.py index 442dc69b7f..8689176810 100644 --- a/tests/fixtures/saas/connection_template_fixtures.py +++ b/tests/fixtures/saas/connection_template_fixtures.py @@ -19,20 +19,20 @@ @pytest.fixture(scope="function") -def secondary_sendgrid_instance(db): +def secondary_hubspot_instance(db): """ - Instantiate a `sendgrid` SaaS connector instance + Instantiate a `hubspot` SaaS connector instance Yields a tuple of the `ConnectionConfig` and `DatasetConfig` """ secrets = { - "domain": "test_sendgrid_domain", - "api_key": "test_sendgrid_api_key", + "domain": "test_hubspot_domain", + "private_app_token": "test_hubspot_api_key", } connection_config, dataset_config = instantiate_connector( db, - "sendgrid", - "secondary_sendgrid_instance", - "Sendgrid ConnectionConfig description", + "hubspot", + "secondary_hubspot_instance", + "Hubspot ConnectionConfig description", secrets, ) yield connection_config, dataset_config diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index 2efd650830..0d3df03108 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -1828,18 +1828,18 @@ def test_get_only_unlinked_datasets( assert response.status_code == 200 assert len(response.json()) == 2 - @pytest.mark.skip(reason="move to plus in progress") + def test_saas_dataset_filter( self, generate_auth_header, api_client, url, - secondary_sendgrid_instance, + secondary_hubspot_instance, linked_dataset, unlinked_dataset, ) -> None: auth_header = generate_auth_header(scopes=[DATASET_READ]) - saas_fides_key = secondary_sendgrid_instance[1].fides_key + saas_fides_key = secondary_hubspot_instance[1].fides_key # Should filter out saas datasets by default response = api_client.get(url, headers=auth_header) assert response.status_code == 200 @@ -1862,14 +1862,14 @@ def test_saas_dataset_filter( assert len(response.json()) == 3 assert saas_fides_key in [d["fides_key"] for d in response.json()] - @pytest.mark.skip(reason="move to plus in progress") + def test_unlinked_and_no_saas_datasets( self, generate_auth_header, api_client, url, unlinked_dataset, - secondary_sendgrid_instance, + secondary_hubspot_instance, linked_dataset, ) -> None: auth_header = generate_auth_header(scopes=[DATASET_READ]) @@ -2135,13 +2135,13 @@ def dataset_with_categories(self, db: Session) -> Generator[CtlDataset, None, No yield dataset db.delete(dataset) - @pytest.mark.skip(reason="move to plus in progress") + def test_list_dataset_no_pagination( self, api_client: TestClient, generate_auth_header, ctl_dataset, - secondary_sendgrid_instance, + secondary_hubspot_instance, ): auth_header = generate_auth_header(scopes=[DATASET_READ]) response = api_client.get(f"{V1_URL_PREFIX}/dataset", headers=auth_header) @@ -2153,15 +2153,15 @@ def test_list_dataset_no_pagination( sorted_items = sorted(response_json, key=lambda x: x["fides_key"]) assert sorted_items[0]["fides_key"] == ctl_dataset.fides_key - assert sorted_items[1]["fides_key"] == secondary_sendgrid_instance[1].fides_key + assert sorted_items[1]["fides_key"] == secondary_hubspot_instance[1].fides_key + - @pytest.mark.skip(reason="move to plus in progress") def test_list_dataset_no_pagination_exclude_saas( self, api_client: TestClient, generate_auth_header, ctl_dataset, - secondary_sendgrid_instance, + secondary_hubspot_instance, ): auth_header = generate_auth_header(scopes=[DATASET_READ]) response = api_client.get( @@ -2174,7 +2174,7 @@ def test_list_dataset_no_pagination_exclude_saas( assert len(response_json) == 1 assert response_json[0]["fides_key"] == ctl_dataset.fides_key - @pytest.mark.skip(reason="move to plus in progress") + def test_list_dataset_no_pagination_only_unlinked_datasets( self, api_client: TestClient, @@ -2193,13 +2193,13 @@ def test_list_dataset_no_pagination_only_unlinked_datasets( assert len(response_json) == 1 assert response_json[0]["fides_key"] == unlinked_dataset.fides_key - @pytest.mark.skip(reason="move to plus in progress") + def test_list_dataset_with_pagination( self, api_client: TestClient, generate_auth_header, ctl_dataset, - secondary_sendgrid_instance, + secondary_hubspot_instance, ): auth_header = generate_auth_header(scopes=[DATASET_READ]) response = api_client.get( @@ -2214,15 +2214,15 @@ def test_list_dataset_with_pagination( assert len(sorted_items) == 2 assert sorted_items[0]["fides_key"] == ctl_dataset.fides_key - assert sorted_items[1]["fides_key"] == secondary_sendgrid_instance[1].fides_key + assert sorted_items[1]["fides_key"] == secondary_hubspot_instance[1].fides_key + - @pytest.mark.skip(reason="move to plus in progress") def test_list_dataset_with_pagination_exclude_saas( self, api_client: TestClient, generate_auth_header, ctl_dataset, - secondary_sendgrid_instance, + secondary_hubspot_instance, ): auth_header = generate_auth_header(scopes=[DATASET_READ]) response = api_client.get( From 3fc25de520ef720415d21d0cf95e117f0277da49 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 11:36:55 -0300 Subject: [PATCH 05/19] fix test_system --- tests/ops/api/v1/endpoints/test_dataset_endpoints.py | 7 ------- tests/ops/api/v1/endpoints/test_system.py | 12 +++++------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index 0d3df03108..6bfc3463cb 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -1828,7 +1828,6 @@ def test_get_only_unlinked_datasets( assert response.status_code == 200 assert len(response.json()) == 2 - def test_saas_dataset_filter( self, generate_auth_header, @@ -1862,7 +1861,6 @@ def test_saas_dataset_filter( assert len(response.json()) == 3 assert saas_fides_key in [d["fides_key"] for d in response.json()] - def test_unlinked_and_no_saas_datasets( self, generate_auth_header, @@ -2135,7 +2133,6 @@ def dataset_with_categories(self, db: Session) -> Generator[CtlDataset, None, No yield dataset db.delete(dataset) - def test_list_dataset_no_pagination( self, api_client: TestClient, @@ -2155,7 +2152,6 @@ def test_list_dataset_no_pagination( assert sorted_items[0]["fides_key"] == ctl_dataset.fides_key assert sorted_items[1]["fides_key"] == secondary_hubspot_instance[1].fides_key - def test_list_dataset_no_pagination_exclude_saas( self, api_client: TestClient, @@ -2174,7 +2170,6 @@ def test_list_dataset_no_pagination_exclude_saas( assert len(response_json) == 1 assert response_json[0]["fides_key"] == ctl_dataset.fides_key - def test_list_dataset_no_pagination_only_unlinked_datasets( self, api_client: TestClient, @@ -2193,7 +2188,6 @@ def test_list_dataset_no_pagination_only_unlinked_datasets( assert len(response_json) == 1 assert response_json[0]["fides_key"] == unlinked_dataset.fides_key - def test_list_dataset_with_pagination( self, api_client: TestClient, @@ -2216,7 +2210,6 @@ def test_list_dataset_with_pagination( assert sorted_items[0]["fides_key"] == ctl_dataset.fides_key assert sorted_items[1]["fides_key"] == secondary_hubspot_instance[1].fides_key - def test_list_dataset_with_pagination_exclude_saas( self, api_client: TestClient, diff --git a/tests/ops/api/v1/endpoints/test_system.py b/tests/ops/api/v1/endpoints/test_system.py index 4850868d33..6bec60e07c 100644 --- a/tests/ops/api/v1/endpoints/test_system.py +++ b/tests/ops/api/v1/endpoints/test_system.py @@ -423,7 +423,6 @@ def test_get_connection_configs_role( assert resp.status_code == expected_status_code -@pytest.mark.skip(reason="move to plus in progress") class TestDeleteSystemConnectionConfig: @pytest.fixture(scope="function") def url(self, system) -> str: @@ -531,19 +530,18 @@ def test_delete_manual_webhook_connection_config( privacy_request_requires_input.status == PrivacyRequestStatus.in_processing ) - @pytest.mark.skip(reason="move to plus in progress") def test_delete_saas_connection_config( self, api_client: TestClient, db: Session, generate_auth_header, system ) -> None: secrets = { - "domain": "test_sendgrid_domain", - "api_key": "test_sendgrid_api_key", + "domain": "test_hubspot_domain", + "private_app_token": "test_hubspot_api_key", } connection_config, dataset_config = instantiate_connector( db, - "sendgrid", - "secondary_sendgrid_instance", - "Sendgrid ConnectionConfig description", + "hubspot", + "secondary_hubspot_instance", + "Hubspot ConnectionConfig description", secrets, system, ) From 68b43527b1153bb68f8cb4b944fcacf380346f0b Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 12:29:10 -0300 Subject: [PATCH 06/19] fix test_authenticated_client --- .../connectors/saas/test_authenticated_client.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/ops/service/connectors/saas/test_authenticated_client.py b/tests/ops/service/connectors/saas/test_authenticated_client.py index a4e6047377..16e0aed7f3 100644 --- a/tests/ops/service/connectors/saas/test_authenticated_client.py +++ b/tests/ops/service/connectors/saas/test_authenticated_client.py @@ -21,28 +21,25 @@ from fides.api.util.saas_util import load_config_with_replacement -@pytest.mark.skip(reason="move to plus in progress") @pytest.fixture def test_saas_config() -> Dict[str, Any]: return load_config_with_replacement( - "data/saas/config/segment_config.yml", + "data/saas/config/hubspot_config.yml", "", "test_config", ) -@pytest.mark.skip(reason="move to plus in progress") @pytest.fixture def test_connection_config(test_saas_config) -> ConnectionConfig: return ConnectionConfig( key="test_config", connection_type=ConnectionType.saas, saas_config=test_saas_config, - secrets={"access_token": "test_token"}, + secrets={"private_app_token": "test_token"}, ) -@pytest.mark.skip(reason="move to plus in progress") @pytest.fixture def test_saas_request() -> SaaSRequestParams: return SaaSRequestParams( @@ -52,13 +49,11 @@ def test_saas_request() -> SaaSRequestParams: ) -@pytest.mark.skip(reason="move to plus in progress") @pytest.fixture def test_client_config() -> ClientConfig: return ClientConfig(protocol="https", host="test_host") -@pytest.mark.skip(reason="move to plus in progress") @pytest.fixture def test_authenticated_client( test_connection_config, test_client_config @@ -68,7 +63,6 @@ def test_authenticated_client( ) -@pytest.mark.skip(reason="move to plus in progress") @pytest.fixture def test_http_server() -> Generator[str, None, None]: """ @@ -96,7 +90,6 @@ def simple_app(environ, start_response): server_thread.join() -@pytest.mark.skip(reason="move to plus in progress") @pytest.mark.unit_saas class TestAuthenticatedClient: @mock.patch.object(Session, "send") @@ -197,7 +190,6 @@ def test_sending_special_characters( test_authenticated_client.send(request_params) -@pytest.mark.skip(reason="move to plus in progress") @pytest.mark.unit_saas class TestRetryAfterHeaderParsing: def test_retry_after_parses_seconds_response(self): From 7bff307ef70b95ad1144d4803ff992f52a2874ae Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 14:47:59 -0300 Subject: [PATCH 07/19] fix test_connector_registry_service --- .../test_connector_registry_service.py | 69 +++++++++---------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/tests/ops/service/connectors/test_connector_registry_service.py b/tests/ops/service/connectors/test_connector_registry_service.py index e1b614bfdb..7d47775956 100644 --- a/tests/ops/service/connectors/test_connector_registry_service.py +++ b/tests/ops/service/connectors/test_connector_registry_service.py @@ -42,7 +42,6 @@ } -@pytest.mark.skip(reason="move to plus in progress") class TestConnectionRegistry: def test_get_connector_template(self): assert "mailchimp" in ConnectorRegistry.connector_types() @@ -79,7 +78,7 @@ def test_update_config_additions( db, secondary_mailchimp_instance, tertiary_mailchimp_instance, - secondary_sendgrid_instance, + secondary_hubspot_instance, ): update_config( load_config_from_string_mock_object, @@ -92,7 +91,7 @@ def test_update_config_additions( db, secondary_mailchimp_instance, tertiary_mailchimp_instance, - secondary_sendgrid_instance, + secondary_hubspot_instance, ) @mock.patch( @@ -112,7 +111,7 @@ def test_update_config_removals( db, secondary_mailchimp_instance, tertiary_mailchimp_instance, - secondary_sendgrid_instance, + secondary_hubspot_instance, ): update_config( load_config_from_string_mock_object, @@ -125,7 +124,7 @@ def test_update_config_removals( db, secondary_mailchimp_instance, tertiary_mailchimp_instance, - secondary_sendgrid_instance, + secondary_hubspot_instance, ) @@ -140,18 +139,18 @@ def update_config( db, secondary_mailchimp_instance, tertiary_mailchimp_instance, - secondary_sendgrid_instance, + secondary_hubspot_instance, ): """ Helper function to test config updates. - First, load the original templates for `mailchimp` and `sendgrid`, - and instantiate two `mailchimp` instances and one `sendgrid` instance + First, load the original templates for `mailchimp` and `hubspot`, + and instantiate two `mailchimp` instances and one `hubspot` instance by means of fixtures. We use these 3 instances to test functionality across multiple instances of the same type, as well as multiple types. Then, based on the provided mock objects and functions to override, - "updates" are made to connector templates for `mailchimp` and `sendgrid`. + "updates" are made to connector templates for `mailchimp` and `hubspot`. The nature of those updates are "plugged in" via the override functions. Then, perform the update "script", i.e. invoke `update_saas_configs`. @@ -169,13 +168,13 @@ def update_config( ) mailchimp_version = mailchimp_template_config["version"] - sendgrid_template_config = load_config_from_string( - ConnectorRegistry.get_connector_template("sendgrid").config + hubspot_template_config = load_config_from_string( + ConnectorRegistry.get_connector_template("hubspot").config ) - sendgrid_template_dataset = load_dataset_from_string( - ConnectorRegistry.get_connector_template("sendgrid").dataset + hubspot_template_dataset = load_dataset_from_string( + ConnectorRegistry.get_connector_template("hubspot").dataset ) - sendgrid_version = sendgrid_template_config["version"] + hubspot_version = hubspot_template_config["version"] # confirm original version of template works as expected ( @@ -210,23 +209,23 @@ def update_config( ) ( - secondary_sendgrid_config, - secondary_sendgrid_dataset, - ) = secondary_sendgrid_instance - secondary_sendgrid_saas_config = secondary_sendgrid_config.saas_config - secondary_sendgrid_dataset.ctl_dataset.description = sendgrid_template_dataset[ + secondary_hubspot_config, + secondary_hubspot_dataset, + ) = secondary_hubspot_instance + secondary_hubspot_saas_config = secondary_hubspot_config.saas_config + secondary_hubspot_dataset.ctl_dataset.description = hubspot_template_dataset[ "description" ] - assert secondary_sendgrid_saas_config["version"] == sendgrid_version + assert secondary_hubspot_saas_config["version"] == hubspot_version assert ( - secondary_sendgrid_saas_config["description"] - == sendgrid_template_config["description"] + secondary_hubspot_saas_config["description"] + == hubspot_template_config["description"] ) # mock methods within template instantiation workflow # to produce an updated saas config template # this mimics "updates" made to SaaS config and dataset templates - # for mailchimp and sendgrid + # for mailchimp and hubspot load_config_from_string_mock_object.side_effect = ( load_config_from_string_mock_function ) @@ -265,22 +264,22 @@ def update_config( tertiary_mailchimp_dataset.fides_key, ) - secondary_sendgrid_dataset: DatasetConfig = DatasetConfig.filter( + secondary_hubspot_dataset: DatasetConfig = DatasetConfig.filter( db=db, - conditions=DatasetConfig.fides_key == secondary_sendgrid_dataset.fides_key, + conditions=DatasetConfig.fides_key == secondary_hubspot_dataset.fides_key, ).first() validation_function( - secondary_sendgrid_dataset, - sendgrid_template_config, - sendgrid_template_dataset, - secondary_sendgrid_config.key, - secondary_sendgrid_dataset.fides_key, + secondary_hubspot_dataset, + hubspot_template_config, + hubspot_template_dataset, + secondary_hubspot_config.key, + secondary_hubspot_dataset.fides_key, ) # clean up after ourselves secondary_mailchimp_config.delete(db) tertiary_mailchimp_config.delete(db) - secondary_sendgrid_config.delete(db) + secondary_hubspot_config.delete(db) def increment_ver(version): @@ -317,7 +316,7 @@ def replace_config_placeholders_mocked_additions_function( def update_config_additions(config: Dict): - if config["name"] in ("Mailchimp", "SendGrid"): + if config["name"] in ("Mailchimp", "HubSpot"): config["version"] = increment_ver(config["version"]) config["description"] = NEW_CONFIG_DESCRIPTION config["connector_params"].append(NEW_CONNECTOR_PARAM) @@ -336,7 +335,7 @@ def replace_dataset_placeholders_mocked_additions_function( ) if dataset["name"] in ( "Mailchimp Dataset", - "SendGrid Dataset", + "Hubspot Dataset", ): dataset["description"] = NEW_DATASET_DESCRIPTION dataset["collections"][0]["fields"].append(NEW_FIELD) @@ -416,7 +415,7 @@ def replace_config_placeholders_mocked_removals_function( def update_config_removals(config: Dict): if config["name"] in ( "Mailchimp", - "SendGrid", + "HubSpot", ): config["version"] = increment_ver(config["version"]) config["endpoints"].pop() @@ -435,7 +434,7 @@ def replace_dataset_placeholders_mocked_removals_function( ) if dataset["name"] in ( "Mailchimp Dataset", - "SendGrid Dataset", + "Hubspot Dataset", ): dataset["collections"].pop() return dataset From 5dd7c22dfd63cb4340e408a7bff7c7df887e9845 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 17:01:52 -0300 Subject: [PATCH 08/19] fix test_connector_template_loaders --- .../test_data/replaceable_hubspot_config.yml | 154 ++++++++++++++++++ .../test_data/replaceable_zendesk_config.yml | 120 -------------- .../test_connector_template_loaders.py | 86 +++++----- 3 files changed, 196 insertions(+), 164 deletions(-) create mode 100644 tests/fixtures/saas/test_data/replaceable_hubspot_config.yml delete mode 100644 tests/fixtures/saas/test_data/replaceable_zendesk_config.yml diff --git a/tests/fixtures/saas/test_data/replaceable_hubspot_config.yml b/tests/fixtures/saas/test_data/replaceable_hubspot_config.yml new file mode 100644 index 0000000000..639f2c4beb --- /dev/null +++ b/tests/fixtures/saas/test_data/replaceable_hubspot_config.yml @@ -0,0 +1,154 @@ +saas_config: + fides_key: + name: HubSpot + type: hubspot + description: A sample schema representing the HubSpot connector for Fides + user_guide: https://docs.ethyca.com/user-guides/integrations/saas-integrations/hubspot + version: 0.0.1 + replaceable: True + + connector_params: + - name: domain + default_value: api.hubapi.com + description: Your HubSpot domain + - name: private_app_token + sensitive: True + label: Private app token + description: Your HubSpot Private Apps access token + + client_config: + protocol: https + host: + authentication: + strategy: bearer + configuration: + token: + + test_request: + method: GET + path: /settings/v3/users + + endpoints: + - name: contacts + requests: + read: + path: /crm/v3/objects/contacts/search + method: POST + body: | + { + "filterGroups": [{ + "filters": [{ + "value": "", + "propertyName": "email", + "operator": "EQ" + }] + }] + } + query_params: + - name: limit + value: 100 + param_values: + - name: email + identity: email + data_path: results + pagination: + strategy: link + configuration: + source: body + path: paging.next.link + update: + request_override: hubspot_contacts_update + param_values: + - name: contactId + references: + - dataset: + field: contacts.id + direction: from + - name: owners + requests: + read: + path: /crm/v3/owners + method: GET + query_params: + - name: limit + value: 100 + param_values: + - name: placeholder + identity: email + data_path: results + postprocessors: + - strategy: filter + configuration: + field: email + value: + identity: email + pagination: + strategy: link + configuration: + source: body + path: paging.next.link + - name: subscription_preferences + requests: + read: + path: /communication-preferences/v3/status/email/ + method: GET + param_values: + - name: email + identity: email + update: + path: /communication-preferences/v3/unsubscribe + method: POST + body: | + { + "emailAddress": "", + "subscriptionId": "", + "legalBasis": "LEGITIMATE_INTEREST_CLIENT", + "legalBasisExplanation": "At users request, we opted them out" + } + data_path: subscriptionStatuses + ignore_errors: [400] + param_values: + - name: email + identity: email + - name: subscriptionId + references: + - dataset: + field: subscription_preferences.id + direction: from + postprocessors: + - strategy: filter + configuration: + field: status + value: SUBSCRIBED + - name: users + requests: + read: + path: /settings/v3/users/ + method: GET + query_params: + - name: limit + value: 100 + param_values: + - name: placeholder + identity: email + data_path: results + postprocessors: + - strategy: filter + configuration: + field: email + value: + identity: email + pagination: + strategy: link + configuration: + source: body + path: paging.next.link + delete: + path: /settings/v3/users/ + method: DELETE + param_values: + - name: userId + references: + - dataset: + field: users.id + direction: from diff --git a/tests/fixtures/saas/test_data/replaceable_zendesk_config.yml b/tests/fixtures/saas/test_data/replaceable_zendesk_config.yml deleted file mode 100644 index d7290bb6b3..0000000000 --- a/tests/fixtures/saas/test_data/replaceable_zendesk_config.yml +++ /dev/null @@ -1,120 +0,0 @@ -saas_config: - fides_key: - name: Zendesk - type: zendesk - description: A sample schema representing the Zendesk connector for Fides - version: 0.0.1 - replaceable: True - - connector_params: - - name: domain - - name: username - - name: api_key - - client_config: - protocol: https - host: - authentication: - strategy: basic - configuration: - username: - password: - - test_request: - method: GET - path: /api/v2/users/search.json - query_params: - - name: query - value: test@ethyca - - endpoints: - - name: user - requests: - read: - method: GET - path: /api/v2/users/search.json - query_params: - - name: query - value: - param_values: - - name: email - identity: email - data_path: users - delete: - method: DELETE - path: /api/v2/users/.json - param_values: - - name: user_id - references: - - dataset: - field: user.id - direction: from - - name: user_identities - requests: - read: - method: GET - path: /api/v2/users//identities.json - query_params: - - name: page[size] - value: 100 - param_values: - - name: user_id - references: - - dataset: - field: user.id - direction: from - data_path: identities - pagination: - strategy: link - configuration: - source: body - path: links.next - - name: tickets - requests: - read: - method: GET - path: /api/v2/users//tickets/requested.json - query_params: - - name: page[size] - value: 100 - param_values: - - name: user_id - references: - - dataset: - field: user.id - direction: from - data_path: tickets - pagination: - strategy: link - configuration: - source: body - path: links.next - delete: - method: DELETE - path: /api/v2/tickets/.json - param_values: - - name: ticket_id - references: - - dataset: - field: tickets.id - direction: from - - name: ticket_comments - requests: - read: - method: GET - path: /api/v2/tickets//comments.json - query_params: - - name: page[size] - value: 100 - param_values: - - name: ticket_id - references: - - dataset: - field: tickets.id - direction: from - data_path: comments - pagination: - strategy: link - configuration: - source: body - path: links.next diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index 5bc1dbd1ac..7a6ae900ea 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -30,7 +30,6 @@ from tests.ops.test_helpers.saas_test_utils import create_zip_file -@pytest.mark.skip(reason="move to plus in progress") class TestFileConnectorTemplateLoader: def test_file_connector_template_loader(self): loader = FileConnectorTemplateLoader() @@ -58,7 +57,6 @@ def test_file_connector_template_loader_connector_not_found(self): assert connector_templates.get("not_found") is None -@pytest.mark.skip(reason="move to plus in progress") class TestCustomConnectorTemplateLoader: @pytest.fixture(autouse=True) def reset_connector_template_loaders(self): @@ -69,17 +67,17 @@ def reset_connector_template_loaders(self): CustomConnectorTemplateLoader._instance = None @pytest.fixture - def zendesk_config(self) -> str: - return load_yaml_as_string("data/saas/config/zendesk_config.yml") + def hubspot_config(self) -> str: + return load_yaml_as_string("data/saas/config/hubspot_config.yml") @pytest.fixture - def zendesk_dataset(self) -> str: - return load_yaml_as_string("data/saas/dataset/zendesk_dataset.yml") + def hubspot_dataset(self) -> str: + return load_yaml_as_string("data/saas/dataset/hubspot_dataset.yml") @pytest.fixture - def replaceable_zendesk_config(self) -> str: + def replaceable_hubspot_config(self) -> str: return load_yaml_as_string( - "tests/fixtures/saas/test_data/replaceable_zendesk_config.yml" + "tests/fixtures/saas/test_data/replaceable_hubspot_config.yml" ) @pytest.fixture @@ -89,22 +87,22 @@ def replaceable_planet_express_config(self) -> str: ) @pytest.fixture - def replaceable_zendesk_zip( - self, replaceable_zendesk_config, zendesk_dataset + def replaceable_hubspot_zip( + self, replaceable_hubspot_config, hubspot_dataset ) -> BytesIO: return create_zip_file( { - "config.yml": replace_version(replaceable_zendesk_config, "0.0.0"), - "dataset.yml": zendesk_dataset, + "config.yml": replace_version(replaceable_hubspot_config, "0.0.0"), + "dataset.yml": hubspot_dataset, } ) @pytest.fixture - def non_replaceable_zendesk_zip(self, zendesk_config, zendesk_dataset) -> BytesIO: + def non_replaceable_hubspot_zip(self, hubspot_config, hubspot_dataset) -> BytesIO: return create_zip_file( { - "config.yml": replace_version(zendesk_config, "0.0.0"), - "dataset.yml": zendesk_dataset, + "config.yml": replace_version(hubspot_config, "0.0.0"), + "dataset.yml": hubspot_dataset, } ) @@ -124,11 +122,11 @@ def replaceable_planet_express_zip( ) @pytest.fixture - def non_replaceable_zendesk_zip(self, zendesk_config, zendesk_dataset) -> BytesIO: + def non_replaceable_hubspot_zip(self, hubspot_config, hubspot_dataset) -> BytesIO: return create_zip_file( { - "config.yml": replace_version(zendesk_config, "0.0.0"), - "dataset.yml": zendesk_dataset, + "config.yml": replace_version(hubspot_config, "0.0.0"), + "dataset.yml": hubspot_dataset, } ) @@ -315,8 +313,8 @@ def test_custom_connector_replacement_replaceable_with_update_available( self, mock_all: MagicMock, mock_delete: MagicMock, - zendesk_config, - zendesk_dataset, + hubspot_config, + hubspot_dataset, ): """ Verify that an existing connector template flagged as replaceable is @@ -326,19 +324,19 @@ def test_custom_connector_replacement_replaceable_with_update_available( mock_all.return_value = [ CustomConnectorTemplate( - key="zendesk", - name="Zendesk", - config=replace_version(zendesk_config, "0.0.0"), - dataset=zendesk_dataset, + key="hubspot", + name="HubSpot", + config=replace_version(hubspot_config, "0.0.0"), + dataset=hubspot_dataset, replaceable=True, ) ] - template = ConnectorRegistry.get_connector_template("zendesk") + template = ConnectorRegistry.get_connector_template("hubspot") assert template saas_config = load_config_from_string(template.config) assert ( - saas_config["version"] == load_config_from_string(zendesk_config)["version"] + saas_config["version"] == load_config_from_string(hubspot_config)["version"] ) assert CustomConnectorTemplateLoader.get_connector_templates() == {} mock_delete.assert_called_once() @@ -399,37 +397,37 @@ def test_custom_connector_replacement_not_replaceable( self, mock_all: MagicMock, mock_delete: MagicMock, - zendesk_config, - zendesk_dataset, + hubspot_config, + hubspot_dataset, ): """ Verify that an existing custom connector template flagged as not replaceable is not deleted even if a newer version of the connector template is found in the FileConnectorTemplateLoader. """ - zendesk_config = replace_version(zendesk_config, "0.0.0") + hubspot_config = replace_version(hubspot_config, "0.0.0") mock_all.return_value = [ CustomConnectorTemplate( - key="zendesk", - name="Zendesk", - config=zendesk_config, - dataset=zendesk_dataset, + key="hubspot", + name="HubSpot", + config=hubspot_config, + dataset=hubspot_dataset, replaceable=False, ) ] - template = ConnectorRegistry.get_connector_template("zendesk") + template = ConnectorRegistry.get_connector_template("hubspot") assert template saas_config = load_config_from_string(template.config) assert saas_config["version"] == "0.0.0" assert CustomConnectorTemplateLoader.get_connector_templates() == { - "zendesk": ConnectorTemplate( - config=zendesk_config, - dataset=zendesk_dataset, - human_readable="Zendesk", + "hubspot": ConnectorTemplate( + config=hubspot_config, + dataset=hubspot_dataset, + human_readable="HubSpot", authorization_required=False, - user_guide="https://docs.ethyca.com/user-guides/integrations/saas-integrations/zendesk", + user_guide="https://docs.ethyca.com/user-guides/integrations/saas-integrations/hubspot", supported_actions=[ActionType.access, ActionType.erasure], ) } @@ -439,20 +437,20 @@ def test_custom_connector_replacement_not_replaceable( "fides.api.models.custom_connector_template.CustomConnectorTemplate.create_or_update" ) def test_replaceable_template_for_existing_template( - self, mock_create_or_update: MagicMock, zendesk_config, replaceable_zendesk_zip + self, mock_create_or_update: MagicMock, hubspot_config, replaceable_hubspot_zip ): """ Verify that a replaceable custom connector template takes on the version of the existing connector template. """ CustomConnectorTemplateLoader.save_template( - db=MagicMock(), zip_file=ZipFile(replaceable_zendesk_zip) + db=MagicMock(), zip_file=ZipFile(replaceable_hubspot_zip) ) assert mock_create_or_update.call_args.kwargs["data"]["replaceable"] config_contents = mock_create_or_update.call_args.kwargs["data"]["config"] custom_config = load_config_from_string(config_contents) - existing_config = load_config_from_string(zendesk_config) + existing_config = load_config_from_string(hubspot_config) assert custom_config["version"] == existing_config["version"] @mock.patch( @@ -480,13 +478,13 @@ def test_replaceable_template_for_new_template( def test_non_replaceable_template( self, mock_create_or_update: MagicMock, - non_replaceable_zendesk_zip, + non_replaceable_hubspot_zip, ): """ Verify that a non replaceable connector template keeps its version even if there is an existing connector template. """ CustomConnectorTemplateLoader.save_template( - db=MagicMock(), zip_file=ZipFile(non_replaceable_zendesk_zip) + db=MagicMock(), zip_file=ZipFile(non_replaceable_hubspot_zip) ) assert not mock_create_or_update.call_args.kwargs["data"]["replaceable"] config_contents = mock_create_or_update.call_args.kwargs["data"]["config"] From cf7b1903ee0e31cdfd240d76f0bf4ec9a313a34d Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 18:12:32 -0300 Subject: [PATCH 09/19] removed skip from working tests --- tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py | 2 +- tests/ops/service/connectors/test_fides_connector.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index db2da05573..a88fa70d45 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -7099,7 +7099,7 @@ def test_create_privacy_request_creates_system_audit_log( @pytest.mark.integration class TestPrivacyRequestDataTransfer: - @pytest.mark.skip(reason="move to plus in progress") + @pytest.mark.usefixtures("postgres_integration_db") async def test_privacy_request_data_transfer( self, diff --git a/tests/ops/service/connectors/test_fides_connector.py b/tests/ops/service/connectors/test_fides_connector.py index 269d4bf34c..063b8881b7 100644 --- a/tests/ops/service/connectors/test_fides_connector.py +++ b/tests/ops/service/connectors/test_fides_connector.py @@ -25,7 +25,7 @@ from tests.ops.graph.graph_test_util import assert_rows_match, generate_node -@pytest.mark.skip(reason="move to plus in progress") + @pytest.mark.unit class TestFidesConnectorUnit: """ @@ -81,7 +81,7 @@ def test_filter_fides_connector_datasets( assert not datasets -@pytest.mark.skip(reason="move to plus in progress") + @pytest.mark.integration class TestFidesConnectorIntegration: """ From 979e59ddf7ce6142a2bd7b2945d632cac2e9b1b5 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 18:13:51 -0300 Subject: [PATCH 10/19] run static checks --- tests/ops/service/connectors/test_fides_connector.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/ops/service/connectors/test_fides_connector.py b/tests/ops/service/connectors/test_fides_connector.py index 063b8881b7..4c46d5db9b 100644 --- a/tests/ops/service/connectors/test_fides_connector.py +++ b/tests/ops/service/connectors/test_fides_connector.py @@ -25,7 +25,6 @@ from tests.ops.graph.graph_test_util import assert_rows_match, generate_node - @pytest.mark.unit class TestFidesConnectorUnit: """ @@ -81,7 +80,6 @@ def test_filter_fides_connector_datasets( assert not datasets - @pytest.mark.integration class TestFidesConnectorIntegration: """ From 3735e991667afd1ddd0a2497c7e0ec2c1b85cf68 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Mon, 28 Oct 2024 19:14:27 -0300 Subject: [PATCH 11/19] remove skip from tests --- tests/ops/service/connectors/test_bigquery_connector.py | 1 - tests/ops/service/connectors/test_queryconfig.py | 7 ------- 2 files changed, 8 deletions(-) diff --git a/tests/ops/service/connectors/test_bigquery_connector.py b/tests/ops/service/connectors/test_bigquery_connector.py index a083e0ae44..7a9ef245b8 100644 --- a/tests/ops/service/connectors/test_bigquery_connector.py +++ b/tests/ops/service/connectors/test_bigquery_connector.py @@ -14,7 +14,6 @@ from fides.api.service.connectors.sql_connector import BigQueryConnector -@pytest.mark.skip(reason="move to plus in progress") @pytest.mark.integration_external @pytest.mark.integration_bigquery class TestBigQueryConnector: diff --git a/tests/ops/service/connectors/test_queryconfig.py b/tests/ops/service/connectors/test_queryconfig.py index 3085ed98bf..4d48e39036 100644 --- a/tests/ops/service/connectors/test_queryconfig.py +++ b/tests/ops/service/connectors/test_queryconfig.py @@ -63,7 +63,6 @@ privacy_request = PrivacyRequest(id="234544") -@pytest.mark.skip(reason="move to plus in progress") class TestSQLQueryConfig: def test_extract_query_components(self): def found_query_keys(node: ExecutionNode, values: Dict[str, Any]) -> Set[str]: @@ -371,7 +370,6 @@ def test_generate_update_stmts_from_multiple_rules( ) # String rewrite masking strategy -@pytest.mark.skip(reason="move to plus in progress") class TestMongoQueryConfig: @pytest.fixture(scope="function") def combined_traversal(self, connection_config, integration_mongodb_config): @@ -629,7 +627,6 @@ def test_generate_update_stmt_multiple_rules( ) -@pytest.mark.skip(reason="move to plus in progress") class TestDynamoDBQueryConfig: @pytest.fixture(scope="function") def identity(self): @@ -744,7 +741,6 @@ def test_put_query_param_formatting_single_key( } -@pytest.mark.skip(reason="move to plus in progress") class TestBigQueryQueryConfig: @pytest.fixture(scope="function") def bigquery_client(self, bigquery_connection_config): @@ -954,7 +950,6 @@ def test_generate_namespaced_delete_stmt( ) -@pytest.mark.skip(reason="move to plus in progress") class TestScyllaDBQueryConfig: @pytest.fixture(scope="function") def complete_execution_node( @@ -994,7 +989,6 @@ def test_query_to_str(self, complete_execution_node): assert query_to_str == "SELECT name FROM users WHERE email = 'test@example.com'" -@pytest.mark.skip(reason="move to plus in progress") @pytest.mark.integration_external @pytest.mark.integration_bigquery class TestBigQueryQueryConfig: @@ -1060,7 +1054,6 @@ def test_generate_query_with_invalid_namespace_meta( assert "Field required" in str(exc) -@pytest.mark.skip(reason="move to plus in progress") class TestSQLLikeQueryConfig: def test_missing_namespace_meta_schema(self): From d3e228a3187d67ffc8c71104028d8bd27b92b5e1 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Tue, 29 Oct 2024 12:29:46 -0300 Subject: [PATCH 12/19] fix test saas connectors --- .../saas/test_data/saas_example_config.yml | 45 +++++-- ...aas_example_consent_preferences_config.yml | 59 ++++++++++ .../saas_example_opt_out_only_config.yml | 44 +++++++ tests/fixtures/saas_example_fixtures.py | 50 ++++++++ .../service/connectors/test_saas_connector.py | 110 +++++++++--------- 5 files changed, 240 insertions(+), 68 deletions(-) create mode 100644 tests/fixtures/saas/test_data/saas_example_consent_preferences_config.yml create mode 100644 tests/fixtures/saas/test_data/saas_example_opt_out_only_config.yml diff --git a/tests/fixtures/saas/test_data/saas_example_config.yml b/tests/fixtures/saas/test_data/saas_example_config.yml index 9854c885b0..487800c6f3 100644 --- a/tests/fixtures/saas/test_data/saas_example_config.yml +++ b/tests/fixtures/saas/test_data/saas_example_config.yml @@ -35,19 +35,25 @@ saas_config: consent_requests: opt_in: - method: POST - path: /allowlists/add - body: | - { - "first_name": "" - } + - method: POST + path: /allowlists/add + body: | + { + "first_name": "" + } opt_out: - method: POST - path: /allowlists/delete - body: | - { - "first_name": "" - } + - method: POST + path: /allowlists/delete + body: | + { + "first_name": "" + } + - method: POST + path: /rejects/add + body: | + { + "first_name": "" + } endpoints: - name: messages @@ -95,6 +101,13 @@ saas_config: - name: email identity: email data_path: exact_matches.members + client_config: + protocol: https + host: + authentication: + strategy: bearer + configuration: + token: - method: GET path: /3.0/search-members query_params: @@ -104,6 +117,14 @@ saas_config: - name: phone identity: phone data_path: exact_matches.members + client_config: + protocol: https + host: + authentication: + strategy: bearer + configuration: + token: + update: method: PUT path: /3.0/lists//members/ diff --git a/tests/fixtures/saas/test_data/saas_example_consent_preferences_config.yml b/tests/fixtures/saas/test_data/saas_example_consent_preferences_config.yml new file mode 100644 index 0000000000..8c2d0b3649 --- /dev/null +++ b/tests/fixtures/saas/test_data/saas_example_consent_preferences_config.yml @@ -0,0 +1,59 @@ +saas_config: + fides_key: saas_connector_consent_preferences_example + name: SaaS Example Consent Preferences Config + type: custom + description: A sample schema representing a SaaS for Fides + version: 0.0.1 + + connector_params: + - name: domain + default_value: localhost + - name: username + - name: api_key + + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: + password: + + test_request: + method: GET + path: /3.0/lists + + consent_requests: + opt_in: + method: POST + path: /allowlists/add + param_values: + - name: email + identity: email + body: | + { + "email": "" + } + + opt_out: + - method: POST + path: /allowlists/delete + param_values: + - name: email + identity: email + body: | + { + "email": "" + } + - method: POST + path: /rejects/add + param_values: + - name: email + identity: email + body: | + { + "email": "" + } + + endpoints: [] diff --git a/tests/fixtures/saas/test_data/saas_example_opt_out_only_config.yml b/tests/fixtures/saas/test_data/saas_example_opt_out_only_config.yml new file mode 100644 index 0000000000..49decee98f --- /dev/null +++ b/tests/fixtures/saas/test_data/saas_example_opt_out_only_config.yml @@ -0,0 +1,44 @@ +saas_config: + fides_key: saas_connector_opt_out_only_example + name: SaaS Example Opt Out Config + type: custom + description: A sample schema representing a SaaS opt out only for Fides + version: 0.0.1 + + connector_params: + - name: domain + default_value: localhost + - name: username + - name: api_key + + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: + password: + + test_request: + method: GET + path: /3.0/lists + + consent_requests: + opt_out: + - method: POST + skip_missing_param_values: true + path: /allowlists/delete + body: | + { + "first_name": "" + } + - method: POST + skip_missing_param_values: true + path: /rejects/add + body: | + { + "first_name": "" + } + + endpoints: [] diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 06c60c7777..ab2f766880 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -60,6 +60,13 @@ def saas_example_secrets(): def saas_example_config() -> Dict: return load_config("tests/fixtures/saas/test_data/saas_example_config.yml") +@pytest.fixture +def saas_example_opt_out_only_config() -> Dict: + return load_config("tests/fixtures/saas/test_data/saas_example_opt_out_only_config.yml") + +@pytest.fixture +def saas_example_consent_preferences_config() -> Dict: + return load_config("tests/fixtures/saas/test_data/saas_example_consent_preferences_config.yml") @pytest.fixture def saas_external_example_config() -> Dict: @@ -106,6 +113,49 @@ def saas_example_connection_config( connection_config.delete(db) +@pytest.fixture(scope="function") +def saas_example_opt_out_only_connection_config( + db: Session, + saas_example_opt_out_only_config: Dict[str, Any], + saas_example_secrets: Dict[str, Any], +) -> Generator: + fides_key = saas_example_opt_out_only_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": saas_example_secrets, + "saas_config": saas_example_opt_out_only_config, + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture(scope="function") +def saas_example_consent_preferences_connection_config( + db: Session, + saas_example_consent_preferences_config: Dict[str, Any], + saas_example_secrets: Dict[str, Any], +) -> Generator: + fides_key = saas_example_consent_preferences_config["fides_key"] + connection_config = ConnectionConfig.create( + db=db, + data={ + "key": fides_key, + "name": fides_key, + "connection_type": ConnectionType.saas, + "access": AccessLevel.write, + "secrets": saas_example_secrets, + "saas_config": saas_example_consent_preferences_config, + }, + ) + yield connection_config + connection_config.delete(db) + @pytest.fixture(scope="function") def saas_external_example_connection_config( db: Session, diff --git a/tests/ops/service/connectors/test_saas_connector.py b/tests/ops/service/connectors/test_saas_connector.py index b55c231a8e..e856d418eb 100644 --- a/tests/ops/service/connectors/test_saas_connector.py +++ b/tests/ops/service/connectors/test_saas_connector.py @@ -66,7 +66,7 @@ def valid_consent_update_override( return ConsentPropagationStatus.executed -@pytest.mark.skip(reason="move to plus in progress") + @pytest.mark.unit_saas class TestSaasConnector: """ @@ -463,7 +463,6 @@ def test_skip_missing_param_values_masking( ) -@pytest.mark.skip(reason="move to plus in progress") @pytest.mark.unit_saas class TestSaaSConnectorOutputTemplate: @mock.patch("fides.api.service.connectors.saas_connector.AuthenticatedClient.send") @@ -589,37 +588,37 @@ def test_request_with_invalid_output_template( assert "Failed to parse value as JSON" in str(exc) -@pytest.mark.skip(reason="move to plus in progress") + @pytest.mark.integration_saas class TestSaaSConnectorMethods: def test_client_config_set_depending_on_state( - self, db: Session, segment_connection_config, segment_dataset_config + self, db: Session, saas_example_connection_config, saas_example_dataset_config ): - connector: SaaSConnector = get_connector(segment_connection_config) + connector: SaaSConnector = get_connector(saas_example_connection_config) connector.set_saas_request_state( SaaSRequest(path="test_path", method=HTTPMethod.GET) ) # Base ClientConfig uses bearer auth - assert connector.get_client_config().authentication.strategy == "bearer" + assert connector.get_client_config().authentication.strategy == "basic" - segment_user_endpoint = next( - end for end in connector.saas_config.endpoints if end.name == "segment_user" + saas_example_member_endpoint = next( + end for end in connector.saas_config.endpoints if end.name == "member" ) - saas_requests: List[SaaSRequest] = segment_user_endpoint.requests.read + saas_requests: List[SaaSRequest] = saas_example_member_endpoint.requests.read for saas_request in saas_requests: connector.set_saas_request_state(saas_request) client = connector.create_client() - # ClientConfig on read segment user request uses basic auth, updating the state should result in the new strategy for client - assert client.client_config.authentication.strategy == "basic" - assert connector.get_client_config().authentication.strategy == "basic" + # ClientConfig on read saas_example member request uses bearer auth, updating the state should result in the new strategy for client + assert client.client_config.authentication.strategy == "bearer" + assert connector.get_client_config().authentication.strategy == "bearer" def test_rate_limit_config_set_depending_on_state( - self, db: Session, segment_connection_config, segment_dataset_config + self, db: Session, saas_example_connection_config, saas_example_dataset_config ): rate_limit_config = {"limits": [{"rate": 1, "period": "second"}]} - segment_connection_config.saas_config["rate_limit_config"] = rate_limit_config - connector: SaaSConnector = get_connector(segment_connection_config) + saas_example_connection_config.saas_config["rate_limit_config"] = rate_limit_config + connector: SaaSConnector = get_connector(saas_example_connection_config) connector.set_saas_request_state( SaaSRequest(path="test_path", method=HTTPMethod.GET) ) @@ -636,14 +635,14 @@ def test_rate_limit_config_set_depending_on_state( assert connector.get_rate_limit_config().enabled is False -@pytest.mark.skip(reason="move to plus in progress") + @pytest.mark.integration_saas class TestConsentRequests: def test_get_consent_requests_by_preference( - self, mailchimp_transactional_connection_config + self, saas_example_connection_config ): connector: SaaSConnector = get_connector( - mailchimp_transactional_connection_config + saas_example_connection_config ) opt_in_request: List[SaaSRequest] = ( @@ -659,16 +658,16 @@ def test_get_consent_requests_by_preference( assert opt_out_request[1].path == "/rejects/add" -@pytest.mark.skip(reason="move to plus in progress") + class TestSaasConnectorRunConsentRequest: def test_no_preferences_to_propagate( self, consent_policy, privacy_request_with_consent_policy, - mailchimp_transactional_connection_config_no_secrets, + saas_example_connection_config, db, ): - connector = get_connector(mailchimp_transactional_connection_config_no_secrets) + connector = get_connector(saas_example_connection_config) with pytest.raises(SkippingConsentPropagation) as exc: traversal_node = TraversalNode(generate_node("a", "b", "c", "c2")) request_task = traversal_node.to_mock_request_task() @@ -689,17 +688,17 @@ def test_data_use_mismatch( system, consent_policy, privacy_request_with_consent_policy, - mailchimp_transactional_connection_config_no_secrets, + saas_example_connection_config, privacy_preference_history_us_ca_provide, ): """System has an advertising data use and this privacy notice for the preference has a provide data use""" - mailchimp_transactional_connection_config_no_secrets.system_id = system.id + saas_example_connection_config.system_id = system.id privacy_preference_history_us_ca_provide.privacy_request_id = ( privacy_request_with_consent_policy.id ) privacy_preference_history_us_ca_provide.save(db) - connector = get_connector(mailchimp_transactional_connection_config_no_secrets) + connector = get_connector(saas_example_connection_config) with pytest.raises(SkippingConsentPropagation) as exc: traversal_node = TraversalNode(generate_node("a", "b", "c", "c2")) request_task = traversal_node.to_mock_request_task() @@ -726,7 +725,7 @@ def test_enforcement_level_not_system_wide( consent_policy, privacy_request_with_consent_policy, privacy_preference_history_fr_provide_service_frontend_only, - mailchimp_transactional_connection_config_no_secrets, + saas_example_connection_config, ): """Can only propagate preferences that have a system wide enforcement level""" privacy_preference_history_fr_provide_service_frontend_only.privacy_request_id = ( @@ -734,7 +733,7 @@ def test_enforcement_level_not_system_wide( ) privacy_preference_history_fr_provide_service_frontend_only.save(db) - connector = get_connector(mailchimp_transactional_connection_config_no_secrets) + connector = get_connector(saas_example_connection_config) with pytest.raises(SkippingConsentPropagation) as exc: traversal_node = TraversalNode(generate_node("a", "b", "c", "c2")) request_task = traversal_node.to_mock_request_task() @@ -760,13 +759,13 @@ def test_missing_identity_data_failure( db, system, consent_policy, - mailchimp_transactional_connection_config_no_secrets, + saas_example_connection_config, privacy_preference_history, ): """We need a matching identity for the connector in order to send the request - Mailchimp Transactional set up to fail if no email supplied + saas_example set up to fail if no first_name supplied """ - mailchimp_transactional_connection_config_no_secrets.system_id = system.id + saas_example_connection_config.system_id = system.id privacy_request = PrivacyRequest( id=f"test_consent_request_task_{random.randint(0, 1000)}", status=PrivacyRequestStatus.pending, @@ -775,7 +774,7 @@ def test_missing_identity_data_failure( privacy_preference_history.privacy_request_id = privacy_request.id privacy_preference_history.save(db) - connector = get_connector(mailchimp_transactional_connection_config_no_secrets) + connector = get_connector(saas_example_connection_config) with pytest.raises(ValueError): traversal_node = TraversalNode(generate_node("a", "b", "c", "c2")) request_task = traversal_node.to_mock_request_task() @@ -799,14 +798,13 @@ def test_missing_identity_data_skipped( db, system, consent_policy, - google_analytics_connection_config_without_secrets, + saas_example_opt_out_only_connection_config, privacy_preference_history, ): - """We need a matching identity for the connector in order to send the google_analytics_connection_config_without_secrets - Google Analytics set up to skip instead of fail if we don't have the ga client id. - There's no guarantee that a ga cookie is in the browser + """We need a matching identity for the connector in order to send the saas_example_connection_config + Saas_Example set up to skip instead of fail if we don't have the ga client id. """ - google_analytics_connection_config_without_secrets.system_id = system.id + saas_example_opt_out_only_connection_config.system_id = system.id privacy_request = PrivacyRequest( id=f"test_consent_request_task_{random.randint(0, 1000)}", @@ -816,7 +814,7 @@ def test_missing_identity_data_skipped( privacy_preference_history.privacy_request_id = privacy_request.id privacy_preference_history.save(db) - connector = get_connector(google_analytics_connection_config_without_secrets) + connector = get_connector(saas_example_opt_out_only_connection_config) with pytest.raises(SkippingConsentPropagation) as exc: traversal_node = TraversalNode(generate_node("a", "b", "c", "c2")) request_task = traversal_node.to_mock_request_task() @@ -840,12 +838,12 @@ def test_no_requests_of_that_type_defined( self, system, consent_policy, - google_analytics_connection_config_without_secrets, + saas_example_opt_out_only_connection_config, privacy_preference_history, db, ): - """User is expressing an opt in preference here but GA only has an opt out preference defined""" - google_analytics_connection_config_without_secrets.system_id = system.id + """User is expressing an opt in preference here but only has an opt out preference defined""" + saas_example_opt_out_only_connection_config.system_id = system.id privacy_request = PrivacyRequest( id=f"test_consent_request_task_{random.randint(0, 1000)}", @@ -857,7 +855,7 @@ def test_no_requests_of_that_type_defined( privacy_preference_history.privacy_request_id = privacy_request.id privacy_preference_history.save(db) - connector = get_connector(google_analytics_connection_config_without_secrets) + connector = get_connector(saas_example_opt_out_only_connection_config) with pytest.raises(SkippingConsentPropagation) as exc: traversal_node = TraversalNode(generate_node("a", "b", "c", "c2")) request_task = traversal_node.to_mock_request_task() @@ -885,14 +883,14 @@ def test_preferences_executable( consent_policy, privacy_request_with_consent_policy, privacy_preference_history, - mailchimp_transactional_connection_config_no_secrets, + saas_example_consent_preferences_connection_config, ): privacy_preference_history.privacy_request_id = ( privacy_request_with_consent_policy.id ) privacy_preference_history.save(db) - connector = get_connector(mailchimp_transactional_connection_config_no_secrets) + connector = get_connector(saas_example_consent_preferences_connection_config) traversal_node = TraversalNode(generate_node("a", "b", "c", "c2")) request_task = traversal_node.to_mock_request_task() execution_node = traversal_node.to_mock_execution_node() @@ -907,7 +905,7 @@ def test_preferences_executable( assert mock_send.called db.refresh(privacy_preference_history) assert privacy_preference_history.affected_system_status == { - mailchimp_transactional_connection_config_no_secrets.system_key: "complete" + saas_example_consent_preferences_connection_config.system_key: "complete" }, "Updated to skipped in graph task, not updated here" def test_preferences_executable_notice_based_consent( @@ -917,7 +915,7 @@ def test_preferences_executable_notice_based_consent( consent_policy, privacy_request_with_consent_policy, privacy_preference_history, - iterable_connection_config_no_secrets, + saas_example_connection_config, ): # Create consentable items linked to Iterable consentable_items = [ @@ -938,13 +936,13 @@ def test_preferences_executable_notice_based_consent( consent_automation = ConsentAutomation.create_or_update( db, data={ - "connection_config_id": iterable_connection_config_no_secrets.id, + "connection_config_id": saas_example_connection_config.id, "consentable_items": consentable_items, }, ) # Register update consent override fn - name = iterable_connection_config_no_secrets.saas_config["type"] + name = saas_example_connection_config.saas_config["type"] register(name, SaaSRequestType.UPDATE_CONSENT)(valid_consent_update_override) assert valid_consent_update_override == SaaSRequestOverrideFactory.get_override( name, SaaSRequestType.UPDATE_CONSENT @@ -958,7 +956,7 @@ def test_preferences_executable_notice_based_consent( privacy_preference_history.save(db) # Build and run consent request - connector = get_connector(iterable_connection_config_no_secrets) + connector = get_connector(saas_example_connection_config) traversal_node = TraversalNode(generate_node("a", "b", "c", "c2")) request_task = traversal_node.to_mock_request_task() execution_node = traversal_node.to_mock_execution_node() @@ -978,26 +976,26 @@ def test_preferences_executable_notice_based_consent( spy.assert_called_once_with(name, SaaSRequestType.UPDATE_CONSENT) db.refresh(privacy_preference_history) assert privacy_preference_history.affected_system_status == { - iterable_connection_config_no_secrets.system_key: "complete" + saas_example_connection_config.system_key: "complete" }, "Updated to skipped in graph task, not updated here" # Cleanup consent_automation.delete(db) -@pytest.mark.skip(reason="move to plus in progress") + class TestRelevantConsentIdentities: def test_no_consent_requests( - self, mailchimp_transactional_connection_config_no_secrets + self, saas_example_connection_config ): - connector = get_connector(mailchimp_transactional_connection_config_no_secrets) + connector = get_connector(saas_example_connection_config) connector.relevant_consent_identities([], {"customer_1@example.com"}) == {} def test_no_identity_data( - self, mailchimp_transactional_connection_config_no_secrets + self, saas_example_connection_config ): - connector = get_connector(mailchimp_transactional_connection_config_no_secrets) + connector = get_connector(saas_example_connection_config) request = SaaSRequest( method=HTTPMethod.POST, @@ -1008,9 +1006,9 @@ def test_no_identity_data( assert connector.relevant_consent_identities([request], {}) == {} def test_get_relevant_identities_only( - self, mailchimp_transactional_connection_config_no_secrets + self, saas_example_connection_config ): - connector = get_connector(mailchimp_transactional_connection_config_no_secrets) + connector = get_connector(saas_example_connection_config) request = SaaSRequest( method=HTTPMethod.POST, @@ -1023,7 +1021,7 @@ def test_get_relevant_identities_only( ) == {"email": "customer-1@example.com"} -@pytest.mark.skip(reason="move to plus in progress") + class TestAsyncConnectors: @pytest.fixture(scope="function") def async_graph(self, saas_example_async_dataset_config, db, privacy_request): From 4284ea2d728af8dede5973dba86416746eb64c95 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Tue, 29 Oct 2024 15:36:37 -0300 Subject: [PATCH 13/19] fix test_create_request_task and test_execute_request_tasks --- .../saas_external_example_config.yml | 31 +++++++++++++++++++ tests/fixtures/saas_example_fixtures.py | 12 +++++-- .../service/connectors/test_saas_connector.py | 30 +++++------------- tests/ops/task/test_create_request_tasks.py | 24 ++++++-------- tests/ops/task/test_execute_request_tasks.py | 9 +++--- 5 files changed, 63 insertions(+), 43 deletions(-) diff --git a/tests/fixtures/saas/test_data/saas_external_example_config.yml b/tests/fixtures/saas/test_data/saas_external_example_config.yml index cb9170da34..e804383611 100644 --- a/tests/fixtures/saas/test_data/saas_external_example_config.yml +++ b/tests/fixtures/saas/test_data/saas_external_example_config.yml @@ -33,3 +33,34 @@ saas_config: param_values: - name: email identity: email + - name: projects + requests: + read: + method: GET + path: /api/0/projects/ + param_values: + - name: placeholder + identity: email + - name: users + after: [saas_connector_external_example.projects] + requests: + read: + method: GET + path: /api/0/projects///user-feedback/ + query_params: + - name: query + value: + grouped_inputs: [organization_slug, project_slug, query] + param_values: + - name: organization_slug + references: + - dataset: saas_connector_external_example + field: projects.organization.slug + direction: from + - name: project_slug + references: + - dataset: saas_connector_external_example + field: projects.slug + direction: from + - name: query + identity: email diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index ab2f766880..a973408372 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -60,13 +60,20 @@ def saas_example_secrets(): def saas_example_config() -> Dict: return load_config("tests/fixtures/saas/test_data/saas_example_config.yml") + @pytest.fixture def saas_example_opt_out_only_config() -> Dict: - return load_config("tests/fixtures/saas/test_data/saas_example_opt_out_only_config.yml") + return load_config( + "tests/fixtures/saas/test_data/saas_example_opt_out_only_config.yml" + ) + @pytest.fixture def saas_example_consent_preferences_config() -> Dict: - return load_config("tests/fixtures/saas/test_data/saas_example_consent_preferences_config.yml") + return load_config( + "tests/fixtures/saas/test_data/saas_example_consent_preferences_config.yml" + ) + @pytest.fixture def saas_external_example_config() -> Dict: @@ -156,6 +163,7 @@ def saas_example_consent_preferences_connection_config( yield connection_config connection_config.delete(db) + @pytest.fixture(scope="function") def saas_external_example_connection_config( db: Session, diff --git a/tests/ops/service/connectors/test_saas_connector.py b/tests/ops/service/connectors/test_saas_connector.py index e856d418eb..02efe14acc 100644 --- a/tests/ops/service/connectors/test_saas_connector.py +++ b/tests/ops/service/connectors/test_saas_connector.py @@ -66,7 +66,6 @@ def valid_consent_update_override( return ConsentPropagationStatus.executed - @pytest.mark.unit_saas class TestSaasConnector: """ @@ -588,7 +587,6 @@ def test_request_with_invalid_output_template( assert "Failed to parse value as JSON" in str(exc) - @pytest.mark.integration_saas class TestSaaSConnectorMethods: def test_client_config_set_depending_on_state( @@ -617,7 +615,9 @@ def test_rate_limit_config_set_depending_on_state( self, db: Session, saas_example_connection_config, saas_example_dataset_config ): rate_limit_config = {"limits": [{"rate": 1, "period": "second"}]} - saas_example_connection_config.saas_config["rate_limit_config"] = rate_limit_config + saas_example_connection_config.saas_config["rate_limit_config"] = ( + rate_limit_config + ) connector: SaaSConnector = get_connector(saas_example_connection_config) connector.set_saas_request_state( SaaSRequest(path="test_path", method=HTTPMethod.GET) @@ -635,15 +635,10 @@ def test_rate_limit_config_set_depending_on_state( assert connector.get_rate_limit_config().enabled is False - @pytest.mark.integration_saas class TestConsentRequests: - def test_get_consent_requests_by_preference( - self, saas_example_connection_config - ): - connector: SaaSConnector = get_connector( - saas_example_connection_config - ) + def test_get_consent_requests_by_preference(self, saas_example_connection_config): + connector: SaaSConnector = get_connector(saas_example_connection_config) opt_in_request: List[SaaSRequest] = ( connector._get_consent_requests_by_preference(opt_in=True) @@ -658,7 +653,6 @@ def test_get_consent_requests_by_preference( assert opt_out_request[1].path == "/rejects/add" - class TestSaasConnectorRunConsentRequest: def test_no_preferences_to_propagate( self, @@ -983,18 +977,13 @@ def test_preferences_executable_notice_based_consent( consent_automation.delete(db) - class TestRelevantConsentIdentities: - def test_no_consent_requests( - self, saas_example_connection_config - ): + def test_no_consent_requests(self, saas_example_connection_config): connector = get_connector(saas_example_connection_config) connector.relevant_consent_identities([], {"customer_1@example.com"}) == {} - def test_no_identity_data( - self, saas_example_connection_config - ): + def test_no_identity_data(self, saas_example_connection_config): connector = get_connector(saas_example_connection_config) request = SaaSRequest( @@ -1005,9 +994,7 @@ def test_no_identity_data( ) assert connector.relevant_consent_identities([request], {}) == {} - def test_get_relevant_identities_only( - self, saas_example_connection_config - ): + def test_get_relevant_identities_only(self, saas_example_connection_config): connector = get_connector(saas_example_connection_config) request = SaaSRequest( @@ -1021,7 +1008,6 @@ def test_get_relevant_identities_only( ) == {"email": "customer-1@example.com"} - class TestAsyncConnectors: @pytest.fixture(scope="function") def async_graph(self, saas_example_async_dataset_config, db, privacy_request): diff --git a/tests/ops/task/test_create_request_tasks.py b/tests/ops/task/test_create_request_tasks.py index ab48157a5b..f3cf0b75f6 100644 --- a/tests/ops/task/test_create_request_tasks.py +++ b/tests/ops/task/test_create_request_tasks.py @@ -637,7 +637,6 @@ def test_persist_initial_erasure_request_tasks( assert not payment_card_task.is_root_task assert not payment_card_task.is_terminator_task - @pytest.mark.skip(reason="move to plus in progress") @pytest.mark.timeout(5) @pytest.mark.integration @pytest.mark.integration_postgres @@ -1106,17 +1105,14 @@ def test_run_erasure_request_with_existing_request_tasks( run_erasure_node_mock.assert_called_with(erasure_request_task, False) -@pytest.mark.skip(reason="move to plus in progress") class TestPersistConsentRequestTasks: def test_persist_new_consent_request_tasks( self, db, privacy_request, - google_analytics_dataset_config_no_secrets, + saas_example_dataset_config, ): - graph = build_consent_dataset_graph( - [google_analytics_dataset_config_no_secrets] - ) + graph = build_consent_dataset_graph([saas_example_dataset_config]) traversal_nodes = {} # Unlike erasure and access graphs, we don't call traversal.traverse, but build a simpler @@ -1135,11 +1131,11 @@ def test_persist_new_consent_request_tasks( assert root_task.action_type == ActionType.consent assert root_task.upstream_tasks == [] assert root_task.downstream_tasks == [ - "google_analytics_instance:google_analytics_instance" + "saas_connector_example:saas_connector_example" ] assert root_task.all_descendant_tasks == [ "__TERMINATE__:__TERMINATE__", - "google_analytics_instance:google_analytics_instance", + "saas_connector_example:saas_connector_example", ] assert root_task.status == ExecutionLogStatus.complete assert root_task.access_data == [{"ga_client_id": "test_id"}] @@ -1151,7 +1147,7 @@ def test_persist_new_consent_request_tasks( assert terminator_task.is_terminator_task assert terminator_task.action_type == ActionType.consent assert terminator_task.upstream_tasks == [ - "google_analytics_instance:google_analytics_instance" + "saas_connector_example:saas_connector_example" ] assert terminator_task.downstream_tasks == [] assert terminator_task.all_descendant_tasks == [] @@ -1159,7 +1155,7 @@ def test_persist_new_consent_request_tasks( ga_task = privacy_request.consent_tasks.filter( RequestTask.collection_address - == "google_analytics_instance:google_analytics_instance", + == "saas_connector_example:saas_connector_example", ).first() assert not ga_task.is_root_task assert not ga_task.is_terminator_task @@ -1173,21 +1169,21 @@ def test_persist_new_consent_request_tasks( # The collection is a fake one for Consent, since requests happen at the dataset level assert ga_task.collection == { - "name": "google_analytics_instance", + "name": "saas_connector_example", "after": [], "fields": [], "erase_after": [], + "partitioning": None, "grouped_inputs": [], - "skip_processing": False, "data_categories": [], + "skip_processing": False, "masking_strategy_override": None, - "partitioning": None, } assert ga_task.traversal_details == { "input_keys": [], "incoming_edges": [], "outgoing_edges": [], - "dataset_connection_key": "google_analytics_instance", + "dataset_connection_key": "saas_connector_example", } @mock.patch( diff --git a/tests/ops/task/test_execute_request_tasks.py b/tests/ops/task/test_execute_request_tasks.py index fadf7d1673..cb3df3fa8f 100644 --- a/tests/ops/task/test_execute_request_tasks.py +++ b/tests/ops/task/test_execute_request_tasks.py @@ -319,13 +319,12 @@ def test_incoming_edges_by_collection(self, address_execution_node): ], } - @pytest.mark.skip(reason="move to plus in progress") - @pytest.mark.usefixtures("sentry_connection_config_without_secrets") + @pytest.mark.usefixtures("saas_external_example_connection_config") def test_grouped_fields( - self, db, privacy_request, sentry_dataset_config_without_secrets + self, db, privacy_request, saas_external_example_dataset_config ): """Test that a config with grouped inputs (sentry saas connector) has grouped inputs persisted""" - merged_graph = sentry_dataset_config_without_secrets.get_graph() + merged_graph = saas_external_example_dataset_config.get_graph() graph = DatasetGraph(merged_graph) identity = {"email": "customer-1@example.com"} @@ -343,7 +342,7 @@ def test_grouped_fields( ) issues_task = privacy_request.access_tasks.filter( - RequestTask.collection_address == "sentry_dataset:issues" + RequestTask.collection_address == "saas_connector_external_example:users" ).first() execution_node = ExecutionNode(issues_task) assert execution_node.grouped_fields == { From 5b2e4918c128f498e9e0eb249c114e754a42800f Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Tue, 29 Oct 2024 16:06:53 -0300 Subject: [PATCH 14/19] fix test_graph_task --- tests/ops/task/test_graph_task.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/ops/task/test_graph_task.py b/tests/ops/task/test_graph_task.py index 03f3b49b6b..ff27818cfa 100644 --- a/tests/ops/task/test_graph_task.py +++ b/tests/ops/task/test_graph_task.py @@ -928,24 +928,24 @@ class TestGraphTaskAffectedConsentSystems: def mock_graph_task( self, db, - mailchimp_transactional_connection_config_no_secrets, + saas_example_connection_config, privacy_request_with_consent_policy, ): task_resources = TaskResources( privacy_request_with_consent_policy, privacy_request_with_consent_policy.policy, - [mailchimp_transactional_connection_config_no_secrets], + [saas_example_connection_config], EMPTY_REQUEST_TASK, db, ) tn = TraversalNode(generate_node("a", "b", "c", "c2")) tn.node.dataset.connection_key = ( - mailchimp_transactional_connection_config_no_secrets.key + saas_example_connection_config.key ) task_resources.privacy_request_task = tn.to_mock_request_task() return GraphTask(task_resources) - @pytest.mark.skip(reason="move to plus in progress") + @mock.patch( "fides.api.service.connectors.saas_connector.SaaSConnector.run_consent_request" ) @@ -979,10 +979,10 @@ def test_skipped_consent_task_for_connector( db.refresh(privacy_preference_history_us_ca_provide) assert privacy_preference_history.affected_system_status == { - "mailchimp_transactional_instance": "skipped" + "saas_connector_example": "skipped" } assert privacy_preference_history_us_ca_provide.affected_system_status == { - "mailchimp_transactional_instance": "skipped" + "saas_connector_example": "skipped" } logs = ( @@ -995,7 +995,7 @@ def test_skipped_consent_task_for_connector( ) assert logs.first().status == ExecutionLogStatus.skipped - @pytest.mark.skip(reason="move to plus in progress") + @mock.patch("fides.api.task.graph_task.mark_current_and_downstream_nodes_as_failed") @mock.patch( "fides.api.service.connectors.saas_connector.SaaSConnector.run_consent_request" @@ -1004,7 +1004,7 @@ def test_errored_consent_task_for_connector_no_relevant_preferences( self, mock_run_consent_request, mark_current_and_downstream_nodes_as_failed_mock, - mailchimp_transactional_connection_config_no_secrets, + saas_example_connection_config, mock_graph_task, db, privacy_request_with_consent_policy, @@ -1025,7 +1025,7 @@ def test_errored_consent_task_for_connector_no_relevant_preferences( cache_initial_status_and_identities_for_consent_reporting( db, privacy_request_with_consent_policy, - mailchimp_transactional_connection_config_no_secrets, + saas_example_connection_config, relevant_preferences=[privacy_preference_history_us_ca_provide], relevant_user_identities={"email": "customer-1@example.com"}, ) @@ -1037,10 +1037,10 @@ def test_errored_consent_task_for_connector_no_relevant_preferences( db.refresh(privacy_preference_history_us_ca_provide) assert privacy_preference_history.affected_system_status == { - "mailchimp_transactional_instance": "skipped" + "saas_connector_example": "skipped" } assert privacy_preference_history_us_ca_provide.affected_system_status == { - "mailchimp_transactional_instance": "error" + "saas_connector_example": "error" } logs = ( From dce1396b09ada27a3c89bee10f3864e83ce4a928 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Tue, 29 Oct 2024 16:44:30 -0300 Subject: [PATCH 15/19] fix test_connection_type --- tests/ops/util/test_connection_type.py | 130 ++++++++----------------- 1 file changed, 38 insertions(+), 92 deletions(-) diff --git a/tests/ops/util/test_connection_type.py b/tests/ops/util/test_connection_type.py index 2c43a960f0..5f0fedbabe 100644 --- a/tests/ops/util/test_connection_type.py +++ b/tests/ops/util/test_connection_type.py @@ -13,7 +13,7 @@ ) -@pytest.mark.skip(reason="move to plus in progress") + def test_get_connection_types(): data = [obj.model_dump(mode="json") for obj in get_connection_types()] assert ( @@ -59,27 +59,21 @@ def test_get_connection_types(): } in data -DOORDASH = "doordash" -GOOGLE_ANALYTICS = "google_analytics" -MAILCHIMP_TRANSACTIONAL = "mailchimp_transactional" -SEGMENT = "segment" +HUBSPOT = "hubspot" +MAILCHIMP = "mailchimp" STRIPE = "stripe" -ZENDESK = "zendesk" @pytest.mark.skip(reason="move to plus in progress") @pytest.fixture def connection_type_objects(): - google_analytics_template = ConnectorRegistry.get_connector_template( - GOOGLE_ANALYTICS + hubspot_template = ConnectorRegistry.get_connector_template( + HUBSPOT ) - mailchimp_transactional_template = ConnectorRegistry.get_connector_template( - MAILCHIMP_TRANSACTIONAL + mailchimp_template = ConnectorRegistry.get_connector_template( + MAILCHIMP ) stripe_template = ConnectorRegistry.get_connector_template("stripe") - zendesk_template = ConnectorRegistry.get_connector_template("zendesk") - doordash_template = ConnectorRegistry.get_connector_template(DOORDASH) - segment_template = ConnectorRegistry.get_connector_template(SEGMENT) return { ConnectionType.postgres.value: { @@ -100,38 +94,27 @@ def connection_type_objects(): "user_guide": None, "supported_actions": [ActionType.access.value, ActionType.erasure.value], }, - GOOGLE_ANALYTICS: { - "identifier": GOOGLE_ANALYTICS, + HUBSPOT: { + "identifier": HUBSPOT, "type": SystemType.saas.value, - "human_readable": google_analytics_template.human_readable, - "encoded_icon": google_analytics_template.icon, - "authorization_required": google_analytics_template.authorization_required, - "user_guide": google_analytics_template.user_guide, + "human_readable": hubspot_template.human_readable, + "encoded_icon": hubspot_template.icon, + "authorization_required": hubspot_template.authorization_required, + "user_guide": hubspot_template.user_guide, "supported_actions": [ - action.value for action in google_analytics_template.supported_actions + action.value for action in hubspot_template.supported_actions ], }, - MAILCHIMP_TRANSACTIONAL: { - "identifier": MAILCHIMP_TRANSACTIONAL, + MAILCHIMP: { + "identifier": MAILCHIMP, "type": SystemType.saas.value, - "human_readable": mailchimp_transactional_template.human_readable, - "encoded_icon": mailchimp_transactional_template.icon, - "authorization_required": mailchimp_transactional_template.authorization_required, - "user_guide": mailchimp_transactional_template.user_guide, + "human_readable": mailchimp_template.human_readable, + "encoded_icon": mailchimp_template.icon, + "authorization_required": mailchimp_template.authorization_required, + "user_guide": mailchimp_template.user_guide, "supported_actions": [ action.value - for action in mailchimp_transactional_template.supported_actions - ], - }, - SEGMENT: { - "identifier": SEGMENT, - "type": SystemType.saas.value, - "human_readable": segment_template.human_readable, - "encoded_icon": segment_template.icon, - "authorization_required": segment_template.authorization_required, - "user_guide": segment_template.user_guide, - "supported_actions": [ - action.value for action in segment_template.supported_actions + for action in mailchimp_template.supported_actions ], }, STRIPE: { @@ -145,28 +128,6 @@ def connection_type_objects(): action.value for action in stripe_template.supported_actions ], }, - ZENDESK: { - "identifier": ZENDESK, - "type": SystemType.saas.value, - "human_readable": zendesk_template.human_readable, - "encoded_icon": zendesk_template.icon, - "authorization_required": zendesk_template.authorization_required, - "user_guide": zendesk_template.user_guide, - "supported_actions": [ - action.value for action in zendesk_template.supported_actions - ], - }, - DOORDASH: { - "identifier": DOORDASH, - "type": SystemType.saas.value, - "human_readable": doordash_template.human_readable, - "encoded_icon": doordash_template.icon, - "authorization_required": doordash_template.authorization_required, - "user_guide": doordash_template.user_guide, - "supported_actions": [ - action.value for action in doordash_template.supported_actions - ], - }, ConnectionType.sovrn.value: { "identifier": ConnectionType.sovrn.value, "type": SystemType.email.value, @@ -188,20 +149,19 @@ def connection_type_objects(): } -@pytest.mark.skip(reason="move to plus in progress") + @pytest.mark.parametrize( "action_types, assert_in_data, assert_not_in_data", [ ( [ActionType.consent], - [GOOGLE_ANALYTICS, MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value], + [ConnectionType.sovrn.value], [ ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, + HUBSPOT, + MAILCHIMP, STRIPE, - ZENDESK, - SEGMENT, ConnectionType.attentive_email.value, ], ), @@ -210,14 +170,11 @@ def connection_type_objects(): [ ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, - SEGMENT, + HUBSPOT, + MAILCHIMP, STRIPE, - ZENDESK, ], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value, ConnectionType.attentive_email.value, ], @@ -226,31 +183,25 @@ def connection_type_objects(): [ActionType.erasure], [ ConnectionType.postgres.value, - SEGMENT, # segment has DPR so it is an erasure + HUBSPOT, STRIPE, - ZENDESK, + MAILCHIMP, ConnectionType.attentive_email.value, ConnectionType.manual_webhook.value, ], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, - DOORDASH, # doordash does not have erasures ConnectionType.sovrn.value, ], ), ( [ActionType.consent, ActionType.access], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, + HUBSPOT, + MAILCHIMP, ConnectionType.sovrn.value, ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, - SEGMENT, STRIPE, - ZENDESK, ], [ ConnectionType.attentive_email.value, @@ -259,18 +210,15 @@ def connection_type_objects(): ( [ActionType.consent, ActionType.erasure], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, + MAILCHIMP, + HUBSPOT, ConnectionType.sovrn.value, ConnectionType.postgres.value, - SEGMENT, # segment has DPR so it is an erasure STRIPE, - ZENDESK, ConnectionType.attentive_email.value, ConnectionType.manual_webhook.value, ], [ - DOORDASH, # doordash does not have erasures ], ), ( @@ -278,21 +226,19 @@ def connection_type_objects(): [ ConnectionType.postgres.value, ConnectionType.manual_webhook.value, - DOORDASH, - SEGMENT, + MAILCHIMP, + HUBSPOT, STRIPE, - ZENDESK, ConnectionType.attentive_email.value, ], [ - GOOGLE_ANALYTICS, - MAILCHIMP_TRANSACTIONAL, ConnectionType.sovrn.value, ], ), ], ) -@pytest.mark.skip(reason="move to plus in progress") + + def test_get_connection_types_action_type_filter( action_types, assert_in_data, assert_not_in_data, connection_type_objects ): @@ -310,7 +256,7 @@ def test_get_connection_types_action_type_filter( assert obj not in data -@pytest.mark.skip(reason="move to plus in progress") + def test_get_connection_type_secret_schemas_aws(): """ AWS secret schemas have inheritance from a base class, and have provided some issues in the past. From ea07bb1a86051aad2466dea6dbc5bd555b8f8774 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Tue, 29 Oct 2024 16:48:55 -0300 Subject: [PATCH 16/19] remove skip --- tests/ops/util/test_connection_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ops/util/test_connection_type.py b/tests/ops/util/test_connection_type.py index 5f0fedbabe..ba7fcdfcec 100644 --- a/tests/ops/util/test_connection_type.py +++ b/tests/ops/util/test_connection_type.py @@ -64,7 +64,7 @@ def test_get_connection_types(): STRIPE = "stripe" -@pytest.mark.skip(reason="move to plus in progress") + @pytest.fixture def connection_type_objects(): hubspot_template = ConnectorRegistry.get_connector_template( From 79ac17db9924cb52425f8545f52589fd6e8940bc Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Wed, 30 Oct 2024 00:17:03 -0300 Subject: [PATCH 17/19] run static checks --- tests/ops/task/test_graph_task.py | 6 +----- tests/ops/util/test_connection_type.py | 20 ++++---------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/tests/ops/task/test_graph_task.py b/tests/ops/task/test_graph_task.py index ff27818cfa..5293a29173 100644 --- a/tests/ops/task/test_graph_task.py +++ b/tests/ops/task/test_graph_task.py @@ -939,13 +939,10 @@ def mock_graph_task( db, ) tn = TraversalNode(generate_node("a", "b", "c", "c2")) - tn.node.dataset.connection_key = ( - saas_example_connection_config.key - ) + tn.node.dataset.connection_key = saas_example_connection_config.key task_resources.privacy_request_task = tn.to_mock_request_task() return GraphTask(task_resources) - @mock.patch( "fides.api.service.connectors.saas_connector.SaaSConnector.run_consent_request" ) @@ -995,7 +992,6 @@ def test_skipped_consent_task_for_connector( ) assert logs.first().status == ExecutionLogStatus.skipped - @mock.patch("fides.api.task.graph_task.mark_current_and_downstream_nodes_as_failed") @mock.patch( "fides.api.service.connectors.saas_connector.SaaSConnector.run_consent_request" diff --git a/tests/ops/util/test_connection_type.py b/tests/ops/util/test_connection_type.py index ba7fcdfcec..5cea381a7e 100644 --- a/tests/ops/util/test_connection_type.py +++ b/tests/ops/util/test_connection_type.py @@ -13,7 +13,6 @@ ) - def test_get_connection_types(): data = [obj.model_dump(mode="json") for obj in get_connection_types()] assert ( @@ -64,15 +63,10 @@ def test_get_connection_types(): STRIPE = "stripe" - @pytest.fixture def connection_type_objects(): - hubspot_template = ConnectorRegistry.get_connector_template( - HUBSPOT - ) - mailchimp_template = ConnectorRegistry.get_connector_template( - MAILCHIMP - ) + hubspot_template = ConnectorRegistry.get_connector_template(HUBSPOT) + mailchimp_template = ConnectorRegistry.get_connector_template(MAILCHIMP) stripe_template = ConnectorRegistry.get_connector_template("stripe") return { @@ -113,8 +107,7 @@ def connection_type_objects(): "authorization_required": mailchimp_template.authorization_required, "user_guide": mailchimp_template.user_guide, "supported_actions": [ - action.value - for action in mailchimp_template.supported_actions + action.value for action in mailchimp_template.supported_actions ], }, STRIPE: { @@ -149,7 +142,6 @@ def connection_type_objects(): } - @pytest.mark.parametrize( "action_types, assert_in_data, assert_not_in_data", [ @@ -218,8 +210,7 @@ def connection_type_objects(): ConnectionType.attentive_email.value, ConnectionType.manual_webhook.value, ], - [ - ], + [], ), ( [ActionType.access, ActionType.erasure], @@ -237,8 +228,6 @@ def connection_type_objects(): ), ], ) - - def test_get_connection_types_action_type_filter( action_types, assert_in_data, assert_not_in_data, connection_type_objects ): @@ -256,7 +245,6 @@ def test_get_connection_types_action_type_filter( assert obj not in data - def test_get_connection_type_secret_schemas_aws(): """ AWS secret schemas have inheritance from a base class, and have provided some issues in the past. From 52ef683ba50422c788328fd3185fa144ad091b92 Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Wed, 30 Oct 2024 10:08:12 -0300 Subject: [PATCH 18/19] very minor fix on using stripe constant --- tests/ops/util/test_connection_type.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ops/util/test_connection_type.py b/tests/ops/util/test_connection_type.py index 5cea381a7e..7ab867529c 100644 --- a/tests/ops/util/test_connection_type.py +++ b/tests/ops/util/test_connection_type.py @@ -67,7 +67,7 @@ def test_get_connection_types(): def connection_type_objects(): hubspot_template = ConnectorRegistry.get_connector_template(HUBSPOT) mailchimp_template = ConnectorRegistry.get_connector_template(MAILCHIMP) - stripe_template = ConnectorRegistry.get_connector_template("stripe") + stripe_template = ConnectorRegistry.get_connector_template(STRIPE) return { ConnectionType.postgres.value: { From 19cda7013a8f1f43682d1306074a81b22add648c Mon Sep 17 00:00:00 2001 From: facundo lopez janza Date: Wed, 30 Oct 2024 10:27:56 -0300 Subject: [PATCH 19/19] update comment based on review --- .../ops/api/v1/endpoints/test_connection_template_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 7b81e59e64..bc9e1668f4 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -94,7 +94,7 @@ def test_get_connection_types_size_param( ) -> None: """Test to ensure size param works as expected since it overrides default value""" - # ensure default size is 100 (effectively testing that here since we have > 50 connectors) + # ensure default size is 100 (effectively testing that here since we have > 20 connectors) auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) resp = api_client.get(url, headers=auth_header) data = resp.json()["items"]