From 22cd8aa70aa95545ef1e2f22b82ca6f453344a56 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 14 Jun 2023 18:51:27 -0500 Subject: [PATCH 01/20] Initial commit - add a Cookies table with FK's to PrivacyDeclaration and System and an upsert_cookies method that is called when creating/updating privacy declarations. --- noxfiles/test_docker_nox.py | 11 +- requirements.txt | 2 +- src/fides/api/ctl/database/crud.py | 2 +- src/fides/api/ctl/database/system.py | 71 +++++++- .../versions/2be84e68df32_add_cookie_table.py | 68 ++++++++ ...9c0dac_remove_deprecated_data_uses_for_.py | 2 - src/fides/api/ctl/schemas/system.py | 5 +- src/fides/api/ctl/sql_models.py | 45 +++++ src/fides/api/schemas/privacy_request.py | 3 +- tests/conftest.py | 4 +- tests/ctl/core/test_api.py | 71 ++++++++ tests/ctl/core/test_system.py | 155 ++++++++++++++++++ 12 files changed, 417 insertions(+), 22 deletions(-) create mode 100644 src/fides/api/ctl/migrations/versions/2be84e68df32_add_cookie_table.py diff --git a/noxfiles/test_docker_nox.py b/noxfiles/test_docker_nox.py index e73bf31798..a43ebb8cd4 100644 --- a/noxfiles/test_docker_nox.py +++ b/noxfiles/test_docker_nox.py @@ -1,14 +1,7 @@ import pytest -from docker_nox import ( - generate_multiplatform_buildx_command, - get_buildx_commands, -) -from constants_nox import ( - DEV_TAG_SUFFIX, - PRERELEASE_TAG_SUFFIX, - RC_TAG_SUFFIX, -) +from constants_nox import DEV_TAG_SUFFIX, PRERELEASE_TAG_SUFFIX, RC_TAG_SUFFIX +from docker_nox import generate_multiplatform_buildx_command, get_buildx_commands class TestGenerateMultiplatformBuilxCommand: diff --git a/requirements.txt b/requirements.txt index 68a6c6d551..6116ecaa9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang==1.4.1 +fideslang @ git+https://github.com/ethyca/fideslang.git@ab9a9ea257d7715506b995020d7d5170fe65f41a fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 diff --git a/src/fides/api/ctl/database/crud.py b/src/fides/api/ctl/database/crud.py index 50ac2cf60b..202499508f 100644 --- a/src/fides/api/ctl/database/crud.py +++ b/src/fides/api/ctl/database/crud.py @@ -113,7 +113,7 @@ async def get_resource( raise_not_found: bool = True, ) -> Base: """ - Get a resource from the databse by its FidesKey. + Get a resource from the database by its FidesKey. Returns a SQLAlchemy model of that resource. """ diff --git a/src/fides/api/ctl/database/system.py b/src/fides/api/ctl/database/system.py index 0dc7e97adf..08059bfc10 100644 --- a/src/fides/api/ctl/database/system.py +++ b/src/fides/api/ctl/database/system.py @@ -1,17 +1,23 @@ """ Functions for interacting with System objects in the database. """ -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple from fastapi import HTTPException +from fideslang.models import Cookies as CookieSchema from fideslang.models import System as SystemSchema from loguru import logger as log +from sqlalchemy import and_, delete, select +from sqlalchemy.dialects.postgresql import Insert +from sqlalchemy.engine import ChunkedIteratorResult from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session +from sqlalchemy.sql import Select from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND from fides.api.ctl.database.crud import create_resource, get_resource, update_resource from fides.api.ctl.sql_models import ( # type: ignore[attr-defined] + Cookies, DataUse, PrivacyDeclaration, System, @@ -112,23 +118,71 @@ async def upsert_privacy_declarations( for privacy_declaration in resource.privacy_declarations: # prepare our 'payload' for either create or update data = privacy_declaration.dict() + cookies: List[Dict] = data.pop("cookies", None) data["system_id"] = system.id # include FK back to system # if we find matching declaration, remove it from our map - if existing_declaration := existing_declarations.pop( + if declaration := existing_declarations.pop( privacy_declaration_logical_id(privacy_declaration), None ): # and update existing declaration *in place* - existing_declaration.update(db, data=data) + declaration.update(db, data=data) else: # otherwise, create a new declaration record - PrivacyDeclaration.create(db, data=data) + declaration = PrivacyDeclaration.create(db, data=data) + + await upsert_cookies(db, cookies, declaration, system) # delete any existing privacy declarations that have not been "matched" in the request for existing_declarations in existing_declarations.values(): await db.delete(existing_declarations) +async def upsert_cookies( + async_session: AsyncSession, + cookies: Optional[List[Dict]], # CookieSchema + privacy_declaration: PrivacyDeclaration, + system: System, +) -> None: + """Upsert cookies for the given privacy declaration""" + cookie_list: List[CookieSchema] = cookies or [] + for cookie_data in cookie_list: + query: Select = select(Cookies).where( + and_( + Cookies.name == cookie_data["name"], + Cookies.system_id == system.id, + Cookies.privacy_declaration_id == privacy_declaration.id, + ) + ) + result: ChunkedIteratorResult = await async_session.execute(query) + row: Optional[Cookies] = result.scalars().first() + + if not row: + stmt: Insert = Insert(Cookies).values( + { + "name": cookie_data["name"], + "privacy_declaration_id": privacy_declaration.id, + "system_id": system.id, + } + ) + await async_session.execute(stmt) + + missing_cookies_query: Select = select(Cookies).where( + and_( + Cookies.name.notin_([cookie["name"] for cookie in cookie_list]), + Cookies.system_id == system.id, + Cookies.privacy_declaration_id == privacy_declaration.id, + ) + ) + delete_result: ChunkedIteratorResult = await async_session.execute( + missing_cookies_query + ) + rows: List = delete_result.scalars().unique().all() + + stmt = delete(Cookies).where(Cookies.id.in_([cookie.id for cookie in rows])) + await async_session.execute(stmt) + + async def update_system(resource: SystemSchema, db: AsyncSession) -> Dict: """Helper function to share core system update logic for wrapping endpoint functions""" system: System = await get_resource( @@ -147,6 +201,7 @@ async def update_system(resource: SystemSchema, db: AsyncSession) -> Dict: delattr( resource, "privacy_declarations" ) # remove the attribute on the system since we've already updated declarations + delattr(resource, "cookies") # remove the cookies attribute # perform any updates on the system resource itself updated_system = await update_resource(System, resource.dict(), db) @@ -170,6 +225,8 @@ async def create_system( # remove the attribute on the system update since the declarations will be created separately delattr(resource, "privacy_declarations") + # Removing cookies attribute; they can't be set on the system directly + delattr(resource, "cookies") # create the system resource using generic creation # the system must be created before the privacy declarations so that it can be referenced @@ -184,9 +241,13 @@ async def create_system( for privacy_declaration in privacy_declarations: data = privacy_declaration.dict() data["system_id"] = created_system.id # add FK back to system - PrivacyDeclaration.create( + cookies: List[Dict] = data.pop("cookies", []) + privacy_declaration = PrivacyDeclaration.create( db, data=data ) # create the associated PrivacyDeclaration + await upsert_cookies( + db, cookies, privacy_declaration, created_system + ) # Create the associated cookies except Exception as e: log.error( f"Error adding privacy declarations, reverting system creation: {str(privacy_declaration_exception)}" diff --git a/src/fides/api/ctl/migrations/versions/2be84e68df32_add_cookie_table.py b/src/fides/api/ctl/migrations/versions/2be84e68df32_add_cookie_table.py new file mode 100644 index 0000000000..4ceeb17280 --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/2be84e68df32_add_cookie_table.py @@ -0,0 +1,68 @@ +"""add cookie table + +Revision ID: 2be84e68df32 +Revises: 5307999c0dac +Create Date: 2023-06-13 23:08:35.011377 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "2be84e68df32" +down_revision = "5307999c0dac" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "cookies", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("name", sa.String(), nullable=False), + sa.Column("system_id", sa.String(), nullable=True), + sa.Column("privacy_declaration_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["privacy_declaration_id"], ["privacydeclaration.id"], ondelete="SET NULL" + ), + sa.ForeignKeyConstraint(["system_id"], ["ctl_systems.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "name", "privacy_declaration_id", name="_cookie_name_privacy_declaration_uc" + ), + ) + op.create_index(op.f("ix_cookies_id"), "cookies", ["id"], unique=False) + op.create_index(op.f("ix_cookies_name"), "cookies", ["name"], unique=False) + op.create_index( + op.f("ix_cookies_privacy_declaration_id"), + "cookies", + ["privacy_declaration_id"], + unique=False, + ) + op.create_index( + op.f("ix_cookies_system_id"), "cookies", ["system_id"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_cookies_system_id"), table_name="cookies") + op.drop_index(op.f("ix_cookies_privacy_declaration_id"), table_name="cookies") + op.drop_index(op.f("ix_cookies_name"), table_name="cookies") + op.drop_index(op.f("ix_cookies_id"), table_name="cookies") + op.drop_table("cookies") + # ### end Alembic commands ### diff --git a/src/fides/api/ctl/migrations/versions/5307999c0dac_remove_deprecated_data_uses_for_.py b/src/fides/api/ctl/migrations/versions/5307999c0dac_remove_deprecated_data_uses_for_.py index 0160c1056b..4c60c30db0 100644 --- a/src/fides/api/ctl/migrations/versions/5307999c0dac_remove_deprecated_data_uses_for_.py +++ b/src/fides/api/ctl/migrations/versions/5307999c0dac_remove_deprecated_data_uses_for_.py @@ -6,12 +6,10 @@ """ from alembic import op - from loguru import logger from sqlalchemy import text from sqlalchemy.engine import Connection - # revision identifiers, used by Alembic. revision = "5307999c0dac" down_revision = "76c02f99eec1" diff --git a/src/fides/api/ctl/schemas/system.py b/src/fides/api/ctl/schemas/system.py index 82016e051a..9111005698 100644 --- a/src/fides/api/ctl/schemas/system.py +++ b/src/fides/api/ctl/schemas/system.py @@ -1,6 +1,6 @@ from typing import List, Optional -from fideslang.models import PrivacyDeclaration, System +from fideslang.models import Cookies, PrivacyDeclaration, System from pydantic import Field from fides.api.schemas.connection_configuration.connection_config import ( @@ -14,6 +14,7 @@ class PrivacyDeclarationResponse(PrivacyDeclaration): id: str = Field( description="The database-assigned ID of the privacy declaration on the system. This is meant to be a read-only field, returned only in API responses" ) + cookies: Optional[List[Cookies]] = [] class SystemResponse(System): @@ -26,3 +27,5 @@ class SystemResponse(System): connection_configs: Optional[ConnectionConfigurationResponse] = Field( description=ConnectionConfigurationResponse.__doc__, ) + + cookies: Optional[List[Cookies]] = [] diff --git a/src/fides/api/ctl/sql_models.py b/src/fides/api/ctl/sql_models.py index ecd32fc74e..1c31ebcc27 100644 --- a/src/fides/api/ctl/sql_models.py +++ b/src/fides/api/ctl/sql_models.py @@ -346,6 +346,10 @@ class System(Base, FidesBase): lazy="selectin", ) + cookies = relationship( + "Cookies", back_populates="system", lazy="selectin", uselist=True, viewonly=True + ) + @classmethod def get_data_uses( cls: Type[System], systems: List[System], include_parents: bool = True @@ -392,6 +396,9 @@ class PrivacyDeclaration(Base): index=True, ) system = relationship(System, back_populates="privacy_declarations") + cookies = relationship( + "Cookies", back_populates="privacy_declaration", lazy="joined", uselist=True + ) @classmethod def create( @@ -596,3 +603,41 @@ class AuditLogResource(Base): request_type = Column(String, nullable=True) fides_keys = Column(ARRAY(String), nullable=True) extra_data = Column(JSON, nullable=True) + + +class Cookies(Base): + """ + Stores cookies. Cookies have a FK to system and privacy declaration. If a privacy declaration is deleted, + the cookie can still remain linked to the system but unassociated with a data use. + """ + + name = Column(String, index=True, nullable=False) + system_id = Column( + String, ForeignKey(System.id_field_path, ondelete="CASCADE"), index=True + ) + privacy_declaration_id = Column( + String, + ForeignKey(PrivacyDeclaration.id_field_path, ondelete="SET NULL"), + index=True, + ) + + system = relationship( + "System", + back_populates="cookies", + cascade="all,delete", + uselist=False, + lazy="selectin", + ) + + privacy_declaration = relationship( + "PrivacyDeclaration", + back_populates="cookies", + uselist=False, + lazy="joined", + ) + + __table_args__ = ( + UniqueConstraint( + "name", "privacy_declaration_id", name="_cookie_name_privacy_declaration_uc" + ), + ) diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index b3ecd8d028..d9fe732b95 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -14,7 +14,8 @@ ) from fides.api.schemas.api import BulkResponse, BulkUpdateFailed from fides.api.schemas.base_class import FidesSchema -from fides.api.schemas.policy import ActionType, PolicyResponse as PolicySchema +from fides.api.schemas.policy import ActionType +from fides.api.schemas.policy import PolicyResponse as PolicySchema from fides.api.schemas.redis_cache import Identity from fides.api.schemas.user import PrivacyRequestReviewer from fides.api.util.encryption.aes_gcm_encryption_scheme import verify_encryption_key diff --git a/tests/conftest.py b/tests/conftest.py index 50130f1577..e6d75b30b1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -101,8 +101,6 @@ async def async_session(test_client): async_engine.dispose() - - # TODO: THIS IS A HACKY WORKAROUND. # This is specific for this test: test_get_resource_with_custom_field # this was added to account for weird error that only happens during a @@ -413,6 +411,7 @@ def resources_dict(): system_type="SYSTEM", name="Test System", description="Test Policy", + cookies=[], privacy_declarations=[ models.PrivacyDeclaration( name="declaration-name", @@ -421,6 +420,7 @@ def resources_dict(): data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ) ], ), diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index d16c8aa6b5..7912fa6939 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -449,6 +449,7 @@ def system_create_request_body(self) -> SystemSchema: data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[{"name": "essential_cookie"}], ), models.PrivacyDeclaration( name="declaration-name-2", @@ -551,12 +552,22 @@ async def test_system_create( assert result.status_code == HTTP_201_CREATED assert result.json()["name"] == "Test System" + assert result.json()["cookies"] == [{"name": "essential_cookie"}] + assert result.json()["privacy_declarations"][0]["cookies"] == [ + {"name": "essential_cookie"} + ] + assert result.json()["privacy_declarations"][1]["cookies"] == [] assert len(result.json()["privacy_declarations"]) == 2 systems = System.all(db) assert len(systems) == 1 assert systems[0].name == "Test System" assert len(systems[0].privacy_declarations) == 2 + assert [cookie.name for cookie in systems[0].cookies] == ["essential_cookie"] + assert [ + cookie.name for cookie in systems[0].privacy_declarations[0].cookies + ] == ["essential_cookie"] + assert systems[0].privacy_declarations[1].cookies == [] async def test_system_create_custom_metadata_saas_config( self, @@ -734,6 +745,28 @@ def system_update_request_body(self, system) -> SystemSchema: ], ) + @pytest.fixture(scope="function") + def system_update_request_body_with_cookies(self, system) -> SystemSchema: + return SystemSchema( + organization_fides_key=1, + registryId=1, + fides_key=system.fides_key, + system_type="SYSTEM", + name=self.updated_system_name, + description="Test Policy", + privacy_declarations=[ + models.PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="essential", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + cookies=[{"name": "my_cookie"}, {"name": "my_other_cookie"}], + ) + ], + ) + def test_system_update_not_authenticated( self, test_config, system_update_request_body ): @@ -1093,6 +1126,37 @@ def test_system_update_privacy_declaration_invalid_duplicate( and system.privacy_declarations[1].name == "new declaration 1" ) + def test_system_update_privacy_declaration_cookies( + self, + test_config, + system_update_request_body_with_cookies, + system, + db, + generate_system_manager_header, + ): + assert system.name != self.updated_system_name + + auth_header = generate_system_manager_header([system.id]) + result = _api.update( + url=test_config.cli.server_url, + headers=auth_header, + resource_type="system", + json_resource=system_update_request_body_with_cookies.json( + exclude_none=True + ), + ) + assert result.status_code == HTTP_200_OK + assert result.json()["name"] == self.updated_system_name + assert result.json()["cookies"] == [ + {"name": "my_cookie"}, + {"name": "my_other_cookie"}, + ] + + db.refresh(system) + assert system.name == self.updated_system_name + assert len(system.cookies) == 2 + assert len(system.privacy_declarations[0].cookies) == 2 + @pytest.mark.parametrize( "update_declarations", [ @@ -1105,6 +1169,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ) ] ), @@ -1118,6 +1183,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), models.PrivacyDeclaration( name="declaration-name-2", @@ -1126,6 +1192,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), ] ), @@ -1139,6 +1206,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), models.PrivacyDeclaration( name="Collect data for marketing", @@ -1147,6 +1215,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), ] ), @@ -1160,6 +1229,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), models.PrivacyDeclaration( name="declaration-name-2", @@ -1168,6 +1238,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), ] ), diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index 5ac5236d19..33e90d253b 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -1,14 +1,23 @@ # pylint: disable=missing-docstring, redefined-outer-name import os from typing import Generator, List +from uuid import uuid4 import pytest +from fideslang.models import Cookies as CookieSchema +from fideslang.models import PrivacyDeclaration as PrivacyDeclarationSchema from fideslang.models import System, SystemMetadata from py._path.local import LocalPath +from sqlalchemy import delete +from fides.api.api.v1.scope_registry import SYSTEM_DELETE +from fides.api.ctl.database.crud import create_resource +from fides.api.ctl.database.system import create_system, upsert_cookies +from fides.api.ctl.sql_models import Cookies, PrivacyDeclaration from fides.api.ctl.sql_models import System as sql_System from fides.connectors.models import OktaConfig from fides.core import api +from fides.core import api as _api from fides.core import system as _system from fides.core.config import FidesConfig @@ -332,3 +341,149 @@ def test_scan_system_okta_fail(tmpdir: LocalPath, test_config: FidesConfig) -> N url=test_config.cli.server_url, headers=test_config.user.auth_header, ) + + +class TestUpsertCookies: + @pytest.fixture() + async def test_cookie_system( + self, async_session_temp, generate_auth_header, test_config + ): + resource = System( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name="test_system_1", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="declaration-name", + data_categories=[], + data_use="essential", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ), + PrivacyDeclarationSchema( + name="declaration-name-2", + data_categories=[], + data_use="improve", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + cookies=[{"name": "strawberry"}], + ), + ], + ) + + system = await create_system(resource, async_session_temp) + return system + + async def test_new_cookies(self, test_cookie_system, async_session_temp): + """Test adding a new cookie to a privacy declaration. The other privacy declaration on the + system already has a cookie.""" + + new_cookies = [{"name": "apple"}] + privacy_declaration = test_cookie_system.privacy_declarations[0] + + await upsert_cookies( + async_session_temp, + new_cookies, + test_cookie_system.privacy_declarations[0], + test_cookie_system, + ) + await async_session_temp.refresh(test_cookie_system) + assert len(test_cookie_system.cookies) == 2 + + assert {cookie.name for cookie in test_cookie_system.cookies} == { + "strawberry", + "apple", + } + assert len(privacy_declaration.cookies) == 1 + assert privacy_declaration.cookies[0].name == "apple" + + new_cookie = privacy_declaration.cookies[0] + assert new_cookie.created_at is not None + assert new_cookie.updated_at is not None + assert new_cookie.name == "apple" + assert new_cookie.system_id == test_cookie_system.id + assert new_cookie.privacy_declaration_id == privacy_declaration.id + + async def test_no_change_to_cookies(self, test_cookie_system, async_session_temp): + """Test specified cookies already exist on given privacy declaration, so no change required""" + + new_cookies = [{"name": "strawberry"}] + privacy_declaration = test_cookie_system.privacy_declarations[1] + existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] + assert existing_cookie.name == "strawberry" + + await upsert_cookies( + async_session_temp, + new_cookies, + privacy_declaration, + test_cookie_system, + ) + await async_session_temp.refresh(test_cookie_system) + assert len(test_cookie_system.cookies) == 1 + + assert {cookie.name for cookie in test_cookie_system.cookies} == { + "strawberry", + } + assert len(privacy_declaration.cookies) == 1 + assert privacy_declaration.cookies[0].name == "strawberry" + + new_cookie = privacy_declaration.cookies[0] + assert new_cookie.created_at is not None + assert new_cookie.updated_at is not None + assert new_cookie.name == "strawberry" + assert new_cookie.system_id == test_cookie_system.id + assert new_cookie.privacy_declaration_id == privacy_declaration.id + + async def test_update_cookies(self, test_cookie_system, async_session_temp): + """Test cookie list is missing a cookie currently on the privacy declaration so add the new + cookie and we remove the existing one""" + + new_cookies = [{"name": "apple"}] + privacy_declaration = test_cookie_system.privacy_declarations[1] + existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] + assert existing_cookie.name == "strawberry" + + await upsert_cookies( + async_session_temp, + new_cookies, + privacy_declaration, + test_cookie_system, + ) + await async_session_temp.refresh(test_cookie_system) + assert len(test_cookie_system.cookies) == 1 + + assert {cookie.name for cookie in test_cookie_system.cookies} == { + "apple", + } + assert len(privacy_declaration.cookies) == 1 + assert privacy_declaration.cookies[0].name == "apple" + + new_cookie = privacy_declaration.cookies[0] + assert new_cookie.created_at is not None + assert new_cookie.updated_at is not None + assert new_cookie.name == "apple" + assert new_cookie.system_id == test_cookie_system.id + assert new_cookie.privacy_declaration_id == privacy_declaration.id + + async def test_delete_privacy_declaration( + self, test_cookie_system, async_session_temp + ): + """Test if a privacy declaration is deleted, its cookie is still linked to the system""" + + privacy_declaration = test_cookie_system.privacy_declarations[1] + existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] + + assert existing_cookie.privacy_declaration_id == privacy_declaration.id + assert existing_cookie.system_id == test_cookie_system.id + + stmt = delete(PrivacyDeclaration).where( + PrivacyDeclaration.id == privacy_declaration.id + ) + await async_session_temp.execute(stmt) + await async_session_temp.refresh(existing_cookie) + + assert existing_cookie.privacy_declaration_id is None + assert existing_cookie.system_id == test_cookie_system.id From bb4282df21f36ae7f382cae27403ff130e7ea3eb Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 15 Jun 2023 09:23:26 -0500 Subject: [PATCH 02/20] Add optional path and domain to Cookies and allow upsert_cookies to update cookies as well as insert/delete. --- requirements.txt | 2 +- src/fides/api/ctl/database/system.py | 24 ++++++++------ .../versions/2be84e68df32_add_cookie_table.py | 2 ++ src/fides/api/ctl/sql_models.py | 3 ++ tests/ctl/core/test_system.py | 31 +++++++++++++++++++ 5 files changed, 52 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6116ecaa9e..7b76736b06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@ab9a9ea257d7715506b995020d7d5170fe65f41a +fideslang @ git+https://github.com/ethyca/fideslang.git@c519be39265ba5947980d2113b7a3b9675024dc0 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 diff --git a/src/fides/api/ctl/database/system.py b/src/fides/api/ctl/database/system.py index 08059bfc10..069b016cd2 100644 --- a/src/fides/api/ctl/database/system.py +++ b/src/fides/api/ctl/database/system.py @@ -7,7 +7,7 @@ from fideslang.models import Cookies as CookieSchema from fideslang.models import System as SystemSchema from loguru import logger as log -from sqlalchemy import and_, delete, select +from sqlalchemy import and_, delete, select, update from sqlalchemy.dialects.postgresql import Insert from sqlalchemy.engine import ChunkedIteratorResult from sqlalchemy.ext.asyncio import AsyncSession @@ -157,15 +157,21 @@ async def upsert_cookies( result: ChunkedIteratorResult = await async_session.execute(query) row: Optional[Cookies] = result.scalars().first() - if not row: - stmt: Insert = Insert(Cookies).values( - { - "name": cookie_data["name"], - "privacy_declaration_id": privacy_declaration.id, - "system_id": system.id, - } + if row: + await async_session.execute( + update(Cookies).where(Cookies.id == row.id).values(cookie_data) + ) + + else: + await async_session.execute( + Insert(Cookies).values( + { + "name": cookie_data["name"], + "privacy_declaration_id": privacy_declaration.id, + "system_id": system.id, + } + ) ) - await async_session.execute(stmt) missing_cookies_query: Select = select(Cookies).where( and_( diff --git a/src/fides/api/ctl/migrations/versions/2be84e68df32_add_cookie_table.py b/src/fides/api/ctl/migrations/versions/2be84e68df32_add_cookie_table.py index 4ceeb17280..e07fc45e58 100644 --- a/src/fides/api/ctl/migrations/versions/2be84e68df32_add_cookie_table.py +++ b/src/fides/api/ctl/migrations/versions/2be84e68df32_add_cookie_table.py @@ -33,6 +33,8 @@ def upgrade(): nullable=True, ), sa.Column("name", sa.String(), nullable=False), + sa.Column("domain", sa.String(), nullable=True), + sa.Column("path", sa.String(), nullable=True), sa.Column("system_id", sa.String(), nullable=True), sa.Column("privacy_declaration_id", sa.String(), nullable=True), sa.ForeignKeyConstraint( diff --git a/src/fides/api/ctl/sql_models.py b/src/fides/api/ctl/sql_models.py index 1c31ebcc27..ed82c067b0 100644 --- a/src/fides/api/ctl/sql_models.py +++ b/src/fides/api/ctl/sql_models.py @@ -612,6 +612,9 @@ class Cookies(Base): """ name = Column(String, index=True, nullable=False) + path = Column(String, index=True) + domain = Column(String, index=True) + system_id = Column( String, ForeignKey(System.id_field_path, ondelete="CASCADE"), index=True ) diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index 33e90d253b..fb82afc9ef 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -438,6 +438,37 @@ async def test_no_change_to_cookies(self, test_cookie_system, async_session_temp assert new_cookie.privacy_declaration_id == privacy_declaration.id async def test_update_cookies(self, test_cookie_system, async_session_temp): + """Test cookie exists but path has changed""" + """Test specified cookies already exist on given privacy declaration, so no change required""" + + new_cookies = [{"name": "strawberry", "path": "/"}] + privacy_declaration = test_cookie_system.privacy_declarations[1] + existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] + assert existing_cookie.name == "strawberry" + + await upsert_cookies( + async_session_temp, + new_cookies, + privacy_declaration, + test_cookie_system, + ) + await async_session_temp.refresh(test_cookie_system) + assert len(test_cookie_system.cookies) == 1 + assert test_cookie_system.cookies[0].name == "strawberry" + assert test_cookie_system.cookies[0].path == "/" + + assert len(privacy_declaration.cookies) == 1 + assert privacy_declaration.cookies[0].name == "strawberry" + assert privacy_declaration.cookies[0].path == "/" + + new_cookie = privacy_declaration.cookies[0] + assert new_cookie.created_at is not None + assert new_cookie.updated_at is not None + assert new_cookie.name == "strawberry" + assert new_cookie.system_id == test_cookie_system.id + assert new_cookie.privacy_declaration_id == privacy_declaration.id + + async def test_remove_cookies(self, test_cookie_system, async_session_temp): """Test cookie list is missing a cookie currently on the privacy declaration so add the new cookie and we remove the existing one""" From 29e2a2d1ae9566552aceb16167667aa846f3df8a Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 15 Jun 2023 11:15:30 -0500 Subject: [PATCH 03/20] Surface relevant cookies on privacy notices by data use. --- src/fides/api/ctl/database/system.py | 7 +- src/fides/api/models/privacy_notice.py | 28 +++++++- src/fides/api/schemas/privacy_notice.py | 2 + tests/conftest.py | 15 ++++- tests/ctl/core/test_api.py | 15 +++-- .../test_privacy_notice_endpoints.py | 29 +++++++++ tests/ops/models/test_privacy_notice.py | 65 +++++++++++++++++++ 7 files changed, 147 insertions(+), 14 deletions(-) diff --git a/src/fides/api/ctl/database/system.py b/src/fides/api/ctl/database/system.py index 069b016cd2..ee6940566c 100644 --- a/src/fides/api/ctl/database/system.py +++ b/src/fides/api/ctl/database/system.py @@ -9,7 +9,6 @@ from loguru import logger as log from sqlalchemy import and_, delete, select, update from sqlalchemy.dialects.postgresql import Insert -from sqlalchemy.engine import ChunkedIteratorResult from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session from sqlalchemy.sql import Select @@ -154,7 +153,7 @@ async def upsert_cookies( Cookies.privacy_declaration_id == privacy_declaration.id, ) ) - result: ChunkedIteratorResult = await async_session.execute(query) + result = await async_session.execute(query) row: Optional[Cookies] = result.scalars().first() if row: @@ -180,9 +179,7 @@ async def upsert_cookies( Cookies.privacy_declaration_id == privacy_declaration.id, ) ) - delete_result: ChunkedIteratorResult = await async_session.execute( - missing_cookies_query - ) + delete_result = await async_session.execute(missing_cookies_query) rows: List = delete_result.scalars().unique().all() stmt = delete(Cookies).where(Cookies.id.in_([cookie.id for cookie in rows])) diff --git a/src/fides/api/models/privacy_notice.py b/src/fides/api/models/privacy_notice.py index b124032217..661b094bd4 100644 --- a/src/fides/api/models/privacy_notice.py +++ b/src/fides/api/models/privacy_notice.py @@ -8,13 +8,17 @@ from fideslang.validation import FidesKey from sqlalchemy import Boolean, Column from sqlalchemy import Enum as EnumColumn -from sqlalchemy import Float, ForeignKey, String +from sqlalchemy import Float, ForeignKey, String, or_ from sqlalchemy.dialects.postgresql import ARRAY from sqlalchemy.orm import Session, relationship from sqlalchemy.util import hybridproperty from fides.api.common_exceptions import ValidationError -from fides.api.ctl.sql_models import System # type: ignore[attr-defined] +from fides.api.ctl.sql_models import ( # type: ignore[attr-defined] + Cookies, + PrivacyDeclaration, + System, +) from fides.api.db.base_class import Base, FidesBase @@ -255,6 +259,26 @@ def default_preference(self) -> UserConsentPreference: raise Exception("Invalid notice consent mechanism.") + @property + def cookies(self) -> List[Cookies]: + """Return relevant cookie names (via the data use)""" + db = Session.object_session(self) + return ( + db.query(Cookies) + .join( + PrivacyDeclaration, + PrivacyDeclaration.id == Cookies.privacy_declaration_id, + ) + .filter( + or_( + *[ + PrivacyDeclaration.data_use.like(f"{notice_use}%") + for notice_use in self.data_uses + ] + ) + ) + ).all() + @classmethod def create( cls: Type[PrivacyNotice], diff --git a/src/fides/api/schemas/privacy_notice.py b/src/fides/api/schemas/privacy_notice.py index f8e97aef88..ae8a229356 100644 --- a/src/fides/api/schemas/privacy_notice.py +++ b/src/fides/api/schemas/privacy_notice.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional +from fideslang.models import Cookies as CookieSchema from fideslang.validation import FidesKey from pydantic import Extra, conlist, root_validator, validator @@ -140,6 +141,7 @@ class PrivacyNoticeResponse(PrivacyNoticeWithId): updated_at: datetime version: float privacy_notice_history_id: str + cookies: List[CookieSchema] class PrivacyNoticeResponseWithUserPreferences(PrivacyNoticeResponse): diff --git a/tests/conftest.py b/tests/conftest.py index e6d75b30b1..d8196e1c17 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,7 @@ JWE_PAYLOAD_SYSTEMS, ) from fides.api.ctl.database.session import sync_engine -from fides.api.ctl.sql_models import DataUse, PrivacyDeclaration +from fides.api.ctl.sql_models import Cookies, DataUse, PrivacyDeclaration from fides.api.main import app from fides.api.models.privacy_request import generate_request_callback_jwe from fides.api.oauth.jwt import generate_jwe @@ -1020,7 +1020,7 @@ def system(db: Session) -> System: }, ) - PrivacyDeclaration.create( + privacy_declaration = PrivacyDeclaration.create( db=db, data={ "name": "Collect data for marketing", @@ -1035,6 +1035,17 @@ def system(db: Session) -> System: }, ) + Cookies.create( + db=db, + data={ + "name": "test_cookie", + "path": "/", + "privacy_declaration_id": privacy_declaration.id, + "system_id": system.id, + }, + check_name=False, + ) + db.refresh(system) return system diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 7912fa6939..4cac0615e9 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -552,9 +552,11 @@ async def test_system_create( assert result.status_code == HTTP_201_CREATED assert result.json()["name"] == "Test System" - assert result.json()["cookies"] == [{"name": "essential_cookie"}] + assert result.json()["cookies"] == [ + {"name": "essential_cookie", "path": None, "domain": None} + ] assert result.json()["privacy_declarations"][0]["cookies"] == [ - {"name": "essential_cookie"} + {"name": "essential_cookie", "path": None, "domain": None} ] assert result.json()["privacy_declarations"][1]["cookies"] == [] assert len(result.json()["privacy_declarations"]) == 2 @@ -1148,13 +1150,16 @@ def test_system_update_privacy_declaration_cookies( assert result.status_code == HTTP_200_OK assert result.json()["name"] == self.updated_system_name assert result.json()["cookies"] == [ - {"name": "my_cookie"}, - {"name": "my_other_cookie"}, + {"name": "my_cookie", "path": None, "domain": None}, + {"name": "my_other_cookie", "path": None, "domain": None}, + {"name": "test_cookie", "path": "/", "domain": None}, ] db.refresh(system) assert system.name == self.updated_system_name - assert len(system.cookies) == 2 + assert ( + len(system.cookies) == 3 + ) # Two from the current privacy declaration, one from the previous privacy declaration that was deleted, but still linked to the system assert len(system.privacy_declarations[0].cookies) == 2 @pytest.mark.parametrize( diff --git a/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py index 027ce3a2e8..f9e62af13a 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py @@ -5,6 +5,7 @@ from uuid import uuid4 import pytest +from fideslang.models import Cookies as CookieSchema from sqlalchemy.orm import Session from starlette.testclient import TestClient @@ -87,6 +88,7 @@ def test_get_privacy_notices_defaults( assert "created_at" in notice_detail assert "updated_at" in notice_detail assert "name" in notice_detail + assert "name" in notice_detail assert "description" in notice_detail assert "regions" in notice_detail assert "consent_mechanism" in notice_detail @@ -756,6 +758,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ) ], }, @@ -825,6 +830,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-2", @@ -845,6 +853,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), ], }, @@ -912,6 +923,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( version=1.0, privacy_notice_history_id="placeholder_id", displayed_in_overlay=True, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-2", @@ -930,6 +944,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( version=1.0, privacy_notice_history_id="placeholder_id", displayed_in_overlay=True, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), ], "third_party_sharing": [ @@ -951,6 +968,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( version=1.0, privacy_notice_history_id="placeholder_id", displayed_in_overlay=True, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), ], }, @@ -1020,6 +1040,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), ], "third_party_sharing": [], @@ -1154,6 +1177,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ) ], "essential.service.operations.support.optimization": [ @@ -1178,6 +1204,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[], ), PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-3", @@ -1198,6 +1225,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[], ), PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-2", @@ -1218,6 +1246,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[], ), ], }, diff --git a/tests/ops/models/test_privacy_notice.py b/tests/ops/models/test_privacy_notice.py index 6c519f4ced..d8b52ae5bf 100644 --- a/tests/ops/models/test_privacy_notice.py +++ b/tests/ops/models/test_privacy_notice.py @@ -1,8 +1,10 @@ import pytest +from fideslang.models import Cookies as CookieSchema from fideslang.validation import FidesValidationError from sqlalchemy.orm import Session from fides.api.common_exceptions import ValidationError +from fides.api.ctl.sql_models import Cookies from fides.api.models.privacy_notice import ( ConsentMechanism, PrivacyNotice, @@ -697,6 +699,69 @@ def test_conflicting_data_uses( existing_privacy_notices=existing_privacy_notices, ) + @pytest.mark.parametrize( + "privacy_notice_data_use,declaration_cookies,expected_cookies,description", + [ + ( + ["marketing.advertising", "third_party_sharing"], + [{"name": "test_cookie"}], + [CookieSchema(name="test_cookie")], + "Data uses overlap exactly", + ), + ( + ["marketing.advertising.first_party", "third_party_sharing"], + [{"name": "test_cookie"}], + [], + "Privacy notice use more specific than system's. Too big a leap to assume system should be adjusted here.", + ), + ( + ["marketing", "third_party_sharing"], + [{"name": "test_cookie"}], + [CookieSchema(name="test_cookie")], + "Privacy notice use more general than system's, so system's data use is under the scope of the notice", + ), + ( + ["marketing.advertising", "third_party_sharing"], + [{"name": "test_cookie"}, {"name": "another_cookie"}], + [CookieSchema(name="test_cookie"), CookieSchema(name="another_cookie")], + "Test multiple cookies", + ), + (["marketing.advertising"], [], [], "No cookies returns an empty set"), + ], + ) + def test_relevant_cookies( + self, + privacy_notice_data_use, + declaration_cookies, + expected_cookies, + description, + privacy_notice, + db, + system, + ): + """Test different combinations of data uses and cookies between the Privacy Notice and the Privacy Declaration""" + db.query(Cookies).delete() + privacy_notice.data_uses = privacy_notice_data_use + privacy_notice.save(db) + + privacy_declaration = system.privacy_declarations[0] + assert privacy_declaration.data_use == "marketing.advertising" + + for cookie in declaration_cookies: + Cookies.create( + db, + data={ + "name": cookie["name"], + "privacy_declaration_id": privacy_declaration.id, + "system_id": system.id, + }, + check_name=False, + ) + + assert [ + CookieSchema.from_orm(cookie) for cookie in privacy_notice.cookies + ] == expected_cookies, description + def test_calculate_relevant_systems( self, db, From f4a693644484f9f86ae7a8fa0bdea3313020f4e1 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 15 Jun 2023 13:56:27 -0500 Subject: [PATCH 04/20] Update fideslang version which removes cookies from System request, as this can't be specified on the system directly currently. - Fix bug where path/domain aren't being saved on create. - Add system endpoints to postman collection with new cookie fields. - Add database annotations. --- .fides/db_dataset.yml | 37 +++++ .../postman/Fides.postman_collection.json | 139 ++++++++++++++++++ requirements.txt | 2 +- src/fides/api/ctl/database/system.py | 5 +- tests/ctl/core/test_api.py | 19 ++- 5 files changed, 193 insertions(+), 9 deletions(-) diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 001fabf84a..a73b4c3261 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -2240,4 +2240,41 @@ dataset: description: 'The name of the organization this Fides deployment belongs to' data_categories: - user.workplace + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: cookies + description: 'Fides Generated Description for Table: cookies' + data_categories: [] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: created_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: domain + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: name + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: path + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_declaration_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: system_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + data_categories: + - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified \ No newline at end of file diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index 27173fd320..f4a7c94c1e 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -4922,6 +4922,145 @@ } ] }, + { + "name": "Systems", + "item": [ + { + "name": "Get System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/system/", + "host": [ + "{{host}}" + ], + "path": [ + "system", + "" + ] + } + }, + "response": [] + }, + { + "name": "Create System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data_responsibility_title\":\"Processor\",\n \"description\":\"Collect data about our users for marketing.\",\n \"egress\":[\n {\n \"fides_key\":\"demo_analytics_system\",\n \"type\":\"system\",\n \"data_categories\":null\n }\n ],\n \"fides_key\":\"test_system\",\n \"ingress\":null,\n \"name\":\"Test system\",\n \"organization_fides_key\":\"default_organization\",\n \"privacy_declarations\":[\n {\n \"name\":\"Collect data for marketing\",\n \"data_categories\":[\n \"user.device.cookie_id\"\n ],\n \"data_use\":\"personalize\",\n \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"data_subjects\":[\n \"customer\"\n ],\n \"dataset_references\":null,\n \"egress\":null,\n \"ingress\":null,\n \"cookies\":[\n {\n \"name\":\"test_cookie\",\n \"path\":\"/\"\n }\n ]\n }\n ],\n \"system_dependencies\":[\n \"demo_analytics_system\"\n ],\n \"system_type\":\"Service\",\n \"tags\":null,\n \"third_country_transfers\":null,\n \"administrating_department\":\"Marketing\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/system", + "host": [ + "{{host}}" + ], + "path": [ + "system" + ] + } + }, + "response": [] + }, + { + "name": "Update System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data_responsibility_title\":\"Processor\",\n \"description\":\"Collect data about our users for marketing.\",\n \"egress\":[\n {\n \"fides_key\":\"demo_analytics_system\",\n \"type\":\"system\",\n \"data_categories\":null\n }\n ],\n \"fides_key\":\"test_system\",\n \"ingress\":null,\n \"name\":\"Test system\",\n \"organization_fides_key\":\"default_organization\",\n \"privacy_declarations\":[\n {\n \"name\":\"Collect data for marketing\",\n \"data_categories\":[\n \"user.device.cookie_id\"\n ],\n \"data_use\":\"marketing.advertising\",\n \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"data_subjects\":[\n \"customer\"\n ],\n \"dataset_references\":null,\n \"egress\":null,\n \"ingress\":null,\n \"cookies\":[\n {\n \"name\":\"another_cookie\",\n \"path\":\"/\"\n }\n ]\n }\n ],\n \"system_dependencies\":[\n \"demo_analytics_system\"\n ],\n \"system_type\":\"Service\",\n \"tags\":null,\n \"third_country_transfers\":null,\n \"administrating_department\":\"Marketing\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/system?", + "host": [ + "{{host}}" + ], + "path": [ + "system" + ], + "query": [ + { + "key": "", + "value": null + } + ] + } + }, + "response": [] + }, + { + "name": "Delete System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "{{host}}/system/test_system", + "host": [ + "{{host}}" + ], + "path": [ + "system", + "test_system" + ] + } + }, + "response": [] + } + ] + }, { "name": "Roles", "item": [ diff --git a/requirements.txt b/requirements.txt index 7b76736b06..5a5355bbe5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@c519be39265ba5947980d2113b7a3b9675024dc0 +fideslang @ git+https://github.com/ethyca/fideslang.git@6e93ab4ea7d35713201d48b3314b8d25df0815cb fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 diff --git a/src/fides/api/ctl/database/system.py b/src/fides/api/ctl/database/system.py index ee6940566c..25bd9b89bb 100644 --- a/src/fides/api/ctl/database/system.py +++ b/src/fides/api/ctl/database/system.py @@ -166,6 +166,8 @@ async def upsert_cookies( Insert(Cookies).values( { "name": cookie_data["name"], + "path": cookie_data["path"], + "domain": cookie_data["domain"], "privacy_declaration_id": privacy_declaration.id, "system_id": system.id, } @@ -204,7 +206,6 @@ async def update_system(resource: SystemSchema, db: AsyncSession) -> Dict: delattr( resource, "privacy_declarations" ) # remove the attribute on the system since we've already updated declarations - delattr(resource, "cookies") # remove the cookies attribute # perform any updates on the system resource itself updated_system = await update_resource(System, resource.dict(), db) @@ -228,8 +229,6 @@ async def create_system( # remove the attribute on the system update since the declarations will be created separately delattr(resource, "privacy_declarations") - # Removing cookies attribute; they can't be set on the system directly - delattr(resource, "cookies") # create the system resource using generic creation # the system must be created before the privacy declarations so that it can be referenced diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 4cac0615e9..db943874eb 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -449,7 +449,13 @@ def system_create_request_body(self) -> SystemSchema: data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], - cookies=[{"name": "essential_cookie"}], + cookies=[ + { + "name": "essential_cookie", + "path": "/", + "domain": "example.com", + } + ], ), models.PrivacyDeclaration( name="declaration-name-2", @@ -553,10 +559,10 @@ async def test_system_create( assert result.status_code == HTTP_201_CREATED assert result.json()["name"] == "Test System" assert result.json()["cookies"] == [ - {"name": "essential_cookie", "path": None, "domain": None} + {"name": "essential_cookie", "path": "/", "domain": "example.com"} ] assert result.json()["privacy_declarations"][0]["cookies"] == [ - {"name": "essential_cookie", "path": None, "domain": None} + {"name": "essential_cookie", "path": "/", "domain": "example.com"} ] assert result.json()["privacy_declarations"][1]["cookies"] == [] assert len(result.json()["privacy_declarations"]) == 2 @@ -764,7 +770,10 @@ def system_update_request_body_with_cookies(self, system) -> SystemSchema: data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], - cookies=[{"name": "my_cookie"}, {"name": "my_other_cookie"}], + cookies=[ + {"name": "my_cookie", "domain": "example.com"}, + {"name": "my_other_cookie"}, + ], ) ], ) @@ -1150,7 +1159,7 @@ def test_system_update_privacy_declaration_cookies( assert result.status_code == HTTP_200_OK assert result.json()["name"] == self.updated_system_name assert result.json()["cookies"] == [ - {"name": "my_cookie", "path": None, "domain": None}, + {"name": "my_cookie", "path": None, "domain": "example.com"}, {"name": "my_other_cookie", "path": None, "domain": None}, {"name": "test_cookie", "path": "/", "domain": None}, ] From f5cca575108d8dd1798cc47e885f1af45907599b Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 15 Jun 2023 14:16:45 -0500 Subject: [PATCH 05/20] Remove the index from path and domain. --- src/fides/api/ctl/database/system.py | 57 ++++++++++++++++------------ src/fides/api/ctl/sql_models.py | 11 +++--- 2 files changed, 39 insertions(+), 29 deletions(-) diff --git a/src/fides/api/ctl/database/system.py b/src/fides/api/ctl/database/system.py index 25bd9b89bb..57dcf13ea8 100644 --- a/src/fides/api/ctl/database/system.py +++ b/src/fides/api/ctl/database/system.py @@ -7,11 +7,9 @@ from fideslang.models import Cookies as CookieSchema from fideslang.models import System as SystemSchema from loguru import logger as log -from sqlalchemy import and_, delete, select, update -from sqlalchemy.dialects.postgresql import Insert +from sqlalchemy import and_, delete, insert, select, update from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session -from sqlalchemy.sql import Select from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND from fides.api.ctl.database.crud import create_resource, get_resource, update_resource @@ -130,6 +128,7 @@ async def upsert_privacy_declarations( # otherwise, create a new declaration record declaration = PrivacyDeclaration.create(db, data=data) + # Upsert cookies for the given privacy declaration await upsert_cookies(db, cookies, declaration, system) # delete any existing privacy declarations that have not been "matched" in the request @@ -143,19 +142,22 @@ async def upsert_cookies( privacy_declaration: PrivacyDeclaration, system: System, ) -> None: - """Upsert cookies for the given privacy declaration""" + """Upsert cookies for the given privacy declaration: retrieve cookies by name/system/privacy declaration + Remove any existing cookies that aren't specified here. + """ cookie_list: List[CookieSchema] = cookies or [] for cookie_data in cookie_list: - query: Select = select(Cookies).where( - and_( - Cookies.name == cookie_data["name"], - Cookies.system_id == system.id, - Cookies.privacy_declaration_id == privacy_declaration.id, + # Check if cookie exists for this name/system/privacy declaration + result = await async_session.execute( + select(Cookies).where( + and_( + Cookies.name == cookie_data["name"], + Cookies.system_id == system.id, + Cookies.privacy_declaration_id == privacy_declaration.id, + ) ) ) - result = await async_session.execute(query) row: Optional[Cookies] = result.scalars().first() - if row: await async_session.execute( update(Cookies).where(Cookies.id == row.id).values(cookie_data) @@ -163,29 +165,36 @@ async def upsert_cookies( else: await async_session.execute( - Insert(Cookies).values( + insert(Cookies).values( { - "name": cookie_data["name"], - "path": cookie_data["path"], - "domain": cookie_data["domain"], + "name": cookie_data.get("name"), + "path": cookie_data.get("path"), + "domain": cookie_data.get("domain"), "privacy_declaration_id": privacy_declaration.id, "system_id": system.id, } ) ) - missing_cookies_query: Select = select(Cookies).where( - and_( - Cookies.name.notin_([cookie["name"] for cookie in cookie_list]), - Cookies.system_id == system.id, - Cookies.privacy_declaration_id == privacy_declaration.id, + # Select cookies which are currently on the privacy declaration but not included in this request + delete_result = await async_session.execute( + select(Cookies).where( + and_( + Cookies.name.notin_([cookie["name"] for cookie in cookie_list]), + Cookies.system_id == system.id, + Cookies.privacy_declaration_id == privacy_declaration.id, + ) ) ) - delete_result = await async_session.execute(missing_cookies_query) - rows: List = delete_result.scalars().unique().all() - stmt = delete(Cookies).where(Cookies.id.in_([cookie.id for cookie in rows])) - await async_session.execute(stmt) + # Remove those cookies altogether + await async_session.execute( + delete(Cookies).where( + Cookies.id.in_( + [cookie.id for cookie in delete_result.scalars().unique().all()] + ) + ) + ) async def update_system(resource: SystemSchema, db: AsyncSession) -> Dict: diff --git a/src/fides/api/ctl/sql_models.py b/src/fides/api/ctl/sql_models.py index ed82c067b0..5a90097b8b 100644 --- a/src/fides/api/ctl/sql_models.py +++ b/src/fides/api/ctl/sql_models.py @@ -612,17 +612,18 @@ class Cookies(Base): """ name = Column(String, index=True, nullable=False) - path = Column(String, index=True) - domain = Column(String, index=True) + path = Column(String) + domain = Column(String) system_id = Column( String, ForeignKey(System.id_field_path, ondelete="CASCADE"), index=True - ) + ) # If system is deleted, remove the associated cookies. + privacy_declaration_id = Column( String, ForeignKey(PrivacyDeclaration.id_field_path, ondelete="SET NULL"), index=True, - ) + ) # If privacy declaration is deleted, just set to null and still keep this connected to the system. system = relationship( "System", @@ -636,7 +637,7 @@ class Cookies(Base): "PrivacyDeclaration", back_populates="cookies", uselist=False, - lazy="joined", + lazy="joined", # Joined is intentional, instead of selectin ) __table_args__ = ( From 561290eb547d59740171d7d6df572a2f2a56f11b Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 16 Jun 2023 13:17:45 -0500 Subject: [PATCH 06/20] Try to make tests more predictable. --- .../api/v1/endpoints/connection_endpoints.py | 4 +-- .../api/api/v1/endpoints/dataset_endpoints.py | 4 +-- src/fides/api/models/datasetconfig.py | 4 +-- src/fides/api/util/data_category.py | 4 +-- tests/ctl/core/test_system.py | 30 ++++++++++++++----- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/connection_endpoints.py b/src/fides/api/api/v1/endpoints/connection_endpoints.py index 07b482cbf9..a39459ee2b 100644 --- a/src/fides/api/api/v1/endpoints/connection_endpoints.py +++ b/src/fides/api/api/v1/endpoints/connection_endpoints.py @@ -35,8 +35,8 @@ ConnectionType, ) from fides.api.models.datasetconfig import DatasetConfig -from fides.api.models.sql_models import ( # type: ignore[attr-defined] - Dataset as CtlDataset, +from fides.api.models.sql_models import ( + Dataset as CtlDataset, # type: ignore[attr-defined] ) from fides.api.oauth.utils import verify_oauth_client from fides.api.schemas.connection_configuration import connection_secrets_schemas diff --git a/src/fides/api/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/api/v1/endpoints/dataset_endpoints.py index 8355c676a7..439d6f09fe 100644 --- a/src/fides/api/api/v1/endpoints/dataset_endpoints.py +++ b/src/fides/api/api/v1/endpoints/dataset_endpoints.py @@ -52,8 +52,8 @@ convert_dataset_to_graph, to_graph_field, ) -from fides.api.models.sql_models import ( # type: ignore[attr-defined] - Dataset as CtlDataset, +from fides.api.models.sql_models import ( + Dataset as CtlDataset, # type: ignore[attr-defined] ) from fides.api.oauth.utils import verify_oauth_client from fides.api.schemas.api import BulkUpdateFailed diff --git a/src/fides/api/models/datasetconfig.py b/src/fides/api/models/datasetconfig.py index 286cebd70e..aa664ba835 100644 --- a/src/fides/api/models/datasetconfig.py +++ b/src/fides/api/models/datasetconfig.py @@ -19,8 +19,8 @@ ) from fides.api.graph.data_type import parse_data_type_string from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType -from fides.api.models.sql_models import ( # type: ignore[attr-defined] - Dataset as CtlDataset, +from fides.api.models.sql_models import ( + Dataset as CtlDataset, # type: ignore[attr-defined] ) from fides.api.util.saas_util import merge_datasets diff --git a/src/fides/api/util/data_category.py b/src/fides/api/util/data_category.py index ccb0e60670..8dfbd492b0 100644 --- a/src/fides/api/util/data_category.py +++ b/src/fides/api/util/data_category.py @@ -6,8 +6,8 @@ from sqlalchemy.orm import Session from fides.api import common_exceptions -from fides.api.models.sql_models import ( # type: ignore[attr-defined] - DataCategory as DataCategoryDbModel, +from fides.api.models.sql_models import ( + DataCategory as DataCategoryDbModel, # type: ignore[attr-defined] ) diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index c17950f896..c5e5e9b4c6 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -4,17 +4,18 @@ from uuid import uuid4 import pytest -from fideslang.models import Cookies as CookieSchema from fideslang.models import PrivacyDeclaration as PrivacyDeclarationSchema from fideslang.models import System, SystemMetadata from py._path.local import LocalPath from sqlalchemy import delete from fides.api.db.system import create_system, upsert_cookies -from fides.api.models.sql_models import PrivacyDeclaration +from fides.api.models.sql_models import Cookies, PrivacyDeclaration from fides.api.models.sql_models import System as sql_System +from fides.api.oauth.roles import OWNER from fides.connectors.models import OktaConfig from fides.core import api +from fides.core import api as _api from fides.core import system as _system from fides.core.config import FidesConfig @@ -343,7 +344,11 @@ def test_scan_system_okta_fail(tmpdir: LocalPath, test_config: FidesConfig) -> N class TestUpsertCookies: @pytest.fixture() async def test_cookie_system( - self, async_session_temp, generate_auth_header, test_config + self, + async_session_temp, + generate_auth_header, + test_config, + generate_role_header, ): resource = System( fides_key=str(uuid4()), @@ -366,13 +371,25 @@ async def test_cookie_system( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], - cookies=[{"name": "strawberry"}], ), ], ) system = await create_system(resource, async_session_temp) - return system + + Cookies.create( + db=db, + data={ + "name": "strawberry", + "path": "/", + "privacy_declaration_id": system.privacy_declarations[1].id, + "system_id": system.id, + }, + check_name=False, + ) + await async_session_temp.refresh(system) + yield system + delete(sql_System).where(sql_System.id == system.id) async def test_new_cookies(self, test_cookie_system, async_session_temp): """Test adding a new cookie to a privacy declaration. The other privacy declaration on the @@ -406,7 +423,6 @@ async def test_new_cookies(self, test_cookie_system, async_session_temp): async def test_no_change_to_cookies(self, test_cookie_system, async_session_temp): """Test specified cookies already exist on given privacy declaration, so no change required""" - new_cookies = [{"name": "strawberry"}] privacy_declaration = test_cookie_system.privacy_declarations[1] existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] @@ -502,7 +518,7 @@ async def test_delete_privacy_declaration( """Test if a privacy declaration is deleted, its cookie is still linked to the system""" privacy_declaration = test_cookie_system.privacy_declarations[1] - existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] + existing_cookie = privacy_declaration.cookies[0] assert existing_cookie.privacy_declaration_id == privacy_declaration.id assert existing_cookie.system_id == test_cookie_system.id From df5a4625b8b06edbde4a92f33f164ccdbcb86b8c Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Fri, 16 Jun 2023 13:19:43 -0500 Subject: [PATCH 07/20] Update changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07b0d6a7a4..a603270a4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The types of changes are: - Included optional env vars to have postgres or Redshift connected via bastion host [#3374](https://github.com/ethyca/fides/pull/3374/) - Support for acknowledge button for notice-only Privacy Notices and to disable toggling them off [#3546](https://github.com/ethyca/fides/pull/3546) - HTML format for privacy request storage destinations [#3427](https://github.com/ethyca/fides/pull/3427) +- New Cookies Table for storing cookies associated with systems and privacy declarations [#3572](https://github.com/ethyca/fides/pull/3572) ### Changed From 04ce44a36bdf0956f1b54a1bcb50749ed1a67248 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Mon, 19 Jun 2023 09:05:21 -0500 Subject: [PATCH 08/20] Add missing fixture. --- requirements.txt | 2 +- tests/ctl/core/test_system.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c265fb286f..f7f046189d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@6e93ab4ea7d35713201d48b3314b8d25df0815cb +fideslang @ git+https://github.com/ethyca/fideslang.git@b66cd3555d8d18ec0f81b58a933f5e326cbf9b3b fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index c5e5e9b4c6..f095b5469c 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -349,6 +349,7 @@ async def test_cookie_system( generate_auth_header, test_config, generate_role_header, + db, ): resource = System( fides_key=str(uuid4()), From 9911bf258dc84dec31636b58185a3afe45192880 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 20 Jun 2023 11:03:06 -0500 Subject: [PATCH 09/20] Bump fides lang commit to see if organization relationship key finding has been fixed. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 178e300a69..780a6cf1b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@b66cd3555d8d18ec0f81b58a933f5e326cbf9b3b +fideslang @ git+https://github.com/ethyca/fideslang.git@ae94fe4fc0b1d2904fdbf98f4ecc32309d52829f fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 From ba642047cafe1b668f1548cfb22dc8ff4da32b57 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 20 Jun 2023 11:21:03 -0500 Subject: [PATCH 10/20] Make history tests more reliable - there's no guarantee that these are in the correct order. --- ...est_privacy_experience_config_endpoints.py | 21 ++++++++++++++----- tests/ops/models/test_privacy_experience.py | 6 +++--- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py index 5ea07046cb..708a0447c0 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py @@ -14,6 +14,7 @@ ComponentType, PrivacyExperience, PrivacyExperienceConfig, + PrivacyExperienceConfigHistory, ) from fides.api.models.privacy_notice import PrivacyNoticeRegion @@ -1167,14 +1168,18 @@ def test_update_experience_config_while_ignoring_regions( experience_config = get_experience_config_or_error(db, resp["id"]) assert experience_config.experiences.all() == [privacy_experience_overlay] assert experience_config.histories.count() == 2 - history = experience_config.histories[0] + history = experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[0] assert history.version == 1.0 assert history.component == ComponentType.overlay assert history.banner_enabled == BannerEnabled.enabled_where_required assert history.experience_config_id == experience_config.id assert history.disabled is False - history = experience_config.histories[1] + history = experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[1] assert history.version == 2.0 assert history.disabled is True @@ -1224,14 +1229,18 @@ def test_update_experience_config_with_no_regions( experience_config = get_experience_config_or_error(db, resp["id"]) assert experience_config.experiences.all() == [] assert experience_config.histories.count() == 2 - history = experience_config.histories[0] + history = experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[0] assert history.version == 1.0 assert history.component == ComponentType.overlay assert history.banner_enabled == BannerEnabled.enabled_where_required assert history.experience_config_id == experience_config.id assert history.disabled is False - history = experience_config.histories[1] + history = experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[1] assert history.version == 2.0 assert history.disabled is True @@ -1448,7 +1457,9 @@ def test_update_experience_config_experience_also_updated( db.refresh(overlay_experience_config) # ExperienceConfig was disabled - this is a change, so another historical record is created assert overlay_experience_config.histories.count() == 2 - experience_config_history = overlay_experience_config.histories[1] + experience_config_history = overlay_experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[1] assert experience_config_history.version == 2.0 assert experience_config_history.disabled assert experience_config_history.component == ComponentType.overlay diff --git a/tests/ops/models/test_privacy_experience.py b/tests/ops/models/test_privacy_experience.py index 78e3e87dcd..da5329233d 100644 --- a/tests/ops/models/test_privacy_experience.py +++ b/tests/ops/models/test_privacy_experience.py @@ -8,7 +8,7 @@ PrivacyExperience, PrivacyExperienceConfig, upsert_privacy_experiences_after_config_update, - upsert_privacy_experiences_after_notice_update, + upsert_privacy_experiences_after_notice_update, PrivacyExperienceConfigHistory, ) from fides.api.models.privacy_notice import ( ConsentMechanism, @@ -123,11 +123,11 @@ def test_update_privacy_experience_config(self, db): assert config.updated_at > config_updated_at assert config.histories.count() == 2 - history = config.histories[1] + history = config.histories.order_by(PrivacyExperienceConfigHistory.created_at)[1] assert history.component == ComponentType.privacy_center assert config.experience_config_history_id == history.id - old_history = config.histories[0] + old_history = config.histories.order_by(PrivacyExperienceConfigHistory.created_at)[0] assert old_history.version == 1.0 assert old_history.component == ComponentType.overlay From 47010a90266486e2040a49ef0d05271c9813aa1f Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 20 Jun 2023 11:24:15 -0500 Subject: [PATCH 11/20] Bump fideslang commit --- requirements.txt | 2 +- .../api/api/v1/endpoints/connection_endpoints.py | 4 ++-- src/fides/api/api/v1/endpoints/dataset_endpoints.py | 4 +++- src/fides/api/models/datasetconfig.py | 5 +++-- src/fides/api/util/data_category.py | 4 +++- tests/ops/api/v1/endpoints/test_dataset_endpoints.py | 4 +--- tests/ops/models/test_privacy_experience.py | 11 ++++++++--- 7 files changed, 21 insertions(+), 13 deletions(-) diff --git a/requirements.txt b/requirements.txt index 780a6cf1b5..3929478481 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@ae94fe4fc0b1d2904fdbf98f4ecc32309d52829f +fideslang @ git+https://github.com/ethyca/fideslang.git@63856c40fb5f106f86a9645c4c45ffde844d3a99 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 diff --git a/src/fides/api/api/v1/endpoints/connection_endpoints.py b/src/fides/api/api/v1/endpoints/connection_endpoints.py index 07b482cbf9..a39459ee2b 100644 --- a/src/fides/api/api/v1/endpoints/connection_endpoints.py +++ b/src/fides/api/api/v1/endpoints/connection_endpoints.py @@ -35,8 +35,8 @@ ConnectionType, ) from fides.api.models.datasetconfig import DatasetConfig -from fides.api.models.sql_models import ( # type: ignore[attr-defined] - Dataset as CtlDataset, +from fides.api.models.sql_models import ( + Dataset as CtlDataset, # type: ignore[attr-defined] ) from fides.api.oauth.utils import verify_oauth_client from fides.api.schemas.connection_configuration import connection_secrets_schemas diff --git a/src/fides/api/api/v1/endpoints/dataset_endpoints.py b/src/fides/api/api/v1/endpoints/dataset_endpoints.py index 6421e8cd7c..439d6f09fe 100644 --- a/src/fides/api/api/v1/endpoints/dataset_endpoints.py +++ b/src/fides/api/api/v1/endpoints/dataset_endpoints.py @@ -52,7 +52,9 @@ convert_dataset_to_graph, to_graph_field, ) -from fides.api.models.sql_models import Dataset as CtlDataset # type: ignore[attr-defined] +from fides.api.models.sql_models import ( + Dataset as CtlDataset, # type: ignore[attr-defined] +) from fides.api.oauth.utils import verify_oauth_client from fides.api.schemas.api import BulkUpdateFailed from fides.api.schemas.dataset import ( diff --git a/src/fides/api/models/datasetconfig.py b/src/fides/api/models/datasetconfig.py index 42cfd0aec0..aa664ba835 100644 --- a/src/fides/api/models/datasetconfig.py +++ b/src/fides/api/models/datasetconfig.py @@ -19,8 +19,9 @@ ) from fides.api.graph.data_type import parse_data_type_string from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType -from fides.api.models.sql_models import Dataset as CtlDataset # type: ignore[attr-defined] - +from fides.api.models.sql_models import ( + Dataset as CtlDataset, # type: ignore[attr-defined] +) from fides.api.util.saas_util import merge_datasets diff --git a/src/fides/api/util/data_category.py b/src/fides/api/util/data_category.py index ae2e8d5757..8dfbd492b0 100644 --- a/src/fides/api/util/data_category.py +++ b/src/fides/api/util/data_category.py @@ -6,7 +6,9 @@ from sqlalchemy.orm import Session from fides.api import common_exceptions -from fides.api.models.sql_models import DataCategory as DataCategoryDbModel # type: ignore[attr-defined] +from fides.api.models.sql_models import ( + DataCategory as DataCategoryDbModel, # type: ignore[attr-defined] +) def generate_fides_data_categories() -> Type[EnumType]: diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index 2348a1d374..3fee8de0e5 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -19,7 +19,6 @@ DATASET_CREATE_OR_UPDATE, DATASET_DELETE, DATASET_READ, - CTL_DATASET_READ, ) from fides.api.api.v1.urn_registry import ( CONNECTION_DATASETS, @@ -27,10 +26,9 @@ DATASET_CONFIGS, DATASET_VALIDATE, DATASETCONFIG_BY_KEY, - CONNECTION_DATASETS, + DATASETS, V1_URL_PREFIX, YAML_DATASETS, - DATASETS, ) from fides.api.models.connectionconfig import ConnectionConfig from fides.api.models.datasetconfig import DatasetConfig diff --git a/tests/ops/models/test_privacy_experience.py b/tests/ops/models/test_privacy_experience.py index da5329233d..27f6ee539a 100644 --- a/tests/ops/models/test_privacy_experience.py +++ b/tests/ops/models/test_privacy_experience.py @@ -7,8 +7,9 @@ ComponentType, PrivacyExperience, PrivacyExperienceConfig, + PrivacyExperienceConfigHistory, upsert_privacy_experiences_after_config_update, - upsert_privacy_experiences_after_notice_update, PrivacyExperienceConfigHistory, + upsert_privacy_experiences_after_notice_update, ) from fides.api.models.privacy_notice import ( ConsentMechanism, @@ -123,11 +124,15 @@ def test_update_privacy_experience_config(self, db): assert config.updated_at > config_updated_at assert config.histories.count() == 2 - history = config.histories.order_by(PrivacyExperienceConfigHistory.created_at)[1] + history = config.histories.order_by(PrivacyExperienceConfigHistory.created_at)[ + 1 + ] assert history.component == ComponentType.privacy_center assert config.experience_config_history_id == history.id - old_history = config.histories.order_by(PrivacyExperienceConfigHistory.created_at)[0] + old_history = config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[0] assert old_history.version == 1.0 assert old_history.component == ComponentType.overlay From 3ee9f3a7c5abac690b4e3bfb8ac9579d30b4a801 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 21 Jun 2023 17:42:17 +0800 Subject: [PATCH 12/20] fix: bump fideslang version for testing --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3929478481..0d3ba2895c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@63856c40fb5f106f86a9645c4c45ffde844d3a99 +fideslang @ git+https://github.com/ethyca/fideslang.git@6f9d7a60ccccf46be2fc92d2a001e4c47c8adba7 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 From 805393e0450e0adba99cac63b408e573bbdfcc43 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 21 Jun 2023 18:34:51 +0800 Subject: [PATCH 13/20] fix: pin pydantic to a new version supported by fideslang --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d3ba2895c..0d3105e8ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ passlib[bcrypt]==1.7.4 plotly==5.13.1 pyarrow==6.0.0 psycopg2-binary==2.9.6 -pydantic<1.10.2 +pydantic==1.10.9 pydash==6.0.2 PyJWT==2.4.0 pymongo==3.13.0 From f59709407e4bd9f1a27d66599f398573f24e4b35 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 21 Jun 2023 09:19:35 -0500 Subject: [PATCH 14/20] Try sorting declarations for repeatability in tests. --- tests/ctl/core/test_system.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index 904f27e45a..3a67e1f911 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -395,7 +395,7 @@ async def test_new_cookies(self, test_cookie_system, async_session_temp): system already has a cookie.""" new_cookies = [{"name": "apple"}] - privacy_declaration = test_cookie_system.privacy_declarations[0] + privacy_declaration = sorted(test_cookie_system.privacy_declarations, key=lambda x: x.name)[0] await upsert_cookies( async_session_temp, @@ -423,8 +423,8 @@ async def test_new_cookies(self, test_cookie_system, async_session_temp): async def test_no_change_to_cookies(self, test_cookie_system, async_session_temp): """Test specified cookies already exist on given privacy declaration, so no change required""" new_cookies = [{"name": "strawberry"}] - privacy_declaration = test_cookie_system.privacy_declarations[1] - existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] + privacy_declaration = sorted(test_cookie_system.privacy_declarations, key=lambda x: x.name)[1] + existing_cookie = privacy_declaration.cookies[0] assert existing_cookie.name == "strawberry" await upsert_cookies( @@ -454,8 +454,8 @@ async def test_update_cookies(self, test_cookie_system, async_session_temp): """Test specified cookies already exist on given privacy declaration, so no change required""" new_cookies = [{"name": "strawberry", "path": "/"}] - privacy_declaration = test_cookie_system.privacy_declarations[1] - existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] + privacy_declaration = sorted(test_cookie_system.privacy_declarations, key=lambda x: x.name)[1] + existing_cookie = privacy_declaration.cookies[0] assert existing_cookie.name == "strawberry" await upsert_cookies( @@ -485,8 +485,8 @@ async def test_remove_cookies(self, test_cookie_system, async_session_temp): cookie and we remove the existing one""" new_cookies = [{"name": "apple"}] - privacy_declaration = test_cookie_system.privacy_declarations[1] - existing_cookie = test_cookie_system.privacy_declarations[1].cookies[0] + privacy_declaration = sorted(test_cookie_system.privacy_declarations, key=lambda x: x.name)[1] + existing_cookie = privacy_declaration.cookies[0] assert existing_cookie.name == "strawberry" await upsert_cookies( From c8493db1895cd65ce6fbef4ffa310cb8dc798705 Mon Sep 17 00:00:00 2001 From: Allison King Date: Wed, 21 Jun 2023 14:11:34 -0400 Subject: [PATCH 15/20] Data use cookie field (#3571) --- CHANGELOG.md | 9 +-- clients/admin-ui/cypress/e2e/systems.cy.ts | 2 + .../cypress/fixtures/systems/system.json | 4 +- .../cypress/fixtures/systems/systems.json | 12 ++- .../systems/systems_with_data_uses.json | 8 +- .../src/features/common/form/inputs.tsx | 19 +++-- .../src/features/system/SystemFormTabs.tsx | 2 +- .../PrivacyDeclarationAccordion.tsx | 21 ++--- .../PrivacyDeclarationForm.tsx | 74 ++++++++++-------- .../PrivacyDeclarationManager.tsx | 77 ++++++++----------- .../PrivacyDeclarationStep.tsx | 5 +- .../system/privacy-declarations/types.ts | 11 --- .../src/features/system/system.slice.ts | 7 +- clients/admin-ui/src/types/api/index.ts | 5 +- .../src/types/api/models/ConnectionType.ts | 34 ++++---- .../src/types/api/models/ConnectorParam.ts | 1 + .../src/types/api/models/ConsentReport.ts | 4 +- .../admin-ui/src/types/api/models/Cookies.ts | 12 +++ ...reateConnectionConfigurationWithSecrets.ts | 4 +- .../admin-ui/src/types/api/models/Dataset.ts | 4 +- .../types/api/models/DynamoDBDocsSchema.ts | 2 +- .../src/types/api/models/EdgeDirection.ts | 11 +++ .../src/types/api/models/EmailDocsSchema.ts | 2 +- .../types/api/models/FidesDatasetReference.ts | 11 +-- .../src/types/api/models/IdentityBase.ts | 11 --- .../types/api/models/PostgreSQLDocsSchema.ts | 1 + .../types/api/models/PrivacyDeclaration.ts | 6 ++ .../api/models/PrivacyDeclarationResponse.ts | 3 + .../types/api/models/PrivacyNoticeResponse.ts | 2 + ...rivacyNoticeResponseWithUserPreferences.ts | 2 + .../types/api/models/RedshiftDocsSchema.ts | 1 + .../src/types/api/models/ResponseFormat.ts | 1 + .../src/types/api/models/SaaSRequest.ts | 2 +- .../src/types/api/models/SovrnDocsSchema.ts | 2 +- .../admin-ui/src/types/api/models/System.ts | 4 +- .../src/types/api/models/SystemResponse.ts | 6 +- .../types/api/models/TimescaleDocsSchema.ts | 1 + 37 files changed, 208 insertions(+), 175 deletions(-) delete mode 100644 clients/admin-ui/src/features/system/privacy-declarations/types.ts create mode 100644 clients/admin-ui/src/types/api/models/Cookies.ts create mode 100644 clients/admin-ui/src/types/api/models/EdgeDirection.ts delete mode 100644 clients/admin-ui/src/types/api/models/IdentityBase.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index cbdaad4a18..73f17588a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The types of changes are: - Support for acknowledge button for notice-only Privacy Notices and to disable toggling them off [#3546](https://github.com/ethyca/fides/pull/3546) - HTML format for privacy request storage destinations [#3427](https://github.com/ethyca/fides/pull/3427) - New Cookies Table for storing cookies associated with systems and privacy declarations [#3572](https://github.com/ethyca/fides/pull/3572) +- Cookie input field on system data use tab [#3571](https://github.com/ethyca/fides/pull/3571) - Access and erasure support for SurveyMonkey [#3590](https://github.com/ethyca/fides/pull/3590) ### Changed @@ -48,6 +49,7 @@ The types of changes are: ## [2.15.0](https://github.com/ethyca/fides/compare/2.14.1...2.15.0) ### Added + - Privacy center can now render its consent values based on Privacy Notices and Privacy Experiences [#3411](https://github.com/ethyca/fides/pull/3411) - Add Google Tag Manager and Privacy Center ENV vars to sample app [#2949](https://github.com/ethyca/fides/pull/2949) - Add `notice_key` field to Privacy Notice UI form [#3403](https://github.com/ethyca/fides/pull/3403) @@ -106,7 +108,6 @@ The types of changes are: - Removed the deprecated `system_dependencies` from `System` resources, migrating to `egress` [#3285](https://github.com/ethyca/fides/pull/3285) - ## [2.14.1](https://github.com/ethyca/fides/compare/2.14.0...2.14.1) ### Added @@ -119,7 +120,6 @@ The types of changes are: - Update privacy centre email and phone validation to allow for both to be blank [#3432](https://github.com/ethyca/fides/pull/3432) - ## [2.14.0](https://github.com/ethyca/fides/compare/2.13.0...2.14.0) ### Added @@ -171,7 +171,6 @@ The types of changes are: - Remove `fides export` command and backing code [#3256](https://github.com/ethyca/fides/pull/3256) - ## [2.13.0](https://github.com/ethyca/fides/compare/2.12.1...2.13.0) ### Added @@ -204,7 +203,7 @@ The types of changes are: ### Developer Experience -- Use prettier to format *all* source files in client packages [#3240](https://github.com/ethyca/fides/pull/3240) +- Use prettier to format _all_ source files in client packages [#3240](https://github.com/ethyca/fides/pull/3240) ### Deprecated @@ -269,7 +268,6 @@ The types of changes are: - Fixed unit tests for saas connector type endpoints now that we have >50 [#3101](https://github.com/ethyca/fides/pull/3101) - Fixed nox docs link [#3121](https://github.com/ethyca/fides/pull/3121/files) - ### Developer Experience - Update fides deploy to use a new database.load_samples setting to initialize sample Systems, Datasets, and Connections for testing [#3102](https://github.com/ethyca/fides/pull/3102) @@ -277,7 +275,6 @@ The types of changes are: - Add smoke tests for consent management [#3158](https://github.com/ethyca/fides/pull/3158) - Added nox command that opens dev docs [#3082](https://github.com/ethyca/fides/pull/3082) - ## [2.11.0](https://github.com/ethyca/fides/compare/2.10.0...2.11.0) ### Added diff --git a/clients/admin-ui/cypress/e2e/systems.cy.ts b/clients/admin-ui/cypress/e2e/systems.cy.ts index 7974b0ceab..2361b15a39 100644 --- a/clients/admin-ui/cypress/e2e/systems.cy.ts +++ b/clients/admin-ui/cypress/e2e/systems.cy.ts @@ -172,6 +172,8 @@ describe("System management page", () => { data_categories: declaration.data_categories, data_subjects: declaration.data_subjects, dataset_references: ["demo_users_dataset_2"], + cookies: [], + id: "", }); }); }); diff --git a/clients/admin-ui/cypress/fixtures/systems/system.json b/clients/admin-ui/cypress/fixtures/systems/system.json index 5929eb18dd..09e930e287 100644 --- a/clients/admin-ui/cypress/fixtures/systems/system.json +++ b/clients/admin-ui/cypress/fixtures/systems/system.json @@ -16,7 +16,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["customer"], - "dataset_references": ["demo_users_dataset"] + "dataset_references": ["demo_users_dataset"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": { "name": "Sally Controller" }, diff --git a/clients/admin-ui/cypress/fixtures/systems/systems.json b/clients/admin-ui/cypress/fixtures/systems/systems.json index a73c0d419f..6e9e6aa264 100644 --- a/clients/admin-ui/cypress/fixtures/systems/systems.json +++ b/clients/admin-ui/cypress/fixtures/systems/systems.json @@ -17,7 +17,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["anonymous_user"], - "dataset_references": ["public"] + "dataset_references": ["public"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": null, @@ -59,7 +61,9 @@ "data_subjects": ["customer"], "dataset_references": ["demo_users_dataset"], "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": null, @@ -99,7 +103,9 @@ "data_subjects": ["customer"], "dataset_references": null, "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_06430a1c-1365-422e-90a7-d444ddb32181" } ], "joint_controller": null, diff --git a/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json b/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json index 533a20f7d9..37c89489c3 100644 --- a/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json +++ b/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json @@ -17,7 +17,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["anonymous_user"], - "dataset_references": ["public"] + "dataset_references": ["public"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" }, { "name": "Collect data for marketing", @@ -27,7 +29,9 @@ "data_subjects": ["customer"], "dataset_references": null, "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_bc6e6efe-f122-3e33-ac9a-732ae8b437bb" } ], "joint_controller": null, diff --git a/clients/admin-ui/src/features/common/form/inputs.tsx b/clients/admin-ui/src/features/common/form/inputs.tsx index d242550648..7f6de443f1 100644 --- a/clients/admin-ui/src/features/common/form/inputs.tsx +++ b/clients/admin-ui/src/features/common/form/inputs.tsx @@ -316,21 +316,26 @@ const CreatableSelectInput = ({ size={size} classNamePrefix="custom-creatable-select" chakraStyles={{ - container: (provided) => ({ ...provided, flexGrow: 1 }), + container: (provided) => ({ + ...provided, + flexGrow: 1, + backgroundColor: "white", + }), dropdownIndicator: (provided) => ({ ...provided, - background: "white", + bg: "transparent", + px: 2, + cursor: "inherit", + }), + indicatorSeparator: (provided) => ({ + ...provided, + display: "none", }), multiValue: (provided) => ({ ...provided, background: "primary.400", color: "white", }), - multiValueRemove: (provided) => ({ - ...provided, - display: "none", - visibility: "hidden", - }), }} components={components} isSearchable={isSearchable} diff --git a/clients/admin-ui/src/features/system/SystemFormTabs.tsx b/clients/admin-ui/src/features/system/SystemFormTabs.tsx index 35450b30fa..64171f0a84 100644 --- a/clients/admin-ui/src/features/system/SystemFormTabs.tsx +++ b/clients/admin-ui/src/features/system/SystemFormTabs.tsx @@ -185,7 +185,7 @@ const SystemFormTabs = ({ label: "Data uses", content: activeSystem ? ( - + ) : null, isDisabled: !activeSystem, diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationAccordion.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationAccordion.tsx index a08df39a7d..0df4261190 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationAccordion.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationAccordion.tsx @@ -9,6 +9,7 @@ import { import { Form, Formik } from "formik"; import { FormGuard } from "~/features/common/hooks/useIsAnyFormDirty"; +import { PrivacyDeclarationResponse } from "~/types/api"; import { DataProps, @@ -16,18 +17,18 @@ import { usePrivacyDeclarationForm, ValidationSchema, } from "./PrivacyDeclarationForm"; -import { PrivacyDeclarationWithId } from "./types"; interface AccordionProps extends DataProps { - privacyDeclarations: PrivacyDeclarationWithId[]; + privacyDeclarations: PrivacyDeclarationResponse[]; onEdit: ( - oldDeclaration: PrivacyDeclarationWithId, - newDeclaration: PrivacyDeclarationWithId - ) => Promise; + oldDeclaration: PrivacyDeclarationResponse, + newDeclaration: PrivacyDeclarationResponse + ) => Promise; onDelete: ( - declaration: PrivacyDeclarationWithId - ) => Promise; + declaration: PrivacyDeclarationResponse + ) => Promise; includeCustomFields?: boolean; + includeCookies?: boolean; } const PrivacyDeclarationAccordionItem = ({ @@ -35,12 +36,13 @@ const PrivacyDeclarationAccordionItem = ({ onEdit, onDelete, includeCustomFields, + includeCookies, ...dataProps -}: { privacyDeclaration: PrivacyDeclarationWithId } & Omit< +}: { privacyDeclaration: PrivacyDeclarationResponse } & Omit< AccordionProps, "privacyDeclarations" >) => { - const handleEdit = (values: PrivacyDeclarationWithId) => + const handleEdit = (values: PrivacyDeclarationResponse) => onEdit(privacyDeclaration, values); const { initialValues, renderHeader, handleSubmit } = @@ -85,6 +87,7 @@ const PrivacyDeclarationAccordionItem = ({ privacyDeclarationId={privacyDeclaration.id} onDelete={onDelete} includeCustomFields={includeCustomFields} + includeCookies={includeCookies} {...dataProps} /> diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationForm.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationForm.tsx index e19c757382..124b004685 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationForm.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationForm.tsx @@ -23,19 +23,21 @@ import { useMemo, useState } from "react"; import * as Yup from "yup"; import ConfirmationModal from "~/features/common/ConfirmationModal"; -import { CustomSelect, CustomTextInput } from "~/features/common/form/inputs"; +import { + CustomCreatableSelect, + CustomSelect, + CustomTextInput, +} from "~/features/common/form/inputs"; import { FormGuard } from "~/features/common/hooks/useIsAnyFormDirty"; import { DataCategory, Dataset, DataSubject, DataUse, - PrivacyDeclaration, + PrivacyDeclarationResponse, ResourceTypes, } from "~/types/api"; -import { PrivacyDeclarationWithId } from "./types"; - export const ValidationSchema = Yup.object().shape({ data_categories: Yup.array(Yup.string()) .min(1, "Must assign at least one data category") @@ -46,8 +48,9 @@ export const ValidationSchema = Yup.object().shape({ .label("Data subjects"), }); -export type FormValues = PrivacyDeclarationWithId & { +export type FormValues = Omit & { customFieldValues: CustomFieldValues; + cookies: string[]; }; const defaultInitialValues: FormValues = { @@ -57,30 +60,21 @@ const defaultInitialValues: FormValues = { dataset_references: [], customFieldValues: {}, id: "", + cookies: [], }; -const transformPrivacyDeclarationToHaveId = ( - privacyDeclaration: PrivacyDeclaration -) => { - // TODO: there's a typing problem here: the backend types still show PrivacyDeclaration - // instead of PrivacyDeclarationResponse (which has an id) - // @ts-ignore - const { id, name, data_use: dataUse } = privacyDeclaration; - let declarationId: string | undefined = id; - if (!declarationId) { - declarationId = name ? `${dataUse} - ${name}` : dataUse; - } +const transformFormValueToDeclaration = (values: FormValues) => { + const { customFieldValues, ...declaration } = values; + return { - ...privacyDeclaration, - id: declarationId, + ...declaration, + // Fill in an empty string for name because of https://github.com/ethyca/fideslang/issues/98 + name: values.name ?? "", + // Transform cookies from string back to an object with default values + cookies: declaration.cookies.map((name) => ({ name, path: "/" })), }; }; -export const transformPrivacyDeclarationsToHaveId = ( - privacyDeclarations: PrivacyDeclaration[] -): PrivacyDeclarationWithId[] => - privacyDeclarations.map(transformPrivacyDeclarationToHaveId); - export interface DataProps { allDataCategories: DataCategory[]; allDataUses: DataUse[]; @@ -95,10 +89,12 @@ export const PrivacyDeclarationFormComponents = ({ allDatasets, onDelete, privacyDeclarationId, + includeCookies, includeCustomFields, }: DataProps & Pick & { privacyDeclarationId?: string; + includeCookies?: boolean; includeCustomFields?: boolean; }) => { const { dirty, isSubmitting, isValid, initialValues } = @@ -113,7 +109,7 @@ export const PrivacyDeclarationFormComponents = ({ : []; const handleDelete = async () => { - await onDelete(transformPrivacyDeclarationToHaveId(initialValues)); + await onDelete(transformFormValueToDeclaration(initialValues)); deleteModal.onClose(); }; @@ -164,6 +160,16 @@ export const PrivacyDeclarationFormComponents = ({ isMulti variant="stacked" /> + {includeCookies ? ( + + ) : null} {allDatasets ? ( privacyDeclaration ? { ...privacyDeclaration, customFieldValues: customFieldValues || {}, + cookies: privacyDeclaration.cookies?.map((cookie) => cookie.name) ?? [], } : defaultInitialValues; @@ -264,8 +271,10 @@ export const usePrivacyDeclarationForm = ({ values: FormValues, formikHelpers: FormikHelpers ) => { - const { customFieldValues: formCustomFieldValues, ...declaration } = values; - const success = await onSubmit(declaration, formikHelpers); + const { customFieldValues: formCustomFieldValues } = values; + const declarationToSubmit = transformFormValueToDeclaration(values); + + const success = await onSubmit(declarationToSubmit, formikHelpers); if (success) { // find the matching resource based on data use and name const customFieldResource = success.filter( @@ -321,15 +330,16 @@ export const usePrivacyDeclarationForm = ({ interface Props { onSubmit: ( - values: PrivacyDeclarationWithId, + values: PrivacyDeclarationResponse, formikHelpers: FormikHelpers - ) => Promise; + ) => Promise; onDelete: ( - declaration: PrivacyDeclarationWithId - ) => Promise; - initialValues?: PrivacyDeclarationWithId; + declaration: PrivacyDeclarationResponse + ) => Promise; + initialValues?: PrivacyDeclarationResponse; privacyDeclarationId?: string; includeCustomFields?: boolean; + includeCookies?: boolean; } export const PrivacyDeclarationForm = ({ diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx index bf18ade8c6..3a8a1318db 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx @@ -13,33 +13,21 @@ import { useEffect, useMemo, useState } from "react"; import { getErrorMessage } from "~/features/common/helpers"; import { errorToastParams, successToastParams } from "~/features/common/toast"; import { useUpdateSystemMutation } from "~/features/system/system.slice"; -import { PrivacyDeclaration, System } from "~/types/api"; +import { + PrivacyDeclarationResponse, + System, + SystemResponse, +} from "~/types/api"; import { isErrorResult } from "~/types/errors"; import PrivacyDeclarationAccordion from "./PrivacyDeclarationAccordion"; -import { - DataProps, - PrivacyDeclarationForm, - transformPrivacyDeclarationsToHaveId, -} from "./PrivacyDeclarationForm"; -import { PrivacyDeclarationWithId } from "./types"; - -const transformDeclarationForSubmission = ( - formValues: PrivacyDeclarationWithId -): PrivacyDeclaration => { - // Remove the id which is only a frontend artifact - const { id, ...values } = formValues; - return { - ...values, - // Fill in an empty string for name because of https://github.com/ethyca/fideslang/issues/98 - name: values.name ?? "", - }; -}; +import { DataProps, PrivacyDeclarationForm } from "./PrivacyDeclarationForm"; interface Props { - system: System; + system: SystemResponse; addButtonProps?: ButtonProps; includeCustomFields?: boolean; + includeCookies?: boolean; onSave?: (system: System) => void; } @@ -47,6 +35,7 @@ const PrivacyDeclarationManager = ({ system, addButtonProps, includeCustomFields, + includeCookies, onSave, ...dataProps }: Props & DataProps) => { @@ -55,24 +44,21 @@ const PrivacyDeclarationManager = ({ const [updateSystemMutationTrigger] = useUpdateSystemMutation(); const [showNewForm, setShowNewForm] = useState(false); const [newDeclaration, setNewDeclaration] = useState< - PrivacyDeclarationWithId | undefined + PrivacyDeclarationResponse | undefined >(undefined); - const allDeclarations = useMemo( - () => transformPrivacyDeclarationsToHaveId(system.privacy_declarations), - [system.privacy_declarations] - ); - // Accordion declarations include all declarations but the newly created one (if it exists) const accordionDeclarations = useMemo(() => { if (!newDeclaration) { - return allDeclarations; + return system.privacy_declarations; } - return allDeclarations.filter((pd) => pd.id !== newDeclaration.id); - }, [newDeclaration, allDeclarations]); + return system.privacy_declarations.filter( + (pd) => pd.id !== newDeclaration.id + ); + }, [newDeclaration, system]); - const checkAlreadyExists = (values: PrivacyDeclaration) => { + const checkAlreadyExists = (values: PrivacyDeclarationResponse) => { if ( accordionDeclarations.filter( (d) => d.data_use === values.data_use && d.name === values.name @@ -89,19 +75,16 @@ const PrivacyDeclarationManager = ({ }; const handleSave = async ( - updatedDeclarations: PrivacyDeclarationWithId[], + updatedDeclarations: PrivacyDeclarationResponse[], isDelete?: boolean ) => { - const transformedDeclarations = updatedDeclarations.map((d) => - transformDeclarationForSubmission(d) - ); const systemBodyWithDeclaration = { ...system, - privacy_declarations: transformedDeclarations, + privacy_declarations: updatedDeclarations, }; const handleResult = ( result: - | { data: System } + | { data: SystemResponse } | { error: FetchBaseQueryError | SerializedError } ) => { if (isErrorResult(result)) { @@ -120,7 +103,7 @@ const PrivacyDeclarationManager = ({ if (onSave) { onSave(result.data); } - return result.data.privacy_declarations as PrivacyDeclarationWithId[]; + return result.data.privacy_declarations; }; const updateSystemResult = await updateSystemMutationTrigger( @@ -131,8 +114,8 @@ const PrivacyDeclarationManager = ({ }; const handleEditDeclaration = async ( - oldDeclaration: PrivacyDeclarationWithId, - updatedDeclaration: PrivacyDeclarationWithId + oldDeclaration: PrivacyDeclarationResponse, + updatedDeclaration: PrivacyDeclarationResponse ) => { // Do not allow editing a privacy declaration to have the same data use as one that already exists if ( @@ -143,13 +126,13 @@ const PrivacyDeclarationManager = ({ } // Because the data use can change, we also need a reference to the old declaration in order to // make sure we are replacing the proper one - const updatedDeclarations = allDeclarations.map((dec) => + const updatedDeclarations = system.privacy_declarations.map((dec) => dec.id === oldDeclaration.id ? updatedDeclaration : dec ); return handleSave(updatedDeclarations); }; - const saveNewDeclaration = async (values: PrivacyDeclarationWithId) => { + const saveNewDeclaration = async (values: PrivacyDeclarationResponse) => { if (checkAlreadyExists(values)) { return undefined; } @@ -174,16 +157,16 @@ const PrivacyDeclarationManager = ({ }; const handleDelete = async ( - declarationToDelete: PrivacyDeclarationWithId + declarationToDelete: PrivacyDeclarationResponse ) => { - const updatedDeclarations = transformPrivacyDeclarationsToHaveId( - system.privacy_declarations - ).filter((dec) => dec.id !== declarationToDelete.id); + const updatedDeclarations = system.privacy_declarations.filter( + (dec) => dec.id !== declarationToDelete.id + ); return handleSave(updatedDeclarations, true); }; const handleDeleteNew = async ( - declarationToDelete: PrivacyDeclarationWithId + declarationToDelete: PrivacyDeclarationResponse ) => { const success = await handleDelete(declarationToDelete); if (success) { @@ -209,6 +192,7 @@ const PrivacyDeclarationManager = ({ onEdit={handleEditDeclaration} onDelete={handleDelete} includeCustomFields={includeCustomFields} + includeCookies={includeCookies} {...dataProps} /> {showNewForm ? ( @@ -218,6 +202,7 @@ const PrivacyDeclarationManager = ({ onSubmit={saveNewDeclaration} onDelete={handleDeleteNew} includeCustomFields={includeCustomFields} + includeCookies={includeCookies} {...dataProps} /> diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationStep.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationStep.tsx index dd7d947f01..7634e8aaf1 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationStep.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationStep.tsx @@ -3,13 +3,13 @@ import NextLink from "next/link"; import { useAppDispatch } from "~/app/hooks"; import { setActiveSystem } from "~/features/system"; -import { System } from "~/types/api"; +import { System, SystemResponse } from "~/types/api"; import { usePrivacyDeclarationData } from "./hooks"; import PrivacyDeclarationManager from "./PrivacyDeclarationManager"; interface Props { - system: System; + system: SystemResponse; } const PrivacyDeclarationStep = ({ system }: Props) => { @@ -48,6 +48,7 @@ const PrivacyDeclarationStep = ({ system }: Props) => { system={system} onSave={onSave} includeCustomFields + includeCookies {...dataProps} /> )} diff --git a/clients/admin-ui/src/features/system/privacy-declarations/types.ts b/clients/admin-ui/src/features/system/privacy-declarations/types.ts deleted file mode 100644 index b16500a316..0000000000 --- a/clients/admin-ui/src/features/system/privacy-declarations/types.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { PrivacyDeclaration } from "~/types/api"; - -/** - * This is because privacy declarations do not have an ID on the backend. - * It is very useful for React rendering to have a stable ID. We currently - * make this the composite of data_use - name, but even better may be to - * give it a UUID (or to have the backend actually enforce this!) - */ -export interface PrivacyDeclarationWithId extends PrivacyDeclaration { - id: string; -} diff --git a/clients/admin-ui/src/features/system/system.slice.ts b/clients/admin-ui/src/features/system/system.slice.ts index 1de03f616b..6a53b2a3fc 100644 --- a/clients/admin-ui/src/features/system/system.slice.ts +++ b/clients/admin-ui/src/features/system/system.slice.ts @@ -6,6 +6,7 @@ import { BulkPutConnectionConfiguration, ConnectionConfigurationResponse, System, + SystemResponse, } from "~/types/api"; interface SystemDeleteResponse { @@ -21,11 +22,11 @@ interface UpsertResponse { const systemApi = baseApi.injectEndpoints({ endpoints: (build) => ({ - getAllSystems: build.query({ + getAllSystems: build.query({ query: () => ({ url: `system/` }), providesTags: () => ["System"], }), - getSystemByFidesKey: build.query({ + getSystemByFidesKey: build.query({ query: (fides_key) => ({ url: `system/${fides_key}/` }), providesTags: ["System"], }), @@ -56,7 +57,7 @@ const systemApi = baseApi.injectEndpoints({ invalidatesTags: ["Datamap", "System", "Datastore Connection"], }), updateSystem: build.mutation< - System, + SystemResponse, Partial & Pick >({ query: ({ ...patch }) => ({ diff --git a/clients/admin-ui/src/types/api/index.ts b/clients/admin-ui/src/types/api/index.ts index 404068de1a..d0fc22bab5 100644 --- a/clients/admin-ui/src/types/api/index.ts +++ b/clients/admin-ui/src/types/api/index.ts @@ -72,6 +72,7 @@ export type { ConsentRequestMap } from "./models/ConsentRequestMap"; export type { ConsentRequestResponse } from "./models/ConsentRequestResponse"; export type { ConsentWithExecutableStatus } from "./models/ConsentWithExecutableStatus"; export type { ContactDetails } from "./models/ContactDetails"; +export type { Cookies } from "./models/Cookies"; export { CoreHealthCheck } from "./models/CoreHealthCheck"; export type { CreateConnectionConfigurationWithSecrets } from "./models/CreateConnectionConfigurationWithSecrets"; export type { CurrentPrivacyPreferenceReportingSchema } from "./models/CurrentPrivacyPreferenceReportingSchema"; @@ -111,6 +112,7 @@ export { DrpRegime } from "./models/DrpRegime"; export type { DrpRevokeRequest } from "./models/DrpRevokeRequest"; export type { DryRunDatasetResponse } from "./models/DryRunDatasetResponse"; export type { DynamoDBDocsSchema } from "./models/DynamoDBDocsSchema"; +export { EdgeDirection } from "./models/EdgeDirection"; export type { EmailDocsSchema } from "./models/EmailDocsSchema"; export type { Endpoint } from "./models/Endpoint"; export { EnforcementLevel } from "./models/EnforcementLevel"; @@ -128,7 +130,7 @@ export type { ExternalDatasetReference } from "./models/ExternalDatasetReference export type { fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds } from "./models/fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds"; export type { fides__api__schemas__policy__Policy } from "./models/fides__api__schemas__policy__Policy"; export type { fides__connectors__models__KeyfileCreds } from "./models/fides__connectors__models__KeyfileCreds"; -export { FidesDatasetReference } from "./models/FidesDatasetReference"; +export type { FidesDatasetReference } from "./models/FidesDatasetReference"; export type { FidesDocsSchema } from "./models/FidesDocsSchema"; export type { fideslang__models__Policy } from "./models/fideslang__models__Policy"; export type { FidesMeta } from "./models/FidesMeta"; @@ -143,7 +145,6 @@ export type { HealthCheck } from "./models/HealthCheck"; export { HTTPMethod } from "./models/HTTPMethod"; export type { HTTPValidationError } from "./models/HTTPValidationError"; export type { Identity } from "./models/Identity"; -export type { IdentityBase } from "./models/IdentityBase"; export type { IdentityTypes } from "./models/IdentityTypes"; export type { IdentityVerificationConfigResponse } from "./models/IdentityVerificationConfigResponse"; export { IncludeExcludeEnum } from "./models/IncludeExcludeEnum"; diff --git a/clients/admin-ui/src/types/api/models/ConnectionType.ts b/clients/admin-ui/src/types/api/models/ConnectionType.ts index 493ef4ef43..f85c008449 100644 --- a/clients/admin-ui/src/types/api/models/ConnectionType.ts +++ b/clients/admin-ui/src/types/api/models/ConnectionType.ts @@ -3,26 +3,26 @@ /* eslint-disable */ /** - * Supported types to which we can connect fidesops. + * Supported types to which we can connect Fides. */ export enum ConnectionType { - POSTGRES = "postgres", //DB - MONGODB = "mongodb", //DB - MYSQL = "mysql", // DB + POSTGRES = "postgres", + MONGODB = "mongodb", + MYSQL = "mysql", HTTPS = "https", SAAS = "saas", - REDSHIFT = "redshift", //DB - SNOWFLAKE = "snowflake", //DB - MSSQL = "mssql", //DB - MARIADB = "mariadb", //DB - BIGQUERY = "bigquery", //DB - MANUAL = "manual", // manual - SOVRN = "sovrn", // email - ATTENTIVE = "attentive", // email - DYNAMODB = "dynamodb", //DB - MANUAL_WEBHOOK = "manual_webhook", //manual - TIMESCALE = "timescale", //DB + REDSHIFT = "redshift", + SNOWFLAKE = "snowflake", + MSSQL = "mssql", + MARIADB = "mariadb", + BIGQUERY = "bigquery", + MANUAL = "manual", + SOVRN = "sovrn", + ATTENTIVE = "attentive", + DYNAMODB = "dynamodb", + MANUAL_WEBHOOK = "manual_webhook", + TIMESCALE = "timescale", FIDES = "fides", - GENERIC_ERASURE_EMAIL = "erasure_email", - GENERIC_CONSENT_EMAIL = "consent_email", + GENERIC_ERASURE_EMAIL = "generic_erasure_email", + GENERIC_CONSENT_EMAIL = "generic_consent_email", } diff --git a/clients/admin-ui/src/types/api/models/ConnectorParam.ts b/clients/admin-ui/src/types/api/models/ConnectorParam.ts index 1828a5c043..00e0eb43a4 100644 --- a/clients/admin-ui/src/types/api/models/ConnectorParam.ts +++ b/clients/admin-ui/src/types/api/models/ConnectorParam.ts @@ -12,4 +12,5 @@ export type ConnectorParam = { default_value?: string | Array; multiselect?: boolean; description?: string; + sensitive?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/ConsentReport.ts b/clients/admin-ui/src/types/api/models/ConsentReport.ts index a120d82ba1..f03914ff64 100644 --- a/clients/admin-ui/src/types/api/models/ConsentReport.ts +++ b/clients/admin-ui/src/types/api/models/ConsentReport.ts @@ -2,7 +2,7 @@ /* tslint:disable */ /* eslint-disable */ -import type { IdentityBase } from "./IdentityBase"; +import type { Identity } from "./Identity"; /** * Schema for reporting Consent requests. @@ -14,7 +14,7 @@ export type ConsentReport = { has_gpc_flag?: boolean; conflicts_with_gpc?: boolean; id: string; - identity: IdentityBase; + identity: Identity; created_at: string; updated_at: string; }; diff --git a/clients/admin-ui/src/types/api/models/Cookies.ts b/clients/admin-ui/src/types/api/models/Cookies.ts new file mode 100644 index 0000000000..923ab591c3 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/Cookies.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * The Cookies resource model + */ +export type Cookies = { + name: string; + path?: string; + domain?: string; +}; diff --git a/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts b/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts index 69301c0f42..8096369510 100644 --- a/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts +++ b/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts @@ -2,8 +2,8 @@ /* tslint:disable */ /* eslint-disable */ -import type { ActionType } from "~/features/privacy-requests/types"; import type { AccessLevel } from "./AccessLevel"; +import type { ActionType } from "./ActionType"; import type { BigQueryDocsSchema } from "./BigQueryDocsSchema"; import type { ConnectionType } from "./ConnectionType"; import type { DynamoDBDocsSchema } from "./DynamoDBDocsSchema"; @@ -31,6 +31,7 @@ export type CreateConnectionConfigurationWithSecrets = { access: AccessLevel; disabled?: boolean; description?: string; + enabled_actions?: Array; secrets?: | MongoDBDocsSchema | PostgreSQLDocsSchema @@ -48,5 +49,4 @@ export type CreateConnectionConfigurationWithSecrets = { | SovrnDocsSchema | DynamoDBDocsSchema; saas_connector_type?: string; - enabled_actions?: ActionType[]; }; diff --git a/clients/admin-ui/src/types/api/models/Dataset.ts b/clients/admin-ui/src/types/api/models/Dataset.ts index 0e37fe052b..9d009b8d61 100644 --- a/clients/admin-ui/src/types/api/models/Dataset.ts +++ b/clients/admin-ui/src/types/api/models/Dataset.ts @@ -28,9 +28,9 @@ export type Dataset = { */ description?: string; /** - * An optional object that provides additional information about the Dataset. You can structure the object however you like. It can be a simple set of `key: value` properties or a deeply nested hierarchy of objects. How you use the object is up to you: Fides ignores it. + * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: Record; + meta?: any; /** * Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset. */ diff --git a/clients/admin-ui/src/types/api/models/DynamoDBDocsSchema.ts b/clients/admin-ui/src/types/api/models/DynamoDBDocsSchema.ts index 3c5b33b8fa..f3bb707c35 100644 --- a/clients/admin-ui/src/types/api/models/DynamoDBDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/DynamoDBDocsSchema.ts @@ -8,6 +8,6 @@ export type DynamoDBDocsSchema = { url?: string; region_name: string; - aws_secret_access_key: string; aws_access_key_id: string; + aws_secret_access_key: string; }; diff --git a/clients/admin-ui/src/types/api/models/EdgeDirection.ts b/clients/admin-ui/src/types/api/models/EdgeDirection.ts new file mode 100644 index 0000000000..dcce420fd7 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/EdgeDirection.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Direction of a FidesDataSetReference + */ +export enum EdgeDirection { + FROM = "from", + TO = "to", +} diff --git a/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts b/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts index 51831625d4..dbd507a4ec 100644 --- a/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts @@ -11,5 +11,5 @@ export type EmailDocsSchema = { third_party_vendor_name: string; recipient_email_address: string; test_email_address?: string; - advanced_settings: AdvancedSettings; + advanced_settings?: AdvancedSettings; }; diff --git a/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts b/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts index 8c6f702e24..39e2fd943e 100644 --- a/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts +++ b/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts @@ -2,18 +2,13 @@ /* tslint:disable */ /* eslint-disable */ +import type { EdgeDirection } from "./EdgeDirection"; + /** * Reference to a field from another Collection */ export type FidesDatasetReference = { dataset: string; field: string; - direction?: FidesDatasetReference.direction; + direction?: EdgeDirection; }; - -export namespace FidesDatasetReference { - export enum direction { - FROM = "from", - TO = "to", - } -} diff --git a/clients/admin-ui/src/types/api/models/IdentityBase.ts b/clients/admin-ui/src/types/api/models/IdentityBase.ts deleted file mode 100644 index 93927af410..0000000000 --- a/clients/admin-ui/src/types/api/models/IdentityBase.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* istanbul ignore file */ -/* tslint:disable */ -/* eslint-disable */ - -/** - * The minimum fields required to represent an identity. - */ -export type IdentityBase = { - phone_number?: string; - email?: string; -}; diff --git a/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts b/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts index f7ea1c5b24..1562d96221 100644 --- a/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts @@ -13,4 +13,5 @@ export type PostgreSQLDocsSchema = { db_schema?: string; host?: string; port?: number; + ssh_required?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts b/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts index 1edd7683e7..a826af7206 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts @@ -2,6 +2,8 @@ /* tslint:disable */ /* eslint-disable */ +import type { Cookies } from "./Cookies"; + /** * The PrivacyDeclaration resource model. * @@ -41,4 +43,8 @@ export type PrivacyDeclaration = { * The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied. */ ingress?: Array; + /** + * Cookies associated with this data use to deliver services and functionality + */ + cookies?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts index dbf5327824..53de457c72 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts @@ -2,6 +2,8 @@ /* tslint:disable */ /* eslint-disable */ +import type { Cookies } from "./Cookies"; + /** * Extension of base pydantic model to include DB `id` field in the response */ @@ -38,6 +40,7 @@ export type PrivacyDeclarationResponse = { * The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied. */ ingress?: Array; + cookies?: Array; /** * The database-assigned ID of the privacy declaration on the system. This is meant to be a read-only field, returned only in API responses */ diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts index ab81aacaa6..623b92d76c 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ConsentMechanism } from "./ConsentMechanism"; +import type { Cookies } from "./Cookies"; import type { EnforcementLevel } from "./EnforcementLevel"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; @@ -29,4 +30,5 @@ export type PrivacyNoticeResponse = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts index 82f78dc9d0..6919ee30d5 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ConsentMechanism } from "./ConsentMechanism"; +import type { Cookies } from "./Cookies"; import type { EnforcementLevel } from "./EnforcementLevel"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; import type { UserConsentPreference } from "./UserConsentPreference"; @@ -31,6 +32,7 @@ export type PrivacyNoticeResponseWithUserPreferences = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; default_preference: UserConsentPreference; current_preference?: UserConsentPreference; outdated_preference?: UserConsentPreference; diff --git a/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts b/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts index 6e0eadf300..0b3ce296f6 100644 --- a/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts @@ -13,4 +13,5 @@ export type RedshiftDocsSchema = { user?: string; password?: string; db_schema?: string; + ssh_required?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/ResponseFormat.ts b/clients/admin-ui/src/types/api/models/ResponseFormat.ts index c874d392e6..bf2c4df56a 100644 --- a/clients/admin-ui/src/types/api/models/ResponseFormat.ts +++ b/clients/admin-ui/src/types/api/models/ResponseFormat.ts @@ -8,4 +8,5 @@ export enum ResponseFormat { JSON = "json", CSV = "csv", + HTML = "html", } diff --git a/clients/admin-ui/src/types/api/models/SaaSRequest.ts b/clients/admin-ui/src/types/api/models/SaaSRequest.ts index 5628c8ae22..c522917335 100644 --- a/clients/admin-ui/src/types/api/models/SaaSRequest.ts +++ b/clients/admin-ui/src/types/api/models/SaaSRequest.ts @@ -29,7 +29,7 @@ export type SaaSRequest = { postprocessors?: Array; pagination?: Strategy; grouped_inputs?: Array; - ignore_errors?: boolean; + ignore_errors?: boolean | Array; rate_limit_config?: RateLimitConfig; skip_missing_param_values?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts b/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts index 5825772c40..3e7dfb60c7 100644 --- a/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts @@ -11,5 +11,5 @@ export type SovrnDocsSchema = { third_party_vendor_name?: string; recipient_email_address?: string; test_email_address?: string; - advanced_settings: AdvancedSettingsWithExtendedIdentityTypes; + advanced_settings?: AdvancedSettingsWithExtendedIdentityTypes; }; diff --git a/clients/admin-ui/src/types/api/models/System.ts b/clients/admin-ui/src/types/api/models/System.ts index ebc3a86179..e2ac6a4cb4 100644 --- a/clients/admin-ui/src/types/api/models/System.ts +++ b/clients/admin-ui/src/types/api/models/System.ts @@ -37,9 +37,9 @@ export type System = { */ registry_id?: number; /** - * An optional property to store any extra information for a system. Not used by fidesctl. + * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: Record; + meta?: any; /** * * The SystemMetadata resource model. diff --git a/clients/admin-ui/src/types/api/models/SystemResponse.ts b/clients/admin-ui/src/types/api/models/SystemResponse.ts index 35f9d0ad6d..8b1847f8df 100644 --- a/clients/admin-ui/src/types/api/models/SystemResponse.ts +++ b/clients/admin-ui/src/types/api/models/SystemResponse.ts @@ -4,6 +4,7 @@ import type { ConnectionConfigurationResponse } from "./ConnectionConfigurationResponse"; import type { ContactDetails } from "./ContactDetails"; +import type { Cookies } from "./Cookies"; import type { DataFlow } from "./DataFlow"; import type { DataProtectionImpactAssessment } from "./DataProtectionImpactAssessment"; import type { DataResponsibilityTitle } from "./DataResponsibilityTitle"; @@ -36,9 +37,9 @@ export type SystemResponse = { */ registry_id?: number; /** - * An optional property to store any extra information for a system. Not used by fidesctl. + * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: Record; + meta?: any; /** * * The SystemMetadata resource model. @@ -114,4 +115,5 @@ export type SystemResponse = { * */ connection_configs?: ConnectionConfigurationResponse; + cookies?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts b/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts index b087fffee4..b9f413993e 100644 --- a/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts @@ -13,4 +13,5 @@ export type TimescaleDocsSchema = { db_schema?: string; host?: string; port?: number; + ssh_required?: boolean; }; From 27209144114c291bd689603bda46c28933f03039 Mon Sep 17 00:00:00 2001 From: Allison King Date: Wed, 21 Jun 2023 14:39:34 -0400 Subject: [PATCH 16/20] `fides-js` and privacy center cookie enforcement (#3569) --- CHANGELOG.md | 1 + clients/fides-js/__tests__/lib/cookie.test.ts | 49 +++++++- clients/fides-js/src/components/Overlay.tsx | 6 +- clients/fides-js/src/fides.ts | 3 +- clients/fides-js/src/lib/consent-types.ts | 20 ++-- clients/fides-js/src/lib/cookie.ts | 20 +++- clients/fides-js/src/lib/preferences.ts | 43 ++++--- clients/fides-js/src/services/fides/api.ts | 1 - .../consent/NoticeDrivenConsent.tsx | 63 ++++++++--- .../cypress/e2e/consent-banner.cy.ts | 56 ++++++++++ .../cypress/e2e/consent-notices.cy.ts | 105 ++++++++++++++++++ .../cypress/fixtures/consent/experience.json | 12 +- .../fixtures/consent/overlay_experience.json | 3 +- .../fixtures/consent/test_banner_options.json | 6 +- .../public/fides-js-components-demo.html | 2 + .../types/api/models/Cookies.ts | 12 ++ .../types/api/models/PrivacyNoticeResponse.ts | 2 + ...rivacyNoticeResponseWithUserPreferences.ts | 2 + 18 files changed, 345 insertions(+), 61 deletions(-) create mode 100644 clients/privacy-center/types/api/models/Cookies.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 73f17588a6..105569de20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The types of changes are: - Support for acknowledge button for notice-only Privacy Notices and to disable toggling them off [#3546](https://github.com/ethyca/fides/pull/3546) - HTML format for privacy request storage destinations [#3427](https://github.com/ethyca/fides/pull/3427) - New Cookies Table for storing cookies associated with systems and privacy declarations [#3572](https://github.com/ethyca/fides/pull/3572) +- `fides-js` and privacy center now delete cookies associated with notices that were opted out of [#3569](https://github.com/ethyca/fides/pull/3569) - Cookie input field on system data use tab [#3571](https://github.com/ethyca/fides/pull/3571) - Access and erasure support for SurveyMonkey [#3590](https://github.com/ethyca/fides/pull/3590) diff --git a/clients/fides-js/__tests__/lib/cookie.test.ts b/clients/fides-js/__tests__/lib/cookie.test.ts index 9835335638..407aec55e3 100644 --- a/clients/fides-js/__tests__/lib/cookie.test.ts +++ b/clients/fides-js/__tests__/lib/cookie.test.ts @@ -1,4 +1,6 @@ import * as uuid from "uuid"; + +import { CookieAttributes } from "typescript-cookie/dist/types"; import { CookieKeyConsent, FidesCookie, @@ -6,10 +8,11 @@ import { isNewFidesCookie, makeConsentDefaultsLegacy, makeFidesCookie, + removeCookiesFromBrowser, saveFidesCookie, } from "../../src/lib/cookie"; import type { ConsentContext } from "../../src/lib/consent-context"; -import { LegacyConsentConfig } from "~/lib/consent-types"; +import { Cookies, LegacyConsentConfig } from "../../src/lib/consent-types"; // Setup mock date const MOCK_DATE = "2023-01-01T12:00:00.000Z"; @@ -31,6 +34,10 @@ const mockSetCookie = jest.fn( (name: string, value: string, attributes: object, encoding: object) => `mock setCookie return (value=${value})` ); +const mockRemoveCookie = jest.fn( + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + (name: string, attributes?: CookieAttributes) => undefined +); jest.mock("typescript-cookie", () => ({ getCookie: () => mockGetCookie(), setCookie: ( @@ -39,6 +46,8 @@ jest.mock("typescript-cookie", () => ({ attributes: object, encoding: object ) => mockSetCookie(name, value, attributes, encoding), + removeCookie: (name: string, attributes?: CookieAttributes) => + mockRemoveCookie(name, attributes), })); describe("makeFidesCookie", () => { @@ -276,3 +285,41 @@ describe("isNewFidesCookie", () => { }); }); }); + +describe("removeCookiesFromBrowser", () => { + afterEach(() => mockRemoveCookie.mockClear()); + + it.each([ + { cookies: [], expectedAttributes: [] }, + { cookies: [{ name: "_ga123" }], expectedAttributes: [{ path: "/" }] }, + { + cookies: [{ name: "_ga123", path: "" }], + expectedAttributes: [{ path: "" }], + }, + { + cookies: [{ name: "_ga123", path: "/subpage" }], + expectedAttributes: [{ path: "/subpage" }], + }, + { + cookies: [{ name: "_ga123" }, { name: "shopify" }], + expectedAttributes: [{ path: "/" }, { path: "/" }], + }, + ])( + "should remove a list of cookies", + ({ + cookies, + expectedAttributes, + }: { + cookies: Cookies[]; + expectedAttributes: CookieAttributes[]; + }) => { + removeCookiesFromBrowser(cookies); + expect(mockRemoveCookie.mock.calls).toHaveLength(cookies.length); + cookies.forEach((cookie, idx) => { + const [name, attributes] = mockRemoveCookie.mock.calls[idx]; + expect(name).toEqual(cookie.name); + expect(attributes).toEqual(expectedAttributes[idx]); + }); + } + ); +}); diff --git a/clients/fides-js/src/components/Overlay.tsx b/clients/fides-js/src/components/Overlay.tsx index fc79e6a768..94c9dfbb95 100644 --- a/clients/fides-js/src/components/Overlay.tsx +++ b/clients/fides-js/src/components/Overlay.tsx @@ -117,11 +117,7 @@ const Overlay: FunctionComponent = ({ enabledPrivacyNoticeKeys.includes(notice.notice_key), notice.consent_mechanism ); - return new SaveConsentPreference( - notice.notice_key, - notice.privacy_notice_history_id, - userPreference - ); + return new SaveConsentPreference(notice, userPreference); }); updateConsentPreferences({ consentPreferencesToSave, diff --git a/clients/fides-js/src/fides.ts b/clients/fides-js/src/fides.ts index eb3153b391..c4fc1fa93e 100644 --- a/clients/fides-js/src/fides.ts +++ b/clients/fides-js/src/fides.ts @@ -144,8 +144,7 @@ const automaticallyApplyGPCPreferences = ( if (notice.has_gpc_flag && !notice.current_preference) { consentPreferencesToSave.push( new SaveConsentPreference( - notice.notice_key, - notice.privacy_notice_history_id, + notice, transformConsentToFidesUserPreference(false, notice.consent_mechanism) ) ); diff --git a/clients/fides-js/src/lib/consent-types.ts b/clients/fides-js/src/lib/consent-types.ts index b56b089f65..be126828f7 100644 --- a/clients/fides-js/src/lib/consent-types.ts +++ b/clients/fides-js/src/lib/consent-types.ts @@ -40,17 +40,10 @@ export type FidesOptions = { export class SaveConsentPreference { consentPreference: UserConsentPreference; - noticeHistoryId: string; + notice: PrivacyNotice; - noticeKey: string; - - constructor( - noticeKey: string, - noticeHistoryId: string, - consentPreference: UserConsentPreference - ) { - this.noticeKey = noticeKey; - this.noticeHistoryId = noticeHistoryId; + constructor(notice: PrivacyNotice, consentPreference: UserConsentPreference) { + this.notice = notice; this.consentPreference = consentPreference; } } @@ -88,6 +81,12 @@ export type ExperienceConfig = { regions: Array; }; +export type Cookies = { + name: string; + path?: string; + domain?: string; +}; + export type PrivacyNotice = { name?: string; notice_key: string; @@ -108,6 +107,7 @@ export type PrivacyNotice = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; default_preference: UserConsentPreference; current_preference?: UserConsentPreference; outdated_preference?: UserConsentPreference; diff --git a/clients/fides-js/src/lib/cookie.ts b/clients/fides-js/src/lib/cookie.ts index dee6a4b6cf..4cf8830469 100644 --- a/clients/fides-js/src/lib/cookie.ts +++ b/clients/fides-js/src/lib/cookie.ts @@ -1,12 +1,16 @@ import { v4 as uuidv4 } from "uuid"; -import { getCookie, setCookie, Types } from "typescript-cookie"; +import { getCookie, removeCookie, setCookie, Types } from "typescript-cookie"; import { ConsentContext } from "./consent-context"; import { resolveConsentValue, resolveLegacyConsentValue, } from "./consent-value"; -import { LegacyConsentConfig, PrivacyExperience } from "./consent-types"; +import { + Cookies, + LegacyConsentConfig, + PrivacyExperience, +} from "./consent-types"; import { debugLog } from "./consent-utils"; /** @@ -273,3 +277,15 @@ export const makeConsentDefaultsLegacy = ( debugLog(debug, `Returning defaults for legacy config.`, defaults); return defaults; }; + +/** + * Given a list of cookies, deletes them from the browser + */ +export const removeCookiesFromBrowser = (cookies: Cookies[]) => { + cookies.forEach((cookie) => { + removeCookie(cookie.name, { + path: cookie.path ?? "/", + domain: cookie.domain, + }); + }); +}; diff --git a/clients/fides-js/src/lib/preferences.ts b/clients/fides-js/src/lib/preferences.ts index ce6bbb9932..71646dd17b 100644 --- a/clients/fides-js/src/lib/preferences.ts +++ b/clients/fides-js/src/lib/preferences.ts @@ -3,9 +3,15 @@ import { ConsentOptionCreate, PrivacyPreferencesRequest, SaveConsentPreference, + UserConsentPreference, } from "./consent-types"; import { debugLog, transformUserPreferenceToBoolean } from "./consent-utils"; -import { CookieKeyConsent, FidesCookie, saveFidesCookie } from "./cookie"; +import { + CookieKeyConsent, + FidesCookie, + removeCookiesFromBrowser, + saveFidesCookie, +} from "./cookie"; import { dispatchFidesEvent } from "./events"; import { patchUserPreferenceToFidesServer } from "../services/fides/api"; @@ -14,6 +20,8 @@ import { patchUserPreferenceToFidesServer } from "../services/fides/api"; * 1. Save preferences to Fides API * 2. Update the window.Fides.consent object * 3. Save preferences to the `fides_consent` cookie in the browser + * 4. Remove any cookies from notices that were opted-out from the browser + * 5. Dispatch a "FidesUpdated" event */ export const updateConsentPreferences = ({ consentPreferencesToSave, @@ -34,21 +42,19 @@ export const updateConsentPreferences = ({ }) => { // Derive the CookieKeyConsent object from privacy notices const noticeMap = new Map( - consentPreferencesToSave.map(({ noticeKey, consentPreference }) => [ - noticeKey, + consentPreferencesToSave.map(({ notice, consentPreference }) => [ + notice.notice_key, transformUserPreferenceToBoolean(consentPreference), ]) ); const consentCookieKey: CookieKeyConsent = Object.fromEntries(noticeMap); // Derive the Fides user preferences array from privacy notices - const fidesUserPreferences: Array = []; - consentPreferencesToSave.forEach(({ noticeHistoryId, consentPreference }) => { - fidesUserPreferences.push({ - privacy_notice_history_id: noticeHistoryId, + const fidesUserPreferences: Array = + consentPreferencesToSave.map(({ notice, consentPreference }) => ({ + privacy_notice_history_id: notice.privacy_notice_history_id, preference: consentPreference, - }); - }); + })); // Update the cookie object // eslint-disable-next-line no-param-reassign @@ -63,12 +69,7 @@ export const updateConsentPreferences = ({ user_geography: userLocationString, method: consentMethod, }; - patchUserPreferenceToFidesServer( - privacyPreferenceCreate, - fidesApiUrl, - cookie.identity.fides_user_device_id, - debug - ); + patchUserPreferenceToFidesServer(privacyPreferenceCreate, fidesApiUrl, debug); // 2. Update the window.Fides.consent object debugLog(debug, "Updating window.Fides"); @@ -78,6 +79,16 @@ export const updateConsentPreferences = ({ debugLog(debug, "Saving preferences to cookie"); saveFidesCookie(cookie); - // 4. Dispatch a "FidesUpdated" event + // 4. Remove cookies associated with notices that were opted-out from the browser + consentPreferencesToSave + .filter( + (preference) => + preference.consentPreference === UserConsentPreference.OPT_OUT + ) + .forEach((preference) => { + removeCookiesFromBrowser(preference.notice.cookies); + }); + + // 5. Dispatch a "FidesUpdated" event dispatchFidesEvent("FidesUpdated", cookie); }; diff --git a/clients/fides-js/src/services/fides/api.ts b/clients/fides-js/src/services/fides/api.ts index b219a01db9..1e53c6840b 100644 --- a/clients/fides-js/src/services/fides/api.ts +++ b/clients/fides-js/src/services/fides/api.ts @@ -81,7 +81,6 @@ export const fetchExperience = async ( export const patchUserPreferenceToFidesServer = async ( preferences: PrivacyPreferencesRequest, fidesApiUrl: string, - fidesUserDeviceId: string, debug: boolean ): Promise => { debugLog(debug, "Saving user consent preference...", preferences); diff --git a/clients/privacy-center/components/consent/NoticeDrivenConsent.tsx b/clients/privacy-center/components/consent/NoticeDrivenConsent.tsx index bf16a75a27..73d137d9eb 100644 --- a/clients/privacy-center/components/consent/NoticeDrivenConsent.tsx +++ b/clients/privacy-center/components/consent/NoticeDrivenConsent.tsx @@ -5,6 +5,7 @@ import { CookieKeyConsent, getConsentContext, getOrMakeFidesCookie, + removeCookiesFromBrowser, saveFidesCookie, transformUserPreferenceToBoolean, } from "fides-js"; @@ -121,30 +122,44 @@ const NoticeDrivenConsent = () => { router.push("/"); }; + /** + * When saving, we need to: + * 1. Send PATCH to Fides backend + * 2. Save to cookie and window object + * 3. Delete any cookies that have been opted out of + */ const handleSave = async () => { const browserIdentities = inspectForBrowserIdentities(); const deviceIdentity = { fides_user_device_id: fidesUserDeviceId }; const identities = browserIdentities ? { ...deviceIdentity, ...browserIdentities } : deviceIdentity; + const notices = experience?.privacy_notices ?? []; - const preferences: ConsentOptionCreate[] = Object.entries( - draftPreferences - ).map(([key, value]) => { - const notice = experience?.privacy_notices?.find( - (n) => n.privacy_notice_history_id === key - ); - if (notice?.consent_mechanism === ConsentMechanism.NOTICE_ONLY) { + // Reconnect preferences to notices + const noticePreferences = Object.entries(draftPreferences).map( + ([historyKey, preference]) => { + const notice = notices.find( + (n) => n.privacy_notice_history_id === historyKey + ); + return { historyKey, preference, notice }; + } + ); + + const preferences: ConsentOptionCreate[] = noticePreferences.map( + ({ historyKey, preference, notice }) => { + if (notice?.consent_mechanism === ConsentMechanism.NOTICE_ONLY) { + return { + privacy_notice_history_id: historyKey, + preference: UserConsentPreference.ACKNOWLEDGE, + }; + } return { - privacy_notice_history_id: key, - preference: UserConsentPreference.ACKNOWLEDGE, + privacy_notice_history_id: historyKey, + preference: preference ?? UserConsentPreference.OPT_OUT, }; } - return { - privacy_notice_history_id: key, - preference: value ?? UserConsentPreference.OPT_OUT, - }; - }); + ); const payload: PrivacyPreferencesRequest = { browser_identity: identities, @@ -155,6 +170,7 @@ const NoticeDrivenConsent = () => { code: verificationCode, }; + // 1. Send PATCH to Fides backend const result = await updatePrivacyPreferencesMutationTrigger({ id: consentRequestId, body: payload, @@ -167,6 +183,8 @@ const NoticeDrivenConsent = () => { }); return; } + + // 2. Save the cookie and window obj on success const noticeKeyMap = new Map( result.data.map((preference) => [ preference.privacy_notice_history.notice_key || "", @@ -174,14 +192,23 @@ const NoticeDrivenConsent = () => { ]) ); const consentCookieKey: CookieKeyConsent = Object.fromEntries(noticeKeyMap); + window.Fides.consent = consentCookieKey; + const updatedCookie = { ...cookie, consent: consentCookieKey }; + saveFidesCookie(updatedCookie); toast({ title: "Your consent preferences have been saved", ...SuccessToastOptions, }); - // Save the cookie and window obj on success - window.Fides.consent = consentCookieKey; - const updatedCookie = { ...cookie, consent: consentCookieKey }; - saveFidesCookie(updatedCookie); + + // 3. Delete any cookies that have been opted out of + noticePreferences.forEach((noticePreference) => { + if ( + noticePreference.preference === UserConsentPreference.OPT_OUT && + noticePreference.notice + ) { + removeCookiesFromBrowser(noticePreference.notice.cookies); + } + }); router.push("/"); }; diff --git a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts index d54664fe90..63f0453fbf 100644 --- a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts +++ b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts @@ -126,6 +126,7 @@ const mockPrivacyNotice = (params: Partial) => { version: 1.0, privacy_notice_history_id: "pri_b09058a7-9f54-4360-8da5-4521e8975d4f", notice_key: "advertising", + cookies: [], }; return { ...notice, ...params }; }; @@ -478,6 +479,61 @@ describe("Consent banner", () => { // TODO: add tests for CSS expect(false).is.eql(true); }); + + describe("cookie enforcement", () => { + beforeEach(() => { + const cookies = [ + { name: "cookie1", path: "/" }, + { name: "cookie2", path: "/" }, + ]; + cookies.forEach((cookie) => { + cy.setCookie(cookie.name, "value", { path: cookie.path }); + }); + stubConfig({ + experience: { + privacy_notices: [ + mockPrivacyNotice({ + name: "one", + privacy_notice_history_id: "one", + notice_key: "one", + consent_mechanism: ConsentMechanism.OPT_OUT, + cookies: [cookies[0]], + }), + mockPrivacyNotice({ + name: "two", + privacy_notice_history_id: "two", + notice_key: "second", + consent_mechanism: ConsentMechanism.OPT_OUT, + cookies: [cookies[1]], + }), + ], + }, + options: { + isOverlayEnabled: true, + }, + }); + }); + + it("can remove all cookies when rejecting all", () => { + cy.contains("button", "Reject Test").click(); + cy.getAllCookies().then((allCookies) => { + expect(allCookies.map((c) => c.name)).to.eql([CONSENT_COOKIE_NAME]); + }); + }); + + it("can remove just the cookies associated with notices that were opted out", () => { + cy.contains("button", "Manage preferences").click(); + // opt out of the first notice + cy.getByTestId("toggle-one").click(); + cy.getByTestId("Save test-btn").click(); + cy.getAllCookies().then((allCookies) => { + expect(allCookies.map((c) => c.name)).to.eql([ + CONSENT_COOKIE_NAME, + "cookie2", + ]); + }); + }); + }); }); describe("when there are only notice-only notices", () => { diff --git a/clients/privacy-center/cypress/e2e/consent-notices.cy.ts b/clients/privacy-center/cypress/e2e/consent-notices.cy.ts index d6c62ef382..cc5fac8f6e 100644 --- a/clients/privacy-center/cypress/e2e/consent-notices.cy.ts +++ b/clients/privacy-center/cypress/e2e/consent-notices.cy.ts @@ -87,6 +87,7 @@ describe("Privacy notice driven consent", () => { cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_2}`).within(() => { cy.getRadio().should("be.checked"); }); + // Notice only, so should be checked and disabled cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_3}`).within(() => { cy.getRadio().should("be.checked").should("be.disabled"); }); @@ -107,6 +108,7 @@ describe("Privacy notice driven consent", () => { expect( preferences.map((p: ConsentOptionCreate) => p.preference) ).to.eql(["opt_in", "opt_in", "acknowledge"]); + // Should update the cookie cy.waitUntilCookieExists(CONSENT_COOKIE_NAME).then(() => { cy.getCookie(CONSENT_COOKIE_NAME).then((cookieJson) => { const cookie = JSON.parse( @@ -118,6 +120,7 @@ describe("Privacy notice driven consent", () => { const expectedConsent = { data_sales: true, advertising: true }; const { consent } = cookie; expect(consent).to.eql(expectedConsent); + // Should update the window object cy.window().then((win) => { expect(win.Fides.consent).to.eql(expectedConsent); }); @@ -154,6 +157,108 @@ describe("Privacy notice driven consent", () => { }); }); }); + + describe("cookie enforcement", () => { + beforeEach(() => { + // First seed the browser with the cookies that are listed in the notices + cy.fixture("consent/experience.json").then((data) => { + const notices: PrivacyNoticeResponseWithUserPreferences[] = + data.items[0].privacy_notices; + + const allCookies = notices.map((notice) => notice.cookies).flat(); + allCookies.forEach((cookie) => { + cy.setCookie(cookie.name, "value", { + path: cookie.path ?? "/", + domain: cookie.domain ?? undefined, + }); + }); + cy.getAllCookies().then((cookies) => { + expect( + cookies.filter((c) => c.name !== CONSENT_COOKIE_NAME).length + ).to.eql(allCookies.length); + }); + cy.wrap(notices).as("notices"); + }); + }); + + it("can delete all cookies for when opting out of all notices", () => { + // Opt out of the opt-out notice + cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_2}`).within(() => { + cy.getRadio().should("be.checked"); + cy.get("span").contains("No").click(); + }); + cy.getByTestId("save-btn").click(); + + cy.wait("@patchPrivacyPreference").then(() => { + // Use waitUntil to help with CI + cy.waitUntil(() => + cy.getAllCookies().then((cookies) => cookies.length === 1) + ).then(() => { + // There should be no cookies related to the privacy notices around + cy.getAllCookies().then((cookies) => { + const filteredCookies = cookies.filter( + (c) => c.name !== CONSENT_COOKIE_NAME + ); + expect(filteredCookies.length).to.eql(0); + }); + }); + }); + }); + + it("can delete only the cookies associated with opt-out notices", () => { + // Opt into first notice + cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_1}`).within(() => { + cy.get("span").contains("Yes").click(); + }); + // Opt out of second notice + cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_2}`).within(() => { + cy.getRadio().should("be.checked"); + cy.get("span").contains("No").click(); + }); + cy.getByTestId("save-btn").click(); + + cy.wait("@patchPrivacyPreference").then(() => { + // Use waitUntil to help with CI + cy.waitUntil(() => + cy.getAllCookies().then((cookies) => cookies.length === 2) + ).then(() => { + // The first notice's cookies should still be around + // But there should be none of the second cookie's + cy.getAllCookies().then((cookies) => { + const filteredCookies = cookies.filter( + (c) => c.name !== CONSENT_COOKIE_NAME + ); + expect(filteredCookies.length).to.eql(1); + cy.get("@notices").then((notices: any) => { + expect(filteredCookies[0]).to.have.property( + "name", + notices[0].cookies[0].name + ); + }); + }); + }); + }); + }); + + it("can successfully delete even if cookie does not exist", () => { + cy.clearAllCookies(); + // Opt out of second notice + cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_2}`).within(() => { + cy.getRadio().should("be.checked"); + cy.get("span").contains("No").click(); + }); + cy.getByTestId("save-btn").click(); + + cy.wait("@patchPrivacyPreference").then(() => { + cy.getAllCookies().then((cookies) => { + const filteredCookies = cookies.filter( + (c) => c.name !== CONSENT_COOKIE_NAME + ); + expect(filteredCookies.length).to.eql(0); + }); + }); + }); + }); }); describe("when user has consented before", () => { diff --git a/clients/privacy-center/cypress/fixtures/consent/experience.json b/clients/privacy-center/cypress/fixtures/consent/experience.json index c52570afb2..3282ec2d51 100644 --- a/clients/privacy-center/cypress/fixtures/consent/experience.json +++ b/clients/privacy-center/cypress/fixtures/consent/experience.json @@ -51,7 +51,8 @@ "privacy_notice_history_id": "pri_df14051b-1eaf-4f07-ae63-232bffd2dc3e", "default_preference": "opt_out", "current_preference": null, - "outdated_preference": null + "outdated_preference": null, + "cookies": [{ "name": "sales", "path": null, "domain": null }] }, { "name": "Advertising", @@ -75,7 +76,11 @@ "privacy_notice_history_id": "pri_b2a0a2fa-ef59-4f7d-8e3d-d2e9bd076707", "default_preference": "opt_in", "current_preference": null, - "outdated_preference": null + "outdated_preference": null, + "cookies": [ + { "name": "_ga", "path": null, "domain": null }, + { "name": "advertisingCookie", "path": null, "domain": null } + ] }, { "name": "Essential", @@ -99,7 +104,8 @@ "privacy_notice_history_id": "pri_b09058a7-9f54-4360-8da5-4521e8975d4e", "default_preference": "opt_in", "current_preference": null, - "outdated_preference": null + "outdated_preference": null, + "cookies": [] } ] } diff --git a/clients/privacy-center/cypress/fixtures/consent/overlay_experience.json b/clients/privacy-center/cypress/fixtures/consent/overlay_experience.json index 1e68cd4194..79a78495bd 100644 --- a/clients/privacy-center/cypress/fixtures/consent/overlay_experience.json +++ b/clients/privacy-center/cypress/fixtures/consent/overlay_experience.json @@ -51,7 +51,8 @@ "privacy_notice_history_id": "pri_b2a0a2fa-ef59-4f7d-8e3d-d2e9bd076707", "default_preference": "opt_out", "current_preference": null, - "outdated_preference": null + "outdated_preference": null, + "cookies": [] } ] } diff --git a/clients/privacy-center/cypress/fixtures/consent/test_banner_options.json b/clients/privacy-center/cypress/fixtures/consent/test_banner_options.json index dc830e1fef..6c30154e0e 100644 --- a/clients/privacy-center/cypress/fixtures/consent/test_banner_options.json +++ b/clients/privacy-center/cypress/fixtures/consent/test_banner_options.json @@ -64,7 +64,8 @@ "updated_at": "2023-04-24T21:29:08.870351+00:00", "version": 1.0, "privacy_notice_history_id": "pri_b09058a7-9f54-4360-8da5-4521e8975d4f", - "notice_key": "advertising" + "notice_key": "advertising", + "cookies": [] }, { "name": "Essential", @@ -85,7 +86,8 @@ "updated_at": "2023-04-24T21:29:08.870351+00:00", "version": 1.0, "privacy_notice_history_id": "pri_b09058a7-9f54-4360-8da5-4521e8975d4e", - "notice_key": "essential" + "notice_key": "essential", + "cookies": [] } ] }, diff --git a/clients/privacy-center/public/fides-js-components-demo.html b/clients/privacy-center/public/fides-js-components-demo.html index 4f07dc08f8..3614d6c117 100644 --- a/clients/privacy-center/public/fides-js-components-demo.html +++ b/clients/privacy-center/public/fides-js-components-demo.html @@ -77,6 +77,7 @@ privacy_notice_history_id: "pri_b09058a7-9f54-4360-8da5-4521e8975d4f", notice_key: "advertising", + cookies: [{ name: "testCookie", path: "/", domain: null }], }, { name: "Essential", @@ -100,6 +101,7 @@ privacy_notice_history_id: "pri_b09058a7-9f54-4360-8da5-4521e8975d4e", notice_key: "essential", + cookies: [], }, ], }, diff --git a/clients/privacy-center/types/api/models/Cookies.ts b/clients/privacy-center/types/api/models/Cookies.ts new file mode 100644 index 0000000000..923ab591c3 --- /dev/null +++ b/clients/privacy-center/types/api/models/Cookies.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * The Cookies resource model + */ +export type Cookies = { + name: string; + path?: string; + domain?: string; +}; diff --git a/clients/privacy-center/types/api/models/PrivacyNoticeResponse.ts b/clients/privacy-center/types/api/models/PrivacyNoticeResponse.ts index 8f33d8b38b..de11c2abfe 100644 --- a/clients/privacy-center/types/api/models/PrivacyNoticeResponse.ts +++ b/clients/privacy-center/types/api/models/PrivacyNoticeResponse.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ConsentMechanism } from "./ConsentMechanism"; +import type { Cookies } from "./Cookies"; import type { EnforcementLevel } from "./EnforcementLevel"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; @@ -28,4 +29,5 @@ export type PrivacyNoticeResponse = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; }; diff --git a/clients/privacy-center/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts b/clients/privacy-center/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts index 82f78dc9d0..6919ee30d5 100644 --- a/clients/privacy-center/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts +++ b/clients/privacy-center/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ConsentMechanism } from "./ConsentMechanism"; +import type { Cookies } from "./Cookies"; import type { EnforcementLevel } from "./EnforcementLevel"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; import type { UserConsentPreference } from "./UserConsentPreference"; @@ -31,6 +32,7 @@ export type PrivacyNoticeResponseWithUserPreferences = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; default_preference: UserConsentPreference; current_preference?: UserConsentPreference; outdated_preference?: UserConsentPreference; From a879831be7f22e2eb66beae0ebf45bec9c4df590 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 21 Jun 2023 16:22:10 -0500 Subject: [PATCH 17/20] More attempts to improve reliability of cookie tests --- tests/ctl/core/test_system.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index 3a67e1f911..a6eaaf5fc0 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -381,7 +381,9 @@ async def test_cookie_system( data={ "name": "strawberry", "path": "/", - "privacy_declaration_id": system.privacy_declarations[1].id, + "privacy_declaration_id": sorted( + system.privacy_declarations, key=lambda x: x.name + )[1].id, "system_id": system.id, }, check_name=False, @@ -395,12 +397,14 @@ async def test_new_cookies(self, test_cookie_system, async_session_temp): system already has a cookie.""" new_cookies = [{"name": "apple"}] - privacy_declaration = sorted(test_cookie_system.privacy_declarations, key=lambda x: x.name)[0] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[0] await upsert_cookies( async_session_temp, new_cookies, - test_cookie_system.privacy_declarations[0], + privacy_declaration, test_cookie_system, ) await async_session_temp.refresh(test_cookie_system) @@ -423,7 +427,9 @@ async def test_new_cookies(self, test_cookie_system, async_session_temp): async def test_no_change_to_cookies(self, test_cookie_system, async_session_temp): """Test specified cookies already exist on given privacy declaration, so no change required""" new_cookies = [{"name": "strawberry"}] - privacy_declaration = sorted(test_cookie_system.privacy_declarations, key=lambda x: x.name)[1] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[1] existing_cookie = privacy_declaration.cookies[0] assert existing_cookie.name == "strawberry" @@ -454,7 +460,9 @@ async def test_update_cookies(self, test_cookie_system, async_session_temp): """Test specified cookies already exist on given privacy declaration, so no change required""" new_cookies = [{"name": "strawberry", "path": "/"}] - privacy_declaration = sorted(test_cookie_system.privacy_declarations, key=lambda x: x.name)[1] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[1] existing_cookie = privacy_declaration.cookies[0] assert existing_cookie.name == "strawberry" @@ -485,7 +493,9 @@ async def test_remove_cookies(self, test_cookie_system, async_session_temp): cookie and we remove the existing one""" new_cookies = [{"name": "apple"}] - privacy_declaration = sorted(test_cookie_system.privacy_declarations, key=lambda x: x.name)[1] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[1] existing_cookie = privacy_declaration.cookies[0] assert existing_cookie.name == "strawberry" @@ -516,7 +526,9 @@ async def test_delete_privacy_declaration( ): """Test if a privacy declaration is deleted, its cookie is still linked to the system""" - privacy_declaration = test_cookie_system.privacy_declarations[1] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[1] existing_cookie = privacy_declaration.cookies[0] assert existing_cookie.privacy_declaration_id == privacy_declaration.id From 16d93554e96ef2ff07c955905e3a3f451fede9fe Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 21 Jun 2023 16:22:43 -0500 Subject: [PATCH 18/20] Fix new mypy errors. --- src/fides/api/schemas/dataset.py | 5 +++-- src/fides/api/schemas/privacy_request.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/fides/api/schemas/dataset.py b/src/fides/api/schemas/dataset.py index bc00dd5d56..882b8150c9 100644 --- a/src/fides/api/schemas/dataset.py +++ b/src/fides/api/schemas/dataset.py @@ -1,4 +1,4 @@ -from typing import Any, List, Optional +from typing import Any, List, Optional, Type from fideslang.models import Dataset, DatasetCollection, DatasetField from fideslang.validation import FidesKey @@ -28,9 +28,10 @@ def validate_data_categories_against_db( defined_data_categories = list(DefaultTaxonomyDataCategories.__members__.keys()) class DataCategoryValidationMixin(BaseModel): + @classmethod @validator("data_categories", check_fields=False, allow_reuse=True) def valid_data_categories( - cls, v: Optional[List[FidesKey]] + cls: Type["DataCategoryValidationMixin"], v: Optional[List[FidesKey]] ) -> Optional[List[FidesKey]]: """Validate that all annotated data categories exist in the taxonomy""" return _valid_data_categories(v, defined_data_categories) diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index c4f1228edb..20a09a3628 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum as EnumType -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Type, Union from fideslang.validation import FidesKey from pydantic import Field, validator @@ -82,9 +82,10 @@ class PrivacyRequestCreate(FidesSchema): encryption_key: Optional[str] = None consent_preferences: Optional[List[Consent]] = None # TODO Slated for deprecation + @classmethod @validator("encryption_key") def validate_encryption_key( - cls: "PrivacyRequestCreate", value: Optional[str] = None + cls: Type["PrivacyRequestCreate"], value: Optional[str] = None ) -> Optional[str]: """Validate encryption key where applicable""" if value: From 9e0bb3597fd3d415105aed885ad8045fa81e0a00 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 21 Jun 2023 16:22:56 -0500 Subject: [PATCH 19/20] Bump fideslang to 1.4.2. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d3105e8ae..90ee39b71b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@6f9d7a60ccccf46be2fc92d2a001e4c47c8adba7 +fideslang==1.4.2 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 From d283aab0e211b646938c40908e1e5ecc6293883c Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Wed, 21 Jun 2023 16:53:40 -0500 Subject: [PATCH 20/20] Classmethod placement was preventing validator from running. --- src/fides/api/schemas/dataset.py | 1 - src/fides/api/schemas/privacy_request.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/fides/api/schemas/dataset.py b/src/fides/api/schemas/dataset.py index 882b8150c9..7b4683dea7 100644 --- a/src/fides/api/schemas/dataset.py +++ b/src/fides/api/schemas/dataset.py @@ -28,7 +28,6 @@ def validate_data_categories_against_db( defined_data_categories = list(DefaultTaxonomyDataCategories.__members__.keys()) class DataCategoryValidationMixin(BaseModel): - @classmethod @validator("data_categories", check_fields=False, allow_reuse=True) def valid_data_categories( cls: Type["DataCategoryValidationMixin"], v: Optional[List[FidesKey]] diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index 20a09a3628..5034c33012 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -82,7 +82,6 @@ class PrivacyRequestCreate(FidesSchema): encryption_key: Optional[str] = None consent_preferences: Optional[List[Consent]] = None # TODO Slated for deprecation - @classmethod @validator("encryption_key") def validate_encryption_key( cls: Type["PrivacyRequestCreate"], value: Optional[str] = None