diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 001fabf84a..a73b4c3261 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -2240,4 +2240,41 @@ dataset: description: 'The name of the organization this Fides deployment belongs to' data_categories: - user.workplace + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: cookies + description: 'Fides Generated Description for Table: cookies' + data_categories: [] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: created_at + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: domain + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: name + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: path + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: privacy_declaration_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: system_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + data_categories: + - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index c5d51be76e..921da9ec50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,9 @@ The types of changes are: - HTML format for privacy request storage destinations [#3427](https://github.com/ethyca/fides/pull/3427) - Persistent message showing result and timestamp of last integration test to "Integrations" tab in system view [#3628](https://github.com/ethyca/fides/pull/3628) - Access and erasure support for SurveyMonkey [#3590](https://github.com/ethyca/fides/pull/3590) +- New Cookies Table for storing cookies associated with systems and privacy declarations [#3572](https://github.com/ethyca/fides/pull/3572) +- `fides-js` and privacy center now delete cookies associated with notices that were opted out of [#3569](https://github.com/ethyca/fides/pull/3569) +- Cookie input field on system data use tab [#3571](https://github.com/ethyca/fides/pull/3571) ### Fixed @@ -198,7 +201,7 @@ The types of changes are: ### Developer Experience -- Use prettier to format *all* source files in client packages [#3240](https://github.com/ethyca/fides/pull/3240) +- Use prettier to format _all_ source files in client packages [#3240](https://github.com/ethyca/fides/pull/3240) ### Deprecated diff --git a/clients/admin-ui/cypress/e2e/systems.cy.ts b/clients/admin-ui/cypress/e2e/systems.cy.ts index 7974b0ceab..2361b15a39 100644 --- a/clients/admin-ui/cypress/e2e/systems.cy.ts +++ b/clients/admin-ui/cypress/e2e/systems.cy.ts @@ -172,6 +172,8 @@ describe("System management page", () => { data_categories: declaration.data_categories, data_subjects: declaration.data_subjects, dataset_references: ["demo_users_dataset_2"], + cookies: [], + id: "", }); }); }); diff --git a/clients/admin-ui/cypress/fixtures/systems/system.json b/clients/admin-ui/cypress/fixtures/systems/system.json index 5929eb18dd..09e930e287 100644 --- a/clients/admin-ui/cypress/fixtures/systems/system.json +++ b/clients/admin-ui/cypress/fixtures/systems/system.json @@ -16,7 +16,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["customer"], - "dataset_references": ["demo_users_dataset"] + "dataset_references": ["demo_users_dataset"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": { "name": "Sally Controller" }, diff --git a/clients/admin-ui/cypress/fixtures/systems/systems.json b/clients/admin-ui/cypress/fixtures/systems/systems.json index a73c0d419f..6e9e6aa264 100644 --- a/clients/admin-ui/cypress/fixtures/systems/systems.json +++ b/clients/admin-ui/cypress/fixtures/systems/systems.json @@ -17,7 +17,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["anonymous_user"], - "dataset_references": ["public"] + "dataset_references": ["public"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": null, @@ -59,7 +61,9 @@ "data_subjects": ["customer"], "dataset_references": ["demo_users_dataset"], "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" } ], "joint_controller": null, @@ -99,7 +103,9 @@ "data_subjects": ["customer"], "dataset_references": null, "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_06430a1c-1365-422e-90a7-d444ddb32181" } ], "joint_controller": null, diff --git a/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json b/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json index 533a20f7d9..37c89489c3 100644 --- a/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json +++ b/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json @@ -17,7 +17,9 @@ "data_use": "improve.system", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["anonymous_user"], - "dataset_references": ["public"] + "dataset_references": ["public"], + "cookies": [], + "id": "pri_ac9d4dfb-d033-4b06-bc7f-968df8d125ff" }, { "name": "Collect data for marketing", @@ -27,7 +29,9 @@ "data_subjects": ["customer"], "dataset_references": null, "egress": null, - "ingress": null + "ingress": null, + "cookies": [], + "id": "pri_bc6e6efe-f122-3e33-ac9a-732ae8b437bb" } ], "joint_controller": null, diff --git a/clients/admin-ui/src/features/common/form/inputs.tsx b/clients/admin-ui/src/features/common/form/inputs.tsx index d242550648..7f6de443f1 100644 --- a/clients/admin-ui/src/features/common/form/inputs.tsx +++ b/clients/admin-ui/src/features/common/form/inputs.tsx @@ -316,21 +316,26 @@ const CreatableSelectInput = ({ size={size} classNamePrefix="custom-creatable-select" chakraStyles={{ - container: (provided) => ({ ...provided, flexGrow: 1 }), + container: (provided) => ({ + ...provided, + flexGrow: 1, + backgroundColor: "white", + }), dropdownIndicator: (provided) => ({ ...provided, - background: "white", + bg: "transparent", + px: 2, + cursor: "inherit", + }), + indicatorSeparator: (provided) => ({ + ...provided, + display: "none", }), multiValue: (provided) => ({ ...provided, background: "primary.400", color: "white", }), - multiValueRemove: (provided) => ({ - ...provided, - display: "none", - visibility: "hidden", - }), }} components={components} isSearchable={isSearchable} diff --git a/clients/admin-ui/src/features/system/SystemFormTabs.tsx b/clients/admin-ui/src/features/system/SystemFormTabs.tsx index 35450b30fa..64171f0a84 100644 --- a/clients/admin-ui/src/features/system/SystemFormTabs.tsx +++ b/clients/admin-ui/src/features/system/SystemFormTabs.tsx @@ -185,7 +185,7 @@ const SystemFormTabs = ({ label: "Data uses", content: activeSystem ? ( - + ) : null, isDisabled: !activeSystem, diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationAccordion.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationAccordion.tsx index a08df39a7d..0df4261190 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationAccordion.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationAccordion.tsx @@ -9,6 +9,7 @@ import { import { Form, Formik } from "formik"; import { FormGuard } from "~/features/common/hooks/useIsAnyFormDirty"; +import { PrivacyDeclarationResponse } from "~/types/api"; import { DataProps, @@ -16,18 +17,18 @@ import { usePrivacyDeclarationForm, ValidationSchema, } from "./PrivacyDeclarationForm"; -import { PrivacyDeclarationWithId } from "./types"; interface AccordionProps extends DataProps { - privacyDeclarations: PrivacyDeclarationWithId[]; + privacyDeclarations: PrivacyDeclarationResponse[]; onEdit: ( - oldDeclaration: PrivacyDeclarationWithId, - newDeclaration: PrivacyDeclarationWithId - ) => Promise; + oldDeclaration: PrivacyDeclarationResponse, + newDeclaration: PrivacyDeclarationResponse + ) => Promise; onDelete: ( - declaration: PrivacyDeclarationWithId - ) => Promise; + declaration: PrivacyDeclarationResponse + ) => Promise; includeCustomFields?: boolean; + includeCookies?: boolean; } const PrivacyDeclarationAccordionItem = ({ @@ -35,12 +36,13 @@ const PrivacyDeclarationAccordionItem = ({ onEdit, onDelete, includeCustomFields, + includeCookies, ...dataProps -}: { privacyDeclaration: PrivacyDeclarationWithId } & Omit< +}: { privacyDeclaration: PrivacyDeclarationResponse } & Omit< AccordionProps, "privacyDeclarations" >) => { - const handleEdit = (values: PrivacyDeclarationWithId) => + const handleEdit = (values: PrivacyDeclarationResponse) => onEdit(privacyDeclaration, values); const { initialValues, renderHeader, handleSubmit } = @@ -85,6 +87,7 @@ const PrivacyDeclarationAccordionItem = ({ privacyDeclarationId={privacyDeclaration.id} onDelete={onDelete} includeCustomFields={includeCustomFields} + includeCookies={includeCookies} {...dataProps} /> diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationForm.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationForm.tsx index e19c757382..124b004685 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationForm.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationForm.tsx @@ -23,19 +23,21 @@ import { useMemo, useState } from "react"; import * as Yup from "yup"; import ConfirmationModal from "~/features/common/ConfirmationModal"; -import { CustomSelect, CustomTextInput } from "~/features/common/form/inputs"; +import { + CustomCreatableSelect, + CustomSelect, + CustomTextInput, +} from "~/features/common/form/inputs"; import { FormGuard } from "~/features/common/hooks/useIsAnyFormDirty"; import { DataCategory, Dataset, DataSubject, DataUse, - PrivacyDeclaration, + PrivacyDeclarationResponse, ResourceTypes, } from "~/types/api"; -import { PrivacyDeclarationWithId } from "./types"; - export const ValidationSchema = Yup.object().shape({ data_categories: Yup.array(Yup.string()) .min(1, "Must assign at least one data category") @@ -46,8 +48,9 @@ export const ValidationSchema = Yup.object().shape({ .label("Data subjects"), }); -export type FormValues = PrivacyDeclarationWithId & { +export type FormValues = Omit & { customFieldValues: CustomFieldValues; + cookies: string[]; }; const defaultInitialValues: FormValues = { @@ -57,30 +60,21 @@ const defaultInitialValues: FormValues = { dataset_references: [], customFieldValues: {}, id: "", + cookies: [], }; -const transformPrivacyDeclarationToHaveId = ( - privacyDeclaration: PrivacyDeclaration -) => { - // TODO: there's a typing problem here: the backend types still show PrivacyDeclaration - // instead of PrivacyDeclarationResponse (which has an id) - // @ts-ignore - const { id, name, data_use: dataUse } = privacyDeclaration; - let declarationId: string | undefined = id; - if (!declarationId) { - declarationId = name ? `${dataUse} - ${name}` : dataUse; - } +const transformFormValueToDeclaration = (values: FormValues) => { + const { customFieldValues, ...declaration } = values; + return { - ...privacyDeclaration, - id: declarationId, + ...declaration, + // Fill in an empty string for name because of https://github.com/ethyca/fideslang/issues/98 + name: values.name ?? "", + // Transform cookies from string back to an object with default values + cookies: declaration.cookies.map((name) => ({ name, path: "/" })), }; }; -export const transformPrivacyDeclarationsToHaveId = ( - privacyDeclarations: PrivacyDeclaration[] -): PrivacyDeclarationWithId[] => - privacyDeclarations.map(transformPrivacyDeclarationToHaveId); - export interface DataProps { allDataCategories: DataCategory[]; allDataUses: DataUse[]; @@ -95,10 +89,12 @@ export const PrivacyDeclarationFormComponents = ({ allDatasets, onDelete, privacyDeclarationId, + includeCookies, includeCustomFields, }: DataProps & Pick & { privacyDeclarationId?: string; + includeCookies?: boolean; includeCustomFields?: boolean; }) => { const { dirty, isSubmitting, isValid, initialValues } = @@ -113,7 +109,7 @@ export const PrivacyDeclarationFormComponents = ({ : []; const handleDelete = async () => { - await onDelete(transformPrivacyDeclarationToHaveId(initialValues)); + await onDelete(transformFormValueToDeclaration(initialValues)); deleteModal.onClose(); }; @@ -164,6 +160,16 @@ export const PrivacyDeclarationFormComponents = ({ isMulti variant="stacked" /> + {includeCookies ? ( + + ) : null} {allDatasets ? ( privacyDeclaration ? { ...privacyDeclaration, customFieldValues: customFieldValues || {}, + cookies: privacyDeclaration.cookies?.map((cookie) => cookie.name) ?? [], } : defaultInitialValues; @@ -264,8 +271,10 @@ export const usePrivacyDeclarationForm = ({ values: FormValues, formikHelpers: FormikHelpers ) => { - const { customFieldValues: formCustomFieldValues, ...declaration } = values; - const success = await onSubmit(declaration, formikHelpers); + const { customFieldValues: formCustomFieldValues } = values; + const declarationToSubmit = transformFormValueToDeclaration(values); + + const success = await onSubmit(declarationToSubmit, formikHelpers); if (success) { // find the matching resource based on data use and name const customFieldResource = success.filter( @@ -321,15 +330,16 @@ export const usePrivacyDeclarationForm = ({ interface Props { onSubmit: ( - values: PrivacyDeclarationWithId, + values: PrivacyDeclarationResponse, formikHelpers: FormikHelpers - ) => Promise; + ) => Promise; onDelete: ( - declaration: PrivacyDeclarationWithId - ) => Promise; - initialValues?: PrivacyDeclarationWithId; + declaration: PrivacyDeclarationResponse + ) => Promise; + initialValues?: PrivacyDeclarationResponse; privacyDeclarationId?: string; includeCustomFields?: boolean; + includeCookies?: boolean; } export const PrivacyDeclarationForm = ({ diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx index bf18ade8c6..3a8a1318db 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationManager.tsx @@ -13,33 +13,21 @@ import { useEffect, useMemo, useState } from "react"; import { getErrorMessage } from "~/features/common/helpers"; import { errorToastParams, successToastParams } from "~/features/common/toast"; import { useUpdateSystemMutation } from "~/features/system/system.slice"; -import { PrivacyDeclaration, System } from "~/types/api"; +import { + PrivacyDeclarationResponse, + System, + SystemResponse, +} from "~/types/api"; import { isErrorResult } from "~/types/errors"; import PrivacyDeclarationAccordion from "./PrivacyDeclarationAccordion"; -import { - DataProps, - PrivacyDeclarationForm, - transformPrivacyDeclarationsToHaveId, -} from "./PrivacyDeclarationForm"; -import { PrivacyDeclarationWithId } from "./types"; - -const transformDeclarationForSubmission = ( - formValues: PrivacyDeclarationWithId -): PrivacyDeclaration => { - // Remove the id which is only a frontend artifact - const { id, ...values } = formValues; - return { - ...values, - // Fill in an empty string for name because of https://github.com/ethyca/fideslang/issues/98 - name: values.name ?? "", - }; -}; +import { DataProps, PrivacyDeclarationForm } from "./PrivacyDeclarationForm"; interface Props { - system: System; + system: SystemResponse; addButtonProps?: ButtonProps; includeCustomFields?: boolean; + includeCookies?: boolean; onSave?: (system: System) => void; } @@ -47,6 +35,7 @@ const PrivacyDeclarationManager = ({ system, addButtonProps, includeCustomFields, + includeCookies, onSave, ...dataProps }: Props & DataProps) => { @@ -55,24 +44,21 @@ const PrivacyDeclarationManager = ({ const [updateSystemMutationTrigger] = useUpdateSystemMutation(); const [showNewForm, setShowNewForm] = useState(false); const [newDeclaration, setNewDeclaration] = useState< - PrivacyDeclarationWithId | undefined + PrivacyDeclarationResponse | undefined >(undefined); - const allDeclarations = useMemo( - () => transformPrivacyDeclarationsToHaveId(system.privacy_declarations), - [system.privacy_declarations] - ); - // Accordion declarations include all declarations but the newly created one (if it exists) const accordionDeclarations = useMemo(() => { if (!newDeclaration) { - return allDeclarations; + return system.privacy_declarations; } - return allDeclarations.filter((pd) => pd.id !== newDeclaration.id); - }, [newDeclaration, allDeclarations]); + return system.privacy_declarations.filter( + (pd) => pd.id !== newDeclaration.id + ); + }, [newDeclaration, system]); - const checkAlreadyExists = (values: PrivacyDeclaration) => { + const checkAlreadyExists = (values: PrivacyDeclarationResponse) => { if ( accordionDeclarations.filter( (d) => d.data_use === values.data_use && d.name === values.name @@ -89,19 +75,16 @@ const PrivacyDeclarationManager = ({ }; const handleSave = async ( - updatedDeclarations: PrivacyDeclarationWithId[], + updatedDeclarations: PrivacyDeclarationResponse[], isDelete?: boolean ) => { - const transformedDeclarations = updatedDeclarations.map((d) => - transformDeclarationForSubmission(d) - ); const systemBodyWithDeclaration = { ...system, - privacy_declarations: transformedDeclarations, + privacy_declarations: updatedDeclarations, }; const handleResult = ( result: - | { data: System } + | { data: SystemResponse } | { error: FetchBaseQueryError | SerializedError } ) => { if (isErrorResult(result)) { @@ -120,7 +103,7 @@ const PrivacyDeclarationManager = ({ if (onSave) { onSave(result.data); } - return result.data.privacy_declarations as PrivacyDeclarationWithId[]; + return result.data.privacy_declarations; }; const updateSystemResult = await updateSystemMutationTrigger( @@ -131,8 +114,8 @@ const PrivacyDeclarationManager = ({ }; const handleEditDeclaration = async ( - oldDeclaration: PrivacyDeclarationWithId, - updatedDeclaration: PrivacyDeclarationWithId + oldDeclaration: PrivacyDeclarationResponse, + updatedDeclaration: PrivacyDeclarationResponse ) => { // Do not allow editing a privacy declaration to have the same data use as one that already exists if ( @@ -143,13 +126,13 @@ const PrivacyDeclarationManager = ({ } // Because the data use can change, we also need a reference to the old declaration in order to // make sure we are replacing the proper one - const updatedDeclarations = allDeclarations.map((dec) => + const updatedDeclarations = system.privacy_declarations.map((dec) => dec.id === oldDeclaration.id ? updatedDeclaration : dec ); return handleSave(updatedDeclarations); }; - const saveNewDeclaration = async (values: PrivacyDeclarationWithId) => { + const saveNewDeclaration = async (values: PrivacyDeclarationResponse) => { if (checkAlreadyExists(values)) { return undefined; } @@ -174,16 +157,16 @@ const PrivacyDeclarationManager = ({ }; const handleDelete = async ( - declarationToDelete: PrivacyDeclarationWithId + declarationToDelete: PrivacyDeclarationResponse ) => { - const updatedDeclarations = transformPrivacyDeclarationsToHaveId( - system.privacy_declarations - ).filter((dec) => dec.id !== declarationToDelete.id); + const updatedDeclarations = system.privacy_declarations.filter( + (dec) => dec.id !== declarationToDelete.id + ); return handleSave(updatedDeclarations, true); }; const handleDeleteNew = async ( - declarationToDelete: PrivacyDeclarationWithId + declarationToDelete: PrivacyDeclarationResponse ) => { const success = await handleDelete(declarationToDelete); if (success) { @@ -209,6 +192,7 @@ const PrivacyDeclarationManager = ({ onEdit={handleEditDeclaration} onDelete={handleDelete} includeCustomFields={includeCustomFields} + includeCookies={includeCookies} {...dataProps} /> {showNewForm ? ( @@ -218,6 +202,7 @@ const PrivacyDeclarationManager = ({ onSubmit={saveNewDeclaration} onDelete={handleDeleteNew} includeCustomFields={includeCustomFields} + includeCookies={includeCookies} {...dataProps} /> diff --git a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationStep.tsx b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationStep.tsx index dd7d947f01..7634e8aaf1 100644 --- a/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationStep.tsx +++ b/clients/admin-ui/src/features/system/privacy-declarations/PrivacyDeclarationStep.tsx @@ -3,13 +3,13 @@ import NextLink from "next/link"; import { useAppDispatch } from "~/app/hooks"; import { setActiveSystem } from "~/features/system"; -import { System } from "~/types/api"; +import { System, SystemResponse } from "~/types/api"; import { usePrivacyDeclarationData } from "./hooks"; import PrivacyDeclarationManager from "./PrivacyDeclarationManager"; interface Props { - system: System; + system: SystemResponse; } const PrivacyDeclarationStep = ({ system }: Props) => { @@ -48,6 +48,7 @@ const PrivacyDeclarationStep = ({ system }: Props) => { system={system} onSave={onSave} includeCustomFields + includeCookies {...dataProps} /> )} diff --git a/clients/admin-ui/src/features/system/privacy-declarations/types.ts b/clients/admin-ui/src/features/system/privacy-declarations/types.ts deleted file mode 100644 index b16500a316..0000000000 --- a/clients/admin-ui/src/features/system/privacy-declarations/types.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { PrivacyDeclaration } from "~/types/api"; - -/** - * This is because privacy declarations do not have an ID on the backend. - * It is very useful for React rendering to have a stable ID. We currently - * make this the composite of data_use - name, but even better may be to - * give it a UUID (or to have the backend actually enforce this!) - */ -export interface PrivacyDeclarationWithId extends PrivacyDeclaration { - id: string; -} diff --git a/clients/admin-ui/src/features/system/system.slice.ts b/clients/admin-ui/src/features/system/system.slice.ts index 1de03f616b..6a53b2a3fc 100644 --- a/clients/admin-ui/src/features/system/system.slice.ts +++ b/clients/admin-ui/src/features/system/system.slice.ts @@ -6,6 +6,7 @@ import { BulkPutConnectionConfiguration, ConnectionConfigurationResponse, System, + SystemResponse, } from "~/types/api"; interface SystemDeleteResponse { @@ -21,11 +22,11 @@ interface UpsertResponse { const systemApi = baseApi.injectEndpoints({ endpoints: (build) => ({ - getAllSystems: build.query({ + getAllSystems: build.query({ query: () => ({ url: `system/` }), providesTags: () => ["System"], }), - getSystemByFidesKey: build.query({ + getSystemByFidesKey: build.query({ query: (fides_key) => ({ url: `system/${fides_key}/` }), providesTags: ["System"], }), @@ -56,7 +57,7 @@ const systemApi = baseApi.injectEndpoints({ invalidatesTags: ["Datamap", "System", "Datastore Connection"], }), updateSystem: build.mutation< - System, + SystemResponse, Partial & Pick >({ query: ({ ...patch }) => ({ diff --git a/clients/admin-ui/src/types/api/index.ts b/clients/admin-ui/src/types/api/index.ts index 404068de1a..d0fc22bab5 100644 --- a/clients/admin-ui/src/types/api/index.ts +++ b/clients/admin-ui/src/types/api/index.ts @@ -72,6 +72,7 @@ export type { ConsentRequestMap } from "./models/ConsentRequestMap"; export type { ConsentRequestResponse } from "./models/ConsentRequestResponse"; export type { ConsentWithExecutableStatus } from "./models/ConsentWithExecutableStatus"; export type { ContactDetails } from "./models/ContactDetails"; +export type { Cookies } from "./models/Cookies"; export { CoreHealthCheck } from "./models/CoreHealthCheck"; export type { CreateConnectionConfigurationWithSecrets } from "./models/CreateConnectionConfigurationWithSecrets"; export type { CurrentPrivacyPreferenceReportingSchema } from "./models/CurrentPrivacyPreferenceReportingSchema"; @@ -111,6 +112,7 @@ export { DrpRegime } from "./models/DrpRegime"; export type { DrpRevokeRequest } from "./models/DrpRevokeRequest"; export type { DryRunDatasetResponse } from "./models/DryRunDatasetResponse"; export type { DynamoDBDocsSchema } from "./models/DynamoDBDocsSchema"; +export { EdgeDirection } from "./models/EdgeDirection"; export type { EmailDocsSchema } from "./models/EmailDocsSchema"; export type { Endpoint } from "./models/Endpoint"; export { EnforcementLevel } from "./models/EnforcementLevel"; @@ -128,7 +130,7 @@ export type { ExternalDatasetReference } from "./models/ExternalDatasetReference export type { fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds } from "./models/fides__api__schemas__connection_configuration__connection_secrets_bigquery__KeyfileCreds"; export type { fides__api__schemas__policy__Policy } from "./models/fides__api__schemas__policy__Policy"; export type { fides__connectors__models__KeyfileCreds } from "./models/fides__connectors__models__KeyfileCreds"; -export { FidesDatasetReference } from "./models/FidesDatasetReference"; +export type { FidesDatasetReference } from "./models/FidesDatasetReference"; export type { FidesDocsSchema } from "./models/FidesDocsSchema"; export type { fideslang__models__Policy } from "./models/fideslang__models__Policy"; export type { FidesMeta } from "./models/FidesMeta"; @@ -143,7 +145,6 @@ export type { HealthCheck } from "./models/HealthCheck"; export { HTTPMethod } from "./models/HTTPMethod"; export type { HTTPValidationError } from "./models/HTTPValidationError"; export type { Identity } from "./models/Identity"; -export type { IdentityBase } from "./models/IdentityBase"; export type { IdentityTypes } from "./models/IdentityTypes"; export type { IdentityVerificationConfigResponse } from "./models/IdentityVerificationConfigResponse"; export { IncludeExcludeEnum } from "./models/IncludeExcludeEnum"; diff --git a/clients/admin-ui/src/types/api/models/ConnectionType.ts b/clients/admin-ui/src/types/api/models/ConnectionType.ts index 493ef4ef43..f85c008449 100644 --- a/clients/admin-ui/src/types/api/models/ConnectionType.ts +++ b/clients/admin-ui/src/types/api/models/ConnectionType.ts @@ -3,26 +3,26 @@ /* eslint-disable */ /** - * Supported types to which we can connect fidesops. + * Supported types to which we can connect Fides. */ export enum ConnectionType { - POSTGRES = "postgres", //DB - MONGODB = "mongodb", //DB - MYSQL = "mysql", // DB + POSTGRES = "postgres", + MONGODB = "mongodb", + MYSQL = "mysql", HTTPS = "https", SAAS = "saas", - REDSHIFT = "redshift", //DB - SNOWFLAKE = "snowflake", //DB - MSSQL = "mssql", //DB - MARIADB = "mariadb", //DB - BIGQUERY = "bigquery", //DB - MANUAL = "manual", // manual - SOVRN = "sovrn", // email - ATTENTIVE = "attentive", // email - DYNAMODB = "dynamodb", //DB - MANUAL_WEBHOOK = "manual_webhook", //manual - TIMESCALE = "timescale", //DB + REDSHIFT = "redshift", + SNOWFLAKE = "snowflake", + MSSQL = "mssql", + MARIADB = "mariadb", + BIGQUERY = "bigquery", + MANUAL = "manual", + SOVRN = "sovrn", + ATTENTIVE = "attentive", + DYNAMODB = "dynamodb", + MANUAL_WEBHOOK = "manual_webhook", + TIMESCALE = "timescale", FIDES = "fides", - GENERIC_ERASURE_EMAIL = "erasure_email", - GENERIC_CONSENT_EMAIL = "consent_email", + GENERIC_ERASURE_EMAIL = "generic_erasure_email", + GENERIC_CONSENT_EMAIL = "generic_consent_email", } diff --git a/clients/admin-ui/src/types/api/models/ConnectorParam.ts b/clients/admin-ui/src/types/api/models/ConnectorParam.ts index 1828a5c043..00e0eb43a4 100644 --- a/clients/admin-ui/src/types/api/models/ConnectorParam.ts +++ b/clients/admin-ui/src/types/api/models/ConnectorParam.ts @@ -12,4 +12,5 @@ export type ConnectorParam = { default_value?: string | Array; multiselect?: boolean; description?: string; + sensitive?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/ConsentReport.ts b/clients/admin-ui/src/types/api/models/ConsentReport.ts index a120d82ba1..f03914ff64 100644 --- a/clients/admin-ui/src/types/api/models/ConsentReport.ts +++ b/clients/admin-ui/src/types/api/models/ConsentReport.ts @@ -2,7 +2,7 @@ /* tslint:disable */ /* eslint-disable */ -import type { IdentityBase } from "./IdentityBase"; +import type { Identity } from "./Identity"; /** * Schema for reporting Consent requests. @@ -14,7 +14,7 @@ export type ConsentReport = { has_gpc_flag?: boolean; conflicts_with_gpc?: boolean; id: string; - identity: IdentityBase; + identity: Identity; created_at: string; updated_at: string; }; diff --git a/clients/admin-ui/src/types/api/models/Cookies.ts b/clients/admin-ui/src/types/api/models/Cookies.ts new file mode 100644 index 0000000000..923ab591c3 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/Cookies.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * The Cookies resource model + */ +export type Cookies = { + name: string; + path?: string; + domain?: string; +}; diff --git a/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts b/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts index 69301c0f42..8096369510 100644 --- a/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts +++ b/clients/admin-ui/src/types/api/models/CreateConnectionConfigurationWithSecrets.ts @@ -2,8 +2,8 @@ /* tslint:disable */ /* eslint-disable */ -import type { ActionType } from "~/features/privacy-requests/types"; import type { AccessLevel } from "./AccessLevel"; +import type { ActionType } from "./ActionType"; import type { BigQueryDocsSchema } from "./BigQueryDocsSchema"; import type { ConnectionType } from "./ConnectionType"; import type { DynamoDBDocsSchema } from "./DynamoDBDocsSchema"; @@ -31,6 +31,7 @@ export type CreateConnectionConfigurationWithSecrets = { access: AccessLevel; disabled?: boolean; description?: string; + enabled_actions?: Array; secrets?: | MongoDBDocsSchema | PostgreSQLDocsSchema @@ -48,5 +49,4 @@ export type CreateConnectionConfigurationWithSecrets = { | SovrnDocsSchema | DynamoDBDocsSchema; saas_connector_type?: string; - enabled_actions?: ActionType[]; }; diff --git a/clients/admin-ui/src/types/api/models/Dataset.ts b/clients/admin-ui/src/types/api/models/Dataset.ts index 0e37fe052b..9d009b8d61 100644 --- a/clients/admin-ui/src/types/api/models/Dataset.ts +++ b/clients/admin-ui/src/types/api/models/Dataset.ts @@ -28,9 +28,9 @@ export type Dataset = { */ description?: string; /** - * An optional object that provides additional information about the Dataset. You can structure the object however you like. It can be a simple set of `key: value` properties or a deeply nested hierarchy of objects. How you use the object is up to you: Fides ignores it. + * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: Record; + meta?: any; /** * Array of Data Category resources identified by `fides_key`, that apply to all collections in the Dataset. */ diff --git a/clients/admin-ui/src/types/api/models/DynamoDBDocsSchema.ts b/clients/admin-ui/src/types/api/models/DynamoDBDocsSchema.ts index 3c5b33b8fa..f3bb707c35 100644 --- a/clients/admin-ui/src/types/api/models/DynamoDBDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/DynamoDBDocsSchema.ts @@ -8,6 +8,6 @@ export type DynamoDBDocsSchema = { url?: string; region_name: string; - aws_secret_access_key: string; aws_access_key_id: string; + aws_secret_access_key: string; }; diff --git a/clients/admin-ui/src/types/api/models/EdgeDirection.ts b/clients/admin-ui/src/types/api/models/EdgeDirection.ts new file mode 100644 index 0000000000..dcce420fd7 --- /dev/null +++ b/clients/admin-ui/src/types/api/models/EdgeDirection.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * Direction of a FidesDataSetReference + */ +export enum EdgeDirection { + FROM = "from", + TO = "to", +} diff --git a/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts b/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts index 51831625d4..dbd507a4ec 100644 --- a/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/EmailDocsSchema.ts @@ -11,5 +11,5 @@ export type EmailDocsSchema = { third_party_vendor_name: string; recipient_email_address: string; test_email_address?: string; - advanced_settings: AdvancedSettings; + advanced_settings?: AdvancedSettings; }; diff --git a/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts b/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts index 8c6f702e24..39e2fd943e 100644 --- a/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts +++ b/clients/admin-ui/src/types/api/models/FidesDatasetReference.ts @@ -2,18 +2,13 @@ /* tslint:disable */ /* eslint-disable */ +import type { EdgeDirection } from "./EdgeDirection"; + /** * Reference to a field from another Collection */ export type FidesDatasetReference = { dataset: string; field: string; - direction?: FidesDatasetReference.direction; + direction?: EdgeDirection; }; - -export namespace FidesDatasetReference { - export enum direction { - FROM = "from", - TO = "to", - } -} diff --git a/clients/admin-ui/src/types/api/models/IdentityBase.ts b/clients/admin-ui/src/types/api/models/IdentityBase.ts deleted file mode 100644 index 93927af410..0000000000 --- a/clients/admin-ui/src/types/api/models/IdentityBase.ts +++ /dev/null @@ -1,11 +0,0 @@ -/* istanbul ignore file */ -/* tslint:disable */ -/* eslint-disable */ - -/** - * The minimum fields required to represent an identity. - */ -export type IdentityBase = { - phone_number?: string; - email?: string; -}; diff --git a/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts b/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts index f7ea1c5b24..1562d96221 100644 --- a/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/PostgreSQLDocsSchema.ts @@ -13,4 +13,5 @@ export type PostgreSQLDocsSchema = { db_schema?: string; host?: string; port?: number; + ssh_required?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts b/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts index 1edd7683e7..a826af7206 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyDeclaration.ts @@ -2,6 +2,8 @@ /* tslint:disable */ /* eslint-disable */ +import type { Cookies } from "./Cookies"; + /** * The PrivacyDeclaration resource model. * @@ -41,4 +43,8 @@ export type PrivacyDeclaration = { * The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied. */ ingress?: Array; + /** + * Cookies associated with this data use to deliver services and functionality + */ + cookies?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts index dbf5327824..53de457c72 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyDeclarationResponse.ts @@ -2,6 +2,8 @@ /* tslint:disable */ /* eslint-disable */ +import type { Cookies } from "./Cookies"; + /** * Extension of base pydantic model to include DB `id` field in the response */ @@ -38,6 +40,7 @@ export type PrivacyDeclarationResponse = { * The resources from which data is received. Any `fides_key`s included in this list reference `DataFlow` entries in the `ingress` array of any `System` resources to which this `PrivacyDeclaration` is applied. */ ingress?: Array; + cookies?: Array; /** * The database-assigned ID of the privacy declaration on the system. This is meant to be a read-only field, returned only in API responses */ diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts index ab81aacaa6..623b92d76c 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponse.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ConsentMechanism } from "./ConsentMechanism"; +import type { Cookies } from "./Cookies"; import type { EnforcementLevel } from "./EnforcementLevel"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; @@ -29,4 +30,5 @@ export type PrivacyNoticeResponse = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; }; diff --git a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts index 82f78dc9d0..6919ee30d5 100644 --- a/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts +++ b/clients/admin-ui/src/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ConsentMechanism } from "./ConsentMechanism"; +import type { Cookies } from "./Cookies"; import type { EnforcementLevel } from "./EnforcementLevel"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; import type { UserConsentPreference } from "./UserConsentPreference"; @@ -31,6 +32,7 @@ export type PrivacyNoticeResponseWithUserPreferences = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; default_preference: UserConsentPreference; current_preference?: UserConsentPreference; outdated_preference?: UserConsentPreference; diff --git a/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts b/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts index 6e0eadf300..0b3ce296f6 100644 --- a/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/RedshiftDocsSchema.ts @@ -13,4 +13,5 @@ export type RedshiftDocsSchema = { user?: string; password?: string; db_schema?: string; + ssh_required?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/ResponseFormat.ts b/clients/admin-ui/src/types/api/models/ResponseFormat.ts index c874d392e6..bf2c4df56a 100644 --- a/clients/admin-ui/src/types/api/models/ResponseFormat.ts +++ b/clients/admin-ui/src/types/api/models/ResponseFormat.ts @@ -8,4 +8,5 @@ export enum ResponseFormat { JSON = "json", CSV = "csv", + HTML = "html", } diff --git a/clients/admin-ui/src/types/api/models/SaaSRequest.ts b/clients/admin-ui/src/types/api/models/SaaSRequest.ts index 5628c8ae22..c522917335 100644 --- a/clients/admin-ui/src/types/api/models/SaaSRequest.ts +++ b/clients/admin-ui/src/types/api/models/SaaSRequest.ts @@ -29,7 +29,7 @@ export type SaaSRequest = { postprocessors?: Array; pagination?: Strategy; grouped_inputs?: Array; - ignore_errors?: boolean; + ignore_errors?: boolean | Array; rate_limit_config?: RateLimitConfig; skip_missing_param_values?: boolean; }; diff --git a/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts b/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts index 5825772c40..3e7dfb60c7 100644 --- a/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/SovrnDocsSchema.ts @@ -11,5 +11,5 @@ export type SovrnDocsSchema = { third_party_vendor_name?: string; recipient_email_address?: string; test_email_address?: string; - advanced_settings: AdvancedSettingsWithExtendedIdentityTypes; + advanced_settings?: AdvancedSettingsWithExtendedIdentityTypes; }; diff --git a/clients/admin-ui/src/types/api/models/System.ts b/clients/admin-ui/src/types/api/models/System.ts index ebc3a86179..e2ac6a4cb4 100644 --- a/clients/admin-ui/src/types/api/models/System.ts +++ b/clients/admin-ui/src/types/api/models/System.ts @@ -37,9 +37,9 @@ export type System = { */ registry_id?: number; /** - * An optional property to store any extra information for a system. Not used by fidesctl. + * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: Record; + meta?: any; /** * * The SystemMetadata resource model. diff --git a/clients/admin-ui/src/types/api/models/SystemResponse.ts b/clients/admin-ui/src/types/api/models/SystemResponse.ts index 35f9d0ad6d..8b1847f8df 100644 --- a/clients/admin-ui/src/types/api/models/SystemResponse.ts +++ b/clients/admin-ui/src/types/api/models/SystemResponse.ts @@ -4,6 +4,7 @@ import type { ConnectionConfigurationResponse } from "./ConnectionConfigurationResponse"; import type { ContactDetails } from "./ContactDetails"; +import type { Cookies } from "./Cookies"; import type { DataFlow } from "./DataFlow"; import type { DataProtectionImpactAssessment } from "./DataProtectionImpactAssessment"; import type { DataResponsibilityTitle } from "./DataResponsibilityTitle"; @@ -36,9 +37,9 @@ export type SystemResponse = { */ registry_id?: number; /** - * An optional property to store any extra information for a system. Not used by fidesctl. + * An optional property to store any extra information for a resource. Data can be structured in any way: simple set of `key: value` pairs or deeply nested objects. */ - meta?: Record; + meta?: any; /** * * The SystemMetadata resource model. @@ -114,4 +115,5 @@ export type SystemResponse = { * */ connection_configs?: ConnectionConfigurationResponse; + cookies?: Array; }; diff --git a/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts b/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts index b087fffee4..b9f413993e 100644 --- a/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts +++ b/clients/admin-ui/src/types/api/models/TimescaleDocsSchema.ts @@ -13,4 +13,5 @@ export type TimescaleDocsSchema = { db_schema?: string; host?: string; port?: number; + ssh_required?: boolean; }; diff --git a/clients/fides-js/__tests__/lib/cookie.test.ts b/clients/fides-js/__tests__/lib/cookie.test.ts index 9835335638..407aec55e3 100644 --- a/clients/fides-js/__tests__/lib/cookie.test.ts +++ b/clients/fides-js/__tests__/lib/cookie.test.ts @@ -1,4 +1,6 @@ import * as uuid from "uuid"; + +import { CookieAttributes } from "typescript-cookie/dist/types"; import { CookieKeyConsent, FidesCookie, @@ -6,10 +8,11 @@ import { isNewFidesCookie, makeConsentDefaultsLegacy, makeFidesCookie, + removeCookiesFromBrowser, saveFidesCookie, } from "../../src/lib/cookie"; import type { ConsentContext } from "../../src/lib/consent-context"; -import { LegacyConsentConfig } from "~/lib/consent-types"; +import { Cookies, LegacyConsentConfig } from "../../src/lib/consent-types"; // Setup mock date const MOCK_DATE = "2023-01-01T12:00:00.000Z"; @@ -31,6 +34,10 @@ const mockSetCookie = jest.fn( (name: string, value: string, attributes: object, encoding: object) => `mock setCookie return (value=${value})` ); +const mockRemoveCookie = jest.fn( + /* eslint-disable-next-line @typescript-eslint/no-unused-vars */ + (name: string, attributes?: CookieAttributes) => undefined +); jest.mock("typescript-cookie", () => ({ getCookie: () => mockGetCookie(), setCookie: ( @@ -39,6 +46,8 @@ jest.mock("typescript-cookie", () => ({ attributes: object, encoding: object ) => mockSetCookie(name, value, attributes, encoding), + removeCookie: (name: string, attributes?: CookieAttributes) => + mockRemoveCookie(name, attributes), })); describe("makeFidesCookie", () => { @@ -276,3 +285,41 @@ describe("isNewFidesCookie", () => { }); }); }); + +describe("removeCookiesFromBrowser", () => { + afterEach(() => mockRemoveCookie.mockClear()); + + it.each([ + { cookies: [], expectedAttributes: [] }, + { cookies: [{ name: "_ga123" }], expectedAttributes: [{ path: "/" }] }, + { + cookies: [{ name: "_ga123", path: "" }], + expectedAttributes: [{ path: "" }], + }, + { + cookies: [{ name: "_ga123", path: "/subpage" }], + expectedAttributes: [{ path: "/subpage" }], + }, + { + cookies: [{ name: "_ga123" }, { name: "shopify" }], + expectedAttributes: [{ path: "/" }, { path: "/" }], + }, + ])( + "should remove a list of cookies", + ({ + cookies, + expectedAttributes, + }: { + cookies: Cookies[]; + expectedAttributes: CookieAttributes[]; + }) => { + removeCookiesFromBrowser(cookies); + expect(mockRemoveCookie.mock.calls).toHaveLength(cookies.length); + cookies.forEach((cookie, idx) => { + const [name, attributes] = mockRemoveCookie.mock.calls[idx]; + expect(name).toEqual(cookie.name); + expect(attributes).toEqual(expectedAttributes[idx]); + }); + } + ); +}); diff --git a/clients/fides-js/src/components/Overlay.tsx b/clients/fides-js/src/components/Overlay.tsx index fc79e6a768..94c9dfbb95 100644 --- a/clients/fides-js/src/components/Overlay.tsx +++ b/clients/fides-js/src/components/Overlay.tsx @@ -117,11 +117,7 @@ const Overlay: FunctionComponent = ({ enabledPrivacyNoticeKeys.includes(notice.notice_key), notice.consent_mechanism ); - return new SaveConsentPreference( - notice.notice_key, - notice.privacy_notice_history_id, - userPreference - ); + return new SaveConsentPreference(notice, userPreference); }); updateConsentPreferences({ consentPreferencesToSave, diff --git a/clients/fides-js/src/fides.ts b/clients/fides-js/src/fides.ts index eb3153b391..c4fc1fa93e 100644 --- a/clients/fides-js/src/fides.ts +++ b/clients/fides-js/src/fides.ts @@ -144,8 +144,7 @@ const automaticallyApplyGPCPreferences = ( if (notice.has_gpc_flag && !notice.current_preference) { consentPreferencesToSave.push( new SaveConsentPreference( - notice.notice_key, - notice.privacy_notice_history_id, + notice, transformConsentToFidesUserPreference(false, notice.consent_mechanism) ) ); diff --git a/clients/fides-js/src/lib/consent-types.ts b/clients/fides-js/src/lib/consent-types.ts index b56b089f65..be126828f7 100644 --- a/clients/fides-js/src/lib/consent-types.ts +++ b/clients/fides-js/src/lib/consent-types.ts @@ -40,17 +40,10 @@ export type FidesOptions = { export class SaveConsentPreference { consentPreference: UserConsentPreference; - noticeHistoryId: string; + notice: PrivacyNotice; - noticeKey: string; - - constructor( - noticeKey: string, - noticeHistoryId: string, - consentPreference: UserConsentPreference - ) { - this.noticeKey = noticeKey; - this.noticeHistoryId = noticeHistoryId; + constructor(notice: PrivacyNotice, consentPreference: UserConsentPreference) { + this.notice = notice; this.consentPreference = consentPreference; } } @@ -88,6 +81,12 @@ export type ExperienceConfig = { regions: Array; }; +export type Cookies = { + name: string; + path?: string; + domain?: string; +}; + export type PrivacyNotice = { name?: string; notice_key: string; @@ -108,6 +107,7 @@ export type PrivacyNotice = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; default_preference: UserConsentPreference; current_preference?: UserConsentPreference; outdated_preference?: UserConsentPreference; diff --git a/clients/fides-js/src/lib/cookie.ts b/clients/fides-js/src/lib/cookie.ts index dee6a4b6cf..4cf8830469 100644 --- a/clients/fides-js/src/lib/cookie.ts +++ b/clients/fides-js/src/lib/cookie.ts @@ -1,12 +1,16 @@ import { v4 as uuidv4 } from "uuid"; -import { getCookie, setCookie, Types } from "typescript-cookie"; +import { getCookie, removeCookie, setCookie, Types } from "typescript-cookie"; import { ConsentContext } from "./consent-context"; import { resolveConsentValue, resolveLegacyConsentValue, } from "./consent-value"; -import { LegacyConsentConfig, PrivacyExperience } from "./consent-types"; +import { + Cookies, + LegacyConsentConfig, + PrivacyExperience, +} from "./consent-types"; import { debugLog } from "./consent-utils"; /** @@ -273,3 +277,15 @@ export const makeConsentDefaultsLegacy = ( debugLog(debug, `Returning defaults for legacy config.`, defaults); return defaults; }; + +/** + * Given a list of cookies, deletes them from the browser + */ +export const removeCookiesFromBrowser = (cookies: Cookies[]) => { + cookies.forEach((cookie) => { + removeCookie(cookie.name, { + path: cookie.path ?? "/", + domain: cookie.domain, + }); + }); +}; diff --git a/clients/fides-js/src/lib/preferences.ts b/clients/fides-js/src/lib/preferences.ts index ce6bbb9932..71646dd17b 100644 --- a/clients/fides-js/src/lib/preferences.ts +++ b/clients/fides-js/src/lib/preferences.ts @@ -3,9 +3,15 @@ import { ConsentOptionCreate, PrivacyPreferencesRequest, SaveConsentPreference, + UserConsentPreference, } from "./consent-types"; import { debugLog, transformUserPreferenceToBoolean } from "./consent-utils"; -import { CookieKeyConsent, FidesCookie, saveFidesCookie } from "./cookie"; +import { + CookieKeyConsent, + FidesCookie, + removeCookiesFromBrowser, + saveFidesCookie, +} from "./cookie"; import { dispatchFidesEvent } from "./events"; import { patchUserPreferenceToFidesServer } from "../services/fides/api"; @@ -14,6 +20,8 @@ import { patchUserPreferenceToFidesServer } from "../services/fides/api"; * 1. Save preferences to Fides API * 2. Update the window.Fides.consent object * 3. Save preferences to the `fides_consent` cookie in the browser + * 4. Remove any cookies from notices that were opted-out from the browser + * 5. Dispatch a "FidesUpdated" event */ export const updateConsentPreferences = ({ consentPreferencesToSave, @@ -34,21 +42,19 @@ export const updateConsentPreferences = ({ }) => { // Derive the CookieKeyConsent object from privacy notices const noticeMap = new Map( - consentPreferencesToSave.map(({ noticeKey, consentPreference }) => [ - noticeKey, + consentPreferencesToSave.map(({ notice, consentPreference }) => [ + notice.notice_key, transformUserPreferenceToBoolean(consentPreference), ]) ); const consentCookieKey: CookieKeyConsent = Object.fromEntries(noticeMap); // Derive the Fides user preferences array from privacy notices - const fidesUserPreferences: Array = []; - consentPreferencesToSave.forEach(({ noticeHistoryId, consentPreference }) => { - fidesUserPreferences.push({ - privacy_notice_history_id: noticeHistoryId, + const fidesUserPreferences: Array = + consentPreferencesToSave.map(({ notice, consentPreference }) => ({ + privacy_notice_history_id: notice.privacy_notice_history_id, preference: consentPreference, - }); - }); + })); // Update the cookie object // eslint-disable-next-line no-param-reassign @@ -63,12 +69,7 @@ export const updateConsentPreferences = ({ user_geography: userLocationString, method: consentMethod, }; - patchUserPreferenceToFidesServer( - privacyPreferenceCreate, - fidesApiUrl, - cookie.identity.fides_user_device_id, - debug - ); + patchUserPreferenceToFidesServer(privacyPreferenceCreate, fidesApiUrl, debug); // 2. Update the window.Fides.consent object debugLog(debug, "Updating window.Fides"); @@ -78,6 +79,16 @@ export const updateConsentPreferences = ({ debugLog(debug, "Saving preferences to cookie"); saveFidesCookie(cookie); - // 4. Dispatch a "FidesUpdated" event + // 4. Remove cookies associated with notices that were opted-out from the browser + consentPreferencesToSave + .filter( + (preference) => + preference.consentPreference === UserConsentPreference.OPT_OUT + ) + .forEach((preference) => { + removeCookiesFromBrowser(preference.notice.cookies); + }); + + // 5. Dispatch a "FidesUpdated" event dispatchFidesEvent("FidesUpdated", cookie); }; diff --git a/clients/fides-js/src/services/fides/api.ts b/clients/fides-js/src/services/fides/api.ts index b219a01db9..1e53c6840b 100644 --- a/clients/fides-js/src/services/fides/api.ts +++ b/clients/fides-js/src/services/fides/api.ts @@ -81,7 +81,6 @@ export const fetchExperience = async ( export const patchUserPreferenceToFidesServer = async ( preferences: PrivacyPreferencesRequest, fidesApiUrl: string, - fidesUserDeviceId: string, debug: boolean ): Promise => { debugLog(debug, "Saving user consent preference...", preferences); diff --git a/clients/privacy-center/components/consent/NoticeDrivenConsent.tsx b/clients/privacy-center/components/consent/NoticeDrivenConsent.tsx index bf16a75a27..73d137d9eb 100644 --- a/clients/privacy-center/components/consent/NoticeDrivenConsent.tsx +++ b/clients/privacy-center/components/consent/NoticeDrivenConsent.tsx @@ -5,6 +5,7 @@ import { CookieKeyConsent, getConsentContext, getOrMakeFidesCookie, + removeCookiesFromBrowser, saveFidesCookie, transformUserPreferenceToBoolean, } from "fides-js"; @@ -121,30 +122,44 @@ const NoticeDrivenConsent = () => { router.push("/"); }; + /** + * When saving, we need to: + * 1. Send PATCH to Fides backend + * 2. Save to cookie and window object + * 3. Delete any cookies that have been opted out of + */ const handleSave = async () => { const browserIdentities = inspectForBrowserIdentities(); const deviceIdentity = { fides_user_device_id: fidesUserDeviceId }; const identities = browserIdentities ? { ...deviceIdentity, ...browserIdentities } : deviceIdentity; + const notices = experience?.privacy_notices ?? []; - const preferences: ConsentOptionCreate[] = Object.entries( - draftPreferences - ).map(([key, value]) => { - const notice = experience?.privacy_notices?.find( - (n) => n.privacy_notice_history_id === key - ); - if (notice?.consent_mechanism === ConsentMechanism.NOTICE_ONLY) { + // Reconnect preferences to notices + const noticePreferences = Object.entries(draftPreferences).map( + ([historyKey, preference]) => { + const notice = notices.find( + (n) => n.privacy_notice_history_id === historyKey + ); + return { historyKey, preference, notice }; + } + ); + + const preferences: ConsentOptionCreate[] = noticePreferences.map( + ({ historyKey, preference, notice }) => { + if (notice?.consent_mechanism === ConsentMechanism.NOTICE_ONLY) { + return { + privacy_notice_history_id: historyKey, + preference: UserConsentPreference.ACKNOWLEDGE, + }; + } return { - privacy_notice_history_id: key, - preference: UserConsentPreference.ACKNOWLEDGE, + privacy_notice_history_id: historyKey, + preference: preference ?? UserConsentPreference.OPT_OUT, }; } - return { - privacy_notice_history_id: key, - preference: value ?? UserConsentPreference.OPT_OUT, - }; - }); + ); const payload: PrivacyPreferencesRequest = { browser_identity: identities, @@ -155,6 +170,7 @@ const NoticeDrivenConsent = () => { code: verificationCode, }; + // 1. Send PATCH to Fides backend const result = await updatePrivacyPreferencesMutationTrigger({ id: consentRequestId, body: payload, @@ -167,6 +183,8 @@ const NoticeDrivenConsent = () => { }); return; } + + // 2. Save the cookie and window obj on success const noticeKeyMap = new Map( result.data.map((preference) => [ preference.privacy_notice_history.notice_key || "", @@ -174,14 +192,23 @@ const NoticeDrivenConsent = () => { ]) ); const consentCookieKey: CookieKeyConsent = Object.fromEntries(noticeKeyMap); + window.Fides.consent = consentCookieKey; + const updatedCookie = { ...cookie, consent: consentCookieKey }; + saveFidesCookie(updatedCookie); toast({ title: "Your consent preferences have been saved", ...SuccessToastOptions, }); - // Save the cookie and window obj on success - window.Fides.consent = consentCookieKey; - const updatedCookie = { ...cookie, consent: consentCookieKey }; - saveFidesCookie(updatedCookie); + + // 3. Delete any cookies that have been opted out of + noticePreferences.forEach((noticePreference) => { + if ( + noticePreference.preference === UserConsentPreference.OPT_OUT && + noticePreference.notice + ) { + removeCookiesFromBrowser(noticePreference.notice.cookies); + } + }); router.push("/"); }; diff --git a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts index d54664fe90..63f0453fbf 100644 --- a/clients/privacy-center/cypress/e2e/consent-banner.cy.ts +++ b/clients/privacy-center/cypress/e2e/consent-banner.cy.ts @@ -126,6 +126,7 @@ const mockPrivacyNotice = (params: Partial) => { version: 1.0, privacy_notice_history_id: "pri_b09058a7-9f54-4360-8da5-4521e8975d4f", notice_key: "advertising", + cookies: [], }; return { ...notice, ...params }; }; @@ -478,6 +479,61 @@ describe("Consent banner", () => { // TODO: add tests for CSS expect(false).is.eql(true); }); + + describe("cookie enforcement", () => { + beforeEach(() => { + const cookies = [ + { name: "cookie1", path: "/" }, + { name: "cookie2", path: "/" }, + ]; + cookies.forEach((cookie) => { + cy.setCookie(cookie.name, "value", { path: cookie.path }); + }); + stubConfig({ + experience: { + privacy_notices: [ + mockPrivacyNotice({ + name: "one", + privacy_notice_history_id: "one", + notice_key: "one", + consent_mechanism: ConsentMechanism.OPT_OUT, + cookies: [cookies[0]], + }), + mockPrivacyNotice({ + name: "two", + privacy_notice_history_id: "two", + notice_key: "second", + consent_mechanism: ConsentMechanism.OPT_OUT, + cookies: [cookies[1]], + }), + ], + }, + options: { + isOverlayEnabled: true, + }, + }); + }); + + it("can remove all cookies when rejecting all", () => { + cy.contains("button", "Reject Test").click(); + cy.getAllCookies().then((allCookies) => { + expect(allCookies.map((c) => c.name)).to.eql([CONSENT_COOKIE_NAME]); + }); + }); + + it("can remove just the cookies associated with notices that were opted out", () => { + cy.contains("button", "Manage preferences").click(); + // opt out of the first notice + cy.getByTestId("toggle-one").click(); + cy.getByTestId("Save test-btn").click(); + cy.getAllCookies().then((allCookies) => { + expect(allCookies.map((c) => c.name)).to.eql([ + CONSENT_COOKIE_NAME, + "cookie2", + ]); + }); + }); + }); }); describe("when there are only notice-only notices", () => { diff --git a/clients/privacy-center/cypress/e2e/consent-notices.cy.ts b/clients/privacy-center/cypress/e2e/consent-notices.cy.ts index d6c62ef382..cc5fac8f6e 100644 --- a/clients/privacy-center/cypress/e2e/consent-notices.cy.ts +++ b/clients/privacy-center/cypress/e2e/consent-notices.cy.ts @@ -87,6 +87,7 @@ describe("Privacy notice driven consent", () => { cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_2}`).within(() => { cy.getRadio().should("be.checked"); }); + // Notice only, so should be checked and disabled cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_3}`).within(() => { cy.getRadio().should("be.checked").should("be.disabled"); }); @@ -107,6 +108,7 @@ describe("Privacy notice driven consent", () => { expect( preferences.map((p: ConsentOptionCreate) => p.preference) ).to.eql(["opt_in", "opt_in", "acknowledge"]); + // Should update the cookie cy.waitUntilCookieExists(CONSENT_COOKIE_NAME).then(() => { cy.getCookie(CONSENT_COOKIE_NAME).then((cookieJson) => { const cookie = JSON.parse( @@ -118,6 +120,7 @@ describe("Privacy notice driven consent", () => { const expectedConsent = { data_sales: true, advertising: true }; const { consent } = cookie; expect(consent).to.eql(expectedConsent); + // Should update the window object cy.window().then((win) => { expect(win.Fides.consent).to.eql(expectedConsent); }); @@ -154,6 +157,108 @@ describe("Privacy notice driven consent", () => { }); }); }); + + describe("cookie enforcement", () => { + beforeEach(() => { + // First seed the browser with the cookies that are listed in the notices + cy.fixture("consent/experience.json").then((data) => { + const notices: PrivacyNoticeResponseWithUserPreferences[] = + data.items[0].privacy_notices; + + const allCookies = notices.map((notice) => notice.cookies).flat(); + allCookies.forEach((cookie) => { + cy.setCookie(cookie.name, "value", { + path: cookie.path ?? "/", + domain: cookie.domain ?? undefined, + }); + }); + cy.getAllCookies().then((cookies) => { + expect( + cookies.filter((c) => c.name !== CONSENT_COOKIE_NAME).length + ).to.eql(allCookies.length); + }); + cy.wrap(notices).as("notices"); + }); + }); + + it("can delete all cookies for when opting out of all notices", () => { + // Opt out of the opt-out notice + cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_2}`).within(() => { + cy.getRadio().should("be.checked"); + cy.get("span").contains("No").click(); + }); + cy.getByTestId("save-btn").click(); + + cy.wait("@patchPrivacyPreference").then(() => { + // Use waitUntil to help with CI + cy.waitUntil(() => + cy.getAllCookies().then((cookies) => cookies.length === 1) + ).then(() => { + // There should be no cookies related to the privacy notices around + cy.getAllCookies().then((cookies) => { + const filteredCookies = cookies.filter( + (c) => c.name !== CONSENT_COOKIE_NAME + ); + expect(filteredCookies.length).to.eql(0); + }); + }); + }); + }); + + it("can delete only the cookies associated with opt-out notices", () => { + // Opt into first notice + cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_1}`).within(() => { + cy.get("span").contains("Yes").click(); + }); + // Opt out of second notice + cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_2}`).within(() => { + cy.getRadio().should("be.checked"); + cy.get("span").contains("No").click(); + }); + cy.getByTestId("save-btn").click(); + + cy.wait("@patchPrivacyPreference").then(() => { + // Use waitUntil to help with CI + cy.waitUntil(() => + cy.getAllCookies().then((cookies) => cookies.length === 2) + ).then(() => { + // The first notice's cookies should still be around + // But there should be none of the second cookie's + cy.getAllCookies().then((cookies) => { + const filteredCookies = cookies.filter( + (c) => c.name !== CONSENT_COOKIE_NAME + ); + expect(filteredCookies.length).to.eql(1); + cy.get("@notices").then((notices: any) => { + expect(filteredCookies[0]).to.have.property( + "name", + notices[0].cookies[0].name + ); + }); + }); + }); + }); + }); + + it("can successfully delete even if cookie does not exist", () => { + cy.clearAllCookies(); + // Opt out of second notice + cy.getByTestId(`consent-item-${PRIVACY_NOTICE_ID_2}`).within(() => { + cy.getRadio().should("be.checked"); + cy.get("span").contains("No").click(); + }); + cy.getByTestId("save-btn").click(); + + cy.wait("@patchPrivacyPreference").then(() => { + cy.getAllCookies().then((cookies) => { + const filteredCookies = cookies.filter( + (c) => c.name !== CONSENT_COOKIE_NAME + ); + expect(filteredCookies.length).to.eql(0); + }); + }); + }); + }); }); describe("when user has consented before", () => { diff --git a/clients/privacy-center/cypress/fixtures/consent/experience.json b/clients/privacy-center/cypress/fixtures/consent/experience.json index c52570afb2..3282ec2d51 100644 --- a/clients/privacy-center/cypress/fixtures/consent/experience.json +++ b/clients/privacy-center/cypress/fixtures/consent/experience.json @@ -51,7 +51,8 @@ "privacy_notice_history_id": "pri_df14051b-1eaf-4f07-ae63-232bffd2dc3e", "default_preference": "opt_out", "current_preference": null, - "outdated_preference": null + "outdated_preference": null, + "cookies": [{ "name": "sales", "path": null, "domain": null }] }, { "name": "Advertising", @@ -75,7 +76,11 @@ "privacy_notice_history_id": "pri_b2a0a2fa-ef59-4f7d-8e3d-d2e9bd076707", "default_preference": "opt_in", "current_preference": null, - "outdated_preference": null + "outdated_preference": null, + "cookies": [ + { "name": "_ga", "path": null, "domain": null }, + { "name": "advertisingCookie", "path": null, "domain": null } + ] }, { "name": "Essential", @@ -99,7 +104,8 @@ "privacy_notice_history_id": "pri_b09058a7-9f54-4360-8da5-4521e8975d4e", "default_preference": "opt_in", "current_preference": null, - "outdated_preference": null + "outdated_preference": null, + "cookies": [] } ] } diff --git a/clients/privacy-center/cypress/fixtures/consent/overlay_experience.json b/clients/privacy-center/cypress/fixtures/consent/overlay_experience.json index 1e68cd4194..79a78495bd 100644 --- a/clients/privacy-center/cypress/fixtures/consent/overlay_experience.json +++ b/clients/privacy-center/cypress/fixtures/consent/overlay_experience.json @@ -51,7 +51,8 @@ "privacy_notice_history_id": "pri_b2a0a2fa-ef59-4f7d-8e3d-d2e9bd076707", "default_preference": "opt_out", "current_preference": null, - "outdated_preference": null + "outdated_preference": null, + "cookies": [] } ] } diff --git a/clients/privacy-center/cypress/fixtures/consent/test_banner_options.json b/clients/privacy-center/cypress/fixtures/consent/test_banner_options.json index dc830e1fef..6c30154e0e 100644 --- a/clients/privacy-center/cypress/fixtures/consent/test_banner_options.json +++ b/clients/privacy-center/cypress/fixtures/consent/test_banner_options.json @@ -64,7 +64,8 @@ "updated_at": "2023-04-24T21:29:08.870351+00:00", "version": 1.0, "privacy_notice_history_id": "pri_b09058a7-9f54-4360-8da5-4521e8975d4f", - "notice_key": "advertising" + "notice_key": "advertising", + "cookies": [] }, { "name": "Essential", @@ -85,7 +86,8 @@ "updated_at": "2023-04-24T21:29:08.870351+00:00", "version": 1.0, "privacy_notice_history_id": "pri_b09058a7-9f54-4360-8da5-4521e8975d4e", - "notice_key": "essential" + "notice_key": "essential", + "cookies": [] } ] }, diff --git a/clients/privacy-center/public/fides-js-components-demo.html b/clients/privacy-center/public/fides-js-components-demo.html index 4f07dc08f8..3614d6c117 100644 --- a/clients/privacy-center/public/fides-js-components-demo.html +++ b/clients/privacy-center/public/fides-js-components-demo.html @@ -77,6 +77,7 @@ privacy_notice_history_id: "pri_b09058a7-9f54-4360-8da5-4521e8975d4f", notice_key: "advertising", + cookies: [{ name: "testCookie", path: "/", domain: null }], }, { name: "Essential", @@ -100,6 +101,7 @@ privacy_notice_history_id: "pri_b09058a7-9f54-4360-8da5-4521e8975d4e", notice_key: "essential", + cookies: [], }, ], }, diff --git a/clients/privacy-center/types/api/models/Cookies.ts b/clients/privacy-center/types/api/models/Cookies.ts new file mode 100644 index 0000000000..923ab591c3 --- /dev/null +++ b/clients/privacy-center/types/api/models/Cookies.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +/** + * The Cookies resource model + */ +export type Cookies = { + name: string; + path?: string; + domain?: string; +}; diff --git a/clients/privacy-center/types/api/models/PrivacyNoticeResponse.ts b/clients/privacy-center/types/api/models/PrivacyNoticeResponse.ts index 8f33d8b38b..de11c2abfe 100644 --- a/clients/privacy-center/types/api/models/PrivacyNoticeResponse.ts +++ b/clients/privacy-center/types/api/models/PrivacyNoticeResponse.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ConsentMechanism } from "./ConsentMechanism"; +import type { Cookies } from "./Cookies"; import type { EnforcementLevel } from "./EnforcementLevel"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; @@ -28,4 +29,5 @@ export type PrivacyNoticeResponse = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; }; diff --git a/clients/privacy-center/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts b/clients/privacy-center/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts index 82f78dc9d0..6919ee30d5 100644 --- a/clients/privacy-center/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts +++ b/clients/privacy-center/types/api/models/PrivacyNoticeResponseWithUserPreferences.ts @@ -3,6 +3,7 @@ /* eslint-disable */ import type { ConsentMechanism } from "./ConsentMechanism"; +import type { Cookies } from "./Cookies"; import type { EnforcementLevel } from "./EnforcementLevel"; import type { PrivacyNoticeRegion } from "./PrivacyNoticeRegion"; import type { UserConsentPreference } from "./UserConsentPreference"; @@ -31,6 +32,7 @@ export type PrivacyNoticeResponseWithUserPreferences = { updated_at: string; version: number; privacy_notice_history_id: string; + cookies: Array; default_preference: UserConsentPreference; current_preference?: UserConsentPreference; outdated_preference?: UserConsentPreference; diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index 27173fd320..f4a7c94c1e 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -4922,6 +4922,145 @@ } ] }, + { + "name": "Systems", + "item": [ + { + "name": "Get System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/system/", + "host": [ + "{{host}}" + ], + "path": [ + "system", + "" + ] + } + }, + "response": [] + }, + { + "name": "Create System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data_responsibility_title\":\"Processor\",\n \"description\":\"Collect data about our users for marketing.\",\n \"egress\":[\n {\n \"fides_key\":\"demo_analytics_system\",\n \"type\":\"system\",\n \"data_categories\":null\n }\n ],\n \"fides_key\":\"test_system\",\n \"ingress\":null,\n \"name\":\"Test system\",\n \"organization_fides_key\":\"default_organization\",\n \"privacy_declarations\":[\n {\n \"name\":\"Collect data for marketing\",\n \"data_categories\":[\n \"user.device.cookie_id\"\n ],\n \"data_use\":\"personalize\",\n \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"data_subjects\":[\n \"customer\"\n ],\n \"dataset_references\":null,\n \"egress\":null,\n \"ingress\":null,\n \"cookies\":[\n {\n \"name\":\"test_cookie\",\n \"path\":\"/\"\n }\n ]\n }\n ],\n \"system_dependencies\":[\n \"demo_analytics_system\"\n ],\n \"system_type\":\"Service\",\n \"tags\":null,\n \"third_country_transfers\":null,\n \"administrating_department\":\"Marketing\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/system", + "host": [ + "{{host}}" + ], + "path": [ + "system" + ] + } + }, + "response": [] + }, + { + "name": "Update System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data_responsibility_title\":\"Processor\",\n \"description\":\"Collect data about our users for marketing.\",\n \"egress\":[\n {\n \"fides_key\":\"demo_analytics_system\",\n \"type\":\"system\",\n \"data_categories\":null\n }\n ],\n \"fides_key\":\"test_system\",\n \"ingress\":null,\n \"name\":\"Test system\",\n \"organization_fides_key\":\"default_organization\",\n \"privacy_declarations\":[\n {\n \"name\":\"Collect data for marketing\",\n \"data_categories\":[\n \"user.device.cookie_id\"\n ],\n \"data_use\":\"marketing.advertising\",\n \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"data_subjects\":[\n \"customer\"\n ],\n \"dataset_references\":null,\n \"egress\":null,\n \"ingress\":null,\n \"cookies\":[\n {\n \"name\":\"another_cookie\",\n \"path\":\"/\"\n }\n ]\n }\n ],\n \"system_dependencies\":[\n \"demo_analytics_system\"\n ],\n \"system_type\":\"Service\",\n \"tags\":null,\n \"third_country_transfers\":null,\n \"administrating_department\":\"Marketing\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/system?", + "host": [ + "{{host}}" + ], + "path": [ + "system" + ], + "query": [ + { + "key": "", + "value": null + } + ] + } + }, + "response": [] + }, + { + "name": "Delete System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "{{host}}/system/test_system", + "host": [ + "{{host}}" + ], + "path": [ + "system", + "test_system" + ] + } + }, + "response": [] + } + ] + }, { "name": "Roles", "item": [ diff --git a/requirements.txt b/requirements.txt index 191f9f08a2..90ee39b71b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang==1.4.1 +fideslang==1.4.2 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 @@ -31,7 +31,7 @@ passlib[bcrypt]==1.7.4 plotly==5.13.1 pyarrow==6.0.0 psycopg2-binary==2.9.6 -pydantic<1.10.2 +pydantic==1.10.9 pydash==6.0.2 PyJWT==2.4.0 pymongo==3.13.0 diff --git a/src/fides/api/alembic/migrations/versions/2be84e68df32_add_cookie_table.py b/src/fides/api/alembic/migrations/versions/2be84e68df32_add_cookie_table.py new file mode 100644 index 0000000000..889e466304 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/2be84e68df32_add_cookie_table.py @@ -0,0 +1,70 @@ +"""add cookie table + +Revision ID: 2be84e68df32 +Revises: c1885270b3cc +Create Date: 2023-06-13 23:08:35.011377 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "2be84e68df32" +down_revision = "c1885270b3cc" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "cookies", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("name", sa.String(), nullable=False), + sa.Column("domain", sa.String(), nullable=True), + sa.Column("path", sa.String(), nullable=True), + sa.Column("system_id", sa.String(), nullable=True), + sa.Column("privacy_declaration_id", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["privacy_declaration_id"], ["privacydeclaration.id"], ondelete="SET NULL" + ), + sa.ForeignKeyConstraint(["system_id"], ["ctl_systems.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "name", "privacy_declaration_id", name="_cookie_name_privacy_declaration_uc" + ), + ) + op.create_index(op.f("ix_cookies_id"), "cookies", ["id"], unique=False) + op.create_index(op.f("ix_cookies_name"), "cookies", ["name"], unique=False) + op.create_index( + op.f("ix_cookies_privacy_declaration_id"), + "cookies", + ["privacy_declaration_id"], + unique=False, + ) + op.create_index( + op.f("ix_cookies_system_id"), "cookies", ["system_id"], unique=False + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f("ix_cookies_system_id"), table_name="cookies") + op.drop_index(op.f("ix_cookies_privacy_declaration_id"), table_name="cookies") + op.drop_index(op.f("ix_cookies_name"), table_name="cookies") + op.drop_index(op.f("ix_cookies_id"), table_name="cookies") + op.drop_table("cookies") + # ### end Alembic commands ### diff --git a/src/fides/api/db/crud.py b/src/fides/api/db/crud.py index 10412dd257..eb13fa38de 100644 --- a/src/fides/api/db/crud.py +++ b/src/fides/api/db/crud.py @@ -113,7 +113,7 @@ async def get_resource( raise_not_found: bool = True, ) -> Base: """ - Get a resource from the databse by its FidesKey. + Get a resource from the database by its FidesKey. Returns a SQLAlchemy model of that resource. """ diff --git a/src/fides/api/db/system.py b/src/fides/api/db/system.py index 4d8d7f179e..924f8fb6d6 100644 --- a/src/fides/api/db/system.py +++ b/src/fides/api/db/system.py @@ -1,17 +1,20 @@ """ Functions for interacting with System objects in the database. """ -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple from fastapi import HTTPException +from fideslang.models import Cookies as CookieSchema from fideslang.models import System as SystemSchema from loguru import logger as log +from sqlalchemy import and_, delete, insert, select, update from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND from fides.api.db.crud import create_resource, get_resource, update_resource from fides.api.models.sql_models import ( # type: ignore[attr-defined] + Cookies, DataUse, PrivacyDeclaration, System, @@ -112,23 +115,88 @@ async def upsert_privacy_declarations( for privacy_declaration in resource.privacy_declarations: # prepare our 'payload' for either create or update data = privacy_declaration.dict() + cookies: List[Dict] = data.pop("cookies", None) data["system_id"] = system.id # include FK back to system # if we find matching declaration, remove it from our map - if existing_declaration := existing_declarations.pop( + if declaration := existing_declarations.pop( privacy_declaration_logical_id(privacy_declaration), None ): # and update existing declaration *in place* - existing_declaration.update(db, data=data) + declaration.update(db, data=data) else: # otherwise, create a new declaration record - PrivacyDeclaration.create(db, data=data) + declaration = PrivacyDeclaration.create(db, data=data) + + # Upsert cookies for the given privacy declaration + await upsert_cookies(db, cookies, declaration, system) # delete any existing privacy declarations that have not been "matched" in the request for existing_declarations in existing_declarations.values(): await db.delete(existing_declarations) +async def upsert_cookies( + async_session: AsyncSession, + cookies: Optional[List[Dict]], # CookieSchema + privacy_declaration: PrivacyDeclaration, + system: System, +) -> None: + """Upsert cookies for the given privacy declaration: retrieve cookies by name/system/privacy declaration + Remove any existing cookies that aren't specified here. + """ + cookie_list: List[CookieSchema] = cookies or [] + for cookie_data in cookie_list: + # Check if cookie exists for this name/system/privacy declaration + result = await async_session.execute( + select(Cookies).where( + and_( + Cookies.name == cookie_data["name"], + Cookies.system_id == system.id, + Cookies.privacy_declaration_id == privacy_declaration.id, + ) + ) + ) + row: Optional[Cookies] = result.scalars().first() + if row: + await async_session.execute( + update(Cookies).where(Cookies.id == row.id).values(cookie_data) + ) + + else: + await async_session.execute( + insert(Cookies).values( + { + "name": cookie_data.get("name"), + "path": cookie_data.get("path"), + "domain": cookie_data.get("domain"), + "privacy_declaration_id": privacy_declaration.id, + "system_id": system.id, + } + ) + ) + + # Select cookies which are currently on the privacy declaration but not included in this request + delete_result = await async_session.execute( + select(Cookies).where( + and_( + Cookies.name.notin_([cookie["name"] for cookie in cookie_list]), + Cookies.system_id == system.id, + Cookies.privacy_declaration_id == privacy_declaration.id, + ) + ) + ) + + # Remove those cookies altogether + await async_session.execute( + delete(Cookies).where( + Cookies.id.in_( + [cookie.id for cookie in delete_result.scalars().unique().all()] + ) + ) + ) + + async def update_system(resource: SystemSchema, db: AsyncSession) -> Dict: """Helper function to share core system update logic for wrapping endpoint functions""" system: System = await get_resource( @@ -184,9 +252,13 @@ async def create_system( for privacy_declaration in privacy_declarations: data = privacy_declaration.dict() data["system_id"] = created_system.id # add FK back to system - PrivacyDeclaration.create( + cookies: List[Dict] = data.pop("cookies", []) + privacy_declaration = PrivacyDeclaration.create( db, data=data ) # create the associated PrivacyDeclaration + await upsert_cookies( + db, cookies, privacy_declaration, created_system + ) # Create the associated cookies except Exception as e: log.error( f"Error adding privacy declarations, reverting system creation: {str(privacy_declaration_exception)}" diff --git a/src/fides/api/models/privacy_notice.py b/src/fides/api/models/privacy_notice.py index 903a1222ad..babdee213f 100644 --- a/src/fides/api/models/privacy_notice.py +++ b/src/fides/api/models/privacy_notice.py @@ -8,14 +8,18 @@ from fideslang.validation import FidesKey from sqlalchemy import Boolean, Column from sqlalchemy import Enum as EnumColumn -from sqlalchemy import Float, ForeignKey, String +from sqlalchemy import Float, ForeignKey, String, or_ from sqlalchemy.dialects.postgresql import ARRAY from sqlalchemy.orm import Session, relationship from sqlalchemy.util import hybridproperty from fides.api.common_exceptions import ValidationError from fides.api.db.base_class import Base, FidesBase -from fides.api.models.sql_models import System # type: ignore[attr-defined] +from fides.api.models.sql_models import ( # type: ignore[attr-defined] + Cookies, + PrivacyDeclaration, + System, +) class UserConsentPreference(Enum): @@ -255,6 +259,26 @@ def default_preference(self) -> UserConsentPreference: raise Exception("Invalid notice consent mechanism.") + @property + def cookies(self) -> List[Cookies]: + """Return relevant cookie names (via the data use)""" + db = Session.object_session(self) + return ( + db.query(Cookies) + .join( + PrivacyDeclaration, + PrivacyDeclaration.id == Cookies.privacy_declaration_id, + ) + .filter( + or_( + *[ + PrivacyDeclaration.data_use.like(f"{notice_use}%") + for notice_use in self.data_uses + ] + ) + ) + ).all() + @classmethod def create( cls: Type[PrivacyNotice], diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index 9ed6a8d8e1..0e8b03b310 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -346,6 +346,10 @@ class System(Base, FidesBase): lazy="selectin", ) + cookies = relationship( + "Cookies", back_populates="system", lazy="selectin", uselist=True, viewonly=True + ) + @classmethod def get_data_uses( cls: Type[System], systems: List[System], include_parents: bool = True @@ -392,6 +396,9 @@ class PrivacyDeclaration(Base): index=True, ) system = relationship(System, back_populates="privacy_declarations") + cookies = relationship( + "Cookies", back_populates="privacy_declaration", lazy="joined", uselist=True + ) @classmethod def create( @@ -596,3 +603,45 @@ class AuditLogResource(Base): request_type = Column(String, nullable=True) fides_keys = Column(ARRAY(String), nullable=True) extra_data = Column(JSON, nullable=True) + + +class Cookies(Base): + """ + Stores cookies. Cookies have a FK to system and privacy declaration. If a privacy declaration is deleted, + the cookie can still remain linked to the system but unassociated with a data use. + """ + + name = Column(String, index=True, nullable=False) + path = Column(String) + domain = Column(String) + + system_id = Column( + String, ForeignKey(System.id_field_path, ondelete="CASCADE"), index=True + ) # If system is deleted, remove the associated cookies. + + privacy_declaration_id = Column( + String, + ForeignKey(PrivacyDeclaration.id_field_path, ondelete="SET NULL"), + index=True, + ) # If privacy declaration is deleted, just set to null and still keep this connected to the system. + + system = relationship( + "System", + back_populates="cookies", + cascade="all,delete", + uselist=False, + lazy="selectin", + ) + + privacy_declaration = relationship( + "PrivacyDeclaration", + back_populates="cookies", + uselist=False, + lazy="joined", # Joined is intentional, instead of selectin + ) + + __table_args__ = ( + UniqueConstraint( + "name", "privacy_declaration_id", name="_cookie_name_privacy_declaration_uc" + ), + ) diff --git a/src/fides/api/schemas/dataset.py b/src/fides/api/schemas/dataset.py index bc00dd5d56..7b4683dea7 100644 --- a/src/fides/api/schemas/dataset.py +++ b/src/fides/api/schemas/dataset.py @@ -1,4 +1,4 @@ -from typing import Any, List, Optional +from typing import Any, List, Optional, Type from fideslang.models import Dataset, DatasetCollection, DatasetField from fideslang.validation import FidesKey @@ -30,7 +30,7 @@ def validate_data_categories_against_db( class DataCategoryValidationMixin(BaseModel): @validator("data_categories", check_fields=False, allow_reuse=True) def valid_data_categories( - cls, v: Optional[List[FidesKey]] + cls: Type["DataCategoryValidationMixin"], v: Optional[List[FidesKey]] ) -> Optional[List[FidesKey]]: """Validate that all annotated data categories exist in the taxonomy""" return _valid_data_categories(v, defined_data_categories) diff --git a/src/fides/api/schemas/privacy_notice.py b/src/fides/api/schemas/privacy_notice.py index f8e97aef88..ae8a229356 100644 --- a/src/fides/api/schemas/privacy_notice.py +++ b/src/fides/api/schemas/privacy_notice.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional +from fideslang.models import Cookies as CookieSchema from fideslang.validation import FidesKey from pydantic import Extra, conlist, root_validator, validator @@ -140,6 +141,7 @@ class PrivacyNoticeResponse(PrivacyNoticeWithId): updated_at: datetime version: float privacy_notice_history_id: str + cookies: List[CookieSchema] class PrivacyNoticeResponseWithUserPreferences(PrivacyNoticeResponse): diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index c4f1228edb..5034c33012 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum as EnumType -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Type, Union from fideslang.validation import FidesKey from pydantic import Field, validator @@ -84,7 +84,7 @@ class PrivacyRequestCreate(FidesSchema): @validator("encryption_key") def validate_encryption_key( - cls: "PrivacyRequestCreate", value: Optional[str] = None + cls: Type["PrivacyRequestCreate"], value: Optional[str] = None ) -> Optional[str]: """Validate encryption key where applicable""" if value: diff --git a/src/fides/api/schemas/system.py b/src/fides/api/schemas/system.py index 82016e051a..9111005698 100644 --- a/src/fides/api/schemas/system.py +++ b/src/fides/api/schemas/system.py @@ -1,6 +1,6 @@ from typing import List, Optional -from fideslang.models import PrivacyDeclaration, System +from fideslang.models import Cookies, PrivacyDeclaration, System from pydantic import Field from fides.api.schemas.connection_configuration.connection_config import ( @@ -14,6 +14,7 @@ class PrivacyDeclarationResponse(PrivacyDeclaration): id: str = Field( description="The database-assigned ID of the privacy declaration on the system. This is meant to be a read-only field, returned only in API responses" ) + cookies: Optional[List[Cookies]] = [] class SystemResponse(System): @@ -26,3 +27,5 @@ class SystemResponse(System): connection_configs: Optional[ConnectionConfigurationResponse] = Field( description=ConnectionConfigurationResponse.__doc__, ) + + cookies: Optional[List[Cookies]] = [] diff --git a/tests/conftest.py b/tests/conftest.py index b99d202bff..ebb44ffb27 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,7 @@ from fides.api.db.ctl_session import sync_engine from fides.api.main import app from fides.api.models.privacy_request import generate_request_callback_jwe -from fides.api.models.sql_models import DataUse, PrivacyDeclaration +from fides.api.models.sql_models import Cookies, DataUse, PrivacyDeclaration from fides.api.oauth.jwt import generate_jwe from fides.api.oauth.roles import ( APPROVER, @@ -411,6 +411,7 @@ def resources_dict(): system_type="SYSTEM", name="Test System", description="Test Policy", + cookies=[], privacy_declarations=[ models.PrivacyDeclaration( name="declaration-name", @@ -419,6 +420,7 @@ def resources_dict(): data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ) ], ), @@ -1018,7 +1020,7 @@ def system(db: Session) -> System: }, ) - PrivacyDeclaration.create( + privacy_declaration = PrivacyDeclaration.create( db=db, data={ "name": "Collect data for marketing", @@ -1033,6 +1035,17 @@ def system(db: Session) -> System: }, ) + Cookies.create( + db=db, + data={ + "name": "test_cookie", + "path": "/", + "privacy_declaration_id": privacy_declaration.id, + "system_id": system.id, + }, + check_name=False, + ) + db.refresh(system) return system diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 18e4686c6e..727119e3bd 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -449,6 +449,13 @@ def system_create_request_body(self) -> SystemSchema: data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[ + { + "name": "essential_cookie", + "path": "/", + "domain": "example.com", + } + ], ), models.PrivacyDeclaration( name="declaration-name-2", @@ -551,12 +558,24 @@ async def test_system_create( assert result.status_code == HTTP_201_CREATED assert result.json()["name"] == "Test System" + assert result.json()["cookies"] == [ + {"name": "essential_cookie", "path": "/", "domain": "example.com"} + ] + assert result.json()["privacy_declarations"][0]["cookies"] == [ + {"name": "essential_cookie", "path": "/", "domain": "example.com"} + ] + assert result.json()["privacy_declarations"][1]["cookies"] == [] assert len(result.json()["privacy_declarations"]) == 2 systems = System.all(db) assert len(systems) == 1 assert systems[0].name == "Test System" assert len(systems[0].privacy_declarations) == 2 + assert [cookie.name for cookie in systems[0].cookies] == ["essential_cookie"] + assert [ + cookie.name for cookie in systems[0].privacy_declarations[0].cookies + ] == ["essential_cookie"] + assert systems[0].privacy_declarations[1].cookies == [] async def test_system_create_custom_metadata_saas_config( self, @@ -734,6 +753,31 @@ def system_update_request_body(self, system) -> SystemSchema: ], ) + @pytest.fixture(scope="function") + def system_update_request_body_with_cookies(self, system) -> SystemSchema: + return SystemSchema( + organization_fides_key=1, + registryId=1, + fides_key=system.fides_key, + system_type="SYSTEM", + name=self.updated_system_name, + description="Test Policy", + privacy_declarations=[ + models.PrivacyDeclaration( + name="declaration-name", + data_categories=[], + data_use="essential", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + cookies=[ + {"name": "my_cookie", "domain": "example.com"}, + {"name": "my_other_cookie"}, + ], + ) + ], + ) + def test_system_update_not_authenticated( self, test_config, system_update_request_body ): @@ -1093,6 +1137,40 @@ def test_system_update_privacy_declaration_invalid_duplicate( and system.privacy_declarations[1].name == "new declaration 1" ) + def test_system_update_privacy_declaration_cookies( + self, + test_config, + system_update_request_body_with_cookies, + system, + db, + generate_system_manager_header, + ): + assert system.name != self.updated_system_name + + auth_header = generate_system_manager_header([system.id]) + result = _api.update( + url=test_config.cli.server_url, + headers=auth_header, + resource_type="system", + json_resource=system_update_request_body_with_cookies.json( + exclude_none=True + ), + ) + assert result.status_code == HTTP_200_OK + assert result.json()["name"] == self.updated_system_name + assert result.json()["cookies"] == [ + {"name": "my_cookie", "path": None, "domain": "example.com"}, + {"name": "my_other_cookie", "path": None, "domain": None}, + {"name": "test_cookie", "path": "/", "domain": None}, + ] + + db.refresh(system) + assert system.name == self.updated_system_name + assert ( + len(system.cookies) == 3 + ) # Two from the current privacy declaration, one from the previous privacy declaration that was deleted, but still linked to the system + assert len(system.privacy_declarations[0].cookies) == 2 + @pytest.mark.parametrize( "update_declarations", [ @@ -1105,6 +1183,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ) ] ), @@ -1118,6 +1197,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), models.PrivacyDeclaration( name="declaration-name-2", @@ -1126,6 +1206,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), ] ), @@ -1139,6 +1220,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), models.PrivacyDeclaration( name="Collect data for marketing", @@ -1147,6 +1229,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), ] ), @@ -1160,6 +1243,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), models.PrivacyDeclaration( name="declaration-name-2", @@ -1168,6 +1252,7 @@ def test_system_update_privacy_declaration_invalid_duplicate( data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], + cookies=[], ), ] ), diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index d801987a4f..a6eaaf5fc0 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -1,11 +1,16 @@ # pylint: disable=missing-docstring, redefined-outer-name import os from typing import Generator, List +from uuid import uuid4 import pytest +from fideslang.models import PrivacyDeclaration as PrivacyDeclarationSchema from fideslang.models import System, SystemMetadata from py._path.local import LocalPath +from sqlalchemy import delete +from fides.api.db.system import create_system, upsert_cookies +from fides.api.models.sql_models import Cookies, PrivacyDeclaration from fides.api.models.sql_models import System as sql_System from fides.config import FidesConfig from fides.connectors.models import OktaConfig @@ -332,3 +337,208 @@ def test_scan_system_okta_fail(tmpdir: LocalPath, test_config: FidesConfig) -> N url=test_config.cli.server_url, headers=test_config.user.auth_header, ) + + +class TestUpsertCookies: + @pytest.fixture() + async def test_cookie_system( + self, + async_session_temp, + generate_auth_header, + test_config, + generate_role_header, + db, + ): + resource = System( + fides_key=str(uuid4()), + organization_fides_key="default_organization", + name="test_system_1", + system_type="test", + privacy_declarations=[ + PrivacyDeclarationSchema( + name="declaration-name", + data_categories=[], + data_use="essential", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ), + PrivacyDeclarationSchema( + name="declaration-name-2", + data_categories=[], + data_use="improve", + data_subjects=[], + data_qualifier="aggregated_data", + dataset_references=[], + ), + ], + ) + + system = await create_system(resource, async_session_temp) + + Cookies.create( + db=db, + data={ + "name": "strawberry", + "path": "/", + "privacy_declaration_id": sorted( + system.privacy_declarations, key=lambda x: x.name + )[1].id, + "system_id": system.id, + }, + check_name=False, + ) + await async_session_temp.refresh(system) + yield system + delete(sql_System).where(sql_System.id == system.id) + + async def test_new_cookies(self, test_cookie_system, async_session_temp): + """Test adding a new cookie to a privacy declaration. The other privacy declaration on the + system already has a cookie.""" + + new_cookies = [{"name": "apple"}] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[0] + + await upsert_cookies( + async_session_temp, + new_cookies, + privacy_declaration, + test_cookie_system, + ) + await async_session_temp.refresh(test_cookie_system) + assert len(test_cookie_system.cookies) == 2 + + assert {cookie.name for cookie in test_cookie_system.cookies} == { + "strawberry", + "apple", + } + assert len(privacy_declaration.cookies) == 1 + assert privacy_declaration.cookies[0].name == "apple" + + new_cookie = privacy_declaration.cookies[0] + assert new_cookie.created_at is not None + assert new_cookie.updated_at is not None + assert new_cookie.name == "apple" + assert new_cookie.system_id == test_cookie_system.id + assert new_cookie.privacy_declaration_id == privacy_declaration.id + + async def test_no_change_to_cookies(self, test_cookie_system, async_session_temp): + """Test specified cookies already exist on given privacy declaration, so no change required""" + new_cookies = [{"name": "strawberry"}] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[1] + existing_cookie = privacy_declaration.cookies[0] + assert existing_cookie.name == "strawberry" + + await upsert_cookies( + async_session_temp, + new_cookies, + privacy_declaration, + test_cookie_system, + ) + await async_session_temp.refresh(test_cookie_system) + assert len(test_cookie_system.cookies) == 1 + + assert {cookie.name for cookie in test_cookie_system.cookies} == { + "strawberry", + } + assert len(privacy_declaration.cookies) == 1 + assert privacy_declaration.cookies[0].name == "strawberry" + + new_cookie = privacy_declaration.cookies[0] + assert new_cookie.created_at is not None + assert new_cookie.updated_at is not None + assert new_cookie.name == "strawberry" + assert new_cookie.system_id == test_cookie_system.id + assert new_cookie.privacy_declaration_id == privacy_declaration.id + + async def test_update_cookies(self, test_cookie_system, async_session_temp): + """Test cookie exists but path has changed""" + """Test specified cookies already exist on given privacy declaration, so no change required""" + + new_cookies = [{"name": "strawberry", "path": "/"}] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[1] + existing_cookie = privacy_declaration.cookies[0] + assert existing_cookie.name == "strawberry" + + await upsert_cookies( + async_session_temp, + new_cookies, + privacy_declaration, + test_cookie_system, + ) + await async_session_temp.refresh(test_cookie_system) + assert len(test_cookie_system.cookies) == 1 + assert test_cookie_system.cookies[0].name == "strawberry" + assert test_cookie_system.cookies[0].path == "/" + + assert len(privacy_declaration.cookies) == 1 + assert privacy_declaration.cookies[0].name == "strawberry" + assert privacy_declaration.cookies[0].path == "/" + + new_cookie = privacy_declaration.cookies[0] + assert new_cookie.created_at is not None + assert new_cookie.updated_at is not None + assert new_cookie.name == "strawberry" + assert new_cookie.system_id == test_cookie_system.id + assert new_cookie.privacy_declaration_id == privacy_declaration.id + + async def test_remove_cookies(self, test_cookie_system, async_session_temp): + """Test cookie list is missing a cookie currently on the privacy declaration so add the new + cookie and we remove the existing one""" + + new_cookies = [{"name": "apple"}] + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[1] + existing_cookie = privacy_declaration.cookies[0] + assert existing_cookie.name == "strawberry" + + await upsert_cookies( + async_session_temp, + new_cookies, + privacy_declaration, + test_cookie_system, + ) + await async_session_temp.refresh(test_cookie_system) + assert len(test_cookie_system.cookies) == 1 + + assert {cookie.name for cookie in test_cookie_system.cookies} == { + "apple", + } + assert len(privacy_declaration.cookies) == 1 + assert privacy_declaration.cookies[0].name == "apple" + + new_cookie = privacy_declaration.cookies[0] + assert new_cookie.created_at is not None + assert new_cookie.updated_at is not None + assert new_cookie.name == "apple" + assert new_cookie.system_id == test_cookie_system.id + assert new_cookie.privacy_declaration_id == privacy_declaration.id + + async def test_delete_privacy_declaration( + self, test_cookie_system, async_session_temp + ): + """Test if a privacy declaration is deleted, its cookie is still linked to the system""" + + privacy_declaration = sorted( + test_cookie_system.privacy_declarations, key=lambda x: x.name + )[1] + existing_cookie = privacy_declaration.cookies[0] + + assert existing_cookie.privacy_declaration_id == privacy_declaration.id + assert existing_cookie.system_id == test_cookie_system.id + + stmt = delete(PrivacyDeclaration).where( + PrivacyDeclaration.id == privacy_declaration.id + ) + await async_session_temp.execute(stmt) + await async_session_temp.refresh(existing_cookie) + + assert existing_cookie.privacy_declaration_id is None + assert existing_cookie.system_id == test_cookie_system.id diff --git a/tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py index 4feb6ebc94..6a4f724e00 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py @@ -13,6 +13,7 @@ ComponentType, PrivacyExperience, PrivacyExperienceConfig, + PrivacyExperienceConfigHistory, ) from fides.api.models.privacy_notice import PrivacyNoticeRegion from fides.common.api import scope_registry as scopes @@ -1167,14 +1168,18 @@ def test_update_experience_config_while_ignoring_regions( experience_config = get_experience_config_or_error(db, resp["id"]) assert experience_config.experiences.all() == [privacy_experience_overlay] assert experience_config.histories.count() == 2 - history = experience_config.histories[0] + history = experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[0] assert history.version == 1.0 assert history.component == ComponentType.overlay assert history.banner_enabled == BannerEnabled.enabled_where_required assert history.experience_config_id == experience_config.id assert history.disabled is False - history = experience_config.histories[1] + history = experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[1] assert history.version == 2.0 assert history.disabled is True @@ -1224,14 +1229,18 @@ def test_update_experience_config_with_no_regions( experience_config = get_experience_config_or_error(db, resp["id"]) assert experience_config.experiences.all() == [] assert experience_config.histories.count() == 2 - history = experience_config.histories[0] + history = experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[0] assert history.version == 1.0 assert history.component == ComponentType.overlay assert history.banner_enabled == BannerEnabled.enabled_where_required assert history.experience_config_id == experience_config.id assert history.disabled is False - history = experience_config.histories[1] + history = experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[1] assert history.version == 2.0 assert history.disabled is True @@ -1448,7 +1457,9 @@ def test_update_experience_config_experience_also_updated( db.refresh(overlay_experience_config) # ExperienceConfig was disabled - this is a change, so another historical record is created assert overlay_experience_config.histories.count() == 2 - experience_config_history = overlay_experience_config.histories[1] + experience_config_history = overlay_experience_config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[1] assert experience_config_history.version == 2.0 assert experience_config_history.disabled assert experience_config_history.component == ComponentType.overlay diff --git a/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py index 4815798aa9..9090daa892 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py @@ -5,6 +5,7 @@ from uuid import uuid4 import pytest +from fideslang.models import Cookies as CookieSchema from sqlalchemy.orm import Session from starlette.testclient import TestClient @@ -87,6 +88,7 @@ def test_get_privacy_notices_defaults( assert "created_at" in notice_detail assert "updated_at" in notice_detail assert "name" in notice_detail + assert "name" in notice_detail assert "description" in notice_detail assert "regions" in notice_detail assert "consent_mechanism" in notice_detail @@ -755,6 +757,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ) ], }, @@ -824,6 +829,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-2", @@ -844,6 +852,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), ], }, @@ -911,6 +922,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( version=1.0, privacy_notice_history_id="placeholder_id", displayed_in_overlay=True, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-2", @@ -929,6 +943,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( version=1.0, privacy_notice_history_id="placeholder_id", displayed_in_overlay=True, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), ], "third_party_sharing": [ @@ -950,6 +967,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( version=1.0, privacy_notice_history_id="placeholder_id", displayed_in_overlay=True, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), ], }, @@ -1019,6 +1039,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ), ], "third_party_sharing": [], @@ -1153,6 +1176,9 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[ + CookieSchema(name="test_cookie", path="/", domain=None) + ], ) ], "essential.service.operations.support.optimization": [ @@ -1177,6 +1203,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[], ), PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-3", @@ -1197,6 +1224,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[], ), PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-2", @@ -1217,6 +1245,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( displayed_in_overlay=True, displayed_in_privacy_center=False, displayed_in_api=False, + cookies=[], ), ], }, diff --git a/tests/ops/models/test_privacy_experience.py b/tests/ops/models/test_privacy_experience.py index 78e3e87dcd..27f6ee539a 100644 --- a/tests/ops/models/test_privacy_experience.py +++ b/tests/ops/models/test_privacy_experience.py @@ -7,6 +7,7 @@ ComponentType, PrivacyExperience, PrivacyExperienceConfig, + PrivacyExperienceConfigHistory, upsert_privacy_experiences_after_config_update, upsert_privacy_experiences_after_notice_update, ) @@ -123,11 +124,15 @@ def test_update_privacy_experience_config(self, db): assert config.updated_at > config_updated_at assert config.histories.count() == 2 - history = config.histories[1] + history = config.histories.order_by(PrivacyExperienceConfigHistory.created_at)[ + 1 + ] assert history.component == ComponentType.privacy_center assert config.experience_config_history_id == history.id - old_history = config.histories[0] + old_history = config.histories.order_by( + PrivacyExperienceConfigHistory.created_at + )[0] assert old_history.version == 1.0 assert old_history.component == ComponentType.overlay diff --git a/tests/ops/models/test_privacy_notice.py b/tests/ops/models/test_privacy_notice.py index 6c519f4ced..c56c02cc80 100644 --- a/tests/ops/models/test_privacy_notice.py +++ b/tests/ops/models/test_privacy_notice.py @@ -1,4 +1,5 @@ import pytest +from fideslang.models import Cookies as CookieSchema from fideslang.validation import FidesValidationError from sqlalchemy.orm import Session @@ -12,6 +13,7 @@ check_conflicting_data_uses, new_data_use_conflicts_with_existing_use, ) +from fides.api.models.sql_models import Cookies class TestPrivacyNoticeModel: @@ -697,6 +699,69 @@ def test_conflicting_data_uses( existing_privacy_notices=existing_privacy_notices, ) + @pytest.mark.parametrize( + "privacy_notice_data_use,declaration_cookies,expected_cookies,description", + [ + ( + ["marketing.advertising", "third_party_sharing"], + [{"name": "test_cookie"}], + [CookieSchema(name="test_cookie")], + "Data uses overlap exactly", + ), + ( + ["marketing.advertising.first_party", "third_party_sharing"], + [{"name": "test_cookie"}], + [], + "Privacy notice use more specific than system's. Too big a leap to assume system should be adjusted here.", + ), + ( + ["marketing", "third_party_sharing"], + [{"name": "test_cookie"}], + [CookieSchema(name="test_cookie")], + "Privacy notice use more general than system's, so system's data use is under the scope of the notice", + ), + ( + ["marketing.advertising", "third_party_sharing"], + [{"name": "test_cookie"}, {"name": "another_cookie"}], + [CookieSchema(name="test_cookie"), CookieSchema(name="another_cookie")], + "Test multiple cookies", + ), + (["marketing.advertising"], [], [], "No cookies returns an empty set"), + ], + ) + def test_relevant_cookies( + self, + privacy_notice_data_use, + declaration_cookies, + expected_cookies, + description, + privacy_notice, + db, + system, + ): + """Test different combinations of data uses and cookies between the Privacy Notice and the Privacy Declaration""" + db.query(Cookies).delete() + privacy_notice.data_uses = privacy_notice_data_use + privacy_notice.save(db) + + privacy_declaration = system.privacy_declarations[0] + assert privacy_declaration.data_use == "marketing.advertising" + + for cookie in declaration_cookies: + Cookies.create( + db, + data={ + "name": cookie["name"], + "privacy_declaration_id": privacy_declaration.id, + "system_id": system.id, + }, + check_name=False, + ) + + assert [ + CookieSchema.from_orm(cookie) for cookie in privacy_notice.cookies + ] == expected_cookies, description + def test_calculate_relevant_systems( self, db,