[Backend] Add Cookies and Surface with Privacy Notices

.fides/db_dataset.yml

@@ -2240,4 +2240,41 @@ dataset:
       description: 'The name of the organization this Fides deployment belongs to'
       data_categories:
       - user.workplace
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+  - name: cookies
+    description: 'Fides Generated Description for Table: cookies'
+    data_categories: []
+    data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    fields:
+    - name: created_at
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: domain
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: id
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: name
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: path
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: privacy_declaration_id
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: system_id
+      data_categories:
+      - system.operations
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: updated_at
+      data_categories:
+      - system.operations
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

docs/fides/docs/development/postman/Fides.postman_collection.json

@@ -4922,6 +4922,145 @@
 				}
 			]
 		},
+		{
+			"name": "Systems",
+			"item": [
+				{
+					"name": "Get System",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{host}}/system/",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"system",
+								""
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "Create System",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "POST",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n   \"data_responsibility_title\":\"Processor\",\n   \"description\":\"Collect data about our users for marketing.\",\n   \"egress\":[\n      {\n         \"fides_key\":\"demo_analytics_system\",\n         \"type\":\"system\",\n         \"data_categories\":null\n      }\n   ],\n   \"fides_key\":\"test_system\",\n   \"ingress\":null,\n   \"name\":\"Test system\",\n   \"organization_fides_key\":\"default_organization\",\n   \"privacy_declarations\":[\n      {\n         \"name\":\"Collect data for marketing\",\n         \"data_categories\":[\n            \"user.device.cookie_id\"\n         ],\n         \"data_use\":\"personalize\",\n         \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n         \"data_subjects\":[\n            \"customer\"\n         ],\n         \"dataset_references\":null,\n         \"egress\":null,\n         \"ingress\":null,\n         \"cookies\":[\n            {\n               \"name\":\"test_cookie\",\n               \"path\":\"/\"\n            }\n         ]\n      }\n   ],\n   \"system_dependencies\":[\n      \"demo_analytics_system\"\n   ],\n   \"system_type\":\"Service\",\n   \"tags\":null,\n   \"third_country_transfers\":null,\n   \"administrating_department\":\"Marketing\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{host}}/system",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"system"
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "Update System",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "PUT",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n   \"data_responsibility_title\":\"Processor\",\n   \"description\":\"Collect data about our users for marketing.\",\n   \"egress\":[\n      {\n         \"fides_key\":\"demo_analytics_system\",\n         \"type\":\"system\",\n         \"data_categories\":null\n      }\n   ],\n   \"fides_key\":\"test_system\",\n   \"ingress\":null,\n   \"name\":\"Test system\",\n   \"organization_fides_key\":\"default_organization\",\n   \"privacy_declarations\":[\n      {\n         \"name\":\"Collect data for marketing\",\n         \"data_categories\":[\n            \"user.device.cookie_id\"\n         ],\n         \"data_use\":\"marketing.advertising\",\n         \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n         \"data_subjects\":[\n            \"customer\"\n         ],\n         \"dataset_references\":null,\n         \"egress\":null,\n         \"ingress\":null,\n         \"cookies\":[\n            {\n               \"name\":\"another_cookie\",\n               \"path\":\"/\"\n            }\n         ]\n      }\n   ],\n   \"system_dependencies\":[\n      \"demo_analytics_system\"\n   ],\n   \"system_type\":\"Service\",\n   \"tags\":null,\n   \"third_country_transfers\":null,\n   \"administrating_department\":\"Marketing\"\n}",
+							"options": {
+								"raw": {
+									"language": "json"
+								}
+							}
+						},
+						"url": {
+							"raw": "{{host}}/system?",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"system"
+							],
+							"query": [
+								{
+									"key": "",
+									"value": null
+								}
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "Delete System",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "DELETE",
+						"header": [],
+						"url": {
+							"raw": "{{host}}/system/test_system",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"system",
+								"test_system"
+							]
+						}
+					},
+					"response": []
+				}
+			]
+		},
 		{
 			"name": "Roles",
 			"item": [

noxfiles/test_docker_nox.py

@@ -1,14 +1,7 @@
 import pytest
 
-from docker_nox import (
-    generate_multiplatform_buildx_command,
-    get_buildx_commands,
-)
-from constants_nox import (
-    DEV_TAG_SUFFIX,
-    PRERELEASE_TAG_SUFFIX,
-    RC_TAG_SUFFIX,
-)
+from constants_nox import DEV_TAG_SUFFIX, PRERELEASE_TAG_SUFFIX, RC_TAG_SUFFIX
+from docker_nox import generate_multiplatform_buildx_command, get_buildx_commands
 
 
 class TestGenerateMultiplatformBuilxCommand:

requirements.txt

@@ -12,7 +12,7 @@ expandvars==0.9.0
 fastapi[all]==0.89.1
 fastapi-caching[redis]==0.3.0
 fastapi-pagination[sqlalchemy]~= 0.10.0
-fideslang==1.4.1
+fideslang @ git+https://github.com/ethyca/fideslang.git@6e93ab4ea7d35713201d48b3314b8d25df0815cb
 fideslog==1.2.10
 firebase-admin==5.3.0
 GitPython==3.1.31

src/fides/api/ctl/database/crud.py

@@ -113,7 +113,7 @@ async def get_resource(
     raise_not_found: bool = True,
 ) -> Base:
     """
-    Get a resource from the databse by its FidesKey.
+    Get a resource from the database by its FidesKey.
 
     Returns a SQLAlchemy model of that resource.
     """

src/fides/api/ctl/database/system.py

@@ -1,17 +1,20 @@
 """
 Functions for interacting with System objects in the database.
 """
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from fastapi import HTTPException
+from fideslang.models import Cookies as CookieSchema
 from fideslang.models import System as SystemSchema
 from loguru import logger as log
+from sqlalchemy import and_, delete, insert, select, update
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session
 from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND
 
 from fides.api.ctl.database.crud import create_resource, get_resource, update_resource
 from fides.api.ctl.sql_models import (  # type: ignore[attr-defined]
+    Cookies,
     DataUse,
     PrivacyDeclaration,
     System,
@@ -112,23 +115,88 @@ async def upsert_privacy_declarations(
         for privacy_declaration in resource.privacy_declarations:
             # prepare our 'payload' for either create or update
             data = privacy_declaration.dict()
+            cookies: List[Dict] = data.pop("cookies", None)
             data["system_id"] = system.id  # include FK back to system
 
             # if we find matching declaration, remove it from our map
-            if existing_declaration := existing_declarations.pop(
+            if declaration := existing_declarations.pop(
                 privacy_declaration_logical_id(privacy_declaration), None
             ):
                 # and update existing declaration *in place*
-                existing_declaration.update(db, data=data)
+                declaration.update(db, data=data)
             else:
                 # otherwise, create a new declaration record
-                PrivacyDeclaration.create(db, data=data)
+                declaration = PrivacyDeclaration.create(db, data=data)
+
+            # Upsert cookies for the given privacy declaration
+            await upsert_cookies(db, cookies, declaration, system)
 
         # delete any existing privacy declarations that have not been "matched" in the request
         for existing_declarations in existing_declarations.values():
             await db.delete(existing_declarations)
 
 
+async def upsert_cookies(
+    async_session: AsyncSession,
+    cookies: Optional[List[Dict]],  # CookieSchema
+    privacy_declaration: PrivacyDeclaration,
+    system: System,
+) -> None:
+    """Upsert cookies for the given privacy declaration: retrieve cookies by name/system/privacy declaration
+    Remove any existing cookies that aren't specified here.
+    """
+    cookie_list: List[CookieSchema] = cookies or []
+    for cookie_data in cookie_list:
+        # Check if cookie exists for this name/system/privacy declaration
+        result = await async_session.execute(
+            select(Cookies).where(
+                and_(
+                    Cookies.name == cookie_data["name"],
+                    Cookies.system_id == system.id,
+                    Cookies.privacy_declaration_id == privacy_declaration.id,
+                )
+            )
+        )
+        row: Optional[Cookies] = result.scalars().first()
+        if row:
+            await async_session.execute(
+                update(Cookies).where(Cookies.id == row.id).values(cookie_data)
+            )
+
+        else:
+            await async_session.execute(
+                insert(Cookies).values(
+                    {
+                        "name": cookie_data.get("name"),
+                        "path": cookie_data.get("path"),
+                        "domain": cookie_data.get("domain"),
+                        "privacy_declaration_id": privacy_declaration.id,
+                        "system_id": system.id,
+                    }
+                )
+            )
+
+    # Select cookies which are currently on the privacy declaration but not included in this request
+    delete_result = await async_session.execute(
+        select(Cookies).where(
+            and_(
+                Cookies.name.notin_([cookie["name"] for cookie in cookie_list]),
+                Cookies.system_id == system.id,
+                Cookies.privacy_declaration_id == privacy_declaration.id,
+            )
+        )
+    )
+
+    # Remove those cookies altogether
+    await async_session.execute(
+        delete(Cookies).where(
+            Cookies.id.in_(
+                [cookie.id for cookie in delete_result.scalars().unique().all()]
+            )
+        )
+    )
+
+
 async def update_system(resource: SystemSchema, db: AsyncSession) -> Dict:
     """Helper function to share core system update logic for wrapping endpoint functions"""
     system: System = await get_resource(
@@ -184,9 +252,13 @@ async def create_system(
             for privacy_declaration in privacy_declarations:
                 data = privacy_declaration.dict()
                 data["system_id"] = created_system.id  # add FK back to system
-                PrivacyDeclaration.create(
+                cookies: List[Dict] = data.pop("cookies", [])
+                privacy_declaration = PrivacyDeclaration.create(
                     db, data=data
                 )  # create the associated PrivacyDeclaration
+                await upsert_cookies(
+                    db, cookies, privacy_declaration, created_system
+                )  # Create the associated cookies
     except Exception as e:
         log.error(
             f"Error adding privacy declarations, reverting system creation: {str(privacy_declaration_exception)}"