Performance Refactor for privacy-experience

noxfiles/drill.yml

@@ -1,16 +1,79 @@
 # This file is used by "drill", a load testing utility
+# https://github.com/fcsonline/drill
 
-concurrency: 4
+concurrency: 5 # How many instances of the `plan` get run concurrently
 base: 'http://localhost:8080'
-# There is a rate-limiter on the webserver, default is set to 2000 requests/min
-iterations: 400
-rampup: 3
+iterations: 20 # How many times to run the plan
+rampup: 1 # The number of seconds between starting concurrent runs
 
+# This plan is run in its entirety, sequentially, for each iteration
 plan:
   - name: Health Check
     request:
+      method: GET
       url: /health
 
-  - name: Worker Health Check
+  - name: Login
+    assign: login
     request:
-      url: /health/workers
+      method: POST
+      body: '{"username": "root_user", "password": "VGVzdHBhc3N3b3JkMSE="}'
+      headers:
+        Content-Type: 'application/json'
+      url: /api/v1/login
+
+  - name: Get Privacy-Experience [Basic]
+    request:
+      method: GET
+      url: /api/v1/privacy-experience
+    tags:
+      - privacy-experience
+
+  - name: Get Privacy-Experience [Include GVL]
+    request:
+      method: GET
+      url: /api/v1/privacy-experience?include_gvl=1
+    tags:
+      - privacy-experience
+
+  - name: Get Data Use
+    request:
+      method: GET
+      headers:
+        Authorization: "Bearer {{ login.body.token_data.access_token }}"
+      url: /api/v1/data_use
+
+  - name: Get Data Category
+    request:
+      method: GET
+      headers:
+        Authorization: "Bearer {{ login.body.token_data.access_token }}"
+      url: /api/v1/data_category
+
+  - name: Get Data Qualifier
+    request:
+      method: GET
+      headers:
+        Authorization: "Bearer {{ login.body.token_data.access_token }}"
+      url: /api/v1/data_qualifier
+
+  - name: Get Data Subject
+    request:
+      method: GET
+      headers:
+        Authorization: "Bearer {{ login.body.token_data.access_token }}"
+      url: /api/v1/data_subject
+
+  - name: Get Systems
+    request:
+      method: GET
+      headers:
+        Authorization: "Bearer {{ login.body.token_data.access_token }}"
+      url: /api/v1/system
+
+  - name: Get Datasets
+    request:
+      method: GET
+      headers:
+        Authorization: "Bearer {{ login.body.token_data.access_token }}"
+      url: /api/v1/dataset

requirements.txt

@@ -11,7 +11,6 @@ deepdiff==6.3.0
 defusedxml==0.7.1
 expandvars==0.9.0
 fastapi[all]==0.89.1
-fastapi-caching[redis]==0.3.0
 fastapi-pagination[sqlalchemy]==0.11.4
 fideslang==2.1.0
 fideslog==1.2.10

src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py

@@ -1,3 +1,4 @@
+import asyncio
 import uuid
 from html import escape, unescape
 from typing import Dict, List, Optional
@@ -126,7 +127,7 @@ def _filter_experiences_by_region_or_country(
     response_model=Page[PrivacyExperienceResponse],
 )
 @fides_limiter.limit(CONFIG.security.public_request_rate_limit)
-def privacy_experience_list(
+async def privacy_experience_list(
     *,
     db: Session = Depends(deps.get_db),
     params: Params = Depends(),
@@ -178,6 +179,7 @@ def privacy_experience_list(
             db=db, fides_user_device_id=fides_user_device_id
         )
 
+    await asyncio.sleep(delay=0.001)
     experience_query = db.query(PrivacyExperience)
 
     if show_disabled is False:
@@ -188,11 +190,13 @@ def privacy_experience_list(
             PrivacyExperienceConfig.id == PrivacyExperience.experience_config_id,
         ).filter(PrivacyExperienceConfig.disabled.is_(False))
 
+    await asyncio.sleep(delay=0.001)
     if region is not None:
         experience_query = _filter_experiences_by_region_or_country(
             db=db, region=region, experience_query=experience_query
         )
 
+    await asyncio.sleep(delay=0.001)
     if component is not None:
         # Intentionally relaxes what is returned when querying for "overlay", by returning both types of overlays.
         # This way the frontend doesn't have to know which type of overlay, regular or tcf, just that it is an overlay.
@@ -204,23 +208,28 @@ def privacy_experience_list(
                 component_search_map.get(component, [component])
             )
         )
+    await asyncio.sleep(delay=0.001)
     if has_config is True:
         experience_query = experience_query.filter(
             PrivacyExperience.experience_config_id.isnot(None)
         )
+    await asyncio.sleep(delay=0.001)
     if has_config is False:
         experience_query = experience_query.filter(
             PrivacyExperience.experience_config_id.is_(None)
         )
 
     results: List[PrivacyExperience] = []
     should_unescape: Optional[str] = request.headers.get(UNESCAPE_SAFESTR_HEADER)
+
     # Builds TCF Experience Contents once here, in case multiple TCF Experiences are requested
     base_tcf_contents: TCFExperienceContents = get_tcf_contents(db)
 
+    await asyncio.sleep(delay=0.001)
     for privacy_experience in experience_query.order_by(
         PrivacyExperience.created_at.desc()
     ):
+        await asyncio.sleep(delay=0.001)
         content_exists: bool = embed_experience_details(
             db,
             privacy_experience=privacy_experience,
@@ -237,6 +246,7 @@ def privacy_experience_list(
             continue
 
         # Temporarily save "show_banner" on the privacy experience object
+        await asyncio.sleep(delay=0.001)
         privacy_experience.show_banner = privacy_experience.get_should_show_banner(
             db, show_disabled
         )