Skip to content

Commit

Permalink
fix: paramaterize exclusion number
Browse files Browse the repository at this point in the history
Due to differences in the expected strings for an access or an erasure, it was determined that we may need to remove 2 or 3 items from the leading edge of the dataset. To avoid having the same function twice a parameter is now passed based on the string argument passed. The parameter name was also updated to be more descriptive
  • Loading branch information
SteveDMurphy committed Jun 30, 2023
1 parent 6b2e206 commit b57bf1e
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 6 deletions.
6 changes: 5 additions & 1 deletion src/fides/api/task/graph_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -777,7 +777,11 @@ def get_cached_data_for_erasures(
value_dict = cache.get_encoded_objects_by_prefix(
f"PLACEHOLDER_RESULTS__{privacy_request_id}"
)
return {extract_key_for_address(k): v for k, v in value_dict.items()}
number_of_leading_strings_to_exclude = 3
return {
extract_key_for_address(k, number_of_leading_strings_to_exclude): v
for k, v in value_dict.items()
}


def update_erasure_mapping_from_cache(
Expand Down
9 changes: 7 additions & 2 deletions src/fides/api/task/task_resources.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,11 @@ def get_all_cached_objects(self) -> Dict[str, Optional[List[Row]]]:
f"{self.request.id}__access_request"
)
# extract request id to return a map of address:value
return {extract_key_for_address(k): v for k, v in value_dict.items()}
number_of_leading_strings_to_exclude = 2
return {
extract_key_for_address(k, number_of_leading_strings_to_exclude): v
for k, v in value_dict.items()
}

def cache_erasure(self, key: str, value: int) -> None:
"""Cache that a node's masking is complete. Object will be stored in redis under
Expand All @@ -163,7 +167,8 @@ def get_all_cached_erasures(self) -> Dict[str, int]:
f"{self.request.id}__erasure_request"
)
# extract request id to return a map of address:value
return {extract_key_for_address(k): v for k, v in value_dict.items()} # type: ignore
number_of_leading_strings_to_exclude = 2
return {extract_key_for_address(k, number_of_leading_strings_to_exclude): v for k, v in value_dict.items()} # type: ignore

def write_execution_log( # pylint: disable=too-many-arguments
self,
Expand Down
10 changes: 7 additions & 3 deletions src/fides/api/util/collection_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,19 @@ def filter_nonempty_values(d: Optional[Dict[Any, Any]]) -> Dict[Any, Any]:
return {}


def extract_key_for_address(full_request_id: str) -> str:
def extract_key_for_address(
full_request_id: str, number_of_leading_strings_to_exclude: int
) -> str:
"""
Handles extracting the correct Dataset:Collection to map to extracted
values.
Due to differences in the number of leading strings based on access or
erasure, a parameter is used to ensure the correct values are returned.
Handles an edge case where double underscores exist in either the fides_key
of the Dataset or the Collection name.
"""
request_id_dataset, collection = full_request_id.split(":")
number_of_expected_items_to_remove = 2
dataset = request_id_dataset.split("__", number_of_expected_items_to_remove)[-1]
dataset = request_id_dataset.split("__", number_of_leading_strings_to_exclude)[-1]
return f"{dataset}:{collection}"

0 comments on commit b57bf1e

Please sign in to comment.