Add rasa_events to customizable keys and rename structlog keys

changelog/12516.bugfix.md

@@ -0,0 +1 @@
+Add `rasa_events` to the list of anonymizable structlog keys and rename structlog keys.

rasa/core/actions/forms.py

@@ -1,3 +1,4 @@
+import copy
 from typing import Text, List, Optional, Union, Any, Dict, Set
 import itertools
 import logging
@@ -256,7 +257,9 @@ async def validate_slots(
             Otherwise, returns empty list since the extracted slots already have
             corresponding `SlotSet` events in the tracker.
         """
-        structlogger.debug("forms.slots.validate", slot_candidates=slot_candidates)
+        structlogger.debug(
+            "forms.slots.validate", slot_candidates=copy.deepcopy(slot_candidates)
+        )
         events: List[Union[SlotSet, Event]] = [
             SlotSet(slot_name, value) for slot_name, value in slot_candidates.items()
         ]
@@ -542,7 +545,7 @@ async def _validate_if_required(
         if needs_validation:
             structlogger.debug(
                 "forms.validation.required",
-                tracker_latest_message=tracker.latest_message,
+                tracker_latest_message=copy.deepcopy(tracker.latest_message),
             )
             return await self.validate(tracker, domain, output_channel, nlg)
         else:
@@ -611,7 +614,9 @@ async def activate(
         if not prefilled_slots:
             logger.debug("No pre-filled required slots to validate.")
         else:
-            structlogger.debug("forms.activate.form", prefilled_slots=prefilled_slots)
+            structlogger.debug(
+                "forms.activate.form", prefilled_slots=copy.deepcopy(prefilled_slots)
+            )
 
         validate_name = f"validate_{self.name()}"
 

rasa/core/brokers/kafka.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import os
 import json
 import logging
@@ -242,7 +243,7 @@ def _publish(self, event: Dict[Text, Any]) -> None:
         structlogger.debug(
             "kafka.publish.event",
             topic=self.topic,
-            rasa_event=event,
+            rasa_event=copy.deepcopy(event),
             partition_key=partition_key,
             headers=headers,
         )

rasa/core/brokers/pika.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import json
 import logging
 import structlog
@@ -304,13 +305,13 @@ async def _publish(
                 "pika.events.publish",
                 rabbitmq_exchange=RABBITMQ_EXCHANGE,
                 host=self.host,
-                rasa_event=event,
+                rasa_event=copy.deepcopy(event),
             )
         except Exception as e:
             structlogger.error(
                 "pika.events.publish.failed",
                 host=self.host,
-                rasa_event=event,
+                rasa_event=copy.deepcopy(event),
             )
             if self.should_keep_unpublished_messages:
                 self._unpublished_events.append(event)

rasa/core/channels/facebook.py

@@ -1,3 +1,4 @@
+import copy
 import hashlib
 import hmac
 import logging
@@ -126,7 +127,9 @@ async def message(
             attachment = message["message"]["attachments"][0]
             text = attachment["payload"]["url"]
         else:
-            structlogger.warning("facebook.message.handle", message=message)
+            structlogger.warning(
+                "facebook.message.handle", message=copy.deepcopy(message)
+            )
             return
 
         await self._handle_user_message(text, self.get_user_id(), metadata)

rasa/core/channels/hangouts.py

@@ -1,3 +1,4 @@
+import copy
 import logging
 import structlog
 import google.auth.transport.requests
@@ -321,9 +322,13 @@ async def receive(request: Request) -> HTTPResponse:
                     )
                 )
             except CancelledError:
-                structlogger.error("hangouts.message.blueprint", text=text)
+                structlogger.error(
+                    "hangouts.message.blueprint", text=copy.deepcopy(text)
+                )
             except Exception:
-                structlogger.exception("hangouts.message.blueprint.failure", text=text)
+                structlogger.exception(
+                    "hangouts.message.blueprint.failure", text=copy.deepcopy(text)
+                )
 
             return response.json(collector.messages)
 

rasa/core/channels/rest.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import inspect
 import json
 import logging
@@ -168,9 +169,13 @@ async def receive(request: Request) -> Union[ResponseStream, HTTPResponse]:
                         )
                     )
                 except CancelledError:
-                    structlogger.error("rest.message.received", text=text)
+                    structlogger.error(
+                        "rest.message.received", text=copy.deepcopy(text)
+                    )
                 except Exception:
-                    structlogger.exception("rest.message.received.failure", text=text)
+                    structlogger.exception(
+                        "rest.message.received.failure", text=copy.deepcopy(text)
+                    )
                 return response.json(collector.messages)
 
         return custom_webhook

rasa/core/http_interpreter.py

@@ -1,5 +1,6 @@
 import aiohttp
 
+import copy
 import logging
 import structlog
 
@@ -48,7 +49,7 @@ async def _rasa_http_parse(
         if not self.endpoint_config or self.endpoint_config.url is None:
             structlogger.error(
                 "http.parse.text",
-                text=text,
+                text=copy.deepcopy(text),
             )
             return None
 
@@ -73,12 +74,15 @@ async def _rasa_http_parse(
                         response_text = await resp.text()
                         structlogger.error(
                             "http.parse.text.failure",
-                            text=text,
-                            response_text=response_text,
+                            text=copy.deepcopy(text),
+                            response_text=copy.deepcopy(response_text),
                         )
                         return None
         except Exception:  # skipcq: PYL-W0703
             # need to catch all possible exceptions when doing http requests
             # (timeouts, value errors, parser errors, ...)
-            structlogger.exception("http.parse.text.exception", text=text)
+            structlogger.exception(
+                "http.parse.text.exception",
+                text=copy.deepcopy(text),
+            )
             return None

rasa/core/nlg/interpolator.py

@@ -1,3 +1,4 @@
+import copy
 import re
 import logging
 import structlog
@@ -39,7 +40,7 @@ def interpolate_text(response: Text, values: Dict[Text, Text]) -> Text:
     except KeyError as e:
         structlogger.exception(
             "interpolator.interpolate.text",
-            response=response,
+            response=copy.deepcopy(response),
             placeholder_key=e.args[0],
         )
         return response

rasa/core/policies/memoization.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+import copy
 import zlib
 
 import base64
@@ -253,7 +254,9 @@ def predict_action_probabilities(
         result = self._default_predictions(domain)
 
         states = self._prediction_states(tracker, domain, rule_only_data=rule_only_data)
-        structlogger.debug("memoization.predict.actions", tracker_states=states)
+        structlogger.debug(
+            "memoization.predict.actions", tracker_states=copy.deepcopy(states)
+        )
         predicted_action_name = self.recall(
             states, tracker, domain, rule_only_data=rule_only_data
         )
@@ -428,7 +431,9 @@ def _recall_using_truncation(
                 # check if we like new futures
                 memorised = self._recall_states(states)
                 if memorised is not None:
-                    structlogger.debug("memoization.states_recall", states=states)
+                    structlogger.debug(
+                        "memoization.states_recall", states=copy.deepcopy(states)
+                    )
                     return memorised
                 old_states = states
 
@@ -438,7 +443,9 @@ def _recall_using_truncation(
             )
 
         # No match found
-        structlogger.debug("memoization.states_recall", old_states=old_states)
+        structlogger.debug(
+            "memoization.states_recall", old_states=copy.deepcopy(old_states)
+        )
         return None
 
     def recall(

rasa/core/policies/rule_policy.py

@@ -1,4 +1,5 @@
 from __future__ import annotations
+import copy
 import functools
 import logging
 import structlog
@@ -1022,7 +1023,9 @@ def _find_action_from_rules(
         )
 
         current_states = self.format_tracker_states(states)
-        structlogger.debug("rule_policy.actions.find", current_states=current_states)
+        structlogger.debug(
+            "rule_policy.actions.find", current_states=copy.deepcopy(current_states)
+        )
 
         # Tracks if we are returning after an unhappy loop path. If this becomes `True`
         # the policy returns an event which notifies the loop action that it

rasa/core/processor.py

@@ -1,3 +1,4 @@
+import copy
 import logging
 import structlog
 import os
@@ -199,7 +200,7 @@ async def run_action_extract_slots(
             "processor.extract.slots",
             action_extract_slot=ACTION_EXTRACT_SLOTS,
             len_extraction_events=len(extraction_events),
-            rasa_events=extraction_events,
+            rasa_events=copy.deepcopy(extraction_events),
         )
 
         return tracker
@@ -649,7 +650,9 @@ def _log_slots(tracker: DialogueStateTracker) -> None:
             [f"\t{s.name}: {s.value}" for s in tracker.slots.values()]
         )
         if slot_values.strip():
-            structlogger.debug("processor.slots.log", slot_values=slot_values)
+            structlogger.debug(
+                "processor.slots.log", slot_values=copy.deepcopy(slot_values)
+            )
 
     def _check_for_unseen_features(self, parse_data: Dict[Text, Any]) -> None:
         """Warns the user if the NLU parse data contains unrecognized features.
@@ -720,9 +723,9 @@ async def parse_message(
 
         structlogger.debug(
             "processor.message.parse",
-            parse_data_text=parse_data["text"],
+            parse_data_text=copy.deepcopy(parse_data["text"]),
             parse_data_intent=parse_data["intent"],
-            parse_data_entities=parse_data["entities"],
+            parse_data_entities=copy.deepcopy(parse_data["entities"]),
         )
 
         self._check_for_unseen_features(parse_data)
@@ -1012,15 +1015,17 @@ def _log_action_on_tracker(
         if not action_was_rejected_manually:
             structlogger.debug(
                 "processor.actions.policy_prediction",
-                prediction_events=prediction.events,
+                prediction_events=copy.deepcopy(prediction.events),
             )
             tracker.update_with_events(prediction.events, self.domain)
 
             # log the action and its produced events
             tracker.update(action.event_for_successful_execution(prediction))
 
         structlogger.debug(
-            "processor.actions.log", action_name=action.name(), rasa_events=events
+            "processor.actions.log",
+            action_name=action.name(),
+            rasa_events=copy.deepcopy(events),
         )
         tracker.update_with_events(events, self.domain)
 

rasa/nlu/test.py

@@ -1,3 +1,4 @@
+import copy
 import itertools
 import os
 import logging
@@ -285,7 +286,7 @@ def write_response_successes(
     if successes:
         rasa.shared.utils.io.dump_obj_as_json_to_file(successes_filename, successes)
         logger.info(f"Successful response predictions saved to {successes_filename}.")
-        structlogger.debug("test.write.response", successes=successes)
+        structlogger.debug("test.write.response", successes=copy.deepcopy(successes))
     else:
         logger.info("No successful response predictions found.")
 
@@ -797,7 +798,7 @@ def write_successful_entity_predictions(
     if successes:
         rasa.shared.utils.io.dump_obj_as_json_to_file(successes_filename, successes)
         logger.info(f"Successful entity predictions saved to {successes_filename}.")
-        structlogger.debug("test.write.entities", successes=successes)
+        structlogger.debug("test.write.entities", successes=copy.deepcopy(successes))
     else:
         logger.info("No successful entity prediction found.")
 
@@ -988,7 +989,9 @@ def do_entities_overlap(entities: List[Dict]) -> bool:
             and next_ent["entity"] != curr_ent["entity"]
         ):
             structlogger.warning(
-                "test.overlaping.entities", curr_ent=curr_ent, next_ent=next_ent
+                "test.overlaping.entities",
+                current_entity=copy.deepcopy(curr_ent),
+                next_entity=copy.deepcopy(next_ent),
             )
             return True
 
@@ -1012,10 +1015,10 @@ def find_intersecting_entities(token: Token, entities: List[Dict]) -> List[Dict]
             candidates.append(e)
             structlogger.debug(
                 "test.intersecting.entities",
-                token_text=token.text,
+                token_text=copy.deepcopy(token.text),
                 token_start=token.start,
                 token_end=token.end,
-                entity=e,
+                entity=copy.deepcopy(e),
             )
     return candidates
 

rasa/shared/core/events.py

@@ -1,4 +1,5 @@
 import abc
+import copy
 import json
 import logging
 import structlog
@@ -116,7 +117,9 @@ def deserialise_events(serialized_events: List[Dict[Text, Any]]) -> List["Event"
             if event:
                 deserialised.append(event)
             else:
-                structlogger.warning("event.deserialization.failed", rasa_event=event)
+                structlogger.warning(
+                    "event.deserialization.failed", rasa_event=copy.deepcopy(event)
+                )
 
     return deserialised
 

rasa/shared/core/training_data/story_reader/yaml_story_reader.py

@@ -1,3 +1,4 @@
+import copy
 import functools
 import json
 from json import JSONDecodeError
@@ -628,7 +629,9 @@ def unpack_regex_message(
         # message text did start with the special prefix -- however, a user might
         # just have decided to start their text this way.
         if not match:
-            structlogger.warning("message.parsing.failed", user_text=user_text)
+            structlogger.warning(
+                "message.parsing.failed", user_text=copy.deepcopy(user_text)
+            )
             return message
 
         # Extract attributes from the match - and validate it via the domain.

rasa/utils/log_utils.py

@@ -30,13 +30,14 @@ def _anonymizer(
         "response",
         "slot_candidates",
         "rasa_event",
+        "rasa_events",
         "tracker_states",
         "current_states",
         "old_states",
         "current_states",
         "successes",
-        "curr_ent",
-        "next_ent",
+        "current_entity",
+        "next_entity",
         "states",
         "entity",
         "token_text",