Merge pull request #7423 from hotzenklotz/conf_mat_response_selector

Use response selector keys for confusion matrix labelling

changelog/7423.improvement.md

@@ -0,0 +1 @@
+Use response selector keys (sub-intents) as labels for plotting the confusion matrix during NLU evaluation to improve readability.

rasa/nlu/test.py

@@ -31,6 +31,7 @@
     ENTITY_ATTRIBUTE_TYPE,
     ENTITY_ATTRIBUTE_GROUP,
     ENTITY_ATTRIBUTE_ROLE,
+    RESPONSE_KEY_ATTRIBUTE,
 )
 from rasa.model import get_model
 from rasa.nlu import config, training_data, utils
@@ -62,7 +63,7 @@
 
 ResponseSelectionEvaluationResult = namedtuple(
     "ResponseSelectionEvaluationResult",
-    "intent_target " "response_target " "response_prediction " "message " "confidence",
+    "response_key response_prediction_full_intent message confidence",
 )
 
 EntityEvaluationResult = namedtuple(
@@ -235,10 +236,10 @@ def remove_empty_response_examples(
     for r in response_results:
         # substitute None values with empty string
         # to enable sklearn evaluation
-        if r.response_prediction is None:
-            r = r._replace(response_prediction="")
+        if r.response_prediction_full_intent is None:
+            r = r._replace(response_prediction_full_intent="")
 
-        if r.response_target != "" and r.response_target is not None:
+        if r.response_key != "" and r.response_key is not None:
             filtered.append(r)
 
     return filtered
@@ -373,7 +374,7 @@ def evaluate_response_selections(
     )
 
     target_responses, predicted_responses = _targets_predictions_from(
-        response_selection_results, "response_target", "response_prediction"
+        response_selection_results, "response_key", "response_prediction_full_intent"
     )
 
     if report_folder:
@@ -409,9 +410,8 @@ def evaluate_response_selections(
     predictions = [
         {
             "text": res.message,
-            "intent_target": res.intent_target,
-            "response_target": res.response_target,
-            "response_predicted": res.response_prediction,
+            "response_key": res.response_key,
+            "response_predicted": res.response_prediction_full_intent,
             "confidence": res.confidence,
         }
         for res in response_selection_results
@@ -1050,13 +1050,16 @@ def get_eval_data(
                 response_prediction_key, {}
             ).get(OPEN_UTTERANCE_PREDICTION_KEY, {})
 
-            response_target = example.get("response", "")
+            response_prediction_full_intent = selector_properties.get(
+                response_prediction_key, {}
+            ).get("full_retrieval_intent", {})
+
+            response_key = example.get_combined_intent_response_key()
 
             response_selection_results.append(
                 ResponseSelectionEvaluationResult(
-                    intent_target,
-                    response_target,
-                    response_prediction.get("name"),
+                    response_key,
+                    response_prediction_full_intent,
                     result.get("text", {}),
                     response_prediction.get("confidence"),
                 )
@@ -1486,7 +1489,9 @@ def compute_metrics(
     response_selection_metrics = {}
     if response_selection_results:
         response_selection_metrics = _compute_metrics(
-            response_selection_results, "response_target", "response_prediction"
+            response_selection_results,
+            "response_key",
+            "response_prediction_full_intent",
         )
 
     return (

tests/nlu/test_evaluation.py

@@ -470,18 +470,10 @@ def test_response_evaluation_report(tmpdir_factory):
 
     response_results = [
         ResponseSelectionEvaluationResult(
-            "chitchat",
-            "It's sunny in Berlin",
-            "It's sunny in Berlin",
-            "What's the weather",
-            0.65432,
+            "weather/ask_weather", "weather/ask_weather", "What's the weather", 0.65432,
         ),
         ResponseSelectionEvaluationResult(
-            "chitchat",
-            "My name is Mr.bot",
-            "My name is Mr.bot",
-            "What's your name?",
-            0.98765,
+            "faq/ask_name", "faq/ask_name", "What's your name?", 0.98765,
         ),
     ]
 
@@ -499,14 +491,13 @@ def test_response_evaluation_report(tmpdir_factory):
 
     prediction = {
         "text": "What's your name?",
-        "intent_target": "chitchat",
-        "response_target": "My name is Mr.bot",
-        "response_predicted": "My name is Mr.bot",
+        "response_key": "faq/ask_name",
+        "response_predicted": "faq/ask_name",
         "confidence": 0.98765,
     }
 
     assert len(report.keys()) == 5
-    assert report["My name is Mr.bot"] == name_query_results
+    assert report["faq/ask_name"] == name_query_results
     assert result["predictions"][1] == prediction
 
 
@@ -590,23 +581,16 @@ def test_empty_intent_removal():
 
 def test_empty_response_removal():
     response_results = [
+        ResponseSelectionEvaluationResult(None, None, "What's the weather", 0.65432,),
         ResponseSelectionEvaluationResult(
-            "chitchat", None, "It's sunny in Berlin", "What's the weather", 0.65432
-        ),
-        ResponseSelectionEvaluationResult(
-            "chitchat",
-            "My name is Mr.bot",
-            "My name is Mr.bot",
-            "What's your name?",
-            0.98765,
+            "faq/ask_name", "faq/ask_name", "What's your name?", 0.98765,
         ),
     ]
     response_results = remove_empty_response_examples(response_results)
 
     assert len(response_results) == 1
-    assert response_results[0].intent_target == "chitchat"
-    assert response_results[0].response_target == "My name is Mr.bot"
-    assert response_results[0].response_prediction == "My name is Mr.bot"
+    assert response_results[0].response_key == "faq/ask_name"
+    assert response_results[0].response_prediction_full_intent == "faq/ask_name"
     assert response_results[0].confidence == 0.98765
     assert response_results[0].message == "What's your name?"