Merge pull request #6714 from RasaHQ/bilou-tag-prefix

Remove BILOU tag prefix from role and group labels
RasaHQ · Sep 21, 2020 · 0390605 · 0390605
2 parents 5612584 + 4bd4061
commit 0390605
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 0 deletions.
diff --git a/changelog/6577.bugfix.md b/changelog/6577.bugfix.md
@@ -0,0 +1 @@
+Remove BILOU tag prefix from role and group labels when creating entities.
diff --git a/rasa/nlu/extractors/extractor.py b/rasa/nlu/extractors/extractor.py
@@ -405,7 +405,9 @@ def convert_predictions_into_entities(
                 continue
 
             current_group_tag = self.get_tag_for(tags, ENTITY_ATTRIBUTE_GROUP, idx)
+            current_group_tag = bilou_utils.tag_without_prefix(current_group_tag)
             current_role_tag = self.get_tag_for(tags, ENTITY_ATTRIBUTE_ROLE, idx)
+            current_role_tag = bilou_utils.tag_without_prefix(current_role_tag)
 
             group_or_role_changed = (
                 last_group_tag != current_group_tag or last_role_tag != current_role_tag

diff --git a/tests/nlu/extractors/test_extractor.py b/tests/nlu/extractors/test_extractor.py
@@ -284,6 +284,37 @@ def test_clean_up_entities(
                 },
             ],
         ),
+        (
+            "I am flying from San Fransisco to Amsterdam",
+            {
+                "entity": ["O", "O", "O", "O", "B-city", "L-city", "O", "U-city"],
+                "role": ["O", "O", "O", "O", "B-from", "L-from", "O", "U-to"],
+            },
+            {
+                "entity": [1.0, 1.0, 1.0, 1.0, 0.98, 0.78, 1.0, 0.89],
+                "role": [1.0, 1.0, 1.0, 1.0, 0.98, 0.78, 1.0, 0.89],
+            },
+            [
+                {
+                    "entity": "city",
+                    "start": 17,
+                    "end": 30,
+                    "value": "San Fransisco",
+                    "role": "from",
+                    "confidence_entity": 0.78,
+                    "confidence_role": 0.78,
+                },
+                {
+                    "entity": "city",
+                    "start": 34,
+                    "end": 43,
+                    "value": "Amsterdam",
+                    "role": "to",
+                    "confidence_entity": 0.89,
+                    "confidence_role": 0.89,
+                },
+            ],
+        ),
         (
             "I am flying from San Fransisco to Amsterdam",
             {