Merge branch 'master' into token-pattern

RasaHQ · Jul 6, 2020 · 05866fb · 05866fb
2 parents 7c88a45 + 8f51534
commit 05866fb
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 3 deletions.
diff --git a/changelog/6143.bugfix.rst b/changelog/6143.bugfix.rst
@@ -0,0 +1 @@
+Prevent ``WhitespaceTokenizer`` from outputting empty list of tokens.
diff --git a/rasa/nlu/tokenizers/whitespace_tokenizer.py b/rasa/nlu/tokenizers/whitespace_tokenizer.py
@@ -72,13 +72,13 @@ def tokenize(self, message: Message, attribute: Text) -> List[Token]:
             text,
         ).split()
 
+        words = [self.remove_emoji(w) for w in words]
+        words = [w for w in words if w]
+
         # if we removed everything like smiles `:)`, use the whole text as 1 token
         if not words:
             words = [text]
 
-        words = [self.remove_emoji(w) for w in words]
-        words = [w for w in words if w]
-
         tokens = self._convert_words_to_tokens(words, text)
 
         return self._apply_token_pattern(tokens)
diff --git a/tests/nlu/tokenizers/test_whitespace_tokenizer.py b/tests/nlu/tokenizers/test_whitespace_tokenizer.py
@@ -65,6 +65,7 @@
         ),
         (":)", [":)"], [(0, 2)]),
         ("Hi :-)", ["Hi"], [(0, 2)]),
+        ("👍", ["👍"], [(0, 1)]),
     ],
 )
 def test_whitespace(text, expected_tokens, expected_indices):
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Prevent ``WhitespaceTokenizer`` from outputting empty list of tokens.