Fix BPE for dual learning

Summary: Fix integration test Reviewed By: xianxl Differential Revision: D18040440 fbshipit-source-id: 98c8ab7970d081f17deb54c69aa35669de12d767
facebookresearch · Nov 1, 2019 · 828c1ca · 828c1ca
1 parent 4c6b689
commit 828c1ca
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/fairseq/data/data_utils.py b/fairseq/data/data_utils.py
@@ -234,6 +234,8 @@ def batch_by_size(
 def process_bpe_symbol(sentence: str, bpe_symbol: str):
     if bpe_symbol == 'sentencepiece':
         sentence = sentence.replace(' ', '').replace('\u2581', ' ').strip()
+    elif bpe_symbol == '_EOW':
+        sentence = sentence.replace(' ', '').replace('_EOW', ' ').strip()
     elif bpe_symbol is not None:
         sentence = (sentence + ' ').replace(bpe_symbol, '').rstrip()
     return sentence