avoid byte concatenation

piskvorky · Feb 13, 2019 · 6dc4aef · 6dc4aef
1 parent 1211071
commit 6dc4aef
Showing 1 changed file with 6 additions and 3 deletions.
diff --git a/gensim/models/_fasttext_bin.py b/gensim/models/_fasttext_bin.py
@@ -30,6 +30,7 @@
 """
 
 import collections
+import io
 import logging
 import struct
 
@@ -173,12 +174,14 @@ def _load_vocab(fin, new_format, encoding='utf-8'):
 
     raw_vocab = collections.OrderedDict()
     for i in range(vocab_size):
-        word_bytes = b''
+        word_bytes = io.BytesIO()
         char_byte = fin.read(1)
-        # Read vocab word
+
         while char_byte != _END_OF_WORD_MARKER:
-            word_bytes += char_byte
+            word_bytes.write(char_byte)
             char_byte = fin.read(1)
+
+        word_bytes = word_bytes.getvalue()
         try:
             word = word_bytes.decode(encoding)
         except UnicodeDecodeError: