reduce phraser memory

piskvorky · Oct 26, 2018 · bba2e46 · bba2e46
1 parent 242c80e
commit bba2e46
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 3 deletions.
diff --git a/gensim/models/phrases.py b/gensim/models/phrases.py
@@ -805,7 +805,7 @@ def __init__(self, phrases_model):
         for bigram, score in phrases_model.export_phrases(corpus, self.delimiter, as_tuples=True):
             if bigram in self.phrasegrams:
                 logger.info('Phraser repeat %s', bigram)
-            self.phrasegrams[bigram] = (None, score)
+            self.phrasegrams[bigram] = score
             count += 1
             if not count % 50000:
                 logger.info('Phraser added %i phrasegrams', count)
@@ -848,7 +848,10 @@ def score_item(self, worda, wordb, components, scorer):
 
         """
         try:
-            return self.phrasegrams[tuple(components)][-1]
+            if list(self.phrasegrams.values())[0].__class__ is tuple:
+                return self.phrasegrams[tuple(components)][-1]
+            else:
+                return self.phrasegrams[tuple(components)]
         except KeyError:
             return -1
 

diff --git a/gensim/test/test_data/phraser_model_3dot6 b/gensim/test/test_data/phraser_model_3dot6
diff --git a/gensim/test/test_phrases.py b/gensim/test/test_phrases.py
@@ -12,7 +12,6 @@
 import unittest
 
 import six
-
 import numpy as np
 
 from gensim.utils import to_unicode