New KeyedVectors.vectors_for_all method for vectorizing all words in a dictionary

gensim/corpora/dictionary.py

@@ -10,6 +10,7 @@
 from collections.abc import Mapping
 import logging
 import itertools
+from typing import Optional, List
 
 from gensim import utils
 
@@ -689,6 +690,32 @@ def load_from_text(fname):
                 result.dfs[wordid] = int(docfreq)
         return result
 
+    def most_common(self, n: Optional[int] = None, include_counts: bool = True) -> List[str]:
+        """Return a list of the n most common words and optionally their counts from the most common to the least.
+
+        Words with equal counts are ordered in the increasing order of their ids.
+
+        Parameters
+        ----------
+        n : int or None, optional
+            The number of most common words to be returned. If `None`, all words in the dictionary
+            will be returned. Default is `None`.
+        include_counts : bool
+            Whether the counts of the n most common words will also be returned. Default is `True`.
+
+        Returns
+        -------
+        most_common : list of {(str, int), str}
+            The n most common words and optionally their counts from the most common to the least.
+
+        """
+        most_common = [
+            (self[word], count) if include_counts else self[word]
+            for word, count
+            in sorted(self.cfs.items(), key=lambda x: (-x[1], x[0]))[:n]
+        ]
+        return most_common
+
     @staticmethod
     def from_corpus(corpus, id2word=None):
         """Create :class:`~gensim.corpora.dictionary.Dictionary` from an existing corpus.

gensim/models/keyedvectors.py

@@ -171,6 +171,8 @@
 import itertools
 import warnings
 from numbers import Integral
+from typing import Iterable
+from collections import OrderedDict
 
 from numpy import (
     dot, float32 as REAL, double, array, zeros, vstack,
@@ -1695,6 +1697,64 @@ def intersect_word2vec_format(self, fname, lockf=0.0, binary=False, encoding='ut
             msg=f"merged {overlap_count} vectors into {self.vectors.shape} matrix from {fname}",
         )
 
+    def vectors_for_all(self, keys: Iterable, allow_inference: bool = True,
+                        copy_vecattrs: bool = False) -> 'KeyedVectors':
+        """Produce vectors for all given keys as a new :class:`KeyedVectors` object.
+
+        Notes
+        -----
+        The keys will always be deduplicated. For optimal performance, you should not pass entire
+        corpora to the method. Instead, you should construct a dictionary of unique words in your
+        corpus:
+
+        >>> from collections import Counter
+        >>> import itertools
+        >>> import gensim.downloader as api
+        >>> from gensim.test.utils import common_texts
+        >>>
+        >>> model = api.load('word2vec-google-news-300')
+        >>> corpus = common_texts
+        >>> word_counts = Counter(itertools.chain.from_iterable(corpus))  # count words in your corpus
+        >>> words_by_freq = (k for k, v in word_counts.most_common())
+        >>> word_vectors = model.wv.vectors_for_all(words_by_freq)  # create word-vectors for words in your corpus
+
+        Parameters
+        ----------
+        keys : iterable
+            The keys that will be vectorized.
+        allow_inference : bool, optional
+            In subclasses such as :class:`~gensim.models.fasttext.FastTextKeyedVectors`,
+            vectors for out-of-vocabulary keys (words) may be inferred. Default is True.
+        copy_vecattrs : bool, optional
+            Additional attributes set via the :meth:`KeyedVectors.set_vecattr` method
+            will be preserved in the produced :class:`KeyedVectors` object. Default is False.
+            To ensure that *all* the produced vectors will have vector attributes assigned,
+            you should set `allow_inference=False`.
+
+        Returns
+        -------
+        keyedvectors : :class:`~gensim.models.keyedvectors.KeyedVectors`
+            Vectors for all the given keys.
+
+        """
+        vocab = (key for key in keys if key in (self if allow_inference else self.key_to_index))  # drop undefined keys
+        vocab = list(OrderedDict.fromkeys(vocab))  # deduplicate keys
+
+        datatype = self.vectors.dtype
+        kv = KeyedVectors(self.vector_size, len(vocab), dtype=datatype)  # preallocate new KeyedVectors object
+
+        for key in vocab:  # produce and index vectors for all the given keys
+            weights = self[key]
+            _add_word_to_kv(kv, None, key, weights, len(vocab))
+            if not copy_vecattrs:
+                continue
+            for attr in self.expandos:  # copy vector attributes to the new KeyedVectors object
+                try:
+                    kv.set_vecattr(key, attr, self.get_vecattr(key, attr))
+                except KeyError:
+                    pass
+        return kv
+
     def _upconvert_old_d2vkv(self):
         """Convert a deserialized older Doc2VecKeyedVectors instance to latest generic KeyedVectors"""
         self.vocab = self.doctags

gensim/similarities/termsim.py

@@ -102,6 +102,28 @@ class WordEmbeddingSimilarityIndex(TermSimilarityIndex):
     Computes cosine similarities between word embeddings and retrieves most
     similar terms for a given term.
 
+    Notes
+    -----
+    By fitting the word embeddings to a vocabulary that you will be using, you
+    can eliminate all out-of-vocabulary (OOV) words that you would otherwise
+    receive from the `most_similar` method. In subword models such as fastText,
+    this procedure will also infer word-vectors for words from your vocabulary
+    that previously had no word-vector.
+
+    >>> from gensim.test.utils import common_texts, datapath
+    >>> from gensim.corpora import Dictionary
+    >>> from gensim.models import FastText
+    >>> from gensim.models.word2vec import LineSentence
+    >>> from gensim.similarities import WordEmbeddingSimilarityIndex
+    >>>
+    >>> model = FastText(common_texts, vector_size=20, min_count=1)  # train word-vectors on a corpus
+    >>> different_corpus = LineSentence(datapath('lee_background.cor'))
+    >>> dictionary = Dictionary(different_corpus)  # construct a vocabulary on a different corpus
+    >>> words = dictionary.most_common(include_counts=False)
+    >>> word_vectors = model.wv.vectors_for_all(words)  # remove OOV word-vectors and infer word-vectors for new words
+    >>> assert len(dictionary) == len(word_vectors)  # all words from our vocabulary received their word-vectors
+    >>> termsim_index = WordEmbeddingSimilarityIndex(word_vectors)
+
     Parameters
     ----------
     keyedvectors : :class:`~gensim.models.keyedvectors.KeyedVectors`
@@ -409,20 +431,19 @@ class SparseTermSimilarityMatrix(SaveLoad):
     >>> from gensim.models import Word2Vec
     >>> from gensim.similarities import SoftCosineSimilarity, SparseTermSimilarityMatrix, WordEmbeddingSimilarityIndex
     >>> from gensim.similarities.index import AnnoyIndexer
-    >>> from scikits.sparse.cholmod import cholesky
     >>>
     >>> model = Word2Vec(common_texts, vector_size=20, min_count=1)  # train word-vectors
     >>> annoy = AnnoyIndexer(model, num_trees=2)  # use annoy for faster word similarity lookups
-    >>> termsim_index = WordEmbeddingSimilarityIndex(model.wv, kwargs={'indexer': annoy})
     >>> dictionary = Dictionary(common_texts)
+    >>> words = dictionary.most_common(include_counts=False)
+    >>> word_vectors = model.wv.vectors_for_all(words)
+    >>> termsim_index = WordEmbeddingSimilarityIndex(word_vectors, kwargs={'indexer': annoy})
     >>> bow_corpus = [dictionary.doc2bow(document) for document in common_texts]
     >>> similarity_matrix = SparseTermSimilarityMatrix(termsim_index, dictionary, symmetric=True, dominant=True)
     >>> docsim_index = SoftCosineSimilarity(bow_corpus, similarity_matrix, num_best=10)
     >>>
     >>> query = 'graph trees computer'.split()  # make a query
     >>> sims = docsim_index[dictionary.doc2bow(query)]  # calculate similarity of query to each doc from bow_corpus
-    >>>
-    >>> word_embeddings = cholesky(similarity_matrix.matrix).L()  # obtain word embeddings from similarity matrix
 
     Check out `the Gallery <https://radimrehurek.com/gensim/auto_examples/tutorials/run_scm.html>`_
     for more examples.

gensim/test/test_corpora_dictionary.py

@@ -359,6 +359,30 @@ def test_patch_with_special_tokens(self):
         self.assertNotIn((1, 1), d.doc2bow(corpus_with_special_tokens[0]))
         self.assertIn((1, 1), d.doc2bow(corpus_with_special_tokens[1]))
 
+    def test_most_common_with_n(self):
+        texts = [['human', 'human', 'human', 'computer', 'computer', 'interface', 'interface']]
+        d = Dictionary(texts)
+        expected = [('human', 3), ('computer', 2)]
+        self.assertEqual(d.most_common(n=2), expected)
+
+    def test_most_common_without_n(self):
+        texts = [['human', 'human', 'human', 'computer', 'computer', 'interface', 'interface']]
+        d = Dictionary(texts)
+        expected = [('human', 3), ('computer', 2), ('interface', 2)]
+        self.assertEqual(d.most_common(n=None), expected)
+
+    def test_most_common_with_counts(self):
+        texts = [['human', 'human', 'human', 'computer', 'computer', 'interface', 'interface']]
+        d = Dictionary(texts)
+        expected = [('human', 3), ('computer', 2), ('interface', 2)]
+        self.assertEqual(d.most_common(include_counts=True), expected)
+
+    def test_most_common_without_counts(self):
+        texts = [['human', 'human', 'human', 'computer', 'computer', 'interface', 'interface']]
+        d = Dictionary(texts)
+        expected = ['human', 'computer', 'interface']
+        self.assertEqual(d.most_common(include_counts=False), expected)
+
 
 # endclass TestDictionary
 

gensim/test/test_fasttext.py

@@ -850,6 +850,54 @@ def obsolete_testLoadOldModel(self):
         self.assertEqual(model.wv.vectors_vocab.shape, (12, 100))
         self.assertEqual(model.wv.vectors_ngrams.shape, (2000000, 100))
 
+    def test_vectors_for_all_with_inference(self):
+        """Test vectors_for_all can infer new vectors."""
+        words = [
+            'responding',
+            'approached',
+            'chairman',
+            'an out-of-vocabulary word',
+            'another out-of-vocabulary word',
+        ]
+        vectors_for_all = self.test_model.wv.vectors_for_all(words)
+
+        expected = 5
+        predicted = len(vectors_for_all)
+        self.assertEqual(expected, predicted)
+
+        expected = self.test_model.wv['responding']
+        predicted = vectors_for_all['responding']
+        self.assertTrue(np.allclose(expected, predicted))
+
+        smaller_distance = np.linalg.norm(
+            vectors_for_all['an out-of-vocabulary word']
+            - vectors_for_all['another out-of-vocabulary word']
+        )
+        greater_distance = np.linalg.norm(
+            vectors_for_all['an out-of-vocabulary word']
+            - vectors_for_all['responding']
+        )
+        self.assertGreater(greater_distance, smaller_distance)
+
+    def test_vectors_for_all_without_inference(self):
+        """Test vectors_for_all does not infer new vectors when prohibited."""
+        words = [
+            'responding',
+            'approached',
+            'chairman',
+            'an out-of-vocabulary word',
+            'another out-of-vocabulary word',
+        ]
+        vectors_for_all = self.test_model.wv.vectors_for_all(words, allow_inference=False)
+
+        expected = 3
+        predicted = len(vectors_for_all)
+        self.assertEqual(expected, predicted)
+
+        expected = self.test_model.wv['responding']
+        predicted = vectors_for_all['responding']
+        self.assertTrue(np.allclose(expected, predicted))
+
 
 with open(datapath('toy-data.txt')) as fin:
     TOY_SENTENCES = [fin.read().strip().split(' ')]

gensim/test/test_keyedvectors.py

@@ -39,6 +39,43 @@ def test_most_similar(self):
         predicted = [result[0] for result in self.vectors.most_similar('war', topn=5)]
         self.assertEqual(expected, predicted)
 
+    def test_vectors_for_all_list(self):
+        """Test vectors_for_all returns expected results with a list of keys."""
+        words = [
+            'conflict',
+            'administration',
+            'terrorism',
+            'an out-of-vocabulary word',
+            'another out-of-vocabulary word',
+        ]
+        vectors_for_all = self.vectors.vectors_for_all(words)
+
+        expected = 3
+        predicted = len(vectors_for_all)
+        self.assertEqual(expected, predicted)
+
+        expected = self.vectors['conflict']
+        predicted = vectors_for_all['conflict']
+        self.assertTrue(np.allclose(expected, predicted))
+
+    def test_vectors_for_all_with_copy_vecattrs(self):
+        """Test vectors_for_all returns can copy vector attributes."""
+        words = ['conflict']
+        vectors_for_all = self.vectors.vectors_for_all(words, copy_vecattrs=True)
+
+        expected = self.vectors.get_vecattr('conflict', 'count')
+        predicted = vectors_for_all.get_vecattr('conflict', 'count')
+        self.assertEqual(expected, predicted)
+
+    def test_vectors_for_all_without_copy_vecattrs(self):
+        """Test vectors_for_all returns can copy vector attributes."""
+        words = ['conflict']
+        vectors_for_all = self.vectors.vectors_for_all(words, copy_vecattrs=False)
+
+        not_expected = self.vectors.get_vecattr('conflict', 'count')
+        predicted = vectors_for_all.get_vecattr('conflict', 'count')
+        self.assertNotEqual(not_expected, predicted)
+
     def test_most_similar_topn(self):
         """Test most_similar returns correct results when `topn` is specified."""
         self.assertEqual(len(self.vectors.most_similar('war', topn=5)), 5)