Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add dtype argument for chunkize_serial, fix LdaModel #2027

Merged
merged 5 commits into from
May 1, 2018
Merged

Add dtype argument for chunkize_serial, fix LdaModel #2027

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d76f1d6
Changed to datatype int for distributed
Apr 10, 2018
f66130a
Updated
darindf Apr 23, 2018
cb4900f
Use dask instead of pyro
darindf Apr 23, 2018
a5e4bc4
Revert "Use dask instead of pyro"
darindf Apr 23, 2018
5a5b300
Removed space after paren
darindf Apr 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion gensim/models/ldamodel.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -698,7 +698,8 @@ def rho():
dirty = False

reallen = 0
for chunk_no, chunk in enumerate(utils.grouper(corpus, chunksize, as_numpy=chunks_as_numpy)):
chunks = utils.grouper(corpus, chunksize, as_numpy=chunks_as_numpy, dtype=self.dtype)
for chunk_no, chunk in enumerate(chunks):
reallen += len(chunk) # keep track of how many documents we've processed so far

if eval_every and ((reallen == lencorpus) or ((chunk_no + 1) % (eval_every * self.numworkers) == 0)):
Expand Down
4 changes: 2 additions & 2 deletions gensim/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1119,7 +1119,7 @@ def substitute_entity(match):
return RE_HTML_ENTITY.sub(substitute_entity, text)


def chunkize_serial(iterable, chunksize, as_numpy=False):
def chunkize_serial(iterable, chunksize, as_numpy=False, dtype=np.float32):
"""Give elements from the iterable in `chunksize`-ed lists.
The last returned element may be smaller (if length of collection is not divisible by `chunksize`).

Expand Down Expand Up @@ -1148,7 +1148,7 @@ def chunkize_serial(iterable, chunksize, as_numpy=False):
if as_numpy:
# convert each document to a 2d numpy array (~6x faster when transmitting
# chunk data over the wire, in Pyro)
wrapped_chunk = [[np.array(doc) for doc in itertools.islice(it, int(chunksize))]]
wrapped_chunk = [[np.array(doc, dtype=dtype) for doc in itertools.islice(it, int(chunksize))]]
else:
wrapped_chunk = [list(itertools.islice(it, int(chunksize)))]
if not wrapped_chunk[0]:
Expand Down