Skip to content

Commit

Permalink
Add osx+py38 case for avoid multiprocessing issue (#2800)
Browse files Browse the repository at this point in the history
* add osx+py38 case for avoid multiprocessing issue

* add comment, fix warning

* extend comment

Co-Authored-By: Radim Řehůřek <me@radimrehurek.com>

* Update gensim/utils.py

* Update gensim/utils.py

Co-Authored-By: Michael Penkov <m@penkov.dev>

Co-authored-by: Radim Řehůřek <me@radimrehurek.com>
Co-authored-by: Michael Penkov <m@penkov.dev>
  • Loading branch information
3 people authored Apr 20, 2020
1 parent a2ec4c3 commit 5b5b545
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions gensim/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -1238,7 +1238,11 @@ def run(self):
self.q.put(wrapped_chunk.pop(), block=True)


if os.name == 'nt':
# Multiprocessing on Windows (and on OSX with python3.8+) uses "spawn" mode, which
# causes issues with pickling.
# So for these two platforms, use simpler serial processing in `chunkize`.
# See https://github.com/RaRe-Technologies/gensim/pull/2800#discussion_r410890171
if os.name == 'nt' or (sys.platform == "darwin" and sys.version_info >= (3, 8)):
def chunkize(corpus, chunksize, maxsize=0, as_numpy=False):
"""Split `corpus` into fixed-sized chunks, using :func:`~gensim.utils.chunkize_serial`.
Expand All @@ -1260,7 +1264,8 @@ def chunkize(corpus, chunksize, maxsize=0, as_numpy=False):
"""
if maxsize > 0:
warnings.warn("detected Windows; aliasing chunkize to chunkize_serial")
entity = "Windows" if os.name == 'nt' else "OSX with python3.8+"
warnings.warn("detected %s; aliasing chunkize to chunkize_serial" % entity)
for chunk in chunkize_serial(corpus, chunksize, as_numpy=as_numpy):
yield chunk
else:
Expand Down

0 comments on commit 5b5b545

Please sign in to comment.