diff --git a/tools/baiduspider.py b/tools/baiduspider.py index e109f19..285e20b 100644 --- a/tools/baiduspider.py +++ b/tools/baiduspider.py @@ -8,6 +8,7 @@ from multiprocessing import Pool, cpu_count from pypinyin import pinyin, Style import tqdm +import math host = "https://shurufa.baidu.com" archives_url = host + "/dict" @@ -81,7 +82,7 @@ def processTag(tag): def download_dicts(): total = len(dicts) - with Pool(processes=cpu_count() - 1) as pool: + with Pool(processes=max(cpu_count() - 1, 8)) as pool: r = list(tqdm.tqdm(pool.imap_unordered(download_dict, dicts), total=total)) def download_dict(dict): diff --git a/tools/rime_dict_tool.py b/tools/rime_dict_tool.py index a162840..274f431 100755 --- a/tools/rime_dict_tool.py +++ b/tools/rime_dict_tool.py @@ -61,7 +61,7 @@ def processDict(dictFile, finalDicts): finalDicts = manager.WordDict() processor = functools.partial(processDict, finalDicts=finalDicts) - with Pool(processes=cpu_count() - 1) as pool: + with Pool(processes=max(cpu_count() - 1, 8)) as pool: r = list(tqdm.tqdm(pool.imap_unordered(processor, files), total=total)) if output: filename = os.path.splitext(os.path.basename(output))[0].split('.dict')[0]