feat: add pool processes

mrzzcn · May 11, 2021 · 051a403 · 051a403
1 parent 02db55a
commit 051a403
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/tools/baiduspider.py b/tools/baiduspider.py
@@ -8,6 +8,7 @@
 from multiprocessing import Pool, cpu_count
 from pypinyin import pinyin, Style
 import tqdm
+import math
 
 host = "https://shurufa.baidu.com"
 archives_url = host + "/dict"
@@ -81,7 +82,7 @@ def processTag(tag):
 
 def download_dicts():
     total = len(dicts)
-    with Pool(processes=cpu_count() - 1) as pool:
+    with Pool(processes=max(cpu_count() - 1, 8)) as pool:
         r = list(tqdm.tqdm(pool.imap_unordered(download_dict, dicts), total=total))
 
 def download_dict(dict):

diff --git a/tools/rime_dict_tool.py b/tools/rime_dict_tool.py
@@ -61,7 +61,7 @@ def processDict(dictFile, finalDicts):
     finalDicts = manager.WordDict()
 
     processor = functools.partial(processDict, finalDicts=finalDicts)
-    with Pool(processes=cpu_count() - 1) as pool:
+    with Pool(processes=max(cpu_count() - 1, 8)) as pool:
         r = list(tqdm.tqdm(pool.imap_unordered(processor, files), total=total))
         if output:
             filename = os.path.splitext(os.path.basename(output))[0].split('.dict')[0]