CLI parallel search updates (qmk#22525)

lib/python/qmk/cli/ci/validate_aliases.py

@@ -1,11 +1,8 @@
 """Validates the list of keyboard aliases.
 """
-from pathlib import Path
-
 from milc import cli
 
-from qmk.json_schema import json_load
-from qmk.keyboard import resolve_keyboard, keyboard_folder
+from qmk.keyboard import resolve_keyboard, keyboard_folder, keyboard_alias_definitions
 
 
 def _safe_keyboard_folder(target):
@@ -34,7 +31,7 @@ def _target_keyboard_exists(target):
 
 @cli.subcommand('Validates the list of keyboard aliases.', hidden=True)
 def ci_validate_aliases(cli):
-    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
+    aliases = keyboard_alias_definitions()
 
     success = True
     for alias in aliases.keys():

lib/python/qmk/cli/generate/api.py

@@ -10,7 +10,7 @@
 from qmk.info import info_json
 from qmk.json_schema import json_load
 from qmk.keymap import list_keymaps
-from qmk.keyboard import find_readme, list_keyboards
+from qmk.keyboard import find_readme, list_keyboards, keyboard_alias_definitions
 from qmk.keycodes import load_spec, list_versions, list_languages
 
 DATA_PATH = Path('data')
@@ -166,7 +166,7 @@ def generate_api(cli):
 
     # Generate data for the global files
     keyboard_list = sorted(kb_all)
-    keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
+    keyboard_aliases = keyboard_alias_definitions()
     keyboard_metadata = {
         'last_updated': current_datetime(),
         'keyboards': keyboard_list,

lib/python/qmk/commands.py

@@ -3,12 +3,12 @@
 import os
 import sys
 import shutil
-from pathlib import Path
 
 from milc import cli
 import jsonschema
 
 from qmk.json_schema import json_load, validate
+from qmk.keyboard import keyboard_alias_definitions
 
 
 def find_make():
@@ -53,7 +53,7 @@ def parse_configurator_json(configurator_file):
         exit(1)
 
     keyboard = user_keymap['keyboard']
-    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
+    aliases = keyboard_alias_definitions()
 
     while keyboard in aliases:
         last_keyboard = keyboard

lib/python/qmk/keyboard.py

@@ -70,6 +70,11 @@ def __eq__(self, other):
 base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep
 
 
+@lru_cache(maxsize=1)
+def keyboard_alias_definitions():
+    return json_load(Path('data/mappings/keyboard_aliases.hjson'))
+
+
 def is_all_keyboards(keyboard):
     """Returns True if the keyboard is an AllKeyboards object.
     """
@@ -112,7 +117,7 @@ def keyboard_folder(keyboard):
 
     This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard.
     """
-    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
+    aliases = keyboard_alias_definitions()
 
     while keyboard in aliases:
         last_keyboard = keyboard

lib/python/qmk/search.py

@@ -6,7 +6,7 @@
 import logging
 import re
 from typing import List, Tuple
-from dotty_dict import dotty
+from dotty_dict import dotty, Dotty
 from milc import cli
 
 from qmk.util import parallel_map
@@ -107,13 +107,22 @@ def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str
     return list(sorted(set(overall_targets)))
 
 
+def _construct_build_target_kb_km(e):
+    return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1])
+
+
+def _construct_build_target_kb_km_json(e):
+    return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2])
+
+
 def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
     """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
 
     Optionally includes the values of the queried info.json keys.
     """
     if len(filters) == 0:
-        targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list]
+        cli.log.info('Preparing target list...')
+        targets = list(parallel_map(_construct_build_target_kb_km, target_list))
     else:
         cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
         valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
@@ -172,7 +181,9 @@ def f(e):
                 cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
                 continue
 
-        targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps]
+        cli.log.info('Preparing target list...')
+        valid_keymaps = [(e[0], e[1], e[2].to_dict() if isinstance(e[2], Dotty) else e[2]) for e in valid_keymaps]  # need to convert dotty_dict back to dict because it doesn't survive parallelisation
+        targets = list(parallel_map(_construct_build_target_kb_km_json, list(valid_keymaps)))
 
     return targets