From 54c7f2be23b6b70696f6d8549e38c117d9ceee2c Mon Sep 17 00:00:00 2001 From: Schuyler Martin Date: Tue, 26 Nov 2024 13:47:44 -0700 Subject: [PATCH] 234 refactor canonical sorting work found in the recipeparserconvert class (#259) * Starts refactor of canonical sort ordering work * _sort_subtree_keys() now uses primary root, not the root node stored on the working V1 recipe copy used in recipe_parse_convert * Redacts public canonical sort function for fear of causing more user disruption than it is worth --- conda_recipe_manager/parser/_types.py | 147 +++++++++--------- conda_recipe_manager/parser/recipe_parser.py | 22 +++ .../parser/recipe_parser_convert.py | 58 ++----- 3 files changed, 116 insertions(+), 111 deletions(-) diff --git a/conda_recipe_manager/parser/_types.py b/conda_recipe_manager/parser/_types.py index 61e2440a..83790762 100644 --- a/conda_recipe_manager/parser/_types.py +++ b/conda_recipe_manager/parser/_types.py @@ -23,76 +23,83 @@ # Marker used to temporarily work around some Jinja-template parsing issues RECIPE_MANAGER_SUB_MARKER: Final[str] = "__RECIPE_MANAGER_SUBSTITUTION_MARKER__" -# Ideal sort-order of the top-level YAML keys for human readability and traditionally how we organize our files. This -# should work on both V0 (pre CEP-13) and V1 recipe formats. -TOP_LEVEL_KEY_SORT_ORDER: Final[dict[str, int]] = { - "schema_version": 0, - "context": 10, - "package": 20, - "recipe": 30, # Used in the v1 recipe format - "source": 40, - "files": 50, - "build": 60, - "requirements": 70, - "outputs": 80, - "test": 90, - "tests": 100, # Used in the v1 recipe format - "about": 110, - "extra": 120, -} - -# Canonical sort order for the new "v1" recipe format's `build` block -V1_SOURCE_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = { - # URL source fields - "url": 0, - "sha256": 10, - "md5": 20, - # Local source fields (not including above) - "path": 30, - "use_gitignore": 40, - # Git source fields (not including above) - "git": 50, - "branch": 60, - "tag": 70, - "rev": 80, - "depth": 90, - "lfs": 100, - # Common fields - "target_directory": 120, - "file_name": 130, - "patches": 140, -} - -# Canonical sort order for the new "v1" recipe format's `build` block -V1_BUILD_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = { - "number": 0, - "string": 10, - "skip": 20, - "noarch": 30, - "script": 40, - "merge_build_and_host_envs": 50, - "always_include_files": 60, - "always_copy_files": 70, - "variant": 80, - "python": 90, - "prefix_detection": 100, - "dynamic_linking": 110, -} - -# Canonical sort order for the new "v1" recipe format's `tests` block -V1_TEST_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = { - "script": 0, - "requirements": 10, - "files": 20, - "python": 30, - "downstream": 40, -} - -# Canonical sort order for the V1 Python test element -V1_PYTHON_TEST_KEY_SORT_ORDER: Final[dict[str, int]] = { - "imports": 0, - "pip_check": 10, -} + +class CanonicalSortOrder: + """ + Namespace that contains all canonical sort-ordering look-up tables. + """ + + # Ideal sort-order of the top-level YAML keys for human readability and traditionally how we organize our files. + # This should work on both V0 (pre CEP-13) and V1 recipe formats. + TOP_LEVEL_KEY_SORT_ORDER: Final[dict[str, int]] = { + "schema_version": 0, + "context": 10, + "package": 20, + "recipe": 30, # Used in the v1 recipe format + "source": 40, + "files": 50, + "build": 60, + "requirements": 70, + "outputs": 80, + "test": 90, + "tests": 100, # Used in the v1 recipe format + "about": 110, + "extra": 120, + } + + # Canonical sort order for the new "v1" recipe format's `build` block + V1_SOURCE_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = { + # URL source fields + "url": 0, + "sha256": 10, + "md5": 20, + # Local source fields (not including above) + "path": 30, + "use_gitignore": 40, + # Git source fields (not including above) + "git": 50, + "branch": 60, + "tag": 70, + "rev": 80, + "depth": 90, + "lfs": 100, + # Common fields + "target_directory": 120, + "file_name": 130, + "patches": 140, + } + + # Canonical sort order for the new "v1" recipe format's `build` block + V1_BUILD_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = { + "number": 0, + "string": 10, + "skip": 20, + "noarch": 30, + "script": 40, + "merge_build_and_host_envs": 50, + "always_include_files": 60, + "always_copy_files": 70, + "variant": 80, + "python": 90, + "prefix_detection": 100, + "dynamic_linking": 110, + } + + # Canonical sort order for the new "v1" recipe format's `tests` block + V1_TEST_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = { + "script": 0, + "requirements": 10, + "files": 20, + "python": 30, + "downstream": 40, + } + + # Canonical sort order for the V1 Python test element + V1_PYTHON_TEST_KEY_SORT_ORDER: Final[dict[str, int]] = { + "imports": 0, + "pip_check": 10, + } + #### Private Classes (Not to be used external to the `parser` module) #### diff --git a/conda_recipe_manager/parser/recipe_parser.py b/conda_recipe_manager/parser/recipe_parser.py index aa8de522..19ce469f 100644 --- a/conda_recipe_manager/parser/recipe_parser.py +++ b/conda_recipe_manager/parser/recipe_parser.py @@ -48,6 +48,28 @@ class RecipeParser(RecipeReader): # Static set of patch operations that require `from`. The others require `value` or nothing. _patch_ops_requiring_from = set(["copy", "move"]) + ## Recipe Key Sorting ## + + def _sort_subtree_keys(self, sort_path: str, tbl: dict[str, int], rename: str = "") -> None: + """ + Convenience function that sorts 1 level of keys, given a path. Optionally allows renaming of the target node. + No changes are made if the path provided is invalid/does not exist. + + :param sort_path: Top-level path to target sorting of child keys + :param tbl: Table describing how keys should be sorted. Lower-value key names appear towards the top of the list + :param rename: (Optional) If specified, renames the top-level key + """ + + def _comparison(n: Node) -> int: + return RecipeParser._canonical_sort_keys_comparison(n, tbl) + + node = traverse(self._root, str_to_stack_path(sort_path)) # pylint: disable=protected-access + if node is None: + return + if rename: + node.value = rename + node.children.sort(key=_comparison) + ## Pre-processing Recipe Text Functions ## @staticmethod diff --git a/conda_recipe_manager/parser/recipe_parser_convert.py b/conda_recipe_manager/parser/recipe_parser_convert.py index 4d9e9dbf..14cfc508 100644 --- a/conda_recipe_manager/parser/recipe_parser_convert.py +++ b/conda_recipe_manager/parser/recipe_parser_convert.py @@ -9,22 +9,8 @@ from typing import Final, Optional, cast from conda_recipe_manager.licenses.spdx_utils import SpdxUtils -from conda_recipe_manager.parser._node import Node -from conda_recipe_manager.parser._traverse import traverse -from conda_recipe_manager.parser._types import ( - ROOT_NODE_VALUE, - TOP_LEVEL_KEY_SORT_ORDER, - V1_BUILD_SECTION_KEY_SORT_ORDER, - V1_PYTHON_TEST_KEY_SORT_ORDER, - V1_SOURCE_SECTION_KEY_SORT_ORDER, - Regex, -) -from conda_recipe_manager.parser._utils import ( - search_any_regex, - set_key_conditionally, - stack_path_to_str, - str_to_stack_path, -) +from conda_recipe_manager.parser._types import ROOT_NODE_VALUE, CanonicalSortOrder, Regex +from conda_recipe_manager.parser._utils import search_any_regex, set_key_conditionally, stack_path_to_str from conda_recipe_manager.parser.enums import SchemaVersion from conda_recipe_manager.parser.recipe_parser import RecipeParser from conda_recipe_manager.parser.types import CURRENT_RECIPE_SCHEMA_FORMAT @@ -130,26 +116,6 @@ def _patch_deprecated_fields(self, base_path: str, fields: list[str]) -> None: if self._patch_and_log({"op": "remove", "path": path}): self._msg_tbl.add_message(MessageCategory.WARNING, f"Field at `{path}` is no longer supported.") - def _sort_subtree_keys(self, sort_path: str, tbl: dict[str, int], rename: str = "") -> None: - """ - Convenience function that sorts 1 level of keys, given a path. Optionally allows renaming of the target node. - No changes are made if the path provided is invalid/does not exist. - - :param sort_path: Top-level path to target sorting of child keys - :param tbl: Table describing how keys should be sorted. Lower-value key names appear towards the top of the list - :param rename: (Optional) If specified, renames the top-level key - """ - - def _comparison(n: Node) -> int: - return RecipeParser._canonical_sort_keys_comparison(n, tbl) - - node = traverse(self._v1_recipe._root, str_to_stack_path(sort_path)) # pylint: disable=protected-access - if node is None: - return - if rename: - node.value = rename - node.children.sort(key=_comparison) - ## Upgrade functions ## def _upgrade_jinja_to_context_obj(self) -> None: @@ -340,7 +306,9 @@ def _upgrade_source_section(self, base_package_paths: list[str]) -> None: self._patch_move_base_path(src_path, "/git_depth", "/depth") # Canonically sort this section - self._sort_subtree_keys(src_path, V1_SOURCE_SECTION_KEY_SORT_ORDER) + self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access + src_path, CanonicalSortOrder.V1_SOURCE_SECTION_KEY_SORT_ORDER + ) def _upgrade_build_script_section(self, build_path: str) -> None: """ @@ -480,7 +448,9 @@ def _upgrade_build_section(self, base_package_paths: list[str]) -> None: self._patch_deprecated_fields(build_path, build_deprecated) # Canonically sort this section - self._sort_subtree_keys(build_path, V1_BUILD_SECTION_KEY_SORT_ORDER) + self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access + build_path, CanonicalSortOrder.V1_BUILD_SECTION_KEY_SORT_ORDER + ) def _upgrade_requirements_section(self, base_package_paths: list[str]) -> None: """ @@ -667,7 +637,9 @@ def _upgrade_test_section(self, base_package_paths: list[str]) -> None: self._patch_move_base_path(test_path, "/downstreams", "/downstream") # Canonically sort the python section, if it exists - self._sort_subtree_keys(RecipeParser.append_to_path(test_path, "/python"), V1_PYTHON_TEST_KEY_SORT_ORDER) + self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access + RecipeParser.append_to_path(test_path, "/python"), CanonicalSortOrder.V1_PYTHON_TEST_KEY_SORT_ORDER + ) # Move `test` to `tests` and encapsulate the pre-existing object into a list new_test_path = f"{test_path}s" @@ -715,7 +687,9 @@ def _upgrade_multi_output(self, base_package_paths: list[str]) -> None: # Not all the top-level keys are found in each output section, but all the output section keys are # found at the top-level. So for consistency, we sort on that ordering. - self._sort_subtree_keys(output_path, TOP_LEVEL_KEY_SORT_ORDER) + self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access + output_path, CanonicalSortOrder.TOP_LEVEL_KEY_SORT_ORDER + ) @staticmethod def pre_process_recipe_text(content: str) -> str: @@ -836,7 +810,9 @@ def render_to_v1_recipe_format(self) -> tuple[str, MessageTable, str]: # Sort the top-level keys to a "canonical" ordering. This should make previous patch operations look more # "sensible" to a human reader. - self._sort_subtree_keys("/", TOP_LEVEL_KEY_SORT_ORDER) + self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access + "/", CanonicalSortOrder.TOP_LEVEL_KEY_SORT_ORDER + ) # Override the schema value as the recipe conversion is now complete. self._v1_recipe._schema_version = SchemaVersion.V1 # pylint: disable=protected-access