Skip to content

Commit

Permalink
234 refactor canonical sorting work found in the recipeparserconvert …
Browse files Browse the repository at this point in the history 
…class (#259)

* Starts refactor of canonical sort ordering work

* _sort_subtree_keys() now uses primary root, not the root node stored on the working V1 recipe copy used in recipe_parse_convert

* Redacts public canonical sort function for fear of causing more user disruption than it is worth
  • Loading branch information
@schuylermartin45
schuylermartin45 authored Nov 26, 2024
1 parent 0bab33b commit 54c7f2b
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 111 deletions.
147 changes: 77 additions & 70 deletions conda_recipe_manager/parser/_types.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,76 +23,83 @@
# Marker used to temporarily work around some Jinja-template parsing issues
RECIPE_MANAGER_SUB_MARKER: Final[str] = "__RECIPE_MANAGER_SUBSTITUTION_MARKER__"

# Ideal sort-order of the top-level YAML keys for human readability and traditionally how we organize our files. This
# should work on both V0 (pre CEP-13) and V1 recipe formats.
TOP_LEVEL_KEY_SORT_ORDER: Final[dict[str, int]] = {
"schema_version": 0,
"context": 10,
"package": 20,
"recipe": 30, # Used in the v1 recipe format
"source": 40,
"files": 50,
"build": 60,
"requirements": 70,
"outputs": 80,
"test": 90,
"tests": 100, # Used in the v1 recipe format
"about": 110,
"extra": 120,
}

# Canonical sort order for the new "v1" recipe format's `build` block
V1_SOURCE_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = {
# URL source fields
"url": 0,
"sha256": 10,
"md5": 20,
# Local source fields (not including above)
"path": 30,
"use_gitignore": 40,
# Git source fields (not including above)
"git": 50,
"branch": 60,
"tag": 70,
"rev": 80,
"depth": 90,
"lfs": 100,
# Common fields
"target_directory": 120,
"file_name": 130,
"patches": 140,
}

# Canonical sort order for the new "v1" recipe format's `build` block
V1_BUILD_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = {
"number": 0,
"string": 10,
"skip": 20,
"noarch": 30,
"script": 40,
"merge_build_and_host_envs": 50,
"always_include_files": 60,
"always_copy_files": 70,
"variant": 80,
"python": 90,
"prefix_detection": 100,
"dynamic_linking": 110,
}

# Canonical sort order for the new "v1" recipe format's `tests` block
V1_TEST_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = {
"script": 0,
"requirements": 10,
"files": 20,
"python": 30,
"downstream": 40,
}

# Canonical sort order for the V1 Python test element
V1_PYTHON_TEST_KEY_SORT_ORDER: Final[dict[str, int]] = {
"imports": 0,
"pip_check": 10,
}

class CanonicalSortOrder:
"""
Namespace that contains all canonical sort-ordering look-up tables.
"""

# Ideal sort-order of the top-level YAML keys for human readability and traditionally how we organize our files.
# This should work on both V0 (pre CEP-13) and V1 recipe formats.
TOP_LEVEL_KEY_SORT_ORDER: Final[dict[str, int]] = {
"schema_version": 0,
"context": 10,
"package": 20,
"recipe": 30, # Used in the v1 recipe format
"source": 40,
"files": 50,
"build": 60,
"requirements": 70,
"outputs": 80,
"test": 90,
"tests": 100, # Used in the v1 recipe format
"about": 110,
"extra": 120,
}

# Canonical sort order for the new "v1" recipe format's `build` block
V1_SOURCE_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = {
# URL source fields
"url": 0,
"sha256": 10,
"md5": 20,
# Local source fields (not including above)
"path": 30,
"use_gitignore": 40,
# Git source fields (not including above)
"git": 50,
"branch": 60,
"tag": 70,
"rev": 80,
"depth": 90,
"lfs": 100,
# Common fields
"target_directory": 120,
"file_name": 130,
"patches": 140,
}

# Canonical sort order for the new "v1" recipe format's `build` block
V1_BUILD_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = {
"number": 0,
"string": 10,
"skip": 20,
"noarch": 30,
"script": 40,
"merge_build_and_host_envs": 50,
"always_include_files": 60,
"always_copy_files": 70,
"variant": 80,
"python": 90,
"prefix_detection": 100,
"dynamic_linking": 110,
}

# Canonical sort order for the new "v1" recipe format's `tests` block
V1_TEST_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = {
"script": 0,
"requirements": 10,
"files": 20,
"python": 30,
"downstream": 40,
}

# Canonical sort order for the V1 Python test element
V1_PYTHON_TEST_KEY_SORT_ORDER: Final[dict[str, int]] = {
"imports": 0,
"pip_check": 10,
}


#### Private Classes (Not to be used external to the `parser` module) ####

Expand Down
22 changes: 22 additions & 0 deletions conda_recipe_manager/parser/recipe_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,28 @@ class RecipeParser(RecipeReader):
# Static set of patch operations that require `from`. The others require `value` or nothing.
_patch_ops_requiring_from = set(["copy", "move"])

## Recipe Key Sorting ##

def _sort_subtree_keys(self, sort_path: str, tbl: dict[str, int], rename: str = "") -> None:
"""
Convenience function that sorts 1 level of keys, given a path. Optionally allows renaming of the target node.
No changes are made if the path provided is invalid/does not exist.
:param sort_path: Top-level path to target sorting of child keys
:param tbl: Table describing how keys should be sorted. Lower-value key names appear towards the top of the list
:param rename: (Optional) If specified, renames the top-level key
"""

def _comparison(n: Node) -> int:
return RecipeParser._canonical_sort_keys_comparison(n, tbl)

node = traverse(self._root, str_to_stack_path(sort_path)) # pylint: disable=protected-access
if node is None:
return
if rename:
node.value = rename
node.children.sort(key=_comparison)

## Pre-processing Recipe Text Functions ##

@staticmethod
Expand Down
58 changes: 17 additions & 41 deletions conda_recipe_manager/parser/recipe_parser_convert.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,8 @@
from typing import Final, Optional, cast

from conda_recipe_manager.licenses.spdx_utils import SpdxUtils
from conda_recipe_manager.parser._node import Node
from conda_recipe_manager.parser._traverse import traverse
from conda_recipe_manager.parser._types import (
ROOT_NODE_VALUE,
TOP_LEVEL_KEY_SORT_ORDER,
V1_BUILD_SECTION_KEY_SORT_ORDER,
V1_PYTHON_TEST_KEY_SORT_ORDER,
V1_SOURCE_SECTION_KEY_SORT_ORDER,
Regex,
)
from conda_recipe_manager.parser._utils import (
search_any_regex,
set_key_conditionally,
stack_path_to_str,
str_to_stack_path,
)
from conda_recipe_manager.parser._types import ROOT_NODE_VALUE, CanonicalSortOrder, Regex
from conda_recipe_manager.parser._utils import search_any_regex, set_key_conditionally, stack_path_to_str
from conda_recipe_manager.parser.enums import SchemaVersion
from conda_recipe_manager.parser.recipe_parser import RecipeParser
from conda_recipe_manager.parser.types import CURRENT_RECIPE_SCHEMA_FORMAT
Expand Down Expand Up @@ -130,26 +116,6 @@ def _patch_deprecated_fields(self, base_path: str, fields: list[str]) -> None:
if self._patch_and_log({"op": "remove", "path": path}):
self._msg_tbl.add_message(MessageCategory.WARNING, f"Field at `{path}` is no longer supported.")

def _sort_subtree_keys(self, sort_path: str, tbl: dict[str, int], rename: str = "") -> None:
"""
Convenience function that sorts 1 level of keys, given a path. Optionally allows renaming of the target node.
No changes are made if the path provided is invalid/does not exist.
:param sort_path: Top-level path to target sorting of child keys
:param tbl: Table describing how keys should be sorted. Lower-value key names appear towards the top of the list
:param rename: (Optional) If specified, renames the top-level key
"""

def _comparison(n: Node) -> int:
return RecipeParser._canonical_sort_keys_comparison(n, tbl)

node = traverse(self._v1_recipe._root, str_to_stack_path(sort_path)) # pylint: disable=protected-access
if node is None:
return
if rename:
node.value = rename
node.children.sort(key=_comparison)

## Upgrade functions ##

def _upgrade_jinja_to_context_obj(self) -> None:
Expand Down Expand Up @@ -340,7 +306,9 @@ def _upgrade_source_section(self, base_package_paths: list[str]) -> None:
self._patch_move_base_path(src_path, "/git_depth", "/depth")

# Canonically sort this section
self._sort_subtree_keys(src_path, V1_SOURCE_SECTION_KEY_SORT_ORDER)
self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access
src_path, CanonicalSortOrder.V1_SOURCE_SECTION_KEY_SORT_ORDER
)

def _upgrade_build_script_section(self, build_path: str) -> None:
"""
Expand Down Expand Up @@ -480,7 +448,9 @@ def _upgrade_build_section(self, base_package_paths: list[str]) -> None:
self._patch_deprecated_fields(build_path, build_deprecated)

# Canonically sort this section
self._sort_subtree_keys(build_path, V1_BUILD_SECTION_KEY_SORT_ORDER)
self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access
build_path, CanonicalSortOrder.V1_BUILD_SECTION_KEY_SORT_ORDER
)

def _upgrade_requirements_section(self, base_package_paths: list[str]) -> None:
"""
Expand Down Expand Up @@ -667,7 +637,9 @@ def _upgrade_test_section(self, base_package_paths: list[str]) -> None:
self._patch_move_base_path(test_path, "/downstreams", "/downstream")

# Canonically sort the python section, if it exists
self._sort_subtree_keys(RecipeParser.append_to_path(test_path, "/python"), V1_PYTHON_TEST_KEY_SORT_ORDER)
self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access
RecipeParser.append_to_path(test_path, "/python"), CanonicalSortOrder.V1_PYTHON_TEST_KEY_SORT_ORDER
)

# Move `test` to `tests` and encapsulate the pre-existing object into a list
new_test_path = f"{test_path}s"
Expand Down Expand Up @@ -715,7 +687,9 @@ def _upgrade_multi_output(self, base_package_paths: list[str]) -> None:

# Not all the top-level keys are found in each output section, but all the output section keys are
# found at the top-level. So for consistency, we sort on that ordering.
self._sort_subtree_keys(output_path, TOP_LEVEL_KEY_SORT_ORDER)
self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access
output_path, CanonicalSortOrder.TOP_LEVEL_KEY_SORT_ORDER
)

@staticmethod
def pre_process_recipe_text(content: str) -> str:
Expand Down Expand Up @@ -836,7 +810,9 @@ def render_to_v1_recipe_format(self) -> tuple[str, MessageTable, str]:

# Sort the top-level keys to a "canonical" ordering. This should make previous patch operations look more
# "sensible" to a human reader.
self._sort_subtree_keys("/", TOP_LEVEL_KEY_SORT_ORDER)
self._v1_recipe._sort_subtree_keys( # pylint: disable=protected-access
"/", CanonicalSortOrder.TOP_LEVEL_KEY_SORT_ORDER
)

# Override the schema value as the recipe conversion is now complete.
self._v1_recipe._schema_version = SchemaVersion.V1 # pylint: disable=protected-access
Expand Down

0 comments on commit 54c7f2b

Please sign in to comment.