From 4bfb3c51efefb4bde6f6a2a8999a61a3c7b6eeb7 Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Tue, 19 Nov 2024 09:59:19 -0500 Subject: [PATCH 1/9] depend on conda-recipe-manager --- CHANGELOG.md | 4 + environment.yaml | 1 + percy/commands/aggregate.py | 3 +- percy/commands/convert.py | 21 +- percy/commands/main.py | 1 + percy/commands/recipe.py | 1 + percy/examples/preinstall/conftest.py | 2 +- percy/parser/__init__.py | 1 - percy/parser/_node.py | 157 -- percy/parser/_selector_info.py | 28 - percy/parser/_traverse.py | 190 -- percy/parser/_types.py | 90 - percy/parser/_utils.py | 149 -- percy/parser/enums.py | 17 - percy/parser/exceptions.py | 23 - percy/parser/recipe_parser.py | 1782 ----------------- percy/parser/types.py | 188 -- percy/render/_renderer.py | 12 +- percy/render/aggregate.py | 5 +- percy/render/dumper.py | 4 + percy/render/exceptions.py | 1 + percy/render/recipe.py | 44 +- percy/render/variants.py | 3 +- percy/repodata/repodata.py | 1 + percy/tests/commands/test_aggregate_cli.py | 1 + percy/tests/commands/test_convert_cli.py | 1 + percy/tests/commands/test_main_cli.py | 1 + percy/tests/commands/test_recipe_cli.py | 1 + percy/tests/parser/test_recipe_parser.py | 2017 -------------------- percy/types.py | 1 + pyproject.toml | 2 +- 31 files changed, 60 insertions(+), 4692 deletions(-) delete mode 100644 percy/parser/__init__.py delete mode 100644 percy/parser/_node.py delete mode 100644 percy/parser/_selector_info.py delete mode 100644 percy/parser/_traverse.py delete mode 100644 percy/parser/_types.py delete mode 100644 percy/parser/_utils.py delete mode 100644 percy/parser/enums.py delete mode 100644 percy/parser/exceptions.py delete mode 100644 percy/parser/recipe_parser.py delete mode 100644 percy/parser/types.py delete mode 100644 percy/tests/parser/test_recipe_parser.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af3c69f..1dd11027 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog Note: version releases in the 0.x.y range may introduce breaking changes. +## 0.2.0 +- Remove previous parser tree code and depend on conda-recipe-manager instead. +- Remove `get_read_only_parser()` function. + ## 0.1.7 - Revert exclusion of `clang-compiler-activation-feedstock` from locally loaded feedstocks since it's actually active and needed. diff --git a/environment.yaml b/environment.yaml index 3cb817ab..7133ec2c 100644 --- a/environment.yaml +++ b/environment.yaml @@ -25,3 +25,4 @@ dependencies: - jsonschema - types-jsonschema - pre-commit + - conda-recipe-manager diff --git a/percy/commands/aggregate.py b/percy/commands/aggregate.py index 87df7355..2b0145c8 100644 --- a/percy/commands/aggregate.py +++ b/percy/commands/aggregate.py @@ -2,6 +2,7 @@ File: aggregate.py Description: CLI tool for performing operations against all of aggregate. """ + from __future__ import annotations import functools @@ -111,7 +112,7 @@ def base_options(f: Callable): "-p", type=str, multiple=False, - default="3.10", + default="3.12", help="Python version. E.g. -p 3.10", ) @click.option( diff --git a/percy/commands/convert.py b/percy/commands/convert.py index 3fd7fcfc..adc5692d 100644 --- a/percy/commands/convert.py +++ b/percy/commands/convert.py @@ -2,6 +2,7 @@ File: convert.py Description: CLI for converting an old recipe file to the "new" format. """ + from __future__ import annotations import os @@ -10,12 +11,8 @@ from typing import Final import click - -from percy.parser.recipe_parser import RecipeParser -from percy.parser.types import MessageCategory, MessageTable - -# Required file name for the recipe, specified in CEP-13 -NEW_FORMAT_RECIPE_FILE_NAME: Final[str] = "recipe.yaml" +from conda_recipe_manager.parser.recipe_parser_convert import RecipeParserConvert +from conda_recipe_manager.types import MessageCategory, MessageTable, Primitives, SentinelType class ExitCode(IntEnum): @@ -65,11 +62,10 @@ def convert(file: click.File, output: click.Path) -> None: # pylint: disable=re """ Recipe conversion CLI utility. By default, recipes print to STDOUT. Messages always print to STDERR. """ - recipe_content: Final[str] = file.read() - parser: RecipeParser try: - parser = RecipeParser(recipe_content) + recipe_content: Final[str] = RecipeParserConvert.pre_process_recipe_text(file.read()) + recipe_converter = RecipeParserConvert(recipe_content) except Exception as e: # pylint: disable=broad-exception-caught print_err("An exception occurred while parsing the recipe file:") print_err(e) @@ -78,19 +74,18 @@ def convert(file: click.File, output: click.Path) -> None: # pylint: disable=re result: str msg_tbl: MessageTable try: - result, msg_tbl = parser.render_to_new_recipe_format() + v1_content, msg_tbl, _ = recipe_converter.render_to_v1_recipe_format() except Exception as e: # pylint: disable=broad-exception-caught print_err("An exception occurred while converting to the new recipe file:") print_err(e) sys.exit(ExitCode.RENDER_EXCEPTION) - if output is None: - print_out(result) + print(v1_content) else: if not os.path.basename(output) == "recipe.yaml": print_err("WARNING: File is not called `recipe.yaml`.") with open(output, "w", encoding="utf-8") as fptr: - fptr.write(result) + fptr.write(v1_content) error_count: Final[int] = msg_tbl.get_message_count(MessageCategory.ERROR) warn_count: Final[int] = msg_tbl.get_message_count(MessageCategory.WARNING) diff --git a/percy/commands/main.py b/percy/commands/main.py index a419cab8..04b2f5c0 100644 --- a/percy/commands/main.py +++ b/percy/commands/main.py @@ -2,6 +2,7 @@ File: main.py Description: Primary execution point of the `click` command line interface """ + from __future__ import annotations import os diff --git a/percy/commands/recipe.py b/percy/commands/recipe.py index 1205d9c8..21868502 100644 --- a/percy/commands/recipe.py +++ b/percy/commands/recipe.py @@ -2,6 +2,7 @@ File: recipe.py Description: CLI for interacting with recipe files. """ + from __future__ import annotations import functools diff --git a/percy/examples/preinstall/conftest.py b/percy/examples/preinstall/conftest.py index 84d47588..f460290b 100644 --- a/percy/examples/preinstall/conftest.py +++ b/percy/examples/preinstall/conftest.py @@ -1,5 +1,5 @@ _DEFAULT_AGGREGATE = "~/work/recipes/aggregate/" -_DEFAULT_PYTHON = ["3.8", "3.9", "3.10", "3.11"] +_DEFAULT_PYTHON = ["3.9", "3.10", "3.11", "3.12", "3.13"] _DEFAULT_SUBDIRS = [ "linux-64", "linux-ppc64le", diff --git a/percy/parser/__init__.py b/percy/parser/__init__.py deleted file mode 100644 index cda944ed..00000000 --- a/percy/parser/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# This package contains supporting files for the `RecipeParser` class diff --git a/percy/parser/_node.py b/percy/parser/_node.py deleted file mode 100644 index 005975c4..00000000 --- a/percy/parser/_node.py +++ /dev/null @@ -1,157 +0,0 @@ -""" -File: _node.py -Description: Provides a private node class only used by the parser. This class is fundamental to tree formation. -""" -from __future__ import annotations - -from typing import Optional - -from percy.parser._types import ROOT_NODE_VALUE -from percy.parser.types import MultilineVariant, NodeValue -from percy.types import SentinelType - - -class Node: - """ - Private class representing a node in a recipe parse tree. - - Each level of a path consists of a list of child nodes. Child nodes can recursively store more child nodes until a - final value is found, indicated by having an empty list of children. - - Remember that YAML keys must be strings, but the `value` can another primitive type for leaf nodes. - - Comments on a recipe line are stored separately from the value. - - Variable names are not substituted. In other words, the raw strings from the file are stored as text. - """ - - # Sentinel used to discern a `null` in the YAML file and a defaulted, unset value. For example, comment-only lines - # should always be set to the `_sentinel` object. - _sentinel = SentinelType() - - def __init__( - self, - value: NodeValue | SentinelType = _sentinel, - comment: str = "", - children: Optional[list["Node"]] = None, - list_member_flag: bool = False, - multiline_variant: MultilineVariant = MultilineVariant.NONE, - key_flag: bool = False, - ): - """ - Constructs a node - :param value: Value of the current node - :param comment: Comment on the line this node was found on - :param children: List of children nodes, descendants of this node - :param list_member_flag: Indicates if this node is part of a list - :param multiline_variant: Indicates if the node represents a multiline value AND which syntax variant is used - :param key_flag: Indicates if the node represents a key that points to zero or more subsequent values - """ - self.value = value - self.comment = comment - self.children: list[Node] = children if children else [] - self.list_member_flag = list_member_flag - self.multiline_variant = multiline_variant - self.key_flag = key_flag - - def __eq__(self, other: object) -> bool: - """ - Determine if two nodes are equal. Useful for `assert` statements in tests. - :param other: Other object to check against - :returns: True if the two nodes are identical. False otherwise. - """ - if not isinstance(other, Node): - return False - return ( - self.value == other.value - and self.comment == other.comment - and self.list_member_flag == other.list_member_flag - and self.multiline_variant == other.multiline_variant - # Save recursive (most expensive) check for last - and self.children == other.children - ) - - def __str__(self) -> str: - """ - Renders the Node as a string. Useful for debugging purposes. - :returns: The node, as a string - """ - value = self.value - if self.is_comment(): - value = "Comment node" - if self.is_collection_element(): - value = "Collection node" - return ( - f"Node: {value}\n" - f" - Comment: {self.comment!r}\n" - f" - Child count: {len(self.children)}\n" - f" - List?: {self.list_member_flag}\n" - f" - Multiline?: {self.multiline_variant}\n" - f" - Key?: {self.key_flag}\n" - ) - - def short_str(self) -> str: - """ - Renders the Node as a simple string. Useful for other `__str__()` functions to call. - :returns: The node, as a simplified string. - """ - if self.is_comment(): - return f"" - if self.is_collection_element(): - return "" - return str(self.value) - - def is_leaf(self) -> bool: - """ - Indicates if a node is a leaf node - :returns: True if the node is a leaf. False otherwise. - """ - return not self.children and not self.is_comment() - - def is_root(self) -> bool: - """ - Indicates if a node is a root node - :returns: True if the node is a root node. False otherwise. - """ - return self.value == ROOT_NODE_VALUE - - def is_comment(self) -> bool: - """ - Indicates if a line contains only a comment. When rendered, this will be a comment only-line. - :returns: True if the node represents only a comment. False otherwise. - """ - return self.value == Node._sentinel and bool(self.comment) and not self.children - - def is_empty_key(self) -> bool: - """ - Indicates a line that is just a "label" and contains no child nodes. These are effectively leaf nodes that need - to be rendered specially. - - Example empty key: - foo: - Versus a non-empty key: - foo: - - bar - - When converted into a Pythonic data structure, the key will point to an `None` value. - :returns: True if the node represents an empty key. False otherwise. - """ - return self.key_flag and self.is_leaf() - - def is_single_key(self) -> bool: - """ - Indicates if a node contains a single child node and is a key. - - This special case is used in several edge cases. Namely, it allows the rendering algorithm to print such - key-value pairs on the same line. - :returns: True if the node represents a single key. False otherwise. - """ - return self.key_flag and len(self.children) == 1 and self.children[0].is_leaf() - - def is_collection_element(self) -> bool: - """ - Indicates if the node is a list member that contains other collection types. In other words, this node has no - value itself BUT it contains children that do. - :returns: True if the node represents an element that is a collection. False otherwise. - """ - return self.value == Node._sentinel and self.list_member_flag and bool(self.children) diff --git a/percy/parser/_selector_info.py b/percy/parser/_selector_info.py deleted file mode 100644 index 6a649c21..00000000 --- a/percy/parser/_selector_info.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -File: _selector_info.py -Description: Provides the `SelectorInfo` class, used to store selector information. -""" -from __future__ import annotations - -from typing import NamedTuple - -from percy.parser._node import Node -from percy.parser._types import StrStack -from percy.parser._utils import stack_path_to_str - - -class SelectorInfo(NamedTuple): - """ - Immutable structure that tracks information about how a particular selector is used. - """ - - node: Node - path: StrStack - - def __str__(self) -> str: - """ - Generates the string form of a `SelectorInfo` object. Useful for debugging. - :returns: String representation of a `SelectorInfo` instance - """ - path_str = stack_path_to_str(self.path.copy()) - return f"{self.node.short_str()} -> {path_str}" diff --git a/percy/parser/_traverse.py b/percy/parser/_traverse.py deleted file mode 100644 index 96065baa..00000000 --- a/percy/parser/_traverse.py +++ /dev/null @@ -1,190 +0,0 @@ -""" -File: py -Description: Provides tree traversal functions only used by the parser. -""" -from __future__ import annotations - -from typing import Callable, Final, Optional - -from percy.parser._node import Node -from percy.parser._types import ROOT_NODE_VALUE, StrStack, StrStackImmutable - -# Indicates an array index that is not valid -INVALID_IDX: Final[int] = -1 - - -def remap_child_indices_virt_to_phys(children: list[Node]) -> list[int]: - """ - Given a list of child nodes, generate a look-up table to map the "virtual" index positions with the "physical" - locations. - - A recipe file may have comment lines, represented as child nodes. For rendering, these nodes must be preserved, - in-order. - - For manipulating and accessing list members, however, comments are to be ignored. The calling program should not - rely on our implementation details and should be able to access a member of a list as expected. In other words, - users will not consider comments in a list as indexable list members. - - :param children: Child node list to process. - :returns: A list of indices. Indexing this list with the "virtual" (user-provided) index will return the "physical" - list position. - """ - mapping: list[int] = [] - cntr = 0 - for child in children: - if child.is_comment(): - cntr += 1 - continue - mapping.append(cntr) - cntr += 1 - - return mapping - - -def remap_child_indices_phys_to_virt(children: list[Node]) -> list[int]: - """ - Produces the "inverted" table created by `remap_child_indices_virt_to_phys()`. - See `remap_child_indices_virt_to_phys()` for more details. - :param children: Child node list to process. - :returns: A list of indices. Indexing this list with the "physical" (class-provided) index will return the "virtual" - list position. - """ - mapping: list[int] = remap_child_indices_virt_to_phys(children) - new_mapping: list[int] = [0] * len(children) - for i in range(len(mapping)): - new_mapping[mapping[i]] = i - return new_mapping - - -def _traverse_recurse(node: Node, path: StrStack) -> Optional[Node]: - """ - Recursive helper function for traversing a tree. - :param node: Current node on the tree. - :param path: Path, as a stack, that describes a location in the tree. - :returns: `Node` object if a node is found in the parse tree at that path. Otherwise returns `None`. - """ - if len(path) == 0: - return node - - path_part = path[-1] - # Check if the path is attempting an array index. - if path_part.isdigit(): - # Map virtual to physical indices and perform some out-of-bounds checks. - idx_map = remap_child_indices_virt_to_phys(node.children) - virtual_idx = int(path_part) - max_idx = len(idx_map) - 1 - if virtual_idx < 0 or virtual_idx > max_idx: - return None - - path_idx = idx_map[virtual_idx] - # Edge case: someone attempts to use the index syntax on a non-list member. As children are stored as a list - # per node, this could "work" with unintended consequences. In other words, users could accidentally abuse - # underlying implementation details. - if not node.children[path_idx].list_member_flag: - return None - - path.pop() - return _traverse_recurse(node.children[path_idx], path) - - for child in node.children: - # Remember: for nodes that represent part of the path, the "value" stored in the node is part of the path-name. - if child.value == path_part: - path.pop() - return _traverse_recurse(child, path) - # Path not found - return None - - -def traverse(node: Optional[Node], path: StrStack) -> Optional[Node]: - """ - Given a path in the recipe tree, traverse the tree and return the node at that path. - - If no Node is found at that path, return `None`. - :param node: Starting node of the tree/branch to traverse. - :param path: Path, as a stack, that describes a location in the tree. - :returns: `Node` object if a node is found in the parse tree at that path. Otherwise returns `None`. - """ - # Bootstrap recursive edge cases - if node is None: - return None - if len(path) == 0: - return None - if len(path) == 1: - if path[0] == ROOT_NODE_VALUE: - return node - return None - # Purge `root` from the path - path.pop() - return _traverse_recurse(node, path) - - -def traverse_with_index(root: Node, path: StrStack) -> tuple[Optional[Node], int, int]: - """ - Given a path, return the node of interest OR the parent node with indexing information, if the node is in a list. - - :param root: Starting node of the tree/branch to traverse. - :param path: Path, as a stack, that describes a location in the tree. - :returns: A tuple containing: - - `Node` object if a node is found in the parse tree at that path. Otherwise - returns `None`. If the path terminates in an index, the parent is returned with the index location. - - If the node is a member of a list, the VIRTUAL index returned will be >= 0 - - If the node is a member of a list, the PHYSICAL index returned will be >= 0 - """ - if len(path) == 0: - return None, INVALID_IDX, INVALID_IDX - - node: Optional[Node] - virt_idx: int = INVALID_IDX - phys_idx: int = INVALID_IDX - # Pre-determine if the path is targeting a list position. Patching only applies on the last index provided. - if path[0].isdigit(): - # Find the index position of the target on the parent's list - virt_idx = int(path.pop(0)) - - node = traverse(root, path) - if node is not None and virt_idx >= 0: - phys_idx = remap_child_indices_virt_to_phys(node.children)[virt_idx] - - # If the node in a list is a "Collection Element", we want return that node and not the parent that contains - # the list. Collection Nodes are abstract containers that will contain the rest of - if node.children[phys_idx].is_collection_element(): - return node.children[phys_idx], INVALID_IDX, INVALID_IDX - - return node, virt_idx, phys_idx - - -def traverse_all( - node: Optional[Node], - func: Callable[[Node, StrStack], None], - path: Optional[StrStackImmutable] = None, - idx_num: int = 0, -) -> None: - """ - Given a node, traverse all child nodes and apply a function to each node. Useful for updating or extracting - information on the whole tree. - - NOTE: The paths provided will return virtual indices, not physical indices. In other words, comments in a list do - not count towards the index position of a list member. - - :param node: Node to start with - :param func: Function to apply against all traversed nodes. - :param path: CALLERS: DO NOT SET. This value tracks the current path of a node. This should only be specified in - recursive calls to this function. Tuples are used for their immutability, so paths change based on the current - stack frame. - :param idx_num: CALLERS: DO NOT SET. Used in recursive calls to track the index position of a list-member node. - """ - if node is None: - return - # Initialize, if on the root node. Otherwise build-up the path - if path is None: - path = (ROOT_NODE_VALUE,) - elif node.list_member_flag: - path = (str(idx_num),) + path - # Leafs do not contain their values in the path, unless the leaf is an empty key (as the key is part of the path). - elif node.is_empty_key() or not node.is_leaf(): - path = (str(node.value),) + path - func(node, list(path)) - # Used for paths that contain lists of items - mapping = remap_child_indices_phys_to_virt(node.children) - for i in range(len(node.children)): - traverse_all(node.children[i], func, path, mapping[i]) diff --git a/percy/parser/_types.py b/percy/parser/_types.py deleted file mode 100644 index d4b2a921..00000000 --- a/percy/parser/_types.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -File: _types.py -Description: Provides private types, type aliases, constants, and small classes used by the parser and related files. -""" -from __future__ import annotations - -import re -from typing import Final - -import yaml - -#### Private Types (Not to be used external to the `parser` module) #### - -# Type alias for a list of strings treated as a Pythonic stack -StrStack = list[str] -# Type alias for a `StrStack` that must be immutable. Useful for some recursive operations. -StrStackImmutable = tuple[str, ...] - -#### Private Constants (Not to be used external to the `parser` module) #### - -# String that represents a root node in our path. -ROOT_NODE_VALUE: Final[str] = "/" -# Marker used to temporarily work around some Jinja-template parsing issues -PERCY_SUB_MARKER: Final[str] = "__PERCY_SUBSTITUTION_MARKER__" - -# Ideal sort-order of the top-level YAML keys for human readability and traditionally how we organize our files. This -# should work on both old and new recipe formats. -TOP_LEVEL_KEY_SORT_ORDER: Final[dict[str, int]] = { - "schema_version": 0, - "context": 10, - "package": 20, - "recipe": 30, # Used in the v1 recipe format - "source": 40, - "files": 50, - "build": 60, - "requirements": 70, - "outputs": 80, - "test": 90, - "tests": 100, # Used in the v1 recipe format - "about": 110, - "extra": 120, -} - -# Canonical sort order for the new "v1" recipe format's `tests` block -V1_TEST_SECTION_KEY_SORT_ORDER: Final[dict[str, int]] = { - "script": 0, - "requirements": 10, - "files": 20, - "python": 30, - "downstream": 40, -} - -#### Private Classes (Not to be used external to the `parser` module) #### - -# NOTE: The classes put in this file should be structures (NamedTuples) and very small support classes that don't make -# sense to dedicate a file for. - - -class ForceIndentDumper(yaml.Dumper): - """ - Custom YAML dumper used to include optional indentation for human readability. - Adapted from: https://stackoverflow.com/questions/25108581/python-yaml-dump-bad-indentation - """ - - def increase_indent(self, flow: bool = False, indentless: bool = False) -> None: # pylint: disable=unused-argument - return super().increase_indent(flow, False) - - -class Regex: - """ - Namespace used to organize all regular expressions used by the `parser` module. - """ - - # Pattern to detect Jinja variable names and functions - _JINJA_VAR_FUNCTION_PATTERN: Final[str] = r"[a-zA-Z_][a-zA-Z0-9_\|\'\"\(\)\, =\.\-]*" - - # Jinja regular expressions - JINJA_SUB: Final[re.Pattern[str]] = re.compile(r"{{\s*" + _JINJA_VAR_FUNCTION_PATTERN + r"\s*}}") - JINJA_FUNCTION_LOWER: Final[re.Pattern[str]] = re.compile(r"\|\s*lower") - JINJA_LINE: Final[re.Pattern[str]] = re.compile(r"({%.*%}|{#.*#})\n") - JINJA_SET_LINE: Final[re.Pattern[str]] = re.compile(r"{%\s*set\s*" + _JINJA_VAR_FUNCTION_PATTERN + r"\s*=.*%}\s*\n") - - SELECTOR: Final[re.Pattern[str]] = re.compile(r"\[.*\]") - # Detects the 6 common variants (3 |'s, 3 >'s). See this guide for more info: - # https://stackoverflow.com/questions/3790454/how-do-i-break-a-string-in-yaml-over-multiple-lines/21699210 - MULTILINE: Final[re.Pattern[str]] = re.compile(r"^\s*.*:\s+(\||>)(\+|\-)?(\s*|\s+#.*)") - # Group where the "variant" string is identified - MULTILINE_VARIANT_CAPTURE_GROUP_CHAR: Final[int] = 1 - MULTILINE_VARIANT_CAPTURE_GROUP_SUFFIX: Final[int] = 2 - DETECT_TRAILING_COMMENT: Final[re.Pattern[str]] = re.compile(r"(\s)+(#)") diff --git a/percy/parser/_utils.py b/percy/parser/_utils.py deleted file mode 100644 index 5f2015bb..00000000 --- a/percy/parser/_utils.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -File: _utils.py -Description: Provides private utility functions only used by the parser. -""" -from __future__ import annotations - -import json -from typing import cast - -from percy.parser._types import PERCY_SUB_MARKER, ROOT_NODE_VALUE, Regex, StrStack, StrStackImmutable -from percy.parser.types import TAB_AS_SPACES, MultilineVariant, NodeValue -from percy.types import H, SentinelType - - -def str_to_stack_path(path: str) -> StrStack: - """ - Takes a JSON-patch path as a string and return a path as a stack of strings. String paths are used by callers, - stacks are used internally. - - For example: - "/foo/bar/baz" -> ["baz", "bar", "foo", "/"] - :param path: Path to deconstruct into a stack - :returns: Path, described as a stack of strings. - """ - # TODO: validate the path starts with `/` (root) - - # `PurePath` could be used here, but isn't for performance gains. - # TODO reduce 3 (O)n operations to 1 O(n) operation - - # Wipe the trailing `/`, if provided. It doesn't have meaning here; only the `root` path is tracked. - if path[-1] == ROOT_NODE_VALUE: - path = path[:-1] - parts = path.split("/") - # Replace empty strings with `/` for compatibility in other functions. - for i in range(0, len(parts)): - if parts[i] == "": - parts[i] = "/" - return parts[::-1] - - -def stack_path_to_str(path_stack: StrStack | StrStackImmutable) -> str: - """ - Takes a stack that represents a path and converts it into a string. String paths are used by callers, stacks are - used internally. - - :param path_stack: Stack to construct back into a string. - :returns: Path, described as a string. - """ - # Normalize type if a tuple is given. - if isinstance(path_stack, tuple): - path_stack = list(path_stack) - path = "" - while len(path_stack) > 0: - value = path_stack.pop() - # Special case to bootstrap root; the first element will automatically add the first slash. - if value == ROOT_NODE_VALUE: - continue - path += f"/{value}" - return path - - -def num_tab_spaces(s: str) -> int: - """ - Counts the number of spaces at the start of the string. Used to indicate depth of a field in a YAML file (the YAML - specification dictates only spaces can be used for indenting). - :param s: Target string - :returns: Number of preceding spaces in a string - """ - cntr: int = 0 - for c in s: - if c == " ": - cntr += 1 - else: - break - return cntr - - -def substitute_markers(s: str, subs: list[str]) -> str: - """ - Given a string, replace substitution markers with the original Jinja template from a list of options. - :param s: String to replace substitution markers with - :param subs: List of substitutions to make, in order of appearance - :returns: New string, with substitutions removed - """ - while s.find(PERCY_SUB_MARKER) >= 0 and len(subs): - s = s.replace(PERCY_SUB_MARKER, subs[0], 1) - subs.pop(0) - return s - - -def stringify_yaml( - val: NodeValue | SentinelType, multiline_variant: MultilineVariant = MultilineVariant.NONE -) -> NodeValue: - """ - Special function for handling edge cases when converting values back to YAML. - :param val: Value to check - :param multiline_variant: (Optional) If the value being processed is a multiline string, indicate which YAML - descriptor is in use. - :returns: YAML version of a value, as a string. - """ - # Handled for type-completeness of `Node.value`. A `Node` with a sentinel as its value indicates a special Node - # type that is not directly render-able. - if isinstance(val, SentinelType): - return "" - # None -> null - if val is None: - return "null" - # True -> true - if isinstance(val, bool): - if val: - return "true" - return "false" - # Ensure string quote escaping if quote marks are present. Otherwise this has the unintended consequence of - # quoting all YAML strings. Although not wrong, it does not follow our common practices. Quote escaping is not - # required for multiline strings. We do not escape quotes for Jinja value statements. We make an exception for - # strings containing the NEW recipe format syntax, ${{ }}, which is valid YAML. - if multiline_variant == MultilineVariant.NONE and isinstance(val, str) and not Regex.JINJA_SUB.match(val): - if "${{" not in val and ("'" in val or '"' in val): - # The PyYaml equivalent function injects newlines, hence why we abuse the JSON library to write our YAML - return json.dumps(val) - return val - - -def normalize_multiline_strings(val: NodeValue, variant: MultilineVariant) -> NodeValue: - """ - Utility function that takes in a Node's value and "normalizes" multiline strings so that they can be accurately - interpreted by PyYaml. We use PyYaml to handle the various ways in which a multiline string can be interpreted. - :param val: Value to normalize - :param variant: Multiline variant rules to follow - :returns: If the value is a multiline string, this returns the "normalized" string to be re-evaluated by PyYaml. - Otherwise, returns the original value. - """ - if variant == MultilineVariant.NONE: - return val - - # Prepend the multiline marker to the string to have PyYaml interpret how the whitespace should be handled. JINJA - # substitutions in multi-line strings do not break the PyYaml parser. - multiline_str = f"\n{TAB_AS_SPACES}".join(cast(list[str], val)) - return f"{variant}\n{TAB_AS_SPACES}{multiline_str}" - - -def dedupe_and_preserve_order(l: list[H]) -> list[H]: - """ - Takes a list of strings - See this StackOverflow post: - https://stackoverflow.com/questions/480214/how-do-i-remove-duplicates-from-a-list-while-preserving-order - - """ - return list(cast(dict[H, None], dict.fromkeys(l))) diff --git a/percy/parser/enums.py b/percy/parser/enums.py deleted file mode 100644 index b55f38ba..00000000 --- a/percy/parser/enums.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -File: enums.py -Description: Provides enumerated types used by the parser. -""" -from __future__ import annotations - -from enum import Enum - - -class SelectorConflictMode(Enum): - """ - Defines how to handle the addition of a selector if one already exists. - """ - - AND = 1 # Logically "and" the new selector with the old - OR = 2 # Logically "or" the new selector with the old - REPLACE = 3 # Replace the existing selector diff --git a/percy/parser/exceptions.py b/percy/parser/exceptions.py deleted file mode 100644 index 66eb155a..00000000 --- a/percy/parser/exceptions.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -File: exceptions.py -Description: Provides exceptions thrown by the parser. -""" -from __future__ import annotations - -import json - -from percy.types import JsonPatchType - - -class JsonPatchValidationException(Exception): - """ - Indicates that the calling code has attempted to use an illegal JSON patch payload that does not meet the schema - criteria. - """ - - def __init__(self, patch: JsonPatchType): - """ - Constructs a JSON Patch Validation Exception - :param op: Operation being encountered. - """ - super().__init__(f"Invalid patch was attempted:\n{json.dumps(patch, indent=2)}") diff --git a/percy/parser/recipe_parser.py b/percy/parser/recipe_parser.py deleted file mode 100644 index 087bf6a0..00000000 --- a/percy/parser/recipe_parser.py +++ /dev/null @@ -1,1782 +0,0 @@ -""" -File: recipe_parser.py -Description: Provides a class that takes text from a Jinja-formatted recipe file and parses it. This allows for easy - semantic understanding and manipulation of the file. - - Patching these files is done using a JSON-patch like syntax. This project closely conforms to the - RFC 6902 spec, but deviates in some specific ways to handle the Jinja variables and comments found in - conda recipe files. - - Links: - - https://jsonpatch.com/ - - https://datatracker.ietf.org/doc/html/rfc6902/ -""" -# Allows older versions of python to use newer forms of type annotation. There are major features introduced in >=3.9 -from __future__ import annotations - -import ast -import difflib -import json -import re -import sys -from typing import Callable, Final, Optional, TypeGuard, cast, no_type_check - -import yaml -from jsonschema import validate as schema_validate - -from percy.parser._node import Node -from percy.parser._selector_info import SelectorInfo -from percy.parser._traverse import ( - INVALID_IDX, - remap_child_indices_virt_to_phys, - traverse, - traverse_all, - traverse_with_index, -) -from percy.parser._types import ( - PERCY_SUB_MARKER, - ROOT_NODE_VALUE, - TOP_LEVEL_KEY_SORT_ORDER, - V1_TEST_SECTION_KEY_SORT_ORDER, - ForceIndentDumper, - Regex, - StrStack, -) -from percy.parser._utils import ( - dedupe_and_preserve_order, - normalize_multiline_strings, - num_tab_spaces, - stack_path_to_str, - str_to_stack_path, - stringify_yaml, - substitute_markers, -) -from percy.parser.enums import SelectorConflictMode -from percy.parser.exceptions import JsonPatchValidationException -from percy.parser.types import ( - CURRENT_RECIPE_SCHEMA_FORMAT, - JSON_PATCH_SCHEMA, - TAB_AS_SPACES, - TAB_SPACE_COUNT, - MessageCategory, - MessageTable, - MultilineVariant, -) -from percy.types import PRIMITIVES_TUPLE, JsonPatchType, JsonType, Primitives, SentinelType - - -class RecipeParser: - """ - Class that parses a recipe file string. Provides many useful mechanisms for changing values in the document. - - A quick search for Jinja statements in YAML files shows that the vast majority of statements are in the form of - initializing variables with `set`. - - The next few prevalent kinds of statements are: - - Conditional macros (i.e. if/endif) - - for loops - And even those only show up in a handful out of thousands of recipes. There are also no current examples of Jinja - style comments. - - So that being said, the initial parser will not support these more edge-case recipes as they don't pass the 80/20 - rule. - """ - - # Static set of patch operations that require `from`. The others require `value` or nothing. - _patch_ops_requiring_from = set(["copy", "move"]) - # Sentinel object used for detecting defaulting behavior. - # See here for a good explanation: https://peps.python.org/pep-0661/ - _sentinel = SentinelType() - - @staticmethod - def _parse_yaml(s: str, parser: Optional[RecipeParser] = None) -> JsonType: - """ - Parse a line (or multiple) of YAML into a Pythonic data structure - :param s: String to parse - :param parser: (Optional) If provided, this will substitute Jinja variables with values specified in in the - recipe file. Since `_parse_yaml()` is critical to constructing recipe files, this function must remain - static. Also, during construction, we shouldn't be using a variables until the entire recipe is read/parsed. - :returns: Pythonic data corresponding to the line of YAML - """ - - # Recursive helper function used when we need to perform variable substitutions - def _parse_yaml_recursive_sub(data: JsonType, modifier: Callable[[str], JsonType]) -> JsonType: - # Add the substitutions back in - if isinstance(data, str): - data = modifier(data) - if isinstance(data, dict): - for key in data.keys(): - data[key] = _parse_yaml_recursive_sub(cast(str, data[key]), modifier) - elif isinstance(data, list): - for i in range(len(data)): - data[i] = _parse_yaml_recursive_sub(cast(str, data[i]), modifier) - return data - - output: JsonType = None - try: - output = cast(JsonType, yaml.safe_load(s)) - except Exception: # pylint: disable=broad-exception-caught - # If a construction exception is thrown, attempt to re-parse by replacing Jinja macros (substrings in - # `{{}}`) with friendly string substitution markers, then re-inject the substitutions back in. We classify - # all Jinja substitutions as string values, so we don't have to worry about the type of the actual - # substitution. - sub_list: list[str] = Regex.JINJA_SUB.findall(s) - s = Regex.JINJA_SUB.sub(PERCY_SUB_MARKER, s) - output = _parse_yaml_recursive_sub( - cast(JsonType, yaml.safe_load(s)), lambda d: substitute_markers(d, sub_list) - ) - # Because we leverage PyYaml to parse the data structures, we need to perform a second pass to perform - # variable substitutions. - if parser is not None: - output = _parse_yaml_recursive_sub( - output, parser._render_jinja_vars # pylint: disable=protected-access - ) - return output - - @staticmethod - def _parse_line_node(s: str) -> Node: - """ - Parses a line of conda-formatted YAML into a Node. - - Latest YAML spec can be found here: https://yaml.org/spec/1.2.2/ - - :param s: Pre-stripped (no leading/trailing spaces), non-Jinja line of a recipe file - :returns: A Node representing a line of the conda-formatted YAML. - """ - # Use PyYaml to safely/easily/correctly parse single lines of YAML. - output = RecipeParser._parse_yaml(s) - - # Attempt to parse-out comments. Fully commented lines are not ignored to preserve context when the text is - # rendered. Their order in the list of child nodes will preserve their location. Fully commented lines just have - # a value of "None". - # - # There is an open issue to PyYaml to support comment parsing: - # - https://github.com/yaml/pyyaml/issues/90 - comment = "" - # The full line is a comment - if s.startswith("#"): - return Node(comment=s) - # There is a comment at the end of the line if a `#` symbol is found with leading whitespace before it. If it is - # "touching" a character on the left-side, it is just part of a string. - comment_re_result = Regex.DETECT_TRAILING_COMMENT.search(s) - if comment_re_result is not None: - # Group 0 is the whole match, Group 1 is the leading whitespace, Group 2 locates the `#` - comment = s[comment_re_result.start(2) :] - - # If a dictionary is returned, we have a line containing a key and potentially a value. There should only be 1 - # key/value pairing in 1 line. Nodes representing keys should be flagged for handling edge cases. - if isinstance(output, dict): - children: list[Node] = [] - key = list(output.keys())[0] - # If the value returned is None, there is no leaf node to set - if output[key] is not None: - # As the line is shared by both parent and child, the comment gets tagged to both. - children.append(Node(cast(Primitives, output[key]), comment)) - return Node(key, comment, children, key_flag=True) - # If a list is returned, then this line is a listed member of the parent Node - if isinstance(output, list): - # The full line is a comment - if s.startswith("#"): - # Comments are list members to ensure indentation - return Node(comment=comment, list_member_flag=True) - # Special scenarios that can occur on 1 line: - # 1. Lists can contain lists: - - foo -> [["foo"]] - # 2. Lists can contain keys: - foo: bar -> [{"foo": "bar"}] - # And, of course, there can be n values in each of these collections on 1 line as well. Scenario 2 occurs in - # multi-output recipe files so we need to support the scenario here. - # - # `PKG-3006` tracks an investigation effort into what we need to support for our purposes. - if isinstance(output[0], dict): - # Build up the key-and-potentially-value pair nodes first - key_children: list[Node] = [] - key = list(output[0].keys())[0] - if output[0][key] is not None: - key_children.append(Node(cast(Primitives, output[0][key]), comment)) - key_node = Node(key, comment, key_children, key_flag=True) - - elem_node = Node(comment=comment, list_member_flag=True) - elem_node.children.append(key_node) - return elem_node - return Node(cast(Primitives, output[0]), comment, list_member_flag=True) - # Other types are just leaf nodes. This is scenario should likely not be triggered given our recipe files don't - # have single valid lines of YAML, but we cover this case for the sake of correctness. - return Node(output, comment) - - @staticmethod - def _generate_subtree(value: JsonType) -> list[Node]: - """ - Given a value supported by JSON, use the RecipeParser to generate a list of child nodes. This effectively - creates a new subtree that can be used to patch other parse trees. - """ - # Multiline values can replace the list of children with a single multiline leaf node. - if isinstance(value, str) and "\n" in value: - return [ - Node( - value=value.splitlines(), - # The conversion from JSON-to-YAML is lossy here. Default to the closest equivalent, which preserves - # newlines. - multiline_variant=MultilineVariant.PIPE, - ) - ] - - # For complex types, generate the YAML equivalent and build a new tree. - if not isinstance(value, PRIMITIVES_TUPLE): - # Although not technically required by YAML, we add the optional spacing for human readability. - return RecipeParser( # pylint: disable=protected-access - yaml.dump(value, Dumper=ForceIndentDumper) # type: ignore[misc] - )._root.children - - # Primitives can be safely stringified to generate a parse tree. - return RecipeParser(str(stringify_yaml(value)))._root.children # pylint: disable=protected-access - - def _render_jinja_vars(self, s: str) -> JsonType: - """ - Helper function that replaces Jinja substitutions with their actual set values. - :param s: String to be re-rendered - :returns: The original value, augmented with Jinja substitutions. Types are re-rendered to account for multiline - strings that may have been "normalized" prior to this call. - """ - # Search the string, replacing all substitutions we can recognize - for match in cast(list[str], Regex.JINJA_SUB.findall(s)): - lower_case = False - # The regex guarantees the string starts and ends with double braces - key = match[2:-2].strip() - # A brief search through `aggregate` shows that `|lower` is a commonly used Jinja command. Few, if any, - # other commands are used, as of writing. If others are found, we might need to support them here. - lower_match = Regex.JINJA_FUNCTION_LOWER.search(key) - if lower_match is not None: - lower_case = True - key = key.replace(lower_match.group(), "").strip() - - if key in self._vars_tbl: - # Replace value as a string. Re-interpret the entire value before returning. - value = str(self._vars_tbl[key]) - if lower_case: - value = value.lower() - s = s.replace(match, value) - return cast(JsonType, yaml.safe_load(s)) - - def _rebuild_selectors(self) -> None: - """ - Re-builds the selector look-up table. This table allows quick access to tree nodes that have a selector - specified. This needs to be called when the tree or selectors are modified. - """ - self._selector_tbl: dict[str, list[SelectorInfo]] = {} - - def _collect_selectors(node: Node, path: StrStack) -> None: - # Ignore empty comments - if not node.comment: - return - match = Regex.SELECTOR.search(node.comment) - if match: - selector = match.group(0) - selector_info = SelectorInfo(node, list(path)) - if selector not in self._selector_tbl: - self._selector_tbl[selector] = [selector_info] - else: - self._selector_tbl[selector].append(selector_info) - - traverse_all(self._root, _collect_selectors) - - def __init__(self, content: str): - """ - Constructs a RecipeParser instance. - :param content: conda-build formatted recipe file, as a single text string. - """ - # The initial, raw, text is preserved for diffing and debugging purposes - self._init_content: Final[str] = content - # Indicates if the original content has changed - self._is_modified = False - - # Tracks Jinja variables set by the file - self._vars_tbl: dict[str, JsonType] = {} - # Find all the set statements and record the values - for line in cast(list[str], Regex.JINJA_SET_LINE.findall(self._init_content)): - key = line[line.find("set") + len("set") : line.find("=")].strip() - value = line[line.find("=") + len("=") : line.find("%}")].strip() - try: - self._vars_tbl[key] = ast.literal_eval(value) # type: ignore[misc] - except Exception: # pylint: disable=broad-exception-caught - self._vars_tbl[key] = value - - # Root of the parse tree - self._root = Node(ROOT_NODE_VALUE) - # Start by removing all Jinja lines. Then traverse line-by-line - sanitized_yaml = Regex.JINJA_LINE.sub("", self._init_content) - - # Read the YAML line-by-line, maintaining a stack to manage the last owning node in the tree. - node_stack: list[Node] = [self._root] - # Relative depth is determined by the increase/decrease of indentation marks (spaces) - cur_indent = 0 - last_node = node_stack[-1] - - # Iterate with an index variable, so we can handle multiline values - line_idx = 0 - lines = sanitized_yaml.splitlines() - num_lines = len(lines) - while line_idx < num_lines: - line = lines[line_idx] - # Increment here, so that the inner multiline processing loop doesn't cause a skip of the line following the - # multiline value. - line_idx += 1 - # Ignore empty lines - clean_line = line.strip() - if clean_line == "": - continue - - new_indent = num_tab_spaces(line) - new_node = RecipeParser._parse_line_node(clean_line) - # If the last node ended (pre-comments) with a |, reset the value to be a list of the following, - # extra-indented strings - multiline_re_match = Regex.MULTILINE.match(line) - if multiline_re_match: - # Calculate which multiline symbol is used. The first character must be matched, the second is optional. - variant_capture = cast(str, multiline_re_match.group(Regex.MULTILINE_VARIANT_CAPTURE_GROUP_CHAR)) - variant_sign = cast(str | None, multiline_re_match.group(Regex.MULTILINE_VARIANT_CAPTURE_GROUP_SUFFIX)) - if variant_sign is not None: - variant_capture += variant_sign - # Per YAML spec, multiline statements can't be commented. In other words, the `#` symbol is seen as a - # string character in multiline values. - multiline_node = Node( - [], - multiline_variant=MultilineVariant(variant_capture), - ) - # Type narrow that we assigned `value` as a `list` - assert isinstance(multiline_node.value, list) - multiline = lines[line_idx] - multiline_indent = num_tab_spaces(multiline) - # Add the line to the list once it is verified to be the next line to capture in this node. This means - # that `line_idx` will point to the line of the next node, post-processing. Note that blank lines are - # valid in multi-line strings, occasionally found in `/about/summary` sections. - while multiline_indent > new_indent or multiline == "": - multiline_node.value.append(multiline.strip()) - line_idx += 1 - multiline = lines[line_idx] - multiline_indent = num_tab_spaces(multiline) - # The previous level is the key to this multi-line value, so we can safely reset it. - new_node.children = [multiline_node] - if new_indent > cur_indent: - node_stack.append(last_node) - elif new_indent < cur_indent: - # Multiple levels of depth can change from line to line, so multiple stack nodes must be pop'd. Example: - # foo: - # bar: - # fizz: buzz - # baz: blah - # TODO Figure out tab-depth of the recipe being read. 4 spaces is technically valid in YAML - depth_to_pop = (cur_indent - new_indent) // TAB_SPACE_COUNT - for _ in range(depth_to_pop): - node_stack.pop() - cur_indent = new_indent - # Look at the stack to determine the parent Node and then append the current node to the new parent. - parent = node_stack[-1] - parent.children.append(new_node) - # Update the last node for the next line interpretation - last_node = new_node - - # Now that the tree is built, construct a selector look-up table that tracks all the nodes that use a particular - # selector. This will make it easier to. - # - # This table will have to be re-built or modified when the tree is modified with `patch()`. - self._rebuild_selectors() - - @staticmethod - def _canonical_sort_keys_comparison(n: Node, priority_tbl: dict[str, int]) -> int: - """ - Given a look-up table defining "canonical" sort order, this function provides a way to compare Nodes. - :param n: Node to evaluate - :param priority_tbl: Table that provides a "canonical ordering" of keys - :returns: An integer indicating sort-order priority - """ - # For now, put all comments at the top of the section. Arguably this is better than having them "randomly tag" - # to another top-level key. - if n.is_comment(): - return -sys.maxsize - # Unidentified keys go to the bottom of the section. - if not isinstance(n.value, str) or n.value not in priority_tbl: - return sys.maxsize - return priority_tbl[n.value] - - @staticmethod - def _str_tree_recurse(node: Node, depth: int, lines: list[str]) -> None: - """ - Helper function that renders a parse tree as a text-based dependency tree. Useful for debugging. - :param node: Node of interest - :param depth: Current depth of the node - :param lines: Accumulated list of lines to text to render - """ - spaces = TAB_AS_SPACES * depth - branch = "" if depth == 0 else "|- " - lines.append(f"{spaces}{branch}{node.short_str()}") - for child in node.children: - RecipeParser._str_tree_recurse(child, depth + 1, lines) - - def __str__(self) -> str: - """ - Casts the parser into a string. Useful for debugging. - :returns: String representation of the recipe file - """ - s = "--------------------\n" - tree_lines: list[str] = [] - RecipeParser._str_tree_recurse(self._root, 0, tree_lines) - s += "RecipeParser Instance\n" - s += "- Variables Table:\n" - s += json.dumps(self._vars_tbl, indent=TAB_AS_SPACES) + "\n" - s += "- Selectors Table:\n" - for key, val in self._selector_tbl.items(): - s += f"{TAB_AS_SPACES}{key}\n" - for info in val: - s += f"{TAB_AS_SPACES}{TAB_AS_SPACES}- {info}\n" - s += f"- is_modified?: {self._is_modified}\n" - s += "- Tree:\n" + "\n".join(tree_lines) + "\n" - s += "--------------------\n" - - return s - - def __eq__(self, other: object) -> bool: - """ - Checks if two recipe representations match entirely - :param other: Other recipe parser instance to check against. - :returns: True if both recipes contain the same current state. False otherwise. - """ - if not isinstance(other, RecipeParser): - raise TypeError - return self.render() == other.render() - - def is_modified(self) -> bool: - """ - Indicates if the recipe has been changed since construction. - :returns: True if the recipe has changed. False otherwise. - """ - return self._is_modified - - def has_unsupported_statements(self) -> bool: - """ - Runs a series of checks against the original recipe file. - :returns: True if the recipe has statements we do not currently support. False otherwise. - """ - # TODO complete - raise NotImplementedError - - @staticmethod - def _render_tree(node: Node, depth: int, lines: list[str], parent: Optional[Node] = None) -> None: - """ - Recursive helper function that traverses the parse tree to generate a file. - :param node: Current node in the tree - :param depth: Current depth of the recursion - :param lines: Accumulated list of lines in the recipe file - :param parent: (Optional) Parent node to the current node. Set by recursive calls only. - """ - spaces = TAB_AS_SPACES * depth - - # Edge case: The first element of dictionary in a list has a list `- ` prefix. Subsequent keys in the dictionary - # just have a tab. - is_first_collection_child: Final[bool] = ( - parent is not None and parent.is_collection_element() and node == parent.children[0] - ) - - # Handle same-line printing - if node.is_single_key(): - # Edge case: Handle a list containing 1 member - if node.children[0].list_member_flag: - lines.append(f"{spaces}{node.value}: {node.comment}".rstrip()) - lines.append( - f"{spaces}{TAB_AS_SPACES}- " - f"{stringify_yaml(node.children[0].value, multiline_variant=node.children[0].multiline_variant)} " - f"{node.children[0].comment}".rstrip() - ) - return - - if is_first_collection_child: - lines.append( - f"{TAB_AS_SPACES * (depth-1)}- {node.value}: " - f"{stringify_yaml(node.children[0].value)} " - f"{node.children[0].comment}".rstrip() - ) - return - - # Handle multi-line statements. In theory this will probably only ever be strings, but we'll try to account - # for other types. - # - # By the language spec, # symbols do not indicate comments on multiline strings. - if node.children[0].multiline_variant != MultilineVariant.NONE: - multi_variant: Final[MultilineVariant] = node.children[0].multiline_variant - lines.append(f"{spaces}{node.value}: {multi_variant} {node.comment}".rstrip()) - for val_line in cast(list[str], node.children[0].value): - lines.append( - f"{spaces}{TAB_AS_SPACES}" - f"{stringify_yaml(val_line, multiline_variant=multi_variant)}".rstrip() - ) - return - lines.append( - f"{spaces}{node.value}: " - f"{stringify_yaml(node.children[0].value)} " - f"{node.children[0].comment}".rstrip() - ) - return - - depth_delta = 1 - # Don't render a `:` for the non-visible root node. Also don't render invisible collection nodes. - if depth > -1 and not node.is_collection_element(): - list_prefix = "" - # Handle special cases for the "parent" key - if node.list_member_flag: - list_prefix = "- " - depth_delta += 1 - if is_first_collection_child: - list_prefix = "- " - spaces = spaces[TAB_SPACE_COUNT:] - # Nodes representing collections in a list have nothing to render - lines.append(f"{spaces}{list_prefix}{node.value}: {node.comment}".rstrip()) - - for child in node.children: - # Top-level empty-key edge case: Top level keys should have no additional indentation. - extra_tab = "" if depth < 0 else TAB_AS_SPACES - # Comments in a list are indented to list-level, but do not include a list `-` mark - if child.is_comment(): - lines.append(f"{spaces}{extra_tab}" f"{child.comment}".rstrip()) - # Empty keys can be easily confused for leaf nodes. The difference is these nodes render with a "dangling" - # `:` mark - elif child.is_empty_key(): - lines.append(f"{spaces}{extra_tab}" f"{stringify_yaml(child.value)}: " f"{child.comment}".rstrip()) - # Leaf nodes are rendered as members in a list - elif child.is_leaf(): - lines.append(f"{spaces}{extra_tab}- " f"{stringify_yaml(child.value)} " f"{child.comment}".rstrip()) - else: - RecipeParser._render_tree(child, depth + depth_delta, lines, node) - # By tradition, recipes have a blank line after every top-level section, unless they are a comment. Comments - # should be left where they are. - if depth < 0 and not child.is_comment(): - lines.append("") - - def render(self) -> str: - """ - Takes the current state of the parse tree and returns the recipe file as a string. - :returns: String representation of the recipe file - """ - lines: list[str] = [] - - # Render variable set section - for key, val in self._vars_tbl.items(): - # Double quote strings - if isinstance(val, str): - val = f'"{val}"' - lines.append(f"{{% set {key} = {val} %}}") - # Add spacing if variables have been set - if len(self._vars_tbl): - lines.append("") - - # Render parse-tree, -1 is passed in as the "root-level" is not directly rendered in a YAML file; it is merely - # implied. - RecipeParser._render_tree(self._root, -1, lines) - - return "\n".join(lines) - - @no_type_check - def _render_object_tree(self, node: Node, replace_variables: bool, data: JsonType) -> None: - """ - Recursive helper function that traverses the parse tree to generate a Pythonic data object. - :param node: Current node in the tree - :param replace_variables: If set to True, this replaces all variable substitutions with their set values. - :param data: Accumulated data structure - """ - # Ignore comment-only lines - if node.is_comment(): - return - - key = cast(str, node.value) - for child in node.children: - # Ignore comment-only lines - if child.is_comment(): - continue - - # Handle multiline strings and variable replacement - value = normalize_multiline_strings(child.value, child.multiline_variant) - if isinstance(value, str): - if replace_variables: - value = self._render_jinja_vars(value) - elif child.multiline_variant != MultilineVariant.NONE: - value = cast(str, yaml.safe_load(value)) - - # Empty keys are interpreted to point to `None` - if child.is_empty_key(): - data[key][child.value] = None - continue - - # Collection nodes are skipped as they are placeholders. However, their children are rendered recursively - # and added to a list. - if child.is_collection_element(): - elem_dict = {} - for element in child.children: - self._render_object_tree(element, replace_variables, elem_dict) - if len(data[key]) == 0: - data[key] = [] - data[key].append(elem_dict) - continue - - # List members accumulate values in a list - if child.list_member_flag: - if key not in data: - data[key] = [] - data[key].append(value) - continue - - # Other (non list and non-empty-key) leaf nodes set values directly - if child.is_leaf(): - data[key] = value - continue - - # All other keys prep for containing more dictionaries - data.setdefault(key, {}) - self._render_object_tree(child, replace_variables, data[key]) - - def render_to_object(self, replace_variables: bool = False) -> JsonType: - """ - Takes the underlying state of the parse tree and produces a Pythonic object/dictionary representation. Analogous - to `json.load()`. - :param replace_variables: (Optional) If set to True, this replaces all variable substitutions with their set - values. - :returns: Pythonic data object representation of the recipe. - """ - data: JsonType = {} - # Type narrow after assignment - assert isinstance(data, dict) - - # Bootstrap/flatten the root-level - for child in self._root.children: - if child.is_comment(): - continue - data.setdefault(cast(str, child.value), {}) - self._render_object_tree(child, replace_variables, data) - - return data - - def render_to_new_recipe_format(self) -> tuple[str, MessageTable]: - # pylint: disable=protected-access - """ - Takes the current recipe representation and renders it to the new format WITHOUT modifying the current recipe - state. - - The "new" format is defined in the following CEPs: - - https://github.com/conda-incubator/ceps/blob/main/cep-13.md - - https://github.com/conda-incubator/ceps/blob/main/cep-14.md - - (As of writing there is no official name other than "the new recipe format") - """ - # Approach: In the event that we want to expand support later, this function should be implemented in terms - # of a `RecipeParser` tree. This will make it easier to build an upgrade-path, if we so choose to pursue one. - - msg_tbl = MessageTable() - - # `copy.deepcopy()` produced some bizarre artifacts, namely single-line comments were being incorrectly rendered - # as list members. Although inefficient, we have tests that validate round-tripping the parser and there - # is no development cost in utilizing tools we already must maintain. - new_recipe: RecipeParser = RecipeParser(self.render()) - # Log the original - old_comments: Final[dict[str, str]] = new_recipe.get_comments_table() - - # Convenience wrapper that logs failed patches to the message table - def _patch_and_log(patch: JsonPatchType) -> None: - if not new_recipe.patch(patch): - msg_tbl.add_message(MessageCategory.ERROR, f"Failed to patch: {patch}") - - # Convenience function constructs missing paths. Useful when you have to construct more than 1 path level at - # once (the JSON patch standard only allows the creation of 1 new level at a time) - def _patch_add_missing_path(base_path: str, ext: str, value: JsonType = None) -> None: - temp_path: Final[str] = RecipeParser.append_to_path(base_path, ext) - if new_recipe.contains_value(temp_path): - return - _patch_and_log({"op": "add", "path": temp_path, "value": value}) - - # Convenience function that moves a value under an old path to a new one sharing a common base path BUT only if - # the old path exists. - def _patch_move_base_path(base_path: str, old_ext: str, new_ext: str) -> None: - old_path: Final[str] = RecipeParser.append_to_path(base_path, old_ext) - if not new_recipe.contains_value(old_path): - return - _patch_and_log({"op": "move", "from": old_path, "path": RecipeParser.append_to_path(base_path, new_ext)}) - - # Convenience function that sorts 1 level of keys, given a path. Optionally allows renaming of the target node. - def _sort_subtree_keys(sort_path: str, tbl: dict[str, int], rename: str = "") -> None: - def _comparison(n: Node) -> int: - return RecipeParser._canonical_sort_keys_comparison(n, tbl) - - node = traverse(new_recipe._root, str_to_stack_path(sort_path)) - if node is None: - msg_tbl.add_message(MessageCategory.WARNING, f"Failed to sort members of {sort_path}") - return - if rename: - node.value = rename - node.children.sort(key=_comparison) - - # Convert the JINJA variable table to a `context` section. Empty tables still add the `context` section for - # future developers' convenience. - _patch_and_log({"op": "add", "path": "/context", "value": None}) - # Filter-out any value not covered in the new format - for name, value in new_recipe._vars_tbl.items(): - if not isinstance(value, (str, int, float, bool)): - msg_tbl.add_message(MessageCategory.WARNING, f"The variable `{name}` is an unsupported type.") - continue - _patch_and_log({"op": "add", "path": f"/context/{name}", "value": value}) - - # Similarly, patch-in the new `schema_version` value to the top of the file - _patch_and_log({"op": "add", "path": "/schema_version", "value": CURRENT_RECIPE_SCHEMA_FORMAT}) - - # Swap all JINJA to use the new `${{ }}` format. - jinja_sub_locations: Final[list[str]] = new_recipe.search(Regex.JINJA_SUB) - for path in jinja_sub_locations: - value = new_recipe.get_value(path) - # Values that match the regex should only be strings. This prevents crashes that should not occur. - if not isinstance(value, str): - msg_tbl.add_message( - MessageCategory.WARNING, f"A non-string value was found as a JINJA substitution: {value}" - ) - continue - value = value.replace("{{", "${{") - _patch_and_log({"op": "replace", "path": path, "value": value}) - - # Convert selectors into ternary statements or `if` blocks - for selector, instances in new_recipe._selector_tbl.items(): - for info in instances: - # Selectors can be applied to the parent node if they appear on the same line. We'll ignore these when - # building replacements. - if not info.node.is_leaf(): - continue - - # Strip the []'s around the selector - bool_expression = selector[1:-1] - # Convert to a public-facing path representation - selector_path = stack_path_to_str(info.path) - - # For now, if a selector lands on a boolean value, use a ternary statement. Otherwise use the - # conditional logic. - # TODO `skip` is special and now can be a list of boolean expressions. - patch: JsonPatchType = { - "op": "replace", - "path": selector_path, - "value": "${{ true if " + bool_expression + " }}", - } - if not isinstance(info.node.value, bool): - # TODO: This logic from CEP-13 may be mis-guided - # CEP-13 states that ONLY list members may use the `if/then/else` blocks - if not info.node.list_member_flag: - msg_tbl.add_message( - MessageCategory.WARNING, f"A non-list item had a selector at: {selector_path}" - ) - continue - bool_object = { - "if": bool_expression, - "then": None if isinstance(info.node.value, SentinelType) else info.node.value, - } - patch = { - "op": "replace", - "path": selector_path, - # Hack: Surround the patched value in a list to render as a list member. - # TODO: Figure out if this is a bug in the patch code. - "value": cast(JsonType, [bool_object]), - } - # Apply the patch - _patch_and_log(patch) - new_recipe.remove_selector(selector_path) - - # TODO Complete - # Scan and alert removed fields - # build/ - # about/ - - # Move `run_exports` and `ignore_run_exports` from `build` to `requirements` - # TODO Fix: comments are not preserved with patch operations (add a flag to `patch()`?) - for base_path in new_recipe.get_package_paths(): - # `run_exports` - old_re_path = RecipeParser.append_to_path(base_path, "/build/run_exports") - if new_recipe.contains_value(old_re_path): - requirements_path = RecipeParser.append_to_path(base_path, "/requirements") - new_re_path = RecipeParser.append_to_path(base_path, "/requirements/run_exports") - if not new_recipe.contains_value(requirements_path): - _patch_and_log({"op": "add", "path": requirements_path, "value": None}) - _patch_and_log({"op": "move", "from": old_re_path, "path": new_re_path}) - - # `ignore_run_exports` - old_ire_path = RecipeParser.append_to_path(base_path, "/build/ignore_run_exports") - if new_recipe.contains_value(old_re_path): - requirements_path = RecipeParser.append_to_path(base_path, "/requirements") - new_ire_path = RecipeParser.append_to_path(base_path, "/requirements/ignore_run_exports") - if not new_recipe.contains_value(requirements_path): - _patch_and_log({"op": "add", "path": requirements_path, "value": None}) - _patch_and_log({"op": "move", "from": old_ire_path, "path": new_ire_path}) - - ## `about` section changes and validation ## - # Warn if "required" fields are missing - about_required: Final[list[str]] = [ - "summary", - "description", - "license", - "license_file", - "license_url", - ] - for field in about_required: - path = f"/about/{field}" - if not new_recipe.contains_value(path): - msg_tbl.add_message(MessageCategory.WARNING, f"Required field missing: {path}") - - # Transform renamed fields - about_rename: Final[list[tuple[str, str]]] = [ - ("home", "homepage"), - ("dev_url", "repository"), - ("doc_url", "documentation"), - ] - for old, new in about_rename: - _patch_move_base_path("/about", old, new) - - # TODO validate: /about/license must be SPDX recognized. - - # Remove deprecated `about` fields - about_deprecated: Final[list[str]] = [ - "prelink_message", - "license_family", - "identifiers", - "tags", - "keywords", - "doc_source_url", - ] - for field in about_deprecated: - path = f"/about/{field}" - if new_recipe.contains_value(path): - _patch_and_log({"op": "remove", "path": path}) - - # Cached copy of all of the "outputs" in a recipe. This is useful for easily handling multi and single output - # recipes in 1 loop construct. - base_package_paths: Final[list[str]] = new_recipe.get_package_paths() - - ## Upgrade the testing section(s) ## - test_paths: Final[map[str]] = map( - cast(Callable[[str], str], lambda s: RecipeParser.append_to_path(s, "/test")), base_package_paths - ) - for test_path in test_paths: - if not new_recipe.contains_value(test_path): - continue - - _patch_move_base_path(test_path, "/files", "/files/recipe") - # Edge case: `/source_files` exists but `/files` does not - if new_recipe.contains_value(RecipeParser.append_to_path(test_path, "/source_files")): - _patch_add_missing_path(test_path, "/files") - _patch_move_base_path(test_path, "/source_files", "/files/source") - - if new_recipe.contains_value(RecipeParser.append_to_path(test_path, "/requires")): - _patch_add_missing_path(test_path, "/requirements") - _patch_move_base_path(test_path, "/requires", "/requirements/run") - - # Replace `- pip check` in `commands` with the new flag. If not found, set the flag to `False` (as the - # flag defaults to `True`). - commands = cast(list[str], new_recipe.get_value(RecipeParser.append_to_path(test_path, "/commands"), [])) - pip_check = False - for i, command in enumerate(commands): - if command != "pip check": - continue - # For now, we will only patch-out the first instance when no selector is attached - # TODO Future: handle selector logic/cases with `pip check || ` - _patch_and_log({"op": "remove", "path": RecipeParser.append_to_path(test_path, f"/commands/{i}")}) - pip_check = True - break - _patch_add_missing_path(test_path, "/python") - _patch_and_log( - {"op": "add", "path": RecipeParser.append_to_path(test_path, "/python/pip-check"), "value": pip_check} - ) - - _patch_move_base_path(test_path, "/commands", "/script") - _patch_move_base_path(test_path, "/imports", "/python/imports") - _patch_move_base_path(test_path, "/downstreams", "/downstream") - - # Sort test section for "canonical order" and rename `test` to `tests`. This effectively invalidates - # the `test_path` variable from this point on. - _sort_subtree_keys(test_path, V1_TEST_SECTION_KEY_SORT_ORDER, rename="tests") - - ## Upgrade the multi-output section(s) ## - # TODO Complete - if new_recipe.contains_value("/outputs"): - # On the top-level, `package` -> `recipe` - _patch_move_base_path(ROOT_NODE_VALUE, "/package", "/recipe") - - for output_path in base_package_paths: - if output_path == ROOT_NODE_VALUE: - continue - - # Move `name` and `version` under `package` - if new_recipe.contains_value( - RecipeParser.append_to_path(output_path, "/name") - ) or new_recipe.contains_value(RecipeParser.append_to_path(output_path, "/version")): - _patch_add_missing_path(output_path, "/package") - _patch_move_base_path(output_path, "/name", "/package/name") - _patch_move_base_path(output_path, "/version", "/package/version") - - # Not all the top-level keys are found in each output section, but all the output section keys are - # found at the top-level. So for consistency, we sort on that ordering. - _sort_subtree_keys(output_path, TOP_LEVEL_KEY_SORT_ORDER) - - ## Final clean-up ## - - # TODO: Comment tracking may need improvement. The "correct way" of tracking comments with patch changes is a - # fairly big engineering effort and refactor. - # Alert the user which comments have been dropped. - new_comments: Final[dict[str, str]] = new_recipe.get_comments_table() - diff_comments: Final[dict[str, str]] = {k: v for k, v in old_comments.items() if k not in new_comments} - for path, comment in diff_comments.items(): - if not new_recipe.contains_value(path): - msg_tbl.add_message(MessageCategory.WARNING, f"Could not relocate comment: {comment}") - - # TODO Complete: move operations may result in empty fields we can eliminate. This may require changes to - # `contains_value()` - # TODO Complete: Attempt to combine consecutive If/Then blocks after other modifications. This should reduce the - # risk of screwing up critical list indices and ordering. - - # Hack: Wipe the existing table so the JINJA `set` statements don't render the final form - new_recipe._vars_tbl = {} - - # Sort the top-level keys to a "canonical" ordering. This should make previous patch operations look more - # "sensible" to a human reader. - _sort_subtree_keys("/", TOP_LEVEL_KEY_SORT_ORDER) - - return new_recipe.render(), msg_tbl - - ## YAML Access Functions ## - - def list_value_paths(self) -> list[str]: - """ - Provides a list of all known terminal paths. This can be used by the caller to perform search operations. - :returns: List of all terminal paths in the parse tree. - """ - lst: list[str] = [] - - def _find_paths(node: Node, path_stack: StrStack) -> None: - if node.is_leaf(): - lst.append(stack_path_to_str(path_stack)) - - traverse_all(self._root, _find_paths) - return lst - - def contains_value(self, path: str) -> bool: - """ - Determines if a value (via a path) is contained in this recipe. This also allows the caller to determine if a - path exists. - :param path: JSON patch (RFC 6902)-style path to a value. - :returns: True if the path exists. False otherwise. - """ - path_stack = str_to_stack_path(path) - return traverse(self._root, path_stack) is not None - - def get_value(self, path: str, default: JsonType | SentinelType = _sentinel, sub_vars: bool = False) -> JsonType: - """ - Retrieves a value at a given path. If the value is not found, return a specified default value or throw. - :param path: JSON patch (RFC 6902)-style path to a value. - :param default: (Optional) If the value is not found, return this value instead. - :param sub_vars: (Optional) If set to True and the value contains a Jinja template variable, the Jinja value - will be "rendered". - :raises KeyError: If the value is not found AND no default is specified - :returns: If found, the value in the recipe at that path. Otherwise, the caller-specified default value. - """ - path_stack = str_to_stack_path(path) - node = traverse(self._root, path_stack) - - # Handle if the path was not found - if node is None: - if default == RecipeParser._sentinel or isinstance(default, SentinelType): - raise KeyError(f"No value/key found at path {path!r}") - return default - - return_value: JsonType = None - # Handle unpacking of the last key-value set of nodes. - if node.is_single_key() and not node.is_root(): - # As of writing, Jinja substitutions are not used - if node.children[0].multiline_variant != MultilineVariant.NONE: - multiline_str = cast( - str, - normalize_multiline_strings( - cast(list[str], node.children[0].value), node.children[0].multiline_variant - ), - ) - if sub_vars: - return self._render_jinja_vars(multiline_str) - return cast(JsonType, yaml.safe_load(multiline_str)) - return_value = cast(Primitives, node.children[0].value) - # Leaf nodes can return their value directly - elif node.is_leaf(): - return_value = cast(Primitives, node.value) - else: - # NOTE: Traversing the tree and generating our own data structures will be more efficient than rendering and - # leveraging the YAML parser, BUT this method re-uses code and is easier to maintain. - lst: list[str] = [] - RecipeParser._render_tree(node, -1, lst) - return_value = "\n".join(lst) - - # Collection types are transformed into strings above and will need to be transformed into a proper data type. - # `_parse_yaml()` will also render JINJA variables for us, if requested. - if isinstance(return_value, str): - parser = self if sub_vars else None - parsed_value = RecipeParser._parse_yaml(return_value, parser) - # Lists containing 1 value will drop the surrounding list by the YAML parser. To ensure greater consistency - # and provide better type-safety, we will re-wrap such values. - if len(node.children) == 1 and node.children[0].list_member_flag: - return [parsed_value] - return parsed_value - return return_value - - def find_value(self, value: Primitives) -> list[str]: - """ - Given a value, find all the paths that contain that value. - - NOTE: This only supports searching for "primitive" values, i.e. you cannot search for collections. - - :param value: Value to find in the recipe. - :raises ValueError: If the value provided is not a primitive type. - :returns: List of paths where the value can be found. - """ - if not isinstance(value, PRIMITIVES_TUPLE): - raise ValueError(f"A non-primitive value was provided: {value}") - - paths: list[str] = [] - - def _find_value_paths(node: Node, path_stack: StrStack) -> None: - # Special case: empty keys imply a null value, although they don't contain a null child. - if (value is None and node.is_empty_key()) or (node.is_leaf() and node.value == value): - paths.append(stack_path_to_str(path_stack)) - - traverse_all(self._root, _find_value_paths) - - return paths - - ## General Convenience Functions ## - - def is_multi_output(self) -> bool: - """ - Indicates if a recipe is a "multiple output" recipe. - :returns: True if the recipe produces multiple outputs. False otherwise. - """ - return self.contains_value("/outputs") - - def get_package_paths(self) -> list[str]: - """ - Convenience function that returns the locations of all "outputs" in the `/outputs` directory AND the root/ - top-level of the recipe file. Combined with a call to `get_value()` with a default value and a for loop, this - should easily allow the calling code to handle editing/examining configurations found in: - - "Simple" (non-multi-output) recipe files - - Multi-output recipe files - - Recipes that have both top-level and multi-output sections. An example can be found here: - https://github.com/AnacondaRecipes/curl-feedstock/blob/master/recipe/meta.yaml - """ - paths: list[str] = ["/"] - - outputs: Final[list[str]] = cast(list[str], self.get_value("/outputs", [])) - for i in range(len(outputs)): - paths.append(f"/outputs/{i}") - - return paths - - @staticmethod - def append_to_path(base_path: str, ext_path: str) -> str: - """ - Convenience function meant to be paired with `get_package_paths()` to generate extended paths. This handles - issues that arise when concatenating paths that do or do not include a trailing/leading `/` character. Most - notably, the root path `/` inherently contains a trailing `/`. - :param base_path: Base path, provided by `get_package_paths()` - :param ext_path: Path to append to the end of the `base_path` - :returns: A normalized path constructed by the two provided paths. - """ - # Ensure the base path always ends in a `/` - if not base_path: - base_path = "/" - if base_path[-1] != "/": - base_path += "/" - # Ensure the extended path never starts with a `/` - if ext_path and ext_path[0] == "/": - ext_path = ext_path[1:] - return f"{base_path}{ext_path}" - - def get_dependency_paths(self) -> list[str]: - """ - Convenience function that returns a list of all dependency lines in a recipe. - :returns: A list of all paths in a recipe file that point to dependencies. - """ - paths: list[str] = [] - req_sections: Final[list[str]] = ["build", "host", "run", "run_constrained"] - - # Convenience function that reduces repeated logic between regular and multi-output recipes - def _scan_requirements(path_prefix: str = "") -> None: - for section in req_sections: - section_path = f"{path_prefix}/requirements/{section}" - # Relying on `get_value()` ensures that we will only examine literal values and ignore comments - # in-between dependencies. - dependencies = cast(list[str], self.get_value(section_path, [])) - for i in range(len(dependencies)): - paths.append(f"{section_path}/{i}") - - # Scan for both multi-output and non-multi-output recipes. Here is an example of a recipe that has both: - # https://github.com/AnacondaRecipes/curl-feedstock/blob/master/recipe/meta.yaml - _scan_requirements() - - outputs = cast(list[JsonType], self.get_value("/outputs", [])) - for i in range(len(outputs)): - _scan_requirements(f"/outputs/{i}") - - return paths - - ## Jinja Variable Functions ## - - def list_variables(self) -> list[str]: - """ - Returns variables found in the recipe, sorted by first appearance. - :returns: List of variables found in the recipe. - """ - return list(self._vars_tbl.keys()) - - def contains_variable(self, var: str) -> bool: - """ - Determines if a variable is set in this recipe. - :param var: Variable to check for. - :returns: True if a variable name is found in this recipe. False otherwise. - """ - return var in self._vars_tbl - - def get_variable(self, var: str, default: JsonType | SentinelType = _sentinel) -> JsonType: - """ - Returns the value of a variable set in the recipe. If specified, a default value will be returned if the - variable name is not found. - :param var: Variable of interest check for. - :param default: (Optional) If the value is not found, return this value instead. - :raises KeyError: If the value is not found AND no default is specified - :returns: The value (or specified default value if not found) of the variable name provided. - """ - if var not in self._vars_tbl: - if default == RecipeParser._sentinel or isinstance(default, SentinelType): - raise KeyError - return default - return self._vars_tbl[var] - - def set_variable(self, var: str, value: JsonType) -> None: - """ - Adds or changes an existing Jinja variable. - :param var: Variable to modify - :param value: Value to set - """ - self._vars_tbl[var] = value - self._is_modified = True - - def del_variable(self, var: str) -> None: - """ - Remove a variable from the project. If one is not found, no changes are made. - :param var: Variable to delete - """ - if not var in self._vars_tbl: - return - del self._vars_tbl[var] - self._is_modified = True - - def get_variable_references(self, var: str) -> list[str]: - """ - Returns a list of paths that use particular variables. - :param var: Variable of interest - :returns: List of paths that use a variable, sorted by first appearance. - """ - if var not in self._vars_tbl: - return [] - - path_list: list[str] = [] - # The text between the braces is very forgiving. Just searching for whitespace characters means we will never - # match the very common `{{ name | lower }}` expression, or similar piping functions. - var_re = re.compile(r"{{.*" + var + r".*}}") - - def _collect_var_refs(node: Node, path: StrStack) -> None: - # Variables can only be found inside string values. - if isinstance(node.value, str) and var_re.search(node.value): - path_list.append(stack_path_to_str(path)) - - traverse_all(self._root, _collect_var_refs) - return dedupe_and_preserve_order(path_list) - - ## Selector Functions ## - - def list_selectors(self) -> list[str]: - """ - Returns selectors found in the recipe, sorted by first appearance. - :returns: List of selectors found in the recipe. - """ - return list(self._selector_tbl.keys()) - - def contains_selector(self, selector: str) -> bool: - """ - Determines if a selector expression is present in this recipe. - :param selector: Selector to check for. - :returns: True if a selector is found in this recipe. False otherwise. - """ - return selector in self._selector_tbl - - def get_selector_paths(self, selector: str) -> list[str]: - """ - Given a selector (including the surrounding brackets), provide a list of paths in the parse tree that use that - selector. - - Selector paths will be ordered by the line they appear on in the file. - - :param selector: Selector of interest. - :returns: A list of all known paths that use a particular selector - """ - # We return a tuple so that caller doesn't accidentally modify a private member variable. - if not self.contains_selector(selector): - return [] - path_list: list[str] = [] - for path_stack in self._selector_tbl[selector]: - path_list.append(stack_path_to_str(path_stack.path)) - # The list should be de-duped and maintain order. Duplications occur when key-value pairings mean a selector - # occurs on two nodes with the same path. - # - # For example: - # skip: True # [unix] - # The nodes for both `skip` and `True` contain the comment `[unix]` - return dedupe_and_preserve_order(path_list) - - def contains_selector_at_path(self, path: str) -> bool: - """ - Given a path, determine if a selector exists on that line. - :param path: Target path - :returns: True if the selector exists at that path. False otherwise. - """ - path_stack = str_to_stack_path(path) - node = traverse(self._root, path_stack) - if node is None: - return False - return bool(Regex.SELECTOR.search(node.comment)) - - def get_selector_at_path(self, path: str, default: str | SentinelType = _sentinel) -> str: - """ - Given a path, return the selector that exists on that line. - :param path: Target path - :param default: (Optional) Default value to use if no selector is found. - :raises KeyError: If a selector is not found on the provided path AND no default has been specified. - :raises ValueError: If the default selector provided is malformed - :returns: Selector on the path provided - """ - path_stack = str_to_stack_path(path) - node = traverse(self._root, path_stack) - if node is None: - raise KeyError(f"Path not found: {path}") - - search_results = Regex.SELECTOR.search(node.comment) - if not search_results: - # Use `default` case - if default != RecipeParser._sentinel and not isinstance(default, SentinelType): - if not Regex.SELECTOR.match(default): - raise ValueError(f"Invalid selector provided: {default}") - return default - raise KeyError(f"Selector not found at path: {path}") - return search_results.group(0) - - def add_selector(self, path: str, selector: str, mode: SelectorConflictMode = SelectorConflictMode.REPLACE) -> None: - """ - Given a path, add a selector (include the surrounding brackets) to the line denoted by path. - :param path: Path to add a selector to - :param selector: Selector statement to add - :param mode: (Optional) Indicates how to handle a conflict if a selector already exists at this path. - :raises KeyError: If the path provided is not found - :raises ValueError: If the selector provided is malformed - """ - path_stack = str_to_stack_path(path) - node = traverse(self._root, path_stack) - - if node is None: - raise KeyError(f"Path not found: {path!r}") - if not Regex.SELECTOR.match(selector): - raise ValueError(f"Invalid selector provided: {selector}") - - # Helper function that extracts the outer set of []'s in a selector - def _extract_selector(s: str) -> str: - return s.replace("[", "", 1)[::-1].replace("]", "", 1)[::-1] - - comment = "" - old_selector_found = Regex.SELECTOR.search(node.comment) - if node.comment == "" or mode == SelectorConflictMode.REPLACE: - comment = f"# {selector}" - # "Append" to existing selectors - elif old_selector_found: - logic_op = "and" if mode == SelectorConflictMode.AND else "or" - old_selector = _extract_selector(old_selector_found.group()) - new_selector = _extract_selector(selector) - comment = f"# [{old_selector} {logic_op} {new_selector}]" - # If the comment is not a selector, put the selector first, then append the comment. - else: - # Strip the existing comment of it's leading `#` symbol - comment = f"# {selector} {node.comment.replace('#', '', 1).strip()}" - - node.comment = comment - # Some lines of YAML correspond to multiple nodes. For consistency, we need to ensure that comments are - # duplicate across all nodes on a line. - if node.is_single_key(): - node.children[0].comment = comment - - self._rebuild_selectors() - self._is_modified = True - - def remove_selector(self, path: str) -> Optional[str]: - """ - Given a path, remove a selector to the line denoted by path. - - If a selector does not exist, nothing happens. - - If a comment exists after the selector, keep it, discard the selector. - :param path: Path to add a selector to - :raises KeyError: If the path provided is not found - :returns: If found, the selector removed (includes surrounding brackets). Otherwise, returns None - """ - path_stack = str_to_stack_path(path) - node = traverse(self._root, path_stack) - - if node is None: - raise KeyError(f"Path not found: {path!r}") - - search_results = Regex.SELECTOR.search(node.comment) - if not search_results: - return None - - selector = search_results.group(0) - comment = node.comment.replace(selector, "") - # Sanitize potential edge-case scenarios after a removal - comment = comment.replace("# ", "# ").replace("# # ", "# ") - # Detect and remove empty comments. Other comments should remain intact. - if comment.strip() == "#": - comment = "" - - node.comment = comment - # Some lines of YAML correspond to multiple nodes. For consistency, we need to ensure that comments are - # duplicate across all nodes on a line. - if node.is_single_key(): - node.children[0].comment = comment - - self._rebuild_selectors() - self._is_modified = True - return selector - - ## Comment Functions ## - - def get_comments_table(self) -> dict[str, str]: - """ - Returns a dictionary containing the location of every comment mapped to the value of the comment. - NOTE: - - Selectors are not considered to be comments. - - Lines containing only comments are currently not addressable by our pathing scheme, so they are omitted. - For our current purposes (of upgrading the recipe format) this should be fine. Non-addressable values - should be less likely to be removed from patch operations. - :returns: List of paths where comments can be found. - """ - comments_tbl: dict[str, str] = {} - - def _track_comments(node: Node, path_stack: StrStack) -> None: - if node.is_comment() or node.comment == "": - return - comment = node.comment - # Handle comments found alongside a selector - if Regex.SELECTOR.search(comment): - comment = Regex.SELECTOR.sub("", comment).strip() - # Sanitize common artifacts left from removing the selector - comment = comment.replace("# # ", "# ", 1).replace("# ", "# ", 1) - - # Reject selector-only comments - if comment in {"", "#"}: - return - if comment[0] != "#": - comment = f"# {comment}" - - path = stack_path_to_str(path_stack) - comments_tbl[path] = comment - - traverse_all(self._root, _track_comments) - return comments_tbl - - def add_comment(self, path: str, comment: str) -> None: - """ - Adds a comment to an existing path. If a comment exists, replaces the existing comment. If a selector exists, - comment is appended after the selector component of the comment. - :param path: Target path to add a comment to - :param comment: Comment to add - :raises KeyError: If the path provided is not found - :raises ValueError: If the comment provided is a selector, the empty string, or consists of only whitespace - characters - """ - comment = comment.strip() - if comment == "": - raise ValueError("Comments cannot consist only of whitespace characters") - - if Regex.SELECTOR.match(comment): - raise ValueError(f"Selectors can not be submitted as comments: {comment}") - - node = traverse(self._root, str_to_stack_path(path)) - - if node is None: - raise KeyError(f"Path not found: {path}") - - search_results = Regex.SELECTOR.search(node.comment) - # If a selector is present, append the selector. - if search_results: - selector = search_results.group(0) - if comment[0] == "#": - comment = comment[1:].strip() - comment = f"# {selector} {comment}" - - # Prepend a `#` if it is missing - if comment[0] != "#": - comment = f"# {comment}" - node.comment = comment - # Comments for "single key" nodes apply to both the parent and child. This is because such parent nodes render - # on the same line as their children. - if node.is_single_key(): - node.children[0].comment = comment - self._is_modified = True - - ## YAML Patching Functions ## - - @staticmethod - def _is_valid_patch_node(node: Optional[Node], node_idx: int) -> TypeGuard[Node]: - """ - Indicates if the target node to perform a patch operation against is a valid node. This is based on the RFC spec - for JSON patching paths. - :param node: Target node to validate - :param node_idx: If the caller is evaluating that a list member, exists, this is the VIRTUAL index into that - list. Otherwise this value should be less than 0. - :returns: True if the node can be patched. False otherwise. - """ - # Path not found - if node is None: - return False - - # Leaf nodes contain values and not path information. Paths should not be made that access leaf nodes, with the - # exception of members of a list and keys. Making such a path violates the RFC. - if not node.list_member_flag and not node.key_flag and node.is_leaf(): - return False - - if node_idx >= 0: - # Check the bounds if the target requires the use of an index, remembering to use the virtual look-up table. - idx_map = remap_child_indices_virt_to_phys(node.children) - if node_idx < 0 or node_idx > (len(idx_map) - 1): - return False - # You cannot use the list access feature to access non-lists - if len(node.children) and not node.children[idx_map[node_idx]].list_member_flag: - return False - - return True - - def _patch_add_find_target(self, path_stack: StrStack) -> tuple[Optional[Node], int, int, str, bool]: - """ - Finds the target node of an `add()` operation, along with some supporting information. - - This function does not modify the parse tree. - :param path_stack: Path that describes a location in the tree, as a list, treated like a stack. - :returns: A tuple containing: - The target node, if found (or the parent node if the target is a list member) - - The index of a node if the target is a list member - An additional path that needs to be created, if - applicable - A flag indicating if the new data will be appended to a list - """ - if len(path_stack) == 0: - return None, INVALID_IDX, INVALID_IDX, "", False - - # Special case that only applies to `add`. The `-` character indicates the new element can be added to the end - # of the list. - append_to_list = False - if path_stack[0] == "-": - path_stack.pop(0) - append_to_list = True - - path_stack_copy = path_stack.copy() - node, virt_idx, phys_idx = traverse_with_index(self._root, path_stack) - # Attempt to run a second time, if no node is found. As per the RFC, the containing object/list must exist. That - # allows us to create only 1 level in the path. - path_to_create = "" - if node is None: - path_to_create = path_stack_copy.pop(0) - node, virt_idx, phys_idx = traverse_with_index(self._root, path_stack_copy) - - return node, virt_idx, phys_idx, path_to_create, append_to_list - - def _patch_add(self, path_stack: StrStack, value: JsonType) -> bool: - """ - Performs a JSON patch `add` operation. - :param path_stack: Path that describes a location in the tree, as a list, treated like a stack. - :param value: Value to add. - :returns: True if the operation was successful. False otherwise. - """ - # NOTE from the RFC: - # Because this operation is designed to add to existing objects and arrays, its target location will often - # not exist...However, the object itself or an array containing it does need to exist - # In other words, the patch op will, at most, create 1 new path level. In addition, that also implies that - # trying to append to an existing list only applies if the append operator is at the end of the list. - node, virt_idx, phys_idx, path_to_create, append_to_list = self._patch_add_find_target(path_stack) - - if not RecipeParser._is_valid_patch_node(node, virt_idx): - return False - - # If we couldn't find 1 level in the path, ensure that we re-insert that as the "root" of the sub-tree we are - # about to create. - if path_to_create: - value = {path_to_create: value} - - new_children: Final[list[Node]] = RecipeParser._generate_subtree(value) - # Mark children as list members if they are list members - if append_to_list or phys_idx >= 0: - for child in new_children: - child.list_member_flag = True - - # Insert members if an index is specified. Otherwise, extend the list of child nodes from the existing list. - if phys_idx >= 0: - node.children[phys_idx:phys_idx] = new_children - # Extend the list of children if we're appending or adding a new key. - elif append_to_list or path_to_create: - node.children.extend(new_children) - # NOTE from the RFC: "If the member already exists, it is replaced by the specified value." - else: - node.children = new_children - - return True - - def _patch_remove(self, path_stack: StrStack) -> bool: - """ - Performs a JSON patch `remove` operation. - :param path_stack: Path that describes a location in the tree, as a list, treated like a stack. - :returns: True if the operation was successful. False otherwise. - """ - if len(path_stack) == 0: - return False - - # Removal in all scenarios requires targeting the parent node. - node_idx = -1 if not path_stack[0].isdigit() else int(path_stack[0]) - # `traverse()` is destructive to the stack, so make a copy for the second traversal call. - path_stack_copy = path_stack.copy() - node_to_rm = traverse(self._root, path_stack) - if not RecipeParser._is_valid_patch_node(node_to_rm, -1): - return False - - path_stack_copy.pop(0) - node = traverse(self._root, path_stack_copy) - if not RecipeParser._is_valid_patch_node(node, node_idx): - return False - - if node_idx >= 0: - # Pop the "physical" index, not the "virtual" one to ensure comments have been accounted for. - node.children.pop(remap_child_indices_virt_to_phys(node.children)[node_idx]) - return True - - # In all other cases, the node to be removed must be found before eviction - for i in range(len(node.children)): - if node.children[i] == node_to_rm: - node.children.pop(i) - return True - return False - - def _patch_replace(self, path_stack: StrStack, value: JsonType) -> bool: - """ - Performs a JSON patch `replace` operation. - :param path_stack: Path that describes a location in the tree, as a list, treated like a stack. - :param value: Value to update with. - :returns: True if the operation was successful. False otherwise. - """ - node, virt_idx, phys_idx = traverse_with_index(self._root, path_stack) - if not RecipeParser._is_valid_patch_node(node, virt_idx): - return False - - new_children: Final[list[Node]] = RecipeParser._generate_subtree(value) - # Lists inject all children at the target position. - if phys_idx >= 0: - # Ensure all children are marked as list members - for child in new_children: - child.list_member_flag = True - node.children[phys_idx:phys_idx] = new_children - # Evict the old child, which is now behind the new children - node.children.pop(phys_idx + len(new_children)) - return True - - # Leafs that represent values/paths of values can evict all children, and be replaced with new children, derived - # from a new tree of values. - node.children = new_children - return True - - def _patch_move(self, path_stack: StrStack, value_from: str) -> bool: - """ - Performs a JSON patch `add` operation. - :param path_stack: Path that describes a location in the tree, as a list, treated like a stack. - :param value_from: The "from" value in the JSON payload, i.e. the path the value originates from. - :returns: True if the operation was successful. False otherwise. - """ - # NOTE from the RFC: - # This operation is functionally identical to a "remove" operation on the "from" location, followed - # immediately by an "add" operation at the target location with the value that was just removed. - # So to save on development and maintenance, that is how this op is written. - original_value: JsonType - try: - original_value = self.get_value(value_from) - except KeyError: - return False - - # Validate that `add`` will succeed before we `remove` anything - node, virt_idx, _, _, _ = self._patch_add_find_target(path_stack.copy()) - if not RecipeParser._is_valid_patch_node(node, virt_idx): - return False - - return self._patch_remove(str_to_stack_path(value_from)) and self._patch_add(path_stack, original_value) - - def _patch_copy(self, path_stack: StrStack, value_from: str) -> bool: - """ - Performs a JSON patch `add` operation. - :param path_stack: Path that describes a location in the tree, as a list, treated like a stack. - :param value_from: The "from" value in the JSON payload, i.e. the path the value originates from. - :returns: True if the operation was successful. False otherwise. - """ - # NOTE from the RFC: - # This operation is functionally identical to an "add" operation at the target location using the value - # specified in the "from" member. - # So to save on development and maintenance, that is how this op is written. - original_value: JsonType - try: - original_value = self.get_value(value_from) - except KeyError: - return False - - return self._patch_add(path_stack, original_value) - - def _patch_test(self, path: str, value: JsonType) -> bool: - """ - Performs a JSON patch `test` operation. - :param path: Path as a string. Useful for invoking public class members. - :param value: Value to evaluate against. - :returns: True if the target value is equal to the provided value. False otherwise. - """ - try: - return self.get_value(path) == value - except KeyError: - # Path not found - return False - - def _call_patch_op(self, op: str, path: str, patch: JsonPatchType) -> bool: - """ - Switching function that calls the appropriate JSON patch operation. - :param op: Patch operation, pre-sanitized. - :param path: Path as a string. - :param patch: The original JSON patch. This is passed to conditionally provide extra arguments, per op. - :returns: True if the patch was successful. False otherwise. - """ - path_stack: Final[StrStack] = str_to_stack_path(path) - # NOTE: The `remove` op has no `value` or `from` field to pass in, so it is executed first. - if op == "remove": - return self._patch_remove(path_stack) - - # The supplemental field name is determined by the operation type. - value_from: Final[str] = "from" if op in RecipeParser._patch_ops_requiring_from else "value" - patch_data: Final[JsonType | str] = patch[value_from] - - if op == "add": - return self._patch_add(path_stack, patch_data) - if op == "replace": - return self._patch_replace(path_stack, patch_data) - if op == "move": - return self._patch_move(path_stack, cast(str, patch_data)) - if op == "copy": - return self._patch_copy(path_stack, cast(str, patch_data)) - if op == "test": - return self._patch_test(path, patch_data) - - # This should be unreachable but is kept for completeness. - return False - - def patch(self, patch: JsonPatchType) -> bool: - """ - Given a JSON-patch object, perform a patch operation. - - Modifications from RFC 6902 - - We're using a Jinja-formatted YAML file, not JSON - - To modify comments, specify the `path` AND `comment` - - :param patch: JSON-patch payload to operate with. - :raises JsonPatchValidationException: If the JSON-patch payload does not conform to our schema/spec. - :returns: If the calling code attempts to perform the `test` operation, this indicates the return value of the - `test` request. In other words, if `value` matches the target variable, return True. False otherwise. For - all other operations, this indicates if the operation was successful. - """ - # Validate the patch schema - try: - schema_validate(patch, JSON_PATCH_SCHEMA) - except Exception as e: - raise JsonPatchValidationException(patch) from e - - path: Final[str] = cast(str, patch["path"]) - - # All RFC ops are supported, so the JSON schema validation checks will prevent us from getting this far, if - # there is an issue. - op: Final[str] = cast(str, patch["op"]) - - # A no-op move is silly, but we might as well make it efficient AND ensure a no-op move doesn't corrupt our - # modification flag. - if op == "move" and path == patch["from"]: - return True - - # Both versions of the path are sent over so that the op can easily use both private and public functions - # (without incurring even more conversions between path types). - is_successful = self._call_patch_op(op, path, patch) - - # Update the selector table and modified flag, if the operation succeeded. - if is_successful and op != "test": - # TODO this is not the most efficient way to update the selector table, but for now, it works. - self._rebuild_selectors() - # TODO technically this doesn't handle a no-op. - self._is_modified = True - - return is_successful - - def search(self, regex: str | re.Pattern[str], include_comment: bool = False) -> list[str]: - """ - Given a regex string, return the list of paths that match the regex. - NOTE: This function only searches against primitive values. All variables and selectors can be fully provided by - using their respective `list_*()` functions. - - :param regex: Regular expression to match with - :param include_comment: (Optional) If set to `True`, this function will execute the regular expression on values - WITH their comments provided. For example: `42 # This is a comment` - :returns: Returns a list of paths where the matched value was found. - """ - re_obj = re.compile(regex) - paths: list[str] = [] - - def _search_paths(node: Node, path_stack: StrStack) -> None: - value = str(stringify_yaml(node.value)) - if include_comment and node.comment: - value = f"{value}{TAB_AS_SPACES}{node.comment}" - if node.is_leaf() and re_obj.search(value): - paths.append(stack_path_to_str(path_stack)) - - traverse_all(self._root, _search_paths) - - return paths - - def search_and_patch( - self, regex: str | re.Pattern[str], patch: JsonPatchType, include_comment: bool = False - ) -> bool: - """ - Given a regex string and a JSON patch, apply the patch to any values that match the search expression. - :param regex: Regular expression to match with - :param patch: JSON patch to perform. NOTE: The `path` field will be replaced with the path(s) found, so it does - not need to be provided. - :param include_comment: (Optional) If set to `True`, this function will execute the regular expression on values - WITH their comments provided. For example: `42 # This is a comment` - :returns: Returns a list of paths where the matched value was found. - """ - paths = self.search(regex, include_comment) - summation: bool = True - for path in paths: - patch["path"] = path - summation = summation and self.patch(patch) - return summation - - def diff(self) -> str: - """ - Returns a git-like-styled diff of the current recipe state with original state of the recipe. Useful for - debugging and providing users with some feedback. - :returns: User-friendly displayable string that represents notifications made to the recipe. - """ - if not self.is_modified(): - return "" - # Utilize `difflib` to lower maintenance overhead. - return "\n".join( - difflib.unified_diff( - self._init_content.splitlines(), self.render().splitlines(), fromfile="original", tofile="current" - ) - ) diff --git a/percy/parser/types.py b/percy/parser/types.py deleted file mode 100644 index 650d5031..00000000 --- a/percy/parser/types.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -File: types.py -Description: Provides public types, type aliases, constants, and small classes used by the parser. -""" -from __future__ import annotations - -from enum import StrEnum, auto -from typing import Final - -from percy.types import Primitives, SchemaType - -#### Types #### - -# Nodes can store a single value or a list of strings (for multiline-string nodes) -NodeValue = Primitives | list[str] - - -#### Constants #### - -# The "new" recipe format introduces the concept of a schema version. Presumably the "old" recipe format would be -# considered "0". When converting to the new format, we'll use this constant value. -CURRENT_RECIPE_SCHEMA_FORMAT: Final[int] = 1 - -# Indicates how many spaces are in a level of indentation -TAB_SPACE_COUNT: Final[int] = 2 -TAB_AS_SPACES: Final[str] = " " * TAB_SPACE_COUNT - -# Schema validator for JSON patching -JSON_PATCH_SCHEMA: Final[SchemaType] = { - "type": "object", - "properties": { - "op": {"enum": ["add", "remove", "replace", "move", "copy", "test"]}, - "path": {"type": "string", "minLength": 1}, - "from": {"type": "string"}, - "value": { - "type": [ - "string", - "number", - "object", - "array", - "boolean", - "null", - ], - "items": { - "type": [ - "string", - "number", - "object", - "array", - "boolean", - "null", - ] - }, - }, - }, - "required": [ - "op", - "path", - ], - "allOf": [ - # `value` is required for `add`/`replace`/`test` - { - "if": { - "properties": {"op": {"const": "add"}}, - }, - "then": {"required": ["value"]}, - }, - { - "if": { - "properties": {"op": {"const": "replace"}}, - }, - "then": {"required": ["value"]}, - }, - { - "if": { - "properties": {"op": {"const": "test"}}, - }, - "then": {"required": ["value"]}, - }, - # `from` is required for `move`/`copy` - { - "if": { - "properties": {"op": {"const": "move"}}, - }, - "then": {"required": ["from"]}, - }, - { - "if": { - "properties": {"op": {"const": "copy"}}, - }, - "then": {"required": ["from"]}, - }, - ], - "additionalProperties": False, -} - - -class MultilineVariant(StrEnum): - """ - Captures which "multiline" descriptor was used on a Node, if one was used at all. - - See this guide for details on the YAML spec: - https://stackoverflow.com/questions/3790454/how-do-i-break-a-string-in-yaml-over-multiple-lines/21699210 - """ - - NONE = "" - PIPE = "|" - PIPE_PLUS = "|+" - PIPE_MINUS = "|-" - CARROT = ">" - CARROT_PLUS = ">+" - CARROT_MINUS = ">-" - - -class MessageCategory(StrEnum): - """ - Categories to classify `RecipeParser` messages into. - """ - - ERROR = auto() - WARNING = auto() - - -class MessageTable: - """ - Stores and tags messages that may come up during `RecipeParser` operations. It is up to the client program to - handle the logging of these messages. - """ - - def __init__(self) -> None: - """ - Constructs an empty message table - """ - self._tbl: dict[MessageCategory, list[str]] = {} - - def add_message(self, category: MessageCategory, message: str) -> None: - """ - Adds a message to the table - :param category: - :param message: - """ - if category not in self._tbl: - self._tbl[category] = [] - self._tbl[category].append(message) - - def get_messages(self, category: MessageCategory) -> list[str]: - """ - Returns all the messages stored in a given category - :param category: Category to target - :returns: A list containing all the messages stored in a category. - """ - if category not in self._tbl: - return [] - return self._tbl[category] - - def get_message_count(self, category: MessageCategory) -> int: - """ - Returns how many messages are stored in a given category - :param category: Category to target - :returns: A list containing all the messages stored in a category. - """ - if category not in self._tbl: - return 0 - return len(self._tbl[category]) - - def get_totals_message(self) -> str: - """ - Convenience function that returns a displayable count of the number of warnings and errors contained in the - messaging object. - :returns: A message indicating the number of errors and warnings that have been accumulated. If there are none, - an empty string is returned. - """ - if not self._tbl: - return "" - - def _pluralize(n: int, s: str) -> str: - if n == 1: - return s - return f"{s}s" - - num_errors: Final[int] = 0 if MessageCategory.ERROR not in self._tbl else len(self._tbl[MessageCategory.ERROR]) - errors: Final[str] = f"{num_errors} {_pluralize(num_errors, 'error')}" - num_warnings: Final[int] = ( - 0 if MessageCategory.WARNING not in self._tbl else len(self._tbl[MessageCategory.WARNING]) - ) - warnings: Final[str] = f"{num_warnings} {_pluralize(num_warnings, 'warning')}" - - return f"{errors} and {warnings} were found." diff --git a/percy/render/_renderer.py b/percy/render/_renderer.py index 6c06dc98..f004cea9 100644 --- a/percy/render/_renderer.py +++ b/percy/render/_renderer.py @@ -2,6 +2,7 @@ File: _renderer.py Description: Provides tools for rendering recipe files. """ + from __future__ import annotations import contextlib @@ -12,8 +13,8 @@ import jinja2 import yaml +from conda_recipe_manager.parser.recipe_parser_deps import RecipeParserDeps -from percy.parser.recipe_parser import RecipeParser from percy.render.exceptions import JinjaRenderFailure, YAMLRenderFailure from percy.render.types import SelectorDict @@ -39,7 +40,8 @@ class RendererType(Enum): PYYAML = 1 RUAMEL = 2 CONDA = 3 - PERCY = 4 # Custom parse-tree-based renderer, found in `recipe_parser.py` + CRM = 4 # conda-recipe-manager + JINJA = 5 if has_ruamel: @@ -263,9 +265,9 @@ def expand_compiler(lang): variants=selector_dict, ) return rendered[0][0].meta - elif renderer_type == RendererType.PERCY: - parser = RecipeParser(meta_yaml) - return parser.render_to_object() + elif renderer_type == RendererType.CRM: + rendered = RecipeParserDeps(meta_yaml) + return rendered else: raise YAMLRenderFailure(recipe_dir, message="Unknown renderer type.") except ParserError as exc: diff --git a/percy/render/aggregate.py b/percy/render/aggregate.py index c302b077..15f1a692 100644 --- a/percy/render/aggregate.py +++ b/percy/render/aggregate.py @@ -3,6 +3,7 @@ Description: A representation of aggregate. May be used to get a rough build order of packages or gather health information. """ + from __future__ import annotations import configparser @@ -265,7 +266,7 @@ def _get_feedstock_git_repo(self, feedstock_path_rel: Path) -> Feedstock: def load_local_feedstocks( self, subdir: str = "linux-64", - python: str = "3.10", + python: str = "3.12", others: Optional[dict[str, Any]] = None, renderer: Optional[RendererType] = None, ) -> dict[str, Package]: @@ -277,7 +278,7 @@ def load_local_feedstocks( This populates attributes packages and feedstocks. :param subdir: The subdir for which to load the feedstocks. Defaults to "linux-64". - :param python: The python version for which to load the feedstocks. Defaults to "3.10". + :param python: The python version for which to load the feedstocks. Defaults to "3.12". :param others: A variant dictionary. E.g. {"blas_impl" : "openblas"} Defaults to None. :param renderer: Rendering engine to use to interpret YAML diff --git a/percy/render/dumper.py b/percy/render/dumper.py index f19c0290..9f8269b1 100644 --- a/percy/render/dumper.py +++ b/percy/render/dumper.py @@ -2,6 +2,7 @@ File: dumper.py Description: Provides utilities to dump rendering results """ + from __future__ import annotations import sys @@ -107,5 +108,8 @@ def dump_render_results(render_results: list, out: TextIO = sys.stdout) -> None: """ if render_results and render_results[0].renderer == RendererType.RUAMEL: _dump_render_results_ruamel(render_results, out) + elif render_results and render_results[0].renderer == RendererType.CRM: + for render_result in render_results: + print(render_result.crm.render()) else: _dump_render_results_yaml(render_results, out) diff --git a/percy/render/exceptions.py b/percy/render/exceptions.py index b52ede40..b0203597 100644 --- a/percy/render/exceptions.py +++ b/percy/render/exceptions.py @@ -2,6 +2,7 @@ File: exceptions.py Description: Exceptions emitted by the rendering engine """ + from __future__ import annotations from pathlib import Path diff --git a/percy/render/recipe.py b/percy/render/recipe.py index 845de001..ca4a1b5f 100644 --- a/percy/render/recipe.py +++ b/percy/render/recipe.py @@ -2,6 +2,7 @@ File: recipe.py Description: Recipe renderer. Not as accurate as conda-build render, but faster and architecture independent. """ + from __future__ import annotations import itertools @@ -15,9 +16,10 @@ from urllib.parse import urlparse import jsonschema +from conda_recipe_manager.parser.recipe_parser_deps import RecipeParserDeps +from conda_recipe_manager.types import JsonPatchType import percy.render._renderer as renderer_utils -from percy.parser.recipe_parser import JsonPatchType, RecipeParser from percy.render.exceptions import EmptyRecipe, MissingMetaYaml from percy.render.types import SelectorDict from percy.render.variants import Variant, read_conda_build_config @@ -39,7 +41,7 @@ class OpMode(Enum): """ CLASSIC = 1 # Original operational mode - PARSE_TREE = 2 # Operational mode that uses the work in `recipe_parser.py` + PARSE_TREE = 2 # Operational mode that uses conda-recipe-manager class Recipe: @@ -287,7 +289,16 @@ def render(self) -> None: self.packages: dict[str, Package] = {} # render meta.yaml - self.meta = renderer_utils.render(self.recipe_dir, self.dump(), self.selector_dict, self.renderer) + self.crm = None + try: + self.crm = RecipeParserDeps(self.dump()) + except: + logging.warning("Failed to render using RecipeParserDeps") + self.crm = None + if self.renderer == renderer_utils.RendererType.CRM: + self.meta = self.crm.render_to_object() + else: + self.meta = renderer_utils.render(self.recipe_dir, self.dump(), self.selector_dict, self.renderer) # should this be skipped? bld = self.meta.get("build", {}) @@ -622,7 +633,7 @@ def contains(self, path: str, value: str, default: Any = KeyError) -> bool: return True return False - def patch_with_parser(self, callback: Callable[[RecipeParser], None]) -> bool: + def patch_with_parser(self, callback: Callable[[RecipeParserDeps], None]) -> bool: """ By providing a callback, this function allows calling code to utilize the new parse-tree/percy recipe parser in a way that is backwards compatible with the `Recipe` class. @@ -639,7 +650,7 @@ def patch_with_parser(self, callback: Callable[[RecipeParser], None]) -> bool: """ # Read in the file as a string. Remembering that `recipe` stores # data as a list. - parser = RecipeParser("\n".join(self.meta_yaml)) + parser = RecipeParserDeps("\n".join(self.meta_yaml)) # Execute the callback to perform actions against the parser. callback(parser) @@ -655,25 +666,6 @@ def patch_with_parser(self, callback: Callable[[RecipeParser], None]) -> bool: self.render() return parser.is_modified() - def get_read_only_parser(self) -> RecipeParser: - """ - This function returns a read-only parser that is incapable of committing changes to the original recipe file. - - To put another way, any writes to this parser will only cause changes in memory AND NOT to the underlying recipe - file. - - It is incredibly important to understand the implications of calling this function from a security and thread - safety perspective. This function exists so that we can use the the `RecipeParser` class in the `check_*()` - functions in `anaconda-linter` - - NOTE: Expect this function to be eventually deprecated. It is provided as a stop-gap as we experiment and - potentially transition to primarily use the `RecipeParser`/parse tree implementation. - :returns: An instance of the `RecipeParser` class containing the state of the recipe file at time of call. - """ - # TODO Future: Consider constructing this with an optional flag that disables writes further or throws when - # a write is attempted(?) - return RecipeParser("\n".join(self.meta_yaml)) - def patch( self, operations: list[JsonPatchType], @@ -693,7 +685,7 @@ def patch( # and use the newer parse tree work. if op_mode == OpMode.PARSE_TREE: - def _patch_all(parser: RecipeParser) -> None: + def _patch_all(parser: RecipeParserDeps) -> None: # Perform all requested patches for patch_op in operations: parser.patch(patch_op) @@ -1087,7 +1079,7 @@ def render( else: variants = [] if not python: - python = ["3.8", "3.9", "3.10", "3.11"] + python = ["3.9", "3.10", "3.11", "3.12", "3.13"] for s, p in list(itertools.product(subdir, python)): variant = {"subdir": s, "python": [p]} variant.update(others) diff --git a/percy/render/variants.py b/percy/render/variants.py index 2625211a..f6e7e1fe 100644 --- a/percy/render/variants.py +++ b/percy/render/variants.py @@ -2,6 +2,7 @@ File: variants.py Description: Reads cbc files and gives variants. Largely inspired from conda build. """ + from __future__ import annotations import copy @@ -187,7 +188,7 @@ def read_conda_build_config( else: subdir = _ensure_list(subdir) if python is None: - python = ["3.8", "3.9", "3.10", "3.11"] + python = ["3.9", "3.10", "3.11", "3.12", "3.13"] else: python = _ensure_list(python) diff --git a/percy/repodata/repodata.py b/percy/repodata/repodata.py index 58745e60..6b3aecbe 100644 --- a/percy/repodata/repodata.py +++ b/percy/repodata/repodata.py @@ -2,6 +2,7 @@ File: repodata.py Description: Provides tools for processing information about a repository """ + from __future__ import annotations import json diff --git a/percy/tests/commands/test_aggregate_cli.py b/percy/tests/commands/test_aggregate_cli.py index 869d0ad5..4c74c06b 100644 --- a/percy/tests/commands/test_aggregate_cli.py +++ b/percy/tests/commands/test_aggregate_cli.py @@ -2,6 +2,7 @@ File: test_aggregate_cli.py Description: Tests the `aggregate` CLI """ + from click.testing import CliRunner from percy.commands.aggregate import aggregate diff --git a/percy/tests/commands/test_convert_cli.py b/percy/tests/commands/test_convert_cli.py index 39c7b72a..bd48e779 100644 --- a/percy/tests/commands/test_convert_cli.py +++ b/percy/tests/commands/test_convert_cli.py @@ -2,6 +2,7 @@ File: test_convert_cli.py Description: Tests the `convert` CLI """ + from click.testing import CliRunner from percy.commands.convert import convert diff --git a/percy/tests/commands/test_main_cli.py b/percy/tests/commands/test_main_cli.py index dd75f5b4..13c0f2e0 100644 --- a/percy/tests/commands/test_main_cli.py +++ b/percy/tests/commands/test_main_cli.py @@ -2,6 +2,7 @@ File: test_main_cli.py Description: Tests the primary CLI interface, found under `percy.commands.main` """ + from click.testing import CliRunner from percy.commands.main import cli diff --git a/percy/tests/commands/test_recipe_cli.py b/percy/tests/commands/test_recipe_cli.py index 432ad4d7..d96ab0de 100644 --- a/percy/tests/commands/test_recipe_cli.py +++ b/percy/tests/commands/test_recipe_cli.py @@ -2,6 +2,7 @@ File: test_recipe_cli.py Description: Tests the recipe CLI """ + from click.testing import CliRunner from percy.commands.recipe import recipe diff --git a/percy/tests/parser/test_recipe_parser.py b/percy/tests/parser/test_recipe_parser.py deleted file mode 100644 index 6361d1f3..00000000 --- a/percy/tests/parser/test_recipe_parser.py +++ /dev/null @@ -1,2017 +0,0 @@ -""" -File: test_recipe_parser.py -Description: Unit tests for the recipe parser class and tools -""" -from __future__ import annotations - -from pathlib import Path -from typing import Final - -import pytest - -from percy.parser.enums import SelectorConflictMode -from percy.parser.exceptions import JsonPatchValidationException -from percy.parser.recipe_parser import RecipeParser -from percy.parser.types import MessageCategory -from percy.types import JsonType - -# Path to supplementary files used in test cases -TEST_FILES_PATH: Final[str] = "percy/tests/test_aux_files" - -# Long multi-line description string found in the `simple-recipe.yaml` test file -SIMPLE_DESCRIPTION: Final[ - str -] = "This is a PEP '561 type stub package for the toml package.\nIt can be used by type-checking tools like mypy, pyright,\npytype, PyCharm, etc. to check code that uses toml." # pylint: disable=line-too-long - -# Multiline string used to validate interpretation of the various multiline variations YAML allows -QUICK_FOX_PIPE: Final[str] = "The quick brown\n{{fox}}\n\njumped over the lazy dog\n" -QUICK_FOX_PIPE_PLUS: Final[str] = "The quick brown\n{{fox}}\n\njumped over the lazy dog\n" -QUICK_FOX_PIPE_MINUS: Final[str] = "The quick brown\n{{fox}}\n\njumped over the lazy dog" -QUICK_FOX_CARROT: Final[str] = "The quick brown {{fox}}\njumped over the lazy dog\n" -QUICK_FOX_CARROT_PLUS: Final[str] = "The quick brown {{fox}}\njumped over the lazy dog\n" -QUICK_FOX_CARROT_MINUS: Final[str] = "The quick brown {{fox}}\njumped over the lazy dog" -# Substitution variants of the multiline string -QUICK_FOX_SUB_PIPE: Final[str] = "The quick brown\ntiger\n\njumped over the lazy dog\n" -QUICK_FOX_SUB_PIPE_PLUS: Final[str] = "The quick brown\ntiger\n\njumped over the lazy dog\n" -QUICK_FOX_SUB_PIPE_MINUS: Final[str] = "The quick brown\ntiger\n\njumped over the lazy dog" -QUICK_FOX_SUB_CARROT: Final[str] = "The quick brown tiger\njumped over the lazy dog\n" -QUICK_FOX_SUB_CARROT_PLUS: Final[str] = "The quick brown tiger\njumped over the lazy dog\n" -QUICK_FOX_SUB_CARROT_MINUS: Final[str] = "The quick brown tiger\njumped over the lazy dog" - - -def load_file(file: Path | str) -> str: - """ - Loads a file into a single string - :param file: Filename of the file to read - :returns: Text from the file - """ - with open(Path(file), "r", encoding="utf-8") as f: - return f.read() - - -def load_recipe(file_name: str) -> RecipeParser: - """ - Convenience function that simplifies initializing a recipe parser. - :param file_name: File name of the test recipe to load - :returns: RecipeParser instance, based on the file - """ - recipe = load_file(f"{TEST_FILES_PATH}/{file_name}") - return RecipeParser(recipe) - - -## Construction and rendering sanity checks ## - - -def test_construction() -> None: - """ - Tests the construction of a recipe parser instance with a simple, common example file. - """ - types_toml = load_file(f"{TEST_FILES_PATH}/types-toml.yaml") - parser = RecipeParser(types_toml) - assert parser._init_content == types_toml # pylint: disable=protected-access - assert parser._vars_tbl == { # pylint: disable=protected-access - "name": "types-toml", - "version": "0.10.8.6", - } - assert not parser._is_modified # pylint: disable=protected-access - # TODO assert on tree structure - # TODO assert on selectors table - # assert parser._root == TODO - - -def test_str() -> None: - """ - Tests string casting - """ - parser = load_recipe("simple-recipe.yaml") - assert str(parser) == load_file(f"{TEST_FILES_PATH}/simple-recipe_to_str.out") - # Regression test: Run a function a second time to ensure that `SelectorInfo::__str__()` doesn't accidentally purge - # the underlying stack when the string is being rendered. - assert str(parser) == load_file(f"{TEST_FILES_PATH}/simple-recipe_to_str.out") - assert not parser.is_modified() - - -def test_eq() -> None: - """ - Tests equivalency function - """ - parser0 = load_recipe("simple-recipe.yaml") - parser1 = load_recipe("simple-recipe.yaml") - parser2 = load_recipe("types-toml.yaml") - assert parser0 == parser1 - assert parser0 != parser2 - assert not parser0.is_modified() - assert not parser1.is_modified() - assert not parser2.is_modified() - - -@pytest.mark.skip(reason="To be re-enable when PAT-46 is fixed") -def test_loading_obj_in_list() -> None: - """ - Regression test: at one point, the parser would crash loading this file, containing an object in a list. - """ - replace = load_file(f"{TEST_FILES_PATH}/simple-recipe_test_patch_replace.yaml") - parser = RecipeParser(replace) - assert parser.render() == replace - - -@pytest.mark.parametrize( - "file", - [ - "types-toml.yaml", # "Easy-difficulty" recipe, representative of common/simple recipes. - "simple-recipe.yaml", # "Medium-difficulty" recipe, containing several contrived examples - "multi-output.yaml", # Contains a multi-output recipe - "huggingface_hub.yaml", # Contains a blank lines in a multiline string - "simple-recipe_multiline_strings.yaml", # Contains multiple multiline strings, using various operators - ], -) -def test_round_trip(file: str) -> None: - """ - Test "eating our own dog food"/round-tripping the parser: Take a recipe, construct a parser, re-render and - ensure the output matches the input. - """ - expected: Final[str] = load_file(f"{TEST_FILES_PATH}/{file}") - parser = RecipeParser(expected) - assert parser.render() == expected - - -@pytest.mark.parametrize( - "file,substitute,expected", - [ - ( - "simple-recipe.yaml", - False, - { - "about": { - "description": SIMPLE_DESCRIPTION, - "license": "Apache-2.0 AND MIT", - "summary": "This is a small recipe for testing", - }, - "test_var_usage": { - "foo": "{{ version }}", - "bar": [ - "baz", - "{{ zz_non_alpha_first }}", - "blah", - "This {{ name }} is silly", - "last", - ], - }, - "build": {"is_true": True, "skip": True, "number": 0}, - "package": {"name": "{{ name|lower }}"}, - "requirements": { - "empty_field1": None, - "host": ["setuptools", "fakereq"], - "empty_field2": None, - "run": ["python"], - "empty_field3": None, - }, - "multi_level": { - "list_3": ["ls", "sl", "cowsay"], - "list_2": ["cat", "bat", "mat"], - "list_1": ["foo", "bar"], - }, - }, - ), - ( - "simple-recipe.yaml", - True, - { - "about": { - "description": SIMPLE_DESCRIPTION, - "license": "Apache-2.0 AND MIT", - "summary": "This is a small recipe for testing", - }, - "test_var_usage": { - "foo": "0.10.8.6", - "bar": [ - "baz", - 42, - "blah", - "This types-toml is silly", - "last", - ], - }, - "build": {"is_true": True, "skip": True, "number": 0}, - "package": {"name": "types-toml"}, - "requirements": { - "empty_field1": None, - "host": ["setuptools", "fakereq"], - "empty_field2": None, - "run": ["python"], - "empty_field3": None, - }, - "multi_level": { - "list_3": ["ls", "sl", "cowsay"], - "list_2": ["cat", "bat", "mat"], - "list_1": ["foo", "bar"], - }, - }, - ), - ( - "simple-recipe_multiline_strings.yaml", - False, - { - "about": { - "description0": QUICK_FOX_PIPE, - "description1": QUICK_FOX_PIPE_PLUS, - "description2": QUICK_FOX_PIPE_MINUS, - "description3": QUICK_FOX_CARROT, - "description4": QUICK_FOX_CARROT_PLUS, - "description5": QUICK_FOX_CARROT_MINUS, - "license": "Apache-2.0 AND MIT", - "summary": "This is a small recipe for testing", - }, - "test_var_usage": { - "foo": "{{ version }}", - "bar": [ - "baz", - "{{ zz_non_alpha_first }}", - "blah", - "This {{ name }} is silly", - "last", - ], - }, - "build": {"is_true": True, "skip": True, "number": 0}, - "package": {"name": "{{ name|lower }}"}, - }, - ), - ( - "simple-recipe_multiline_strings.yaml", - True, - { - "about": { - "description0": QUICK_FOX_SUB_PIPE, - "description1": QUICK_FOX_SUB_PIPE_PLUS, - "description2": QUICK_FOX_SUB_PIPE_MINUS, - "description3": QUICK_FOX_SUB_CARROT, - "description4": QUICK_FOX_SUB_CARROT_PLUS, - "description5": QUICK_FOX_SUB_CARROT_MINUS, - "license": "Apache-2.0 AND MIT", - "summary": "This is a small recipe for testing", - }, - "test_var_usage": { - "foo": "0.10.8.6", - "bar": [ - "baz", - 42, - "blah", - "This types-toml is silly", - "last", - ], - }, - "build": {"is_true": True, "skip": True, "number": 0}, - "package": {"name": "types-toml"}, - }, - ), - ], -) -def test_render_to_object(file: str, substitute: bool, expected: JsonType) -> None: - """ - Tests rendering a recipe to an object format. - :param file: File to load and test against - :param substitute: True to run the function with JINJA substitutions on, False for off - :param expected: Expected value to return - """ - parser = load_recipe(file) - assert parser.render_to_object(substitute) == expected - - -def test_render_to_object_multi_output() -> None: - """ - Tests rendering a recipe to an object format. - """ - parser = load_recipe("multi-output.yaml") - assert parser.render_to_object() == { - "outputs": [ - { - "name": "libdb", - "build": { - "run_exports": ["bar"], - }, - "test": { - "commands": [ - "test -f ${PREFIX}/lib/libdb${SHLIB_EXT}", - r"if not exist %LIBRARY_BIN%\libdb%SHLIB_EXT%", - ], - }, - }, - { - "name": "db", - "requirements": { - "build": [ - "foo3", - "foo2", - "{{ compiler('c') }}", - "{{ compiler('cxx') }}", - ], - "run": ["foo"], - }, - "test": { - "commands": [ - "db_archive -m hello", - ] - }, - }, - ] - } - - -@pytest.mark.parametrize( - "file_base,errors,warnings", - [ - ( - "simple-recipe.yaml", - [], - [ - "A non-list item had a selector at: /requirements/empty_field2", - "Required field missing: /about/license_file", - "Required field missing: /about/license_url", - ], - ), - ( - "multi-output.yaml", - [], - [ - "Required field missing: /about/summary", - "Required field missing: /about/description", - "Required field missing: /about/license", - "Required field missing: /about/license_file", - "Required field missing: /about/license_url", - ], - ), - ( - "huggingface_hub.yaml", - [], - [ - "Required field missing: /about/license_url", - ], - ), - # TODO Complete: The `curl.yaml` test is far from perfect. It is very much a work in progress. - # ( - # "curl.yaml", - # [], - # [ - # "A non-list item had a selector at: /outputs/0/build/ignore_run_exports", - # ], - # ), - ], -) -def test_render_to_new_recipe_format(file_base: str, errors: list[str], warnings: list[str]) -> None: - """ - Validates rendering a recipe in the new format. - :param file_base: Base file name for both the input and the expected out - """ - parser = load_recipe(file_base) - result, tbl = parser.render_to_new_recipe_format() - assert result == load_file(f"{TEST_FILES_PATH}/new_format_{file_base}") - assert tbl.get_messages(MessageCategory.ERROR) == errors - assert tbl.get_messages(MessageCategory.WARNING) == warnings - # Ensure that the original file was untouched - assert not parser.is_modified() - assert parser.diff() == "" - - -## Values ## - - -def test_list_value_paths() -> None: - """ - Tests retrieval of all value paths - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.list_value_paths() == [ - "/package/name", - "/build/number", - "/build/skip", - "/build/is_true", - "/requirements/empty_field1", - "/requirements/host/0", - "/requirements/host/1", - "/requirements/empty_field2", - "/requirements/run/0", - "/requirements/empty_field3", - "/about/summary", - "/about/description", - "/about/license", - "/multi_level/list_1/0", - "/multi_level/list_1/1", - "/multi_level/list_2/0", - "/multi_level/list_2/1", - "/multi_level/list_2/2", - "/multi_level/list_3/0", - "/multi_level/list_3/1", - "/multi_level/list_3/2", - "/test_var_usage/foo", - "/test_var_usage/bar/0", - "/test_var_usage/bar/1", - "/test_var_usage/bar/2", - "/test_var_usage/bar/3", - "/test_var_usage/bar/4", - ] - - -@pytest.mark.parametrize( - "file,path,expected", - [ - ## simple-recipe.yaml ## - ("simple-recipe.yaml", "/build/number", True), - ("simple-recipe.yaml", "/build/number/", True), - ("simple-recipe.yaml", "/build", True), - ("simple-recipe.yaml", "/requirements/host/0", True), - ("simple-recipe.yaml", "/requirements/host/1", True), - ("simple-recipe.yaml", "/multi_level/list_1/1", True), # Comments in lists could throw-off array indexing - ("simple-recipe.yaml", "/invalid/fake/path", False), - ## multi-output.yaml ## - ("multi-output.yaml", "/outputs/0/build/run_exports", True), - ("multi-output.yaml", "/outputs/1/build/run_exports", False), - ("multi-output.yaml", "/outputs/1/requirements/0", False), # Should fail as this is an object, not a list - ("multi-output.yaml", "/outputs/1/requirements/build/0", True), - ("multi-output.yaml", "/outputs/1/requirements/build/1", True), - ("multi-output.yaml", "/outputs/1/requirements/build/2", True), - ("multi-output.yaml", "/outputs/1/requirements/build/3", True), - ("multi-output.yaml", "/outputs/1/requirements/build/4", False), - ], -) -def test_contains_value(file: str, path: str, expected: bool) -> None: - """ - Tests if a path exists in a parsed recipe file. - :param file: File to work against - :param path: Target input path - :param expected: Expected result of the test - """ - parser = load_recipe(file) - assert parser.contains_value(path) == expected - assert not parser.is_modified() - - -@pytest.mark.parametrize( - "file,path,sub_vars,expected", - [ - ## simple-recipe.yaml ## - # Return a single value - ("simple-recipe.yaml", "/build/number", False, 0), - ("simple-recipe.yaml", "/build/number/", False, 0), - # Return a compound value - ( - "simple-recipe.yaml", - "/build", - False, - { - "number": 0, - "skip": True, - "is_true": True, - }, - ), - ( - "simple-recipe.yaml", - "/build/", - False, - { - "number": 0, - "skip": True, - "is_true": True, - }, - ), - # Return a Jinja value (substitution flag not in use) - ("simple-recipe.yaml", "/package/name", False, "{{ name|lower }}"), - # Return a value in a list - ("simple-recipe.yaml", "/requirements/host", False, ["setuptools", "fakereq"]), - ("simple-recipe.yaml", "/requirements/host/", False, ["setuptools", "fakereq"]), - ("simple-recipe.yaml", "/requirements/host/0", False, "setuptools"), - ("simple-recipe.yaml", "/requirements/host/1", False, "fakereq"), - # Regression: A list containing 1 value may be interpreted as the base type by YAML parsers. This can wreak - # havoc on type safety. - ("simple-recipe.yaml", "/requirements/run", False, ["python"]), - # Return a multiline string - ("simple-recipe.yaml", "/about/description", False, SIMPLE_DESCRIPTION), - ("simple-recipe.yaml", "/about/description/", False, SIMPLE_DESCRIPTION), - # Return multiline string variants - ("simple-recipe_multiline_strings.yaml", "/about/description0", False, QUICK_FOX_PIPE), - ("simple-recipe_multiline_strings.yaml", "/about/description1", False, QUICK_FOX_PIPE_PLUS), - ("simple-recipe_multiline_strings.yaml", "/about/description2", False, QUICK_FOX_PIPE_MINUS), - ("simple-recipe_multiline_strings.yaml", "/about/description3", False, QUICK_FOX_CARROT), - ("simple-recipe_multiline_strings.yaml", "/about/description4", False, QUICK_FOX_CARROT_PLUS), - ("simple-recipe_multiline_strings.yaml", "/about/description5", False, QUICK_FOX_CARROT_MINUS), - # Return multiline string variants, with substitution - ("simple-recipe_multiline_strings.yaml", "/about/description0", True, QUICK_FOX_SUB_PIPE), - ("simple-recipe_multiline_strings.yaml", "/about/description1", True, QUICK_FOX_SUB_PIPE_PLUS), - ("simple-recipe_multiline_strings.yaml", "/about/description2", True, QUICK_FOX_SUB_PIPE_MINUS), - ("simple-recipe_multiline_strings.yaml", "/about/description3", True, QUICK_FOX_SUB_CARROT), - ("simple-recipe_multiline_strings.yaml", "/about/description4", True, QUICK_FOX_SUB_CARROT_PLUS), - ("simple-recipe_multiline_strings.yaml", "/about/description5", True, QUICK_FOX_SUB_CARROT_MINUS), - # Comments in lists could throw-off array indexing - ("simple-recipe.yaml", "/multi_level/list_1/1", False, "bar"), - # Render a recursive, complex type. - ( - "simple-recipe.yaml", - "/test_var_usage", - True, - { - "foo": "0.10.8.6", - "bar": [ - "baz", - 42, - "blah", - "This types-toml is silly", - "last", - ], - }, - ), - ## multi-output.yaml ## - ("multi-output.yaml", "/outputs/0/build/run_exports/0", False, "bar"), - ("multi-output.yaml", "/outputs/0/build/run_exports", False, ["bar"]), - ("multi-output.yaml", "/outputs/0/build", False, {"run_exports": ["bar"]}), - # TODO FIX: This case - # ( - # "multi-output.yaml", - # "/outputs/1", - # False, - # { - # "name": "db", - # "requirements": { - # "build": ["foo3", "foo2", "{{ compiler('c') }}", "{{ compiler('cxx') }}"], - # "run": ["foo"], - # }, - # "test": {"commands": ["db_archive -m hello"]}, - # }, - # ), - ], -) -def test_get_value(file: str, path: str, sub_vars: bool, expected: JsonType) -> None: - """ - Tests retrieval of a value from a parsed YAML example. - """ - parser = load_recipe(file) - assert parser.get_value(path, sub_vars=sub_vars) == expected - assert not parser.is_modified() - - -def test_get_value_not_found() -> None: - """ - Tests failure to retrieve a value from a parsed YAML example. - """ - parser = load_recipe("simple-recipe.yaml") - # Path not found cases - with pytest.raises(KeyError): - parser.get_value("/invalid/fake/path") - assert parser.get_value("/invalid/fake/path", 42) == 42 - # Tests that a user can pass `None` without throwing - assert parser.get_value("/invalid/fake/path", None) is None - assert not parser.is_modified() - - -def test_get_value_with_var_subs() -> None: - """ - Tests retrieval of a value from a parsed YAML example, with Jinja variable substitutions enabled. - """ - parser = load_recipe("simple-recipe.yaml") - # No change on lines without any variable substitutions - assert parser.get_value("/requirements/host", sub_vars=True) == ["setuptools", "fakereq"] - - ## Test base types - assert parser.get_value("/test_var_usage/foo", sub_vars=True) == "0.10.8.6" - assert parser.get_value("/test_var_usage/bar/1", sub_vars=True) == 42 - # Test string with `|lower` function applied - assert parser.get_value("/package/name", sub_vars=True) == "types-toml" - # Test collection types - assert parser.get_value("/test_var_usage/bar", sub_vars=True) == [ - "baz", - 42, - "blah", - "This types-toml is silly", - "last", - ] - assert parser.get_value("/test_var_usage", sub_vars=True) == { - "foo": "0.10.8.6", - "bar": [ - "baz", - 42, - "blah", - "This types-toml is silly", - "last", - ], - } - assert not parser.is_modified() - - -def test_find_value() -> None: - """ - Tests finding a value from a parsed YAML example. - """ - parser = load_recipe("simple-recipe.yaml") - # Values in the recipe - assert parser.find_value(None) == [ - "/requirements/empty_field1", - "/requirements/empty_field2", - "/requirements/empty_field3", - ] - assert parser.find_value("fakereq") == ["/requirements/host/1"] - assert parser.find_value(True) == ["/build/skip", "/build/is_true"] - assert parser.find_value("foo") == ["/multi_level/list_1/0"] - assert parser.find_value("Apache-2.0 AND MIT") == ["/about/license"] - # Values not in the recipe - assert not parser.find_value(43) - assert not parser.find_value("fooz") - assert not parser.find_value("") - # Values that are not supported for searching - with pytest.raises(ValueError): - parser.find_value(["foo", "bar"]) - with pytest.raises(ValueError): - parser.find_value(("foo", "bar")) - with pytest.raises(ValueError): - parser.find_value({"foo": "bar"}) - # Find does not modify the parser - assert not parser.is_modified() - - -## Dependencies ## - - -@pytest.mark.parametrize("file,expected", [("multi-output.yaml", True), ("simple-recipe.yaml", False)]) -def test_is_multi_output(file: str, expected: bool) -> None: - """ - Validates if a recipe is in the multi-output format - :param file: File to test against - :param expected: Expected output - """ - assert load_recipe(file).is_multi_output() == expected - - -@pytest.mark.parametrize( - "file,expected", - [ - ("multi-output.yaml", ["/", "/outputs/0", "/outputs/1"]), - ("simple-recipe.yaml", ["/"]), - ("simple-recipe_comment_in_requirements.yaml", ["/"]), - ("huggingface_hub.yaml", ["/"]), - ], -) -def test_get_package_paths(file: str, expected: list[str]) -> None: - """ - Validates fetching paths containing recipe dependencies - :param file: File to test against - :param expected: Expected output - """ - assert load_recipe(file).get_package_paths() == expected - - -@pytest.mark.parametrize( - "base,ext,expected", - [ - ("", "", "/"), - ("/", "/foo/bar", "/foo/bar"), - ("/", "foo/bar", "/foo/bar"), - ("/foo/bar", "baz", "/foo/bar/baz"), - ("/foo/bar", "/baz", "/foo/bar/baz"), - ], -) -def test_append_to_path(base: str, ext: str, expected) -> None: - """ - :param base: Base string path - :param ext: Path to extend the base path with - :param expected: Expected output - """ - assert RecipeParser.append_to_path(base, ext) == expected - - -@pytest.mark.parametrize( - "file,expected", - [ - ( - "multi-output.yaml", - [ - "/outputs/1/requirements/build/0", - "/outputs/1/requirements/build/1", - "/outputs/1/requirements/build/2", - "/outputs/1/requirements/build/3", - "/outputs/1/requirements/run/0", - ], - ), - ("simple-recipe.yaml", ["/requirements/host/0", "/requirements/host/1", "/requirements/run/0"]), - ( - "simple-recipe_comment_in_requirements.yaml", - ["/requirements/host/0", "/requirements/host/1", "/requirements/run/0"], - ), - ( - "huggingface_hub.yaml", - [ - "/requirements/host/0", - "/requirements/host/1", - "/requirements/host/2", - "/requirements/host/3", - "/requirements/run/0", - "/requirements/run/1", - "/requirements/run/2", - "/requirements/run/3", - "/requirements/run/4", - "/requirements/run/5", - "/requirements/run/6", - "/requirements/run/7", - "/requirements/run/8", - "/requirements/run_constrained/0", - "/requirements/run_constrained/1", - "/requirements/run_constrained/2", - ], - ), - ], -) -def test_get_dependency_paths(file: str, expected: list[str]) -> None: - """ - Validates fetching paths containing recipe dependencies - :param file: File to test against - :param expected: Expected output - """ - assert load_recipe(file).get_dependency_paths() == expected - - -## Variables ## - - -def test_list_variable() -> None: - """ - Validates the list of variables found - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.list_variables() == ["zz_non_alpha_first", "name", "version"] - assert not parser.is_modified() - - -def test_contains_variable() -> None: - """ - Validates checking if a variable exists in a recipe - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.contains_variable("zz_non_alpha_first") - assert parser.contains_variable("name") - assert parser.contains_variable("version") - assert not parser.contains_variable("fake_var") - assert not parser.is_modified() - - -def test_get_variable() -> None: - """ - Tests the value returned from fetching a variable - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.get_variable("zz_non_alpha_first") == 42 - assert parser.get_variable("name") == "types-toml" - assert parser.get_variable("version") == "0.10.8.6" - with pytest.raises(KeyError): - parser.get_variable("fake_var") - assert parser.get_variable("fake_var", 43) == 43 - # Tests that a user can pass `None` without throwing - assert parser.get_variable("fake_var", None) is None - assert not parser.is_modified() - - -def test_set_variable() -> None: - """ - Tests setting and adding a variable - """ - parser = load_recipe("simple-recipe.yaml") - parser.set_variable("name", "foobar") - parser.set_variable("zz_non_alpha_first", 24) - # Ensure a missing variable gets added - parser.set_variable("DNE", "The limit doesn't exist") - # Validate - assert parser.is_modified() - assert parser.list_variables() == [ - "zz_non_alpha_first", - "name", - "version", - "DNE", - ] - assert parser.get_variable("name") == "foobar" - assert parser.get_variable("zz_non_alpha_first") == 24 - assert parser.get_variable("DNE") == "The limit doesn't exist" - - -def test_del_variable() -> None: - """ - Tests deleting a variable - """ - parser = load_recipe("simple-recipe.yaml") - parser.del_variable("name") - # Ensure a missing var doesn't crash a delete - parser.del_variable("DNE") - # Validate - assert parser.is_modified() - assert parser.list_variables() == ["zz_non_alpha_first", "version"] - with pytest.raises(KeyError): - parser.get_variable("name") - - -def test_get_variable_references() -> None: - """ - Tests generating a list of paths that use a variable - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.get_variable_references("version") == [ - "/test_var_usage/foo", - ] - assert parser.get_variable_references("zz_non_alpha_first") == [ - "/test_var_usage/bar/1", - ] - assert parser.get_variable_references("name") == [ - "/package/name", - "/test_var_usage/bar/3", - ] - assert not parser.is_modified() - - -## Selectors ## - - -def test_list_selectors() -> None: - """ - Validates the list of selectors found - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.list_selectors() == ["[unix]", "[py<37]", "[unix and win]"] - assert not parser.is_modified() - - -def test_contains_selectors() -> None: - """ - Validates checking if a selector exists in a recipe - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.contains_selector("[py<37]") - assert parser.contains_selector("[unix]") - assert not parser.contains_selector("[fake selector]") - assert not parser.is_modified() - - -def test_get_selector_paths() -> None: - """ - Tests the paths returned from fetching a selector - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.get_selector_paths("[py<37]") == ["/build/skip"] - assert parser.get_selector_paths("[unix]") == [ - "/package/name", - "/requirements/host/0", - "/requirements/host/1", - ] - assert not parser.get_selector_paths("[fake selector]") - assert not parser.is_modified() - - -@pytest.mark.parametrize( - "file,path,expected", - [ - ("simple-recipe.yaml", "/build/skip", True), - ("simple-recipe.yaml", "/requirements/host/0", True), - ("simple-recipe.yaml", "/requirements/host/1", True), - ("simple-recipe.yaml", "/requirements/empty_field2", True), - ("simple-recipe.yaml", "/requirements/run/0", False), - ("simple-recipe.yaml", "/requirements/run", False), - ("simple-recipe.yaml", "/fake/path", False), - ], -) -def test_contains_selector_at_path(file: str, path: str, expected: bool) -> None: - """ - Tests checking if a selector exists on a given path - :param file: File to run against - :param path: Path to check - :param expected: Expected value - """ - assert load_recipe(file).contains_selector_at_path(path) == expected - - -@pytest.mark.parametrize( - "file,path,expected", - [ - ("simple-recipe.yaml", "/build/skip", "[py<37]"), - ("simple-recipe.yaml", "/requirements/host/0", "[unix]"), - ("simple-recipe.yaml", "/requirements/host/1", "[unix]"), - ("simple-recipe.yaml", "/requirements/empty_field2", "[unix and win]"), - ], -) -def test_get_selector_at_path_exists(file: str, path: str, expected: bool) -> None: - """ - Tests cases where a selector exists on a path - :param file: File to run against - :param path: Path to check - :param expected: Expected value - """ - assert load_recipe(file).get_selector_at_path(path) == expected - - -def test_get_selector_at_path_dne() -> None: - """ - Tests edge cases where `get_selector_at_path()` should fail correctly OR - handles non-existent selectors gracefully - """ - parser = load_recipe("simple-recipe.yaml") - # Path does not exist - with pytest.raises(KeyError): - parser.get_selector_at_path("/fake/path") - # No default was provided - with pytest.raises(KeyError): - parser.get_selector_at_path("/requirements/run/0") - # Invalid default was provided - with pytest.raises(ValueError): - parser.get_selector_at_path("/requirements/run/0", "not a selector") - - # Valid default was provided - assert parser.get_selector_at_path("/requirements/run/0", "[unix]") == "[unix]" - - -def test_add_selector() -> None: - """ - Tests adding a selector to a recipe - """ - parser = load_recipe("simple-recipe.yaml") - # Test that selector validation is working - with pytest.raises(KeyError): - parser.add_selector("/package/path/to/fake/value", "[unix]") - with pytest.raises(ValueError): - parser.add_selector("/build/number", "bad selector") - assert not parser.is_modified() - - # Add selectors to lines without existing selectors - parser.add_selector("/requirements/empty_field3", "[unix]") - parser.add_selector("/multi_level/list_1/0", "[unix]") - parser.add_selector("/build/number", "[win]") - parser.add_selector("/multi_level/list_2/1", "[win]") - assert parser.get_selector_paths("[unix]") == [ - "/package/name", - "/requirements/host/0", - "/requirements/host/1", - "/requirements/empty_field3", - "/multi_level/list_1/0", - ] - assert parser.get_selector_paths("[win]") == [ - "/build/number", - "/multi_level/list_2/1", - ] - - # Add selectors to existing selectors - parser.add_selector("/requirements/host/0", "[win]", SelectorConflictMode.REPLACE) - assert parser.get_selector_paths("[win]") == [ - "/build/number", - "/requirements/host/0", - "/multi_level/list_2/1", - ] - parser.add_selector("/requirements/host/1", "[win]", SelectorConflictMode.AND) - assert parser.get_selector_paths("[unix and win]") == ["/requirements/host/1", "/requirements/empty_field2"] - parser.add_selector("/build/skip", "[win]", SelectorConflictMode.OR) - assert parser.get_selector_paths("[py<37 or win]") == ["/build/skip"] - parser.add_selector("/requirements/run/0", "[win]", SelectorConflictMode.AND) - assert parser.get_selector_paths("[win]") == [ - "/build/number", - "/requirements/host/0", - "/requirements/run/0", - "/multi_level/list_2/1", - ] - - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe_test_add_selector.yaml") - assert parser.is_modified() - - -def test_remove_selector() -> None: - """ - Tests removing a selector to a recipe - """ - parser = load_recipe("simple-recipe.yaml") - # Test that selector validation is working - with pytest.raises(KeyError): - parser.remove_selector("/package/path/to/fake/value") - - # Don't fail when a selector doesn't exist on a line - assert parser.remove_selector("/build/number") is None - # Don't remove a non-selector comment - assert parser.remove_selector("/requirements/run/0") is None - assert not parser.is_modified() - - # Remove a selector - assert parser.remove_selector("/package/name") == "[unix]" - assert parser.get_selector_paths("[unix]") == [ - "/requirements/host/0", - "/requirements/host/1", - ] - # Remove a selector with a comment - assert parser.remove_selector("/requirements/host/1") == "[unix]" - assert parser.get_selector_paths("[unix]") == [ - "/requirements/host/0", - ] - # Remove a selector with a "double comment (extra `#` symbols used)" - assert parser.remove_selector("/requirements/empty_field2") == "[unix and win]" - assert not parser.get_selector_paths("[unix and win]") - - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe_test_remove_selector.yaml") - assert parser.is_modified() - - -## Comments ## - - -@pytest.mark.parametrize( - "file,expected", - [ - ( - "simple-recipe.yaml", - { - "/requirements/host/1": "# selector with comment", - "/requirements/empty_field2": "# selector with comment with comment symbol", - "/requirements/run/0": "# not a selector", - }, - ), - ("huggingface_hub.yaml", {}), - ("multi-output.yaml", {}), - ( - "curl.yaml", - { - "/outputs/0/requirements/run/2": "# exact pin handled through openssl run_exports", - "/outputs/2/requirements/host/0": "# Only required to produce all openssl variants.", - }, - ), - ], -) -def test_get_comments_table(file: str, expected: dict[str, str]) -> None: - """ - Tests generating a table of comment locations - """ - parser = load_recipe(file) - assert parser.get_comments_table() == expected - - -@pytest.mark.parametrize( - "file,ops,expected", - [ - ( - "simple-recipe.yaml", - [ - ("/package/name", "# Come on Jeffery"), - ("/build/number", "# you can do it!"), - ("/requirements/empty_field1", "Pave the way!"), - ("/multi_level/list_1", "# Put your back into it!"), - ("/multi_level/list_2/1", "# Tell us why"), - ("/multi_level/list_2/2", " Show us how"), - ("/multi_level/list_3/0", "# Look at where you came from"), - ("/test_var_usage/foo", "Look at you now!"), - ], - "simple-recipe_test_add_comment.yaml", - ), - ], -) -def test_add_comment(file: str, ops: list[tuple[str, str]], expected: str) -> None: - parser = load_recipe(file) - for path, comment in ops: - parser.add_comment(path, comment) - assert parser.is_modified() - assert parser.render() == load_file(f"{TEST_FILES_PATH}/{expected}") - - -@pytest.mark.parametrize( - "file,path,comment,exception", - [ - ("simple-recipe.yaml", "/package/path/to/fake/value", "A comment", KeyError), - ("simple-recipe.yaml", "/build/number", "[unix]", ValueError), - ("simple-recipe.yaml", "/build/number", "", ValueError), - ("simple-recipe.yaml", "/build/number", " ", ValueError), - ], -) -def test_add_comment_raises(file: str, path: str, comment: str, exception: Exception) -> None: - """ - Tests scenarios where `add_comment()` should raise an exception - :param file: File to test against - :param path: Path to add a comment - :param comment: Comment to add - :param exception: Exception expected to be raised - """ - parser = load_recipe(file) - with pytest.raises(exception): - parser.add_comment(path, comment) - - -## Patch and Search ## - - -def test_patch_schema_validation() -> None: - """ - Tests edge cases that should trigger an exception on JSON patch schema validation. Valid schemas are inherently - tested in the other patching tests. - """ - parser = load_recipe("simple-recipe.yaml") - # Invalid enum/unknown op - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "fakeop", - "path": "/build/number", - "value": 42, - } - ) - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "", - "path": "/build/number", - "value": 42, - } - ) - # Patch has extra field(s) - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "replace", - "path": "/build/number", - "value": 42, - "extra": "field", - } - ) - # Patch is missing required fields - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "path": "/build/number", - "value": 42, - } - ) - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "replace", - "value": 42, - } - ) - # Patch is missing required fields, based on `op` - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "add", - "path": "/build/number", - } - ) - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "replace", - "path": "/build/number", - } - ) - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "move", - "path": "/build/number", - } - ) - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "copy", - "path": "/build/number", - } - ) - with pytest.raises(JsonPatchValidationException): - parser.patch( - { - "op": "test", - "path": "/build/number", - } - ) - # Patch has invalid types in critical fields - with pytest.raises(JsonPatchValidationException): - parser.patch({"op": "move", "path": 42, "value": 42}) - with pytest.raises(JsonPatchValidationException): - parser.patch({"op": "move", "path": "/build/number", "from": 42}) - - -def test_patch_path_invalid() -> None: - """ - Tests if `patch` returns false on all ops when the path is not found. Also checks if the tree has been modified. - """ - parser = load_recipe("simple-recipe.yaml") - - # Passing an empty path fails at the JSON schema validation layer, so it applies to all patch functions. - with pytest.raises(JsonPatchValidationException): - assert not ( - parser.patch( - { - "op": "test", - "path": "", - "value": 42, - } - ) - ) - - # add - assert not ( - parser.patch( - { - "op": "add", - "path": "/package/path/to/fake/value", - "value": 42, - } - ) - ) - assert not ( - parser.patch( - { - "op": "add", - "path": "/build/number/0", - "value": 42, - } - ) - ) - assert not ( - parser.patch( - { - "op": "add", - "path": "/multi_level/list2/4", - "value": 42, - } - ) - ) - # remove - assert not ( - parser.patch( - { - "op": "remove", - "path": "/package/path/to/fake/value", - } - ) - ) - assert not ( - parser.patch( - { - "op": "remove", - "path": "/build/number/0", - } - ) - ) - assert not ( - parser.patch( - { - "op": "remove", - "path": "/multi_level/list2/4", - } - ) - ) - assert not ( - parser.patch( - { - "op": "remove", - "path": "/build/skip/true", - } - ) - ) - # replace - assert not ( - parser.patch( - { - "op": "replace", - "path": "/build/number/0", - "value": 42, - } - ) - ) - assert not ( - parser.patch( - { - "op": "replace", - "path": "/multi_level/list2/4", - "value": 42, - } - ) - ) - assert not ( - parser.patch( - { - "op": "replace", - "path": "/build/skip/true", - "value": 42, - } - ) - ) - assert not ( - parser.patch( - { - "op": "replace", - "path": "/package/path/to/fake/value", - "value": 42, - } - ) - ) - - # move, `path` is invalid - assert not ( - parser.patch( - { - "op": "move", - "path": "/package/path/to/fake/value", - "from": "/about/summary", - } - ) - ) - assert not ( - parser.patch( - { - "op": "move", - "path": "/build/number/0", - "from": "/about/summary", - } - ) - ) - assert not ( - parser.patch( - { - "op": "move", - "path": "/multi_level/list2/4", - "from": "/about/summary", - } - ) - ) - # move, `from` is invalid - assert not ( - parser.patch( - { - "op": "move", - "from": "/package/path/to/fake/value", - "path": "/about/summary", - } - ) - ) - assert not ( - parser.patch( - { - "op": "move", - "from": "/build/number/0", - "path": "/about/summary", - } - ) - ) - assert not ( - parser.patch( - { - "op": "move", - "from": "/multi_level/list2/4", - "path": "/about/summary", - } - ) - ) - - # copy, `path` is invalid - assert not ( - parser.patch( - { - "op": "copy", - "path": "/package/path/to/fake/value", - "from": "/about/summary", - } - ) - ) - assert not ( - parser.patch( - { - "op": "copy", - "path": "/build/number/0", - "from": "/about/summary", - } - ) - ) - assert not ( - parser.patch( - { - "op": "copy", - "path": "/multi_level/list2/4", - "from": "/about/summary", - } - ) - ) - # copy, `from` is invalid - assert not ( - parser.patch( - { - "op": "copy", - "from": "/package/path/to/fake/value", - "path": "/about/summary", - } - ) - ) - assert not ( - parser.patch( - { - "op": "copy", - "from": "/build/number/0", - "path": "/about/summary", - } - ) - ) - assert not ( - parser.patch( - { - "op": "copy", - "from": "/multi_level/list2/4", - "path": "/about/summary", - } - ) - ) - - # test - assert not ( - parser.patch( - { - "op": "test", - "path": "/package/path/to/fake/value", - "value": 42, - } - ) - ) - - assert not parser.is_modified() - - -def test_patch_test() -> None: - """ - Tests the `test` patch op. The `test` op may be useful for other test assertions, so it is tested before the other - patch operations. - """ - parser = load_recipe("simple-recipe.yaml") - - # Test that values match, as expected - assert parser.patch( - { - "op": "test", - "path": "/build/number", - "value": 0, - } - ) - assert parser.patch( - { - "op": "test", - "path": "/build", - "value": { - "number": 0, - "skip": True, - "is_true": True, - }, - } - ) - assert parser.patch( - { - "op": "test", - "path": "/requirements/host", - "value": ["setuptools", "fakereq"], - } - ) - assert parser.patch( - { - "op": "test", - "path": "/requirements/host/1", - "value": "fakereq", - } - ) - assert parser.patch( - { - "op": "test", - "path": "/about/description", - "value": SIMPLE_DESCRIPTION, - } - ) - # Test that values do not match, as expected - assert not ( - parser.patch( - { - "op": "test", - "path": "/build/number", - "value": 42, - } - ) - ) - assert not ( - parser.patch( - { - "op": "test", - "path": "/build", - "value": { - "number": 42, - "skip": True, - }, - } - ) - ) - assert not ( - parser.patch( - { - "op": "test", - "path": "/requirements/host", - "value": ["not_setuptools", "fakereq"], - } - ) - ) - assert not ( - parser.patch( - { - "op": "test", - "path": "/requirements/host/1", - "value": "other_fake", - } - ) - ) - assert not ( - parser.patch( - { - "op": "test", - "path": "/about/description", - "value": "other_fake\nmultiline", - } - ) - ) - - # Ensure that `test` does not modify the tree - assert not parser.is_modified() - - -def test_patch_add() -> None: - """ - Tests the `add` patch op. - """ - parser = load_recipe("simple-recipe.yaml") - - # As per the RFC, `add` will not construct multiple-levels of non-existing structures. The containing - # object(s)/list(s) must exist. - assert not parser.patch( - { - "op": "add", - "path": "/build/fake/meaning_of_life", - "value": 42, - } - ) - # Similarly, appending to a list - assert not parser.patch( - { - "op": "add", - "path": "/requirements/empty_field1/-/blah", - "value": 42, - } - ) - assert not parser.is_modified() - - # Add primitive values - assert parser.patch( - { - "op": "add", - "path": "/build/meaning_of_life", - "value": 42, - } - ) - assert parser.patch( - { - "op": "add", - "path": "/package/is_cool_name", - "value": True, - } - ) - - # Add to empty-key node - assert parser.patch( - { - "op": "add", - "path": "/requirements/empty_field2", - "value": "Not so empty now", - } - ) - - # Add list items - assert parser.patch( - { - "op": "add", - "path": "/multi_level/list_2/1", - "value": "We got it all on UHF", - } - ) - assert parser.patch( - { - "op": "add", - "path": "/multi_level/list_1/0", - "value": "There's just one place to go for all your spatula needs!", - } - ) - assert parser.patch( - { - "op": "add", - "path": "/multi_level/list_1/-", - "value": "Spatula City!", - } - ) - - # Add a complex value - assert parser.patch( - { - "op": "add", - "path": "/test_var_usage/Stanley", - "value": [ - "Oh, Joel Miller, you've just found the marble in the oatmeal.", - "You're a lucky, lucky, lucky little boy.", - "'Cause you know why?", - "You get to drink from... the FIRE HOOOOOSE!", - ], - } - ) - - # Add a top-level complex value - assert parser.patch( - { - "op": "add", - "path": "/U62", - "value": { - "George": ["How'd you like your own TV show?", "You're on"], - "Stanley": ["Ok"], - }, - } - ) - - # Edge case: adding a value to an existing key (non-list) actually replaces the value at that key, as per the RFC. - assert parser.patch({"op": "add", "path": "/about/summary", "value": 62}) - - # Add a value in a list with a comment - assert parser.patch({"op": "add", "path": "/multi_level/list_1/1", "value": "ken"}) - assert parser.patch({"op": "add", "path": "/multi_level/list_1/3", "value": "barbie"}) - - # Sanity check: validate all modifications - assert parser.is_modified() - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe_test_patch_add.yaml") - - -def test_patch_remove() -> None: - """ - Tests the `remove` patch op. - """ - parser = load_recipe("simple-recipe.yaml") - - # Remove primitive values - assert parser.patch( - { - "op": "remove", - "path": "/build/number", - } - ) - assert parser.patch( - { - "op": "remove", - "path": "/package/name", - } - ) - - # Remove empty-key node - assert parser.patch( - { - "op": "remove", - "path": "/requirements/empty_field2", - } - ) - - # Remove list items - assert parser.patch( - { - "op": "remove", - "path": "/multi_level/list_2/0", - } - ) - # Ensure comments don't get erased - assert parser.patch( - { - "op": "remove", - "path": "/multi_level/list_1/1", - } - ) - - # Remove a complex value - assert parser.patch( - { - "op": "remove", - "path": "/multi_level/list_3", - } - ) - - # Remove a top-level complex value - assert parser.patch( - { - "op": "remove", - "path": "/about", - } - ) - - # Sanity check: validate all modifications - assert parser.is_modified() - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe_test_patch_remove.yaml") - - -def test_patch_replace() -> None: - """ - Tests the `replace` patch op. - """ - parser = load_recipe("simple-recipe.yaml") - # Patch an integer - assert parser.patch( - { - "op": "replace", - "path": "/build/number", - "value": 42, - } - ) - # Patch a bool - assert parser.patch( - { - "op": "replace", - "path": "/build/is_true", - "value": False, - } - ) - # Patch a string - assert parser.patch( - { - "op": "replace", - "path": "/about/license", - "value": "MIT", - } - ) - # Patch an array element - assert parser.patch( - { - "op": "replace", - "path": "/requirements/run/0", - "value": "cpython", - } - ) - # Patch an element to become an array - assert parser.patch( - { - "op": "replace", - "path": "/about/summary", - "value": [ - "The Trial", - "Never Ends", - "Picard", - ], - } - ) - # Patch a multiline string - assert parser.patch( - { - "op": "replace", - "path": "/about/description", - "value": ("This is a PEP 561\ntype stub package\nfor the toml package."), - } - ) - - # Hard mode: replace a string with an object containing multiple types in a complex data structure. - assert parser.patch( - { - "op": "replace", - "path": "/multi_level/list_2/1", - "value": {"build": {"number": 42, "skip": True}}, - } - ) - - # Patch-in strings with quotes - assert parser.patch( - { - "op": "replace", - "path": "/multi_level/list_3/2", - "value": "{{ compiler('c') }}", - } - ) - - # Patch lists with comments - assert parser.patch( - { - "op": "replace", - "path": "/multi_level/list_1/0", - "value": "ken", - } - ) - assert parser.patch( - { - "op": "replace", - "path": "/multi_level/list_1/1", - "value": "barbie", - } - ) - - # Sanity check: validate all modifications - assert parser.is_modified() - # NOTE: That patches, as of writing, cannot preserve selectors - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe_test_patch_replace.yaml") - - -def test_patch_move() -> None: - """ - Tests the `move` patch op. - """ - parser = load_recipe("simple-recipe.yaml") - # No-op moves should not corrupt our modification state. - assert parser.patch( - { - "op": "move", - "path": "/build/number", - "from": "/build/number", - } - ) - # Special failure case: trying to "add" to an illegal path while the "remove" path is still valid - assert not parser.patch( - { - "op": "move", - "path": "/build/number/0", - "from": "/build/number", - } - ) - assert not parser.is_modified() - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe.yaml") - - # Simple move - assert parser.patch( - { - "op": "move", - "path": "/requirements/number", - "from": "/build/number", - } - ) - - # Moving list item to a new key (replaces existing value) - assert parser.patch( - { - "op": "move", - "path": "/build/is_true", - "from": "/multi_level/list_3/0", - } - ) - - # Moving list item to a different list - assert parser.patch( - { - "op": "move", - "path": "/requirements/host/-", - "from": "/multi_level/list_1/1", - } - ) - - # Moving a list entry to another list entry position - assert parser.patch( - { - "op": "move", - "path": "/multi_level/list_2/0", - "from": "/multi_level/list_2/1", - } - ) - - # Moving a compound type - assert parser.patch( - { - "op": "move", - "path": "/multi_level/bar", - "from": "/test_var_usage/bar", - } - ) - - # Sanity check: validate all modifications - assert parser.is_modified() - # NOTE: That patches, as of writing, cannot preserve selectors - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe_test_patch_move.yaml") - - -def test_patch_copy() -> None: - """ - Tests the `copy` patch op. - """ - parser = load_recipe("simple-recipe.yaml") - - # Simple copy - assert parser.patch( - { - "op": "copy", - "path": "/requirements/number", - "from": "/build/number", - } - ) - - # Copying list item to a new key - assert parser.patch( - { - "op": "copy", - "path": "/build/is_true", - "from": "/multi_level/list_3/0", - } - ) - - # Copying list item to a different list - assert parser.patch( - { - "op": "copy", - "path": "/requirements/host/-", - "from": "/multi_level/list_1/1", - } - ) - - # Copying a list entry to another list entry position - assert parser.patch( - { - "op": "copy", - "path": "/multi_level/list_2/0", - "from": "/multi_level/list_2/1", - } - ) - - # Copying a compound type - assert parser.patch( - { - "op": "copy", - "path": "/multi_level/bar", - "from": "/test_var_usage/bar", - } - ) - - # Sanity check: validate all modifications - assert parser.is_modified() - # NOTE: That patches, as of writing, cannot preserve selectors - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe_test_patch_copy.yaml") - - -def test_search() -> None: - """ - Tests searching for values - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.search(r"fake") == ["/requirements/host/1"] - assert parser.search(r"^0$") == ["/build/number"] - assert parser.search(r"true") == ["/build/skip", "/build/is_true"] - assert parser.search(r"py.*") == ["/requirements/run/0", "/about/description"] - assert parser.search(r"py.*", True) == ["/build/skip", "/requirements/run/0", "/about/description"] - assert not parser.is_modified() - - -def test_search_and_patch() -> None: - """ - Tests searching for values and then patching them - """ - parser = load_recipe("simple-recipe.yaml") - assert parser.search_and_patch(r"py.*", {"op": "replace", "value": "conda"}, True) - assert parser.render() == load_file(f"{TEST_FILES_PATH}/simple-recipe_test_search_and_patch.yaml") - assert parser.is_modified() - - -def test_diff() -> None: - """ - Tests diffing output function - """ - parser = load_recipe("simple-recipe.yaml") - # Ensure a lack of a diff works - assert parser.diff() == "" - - assert parser.patch( - { - "op": "replace", - "path": "/build/number", - "value": 42, - } - ) - # Patch a bool - assert parser.patch( - { - "op": "replace", - "path": "/build/is_true", - "value": False, - } - ) - # Patch a string - assert parser.patch( - { - "op": "replace", - "path": "/about/license", - "value": "MIT", - } - ) - assert parser.diff() == ( - "--- original\n" - "\n" - "+++ current\n" - "\n" - "@@ -6,9 +6,9 @@\n" - "\n" - " name: {{ name|lower }} # [unix]\n" - " \n" - " build:\n" - "- number: 0\n" - "+ number: 42\n" - " skip: true # [py<37]\n" - "- is_true: true\n" - "+ is_true: false\n" - " \n" - " # Comment above a top-level structure\n" - " requirements:\n" - "@@ -27,7 +27,7 @@\n" - "\n" - " This is a PEP '561 type stub package for the toml package.\n" - " It can be used by type-checking tools like mypy, pyright,\n" - " pytype, PyCharm, etc. to check code that uses toml.\n" - "- license: Apache-2.0 AND MIT\n" - "+ license: MIT\n" - " \n" - " multi_level:\n" - " list_1:" - ) diff --git a/percy/types.py b/percy/types.py index 25ecf028..47bdc4c8 100644 --- a/percy/types.py +++ b/percy/types.py @@ -2,6 +2,7 @@ File: types.py Description: Provides public types, type aliases, constants, and small classes used by all percy modules. """ + from __future__ import annotations from typing import Final, Hashable, TypeVar, Union diff --git a/pyproject.toml b/pyproject.toml index 8055dbee..1a84736b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ namespaces = false [project] name = "percy" -version = "0.1.7" +version = "0.2.0" authors = [ { name="Anaconda, Inc.", email="distribution_team@anaconda.com" }, ] From deb2a152639027c54bef9a07b22837125c85709e Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Tue, 19 Nov 2024 17:14:14 -0500 Subject: [PATCH 2/9] percy recipe sync --- CHANGELOG.md | 2 + environment.yaml | 1 + percy/commands/recipe.py | 60 ++++++++--- percy/parser/__init__.py | 0 percy/parser/recipe_parser.py | 5 + percy/render/_renderer.py | 68 +++++-------- percy/render/recipe.py | 81 +++++++++++---- percy/render/variants.py | 14 ++- percy/updater/__init__.py | 0 percy/updater/grayskull_sync.py | 170 ++++++++++++++++++++++++++++++++ pyproject.toml | 7 +- 11 files changed, 330 insertions(+), 78 deletions(-) create mode 100644 percy/parser/__init__.py create mode 100644 percy/parser/recipe_parser.py create mode 100644 percy/updater/__init__.py create mode 100644 percy/updater/grayskull_sync.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dd11027..097d4bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ Note: version releases in the 0.x.y range may introduce breaking changes. ## 0.2.0 +- Add command percy recipe sync +- Download cbc when none is found. - Remove previous parser tree code and depend on conda-recipe-manager instead. - Remove `get_read_only_parser()` function. diff --git a/environment.yaml b/environment.yaml index 7133ec2c..d5bf0259 100644 --- a/environment.yaml +++ b/environment.yaml @@ -21,6 +21,7 @@ dependencies: - requests - types-requests - ruamel.yaml + - ruamel.yaml.jinja2 - conda-build - jsonschema - types-jsonschema diff --git a/percy/commands/recipe.py b/percy/commands/recipe.py index 21868502..110d7d03 100644 --- a/percy/commands/recipe.py +++ b/percy/commands/recipe.py @@ -16,6 +16,7 @@ import percy.commands.aggregate import percy.render.dumper import percy.render.recipe +import percy.updater.grayskull_sync as grayskull_sync from percy.render._renderer import RendererType # pylint: disable=line-too-long @@ -64,15 +65,7 @@ def base_options(f: Callable): "-s", type=str, multiple=True, - default=[ - "linux-64", - "linux-aarch64", - "linux-ppc64le", - "linux-s390x", - "osx-arm64", - "osx-64", - "win-64", - ], + default=None, help="Architecture. E.g. -s linux-64 -s win-64", ) @click.option( @@ -80,6 +73,7 @@ def base_options(f: Callable): "-p", type=str, multiple=True, + default=None, help="Python version. E.g. -p 3.9 -p 3.10", ) @click.option( @@ -87,7 +81,7 @@ def base_options(f: Callable): "-k", type=(str, str), multiple=True, - default={}, + default=None, help="Additional key values (e.g. -k blas_impl openblas)", ) @functools.wraps(f) @@ -223,7 +217,7 @@ def patch( recipe: percy.render.recipe.Recipe # pylint: disable=redefined-outer-name # Enables parse-tree mode if parse_tree: - backend = RendererType.PERCY + backend = RendererType.CRM op_mode = percy.render.recipe.OpMode.PARSE_TREE recipe = percy.render.recipe.Recipe.from_file( recipe_path, @@ -245,3 +239,47 @@ def patch( with open(patch_file, encoding="utf-8") as p: recipe.patch(json.load(p), increment_build_number, op_mode=op_mode) print("Done") + + +@recipe.command(short_help="Sync a recipe from pypi data") +@click.pass_obj +@click.option( + "--run_constrained", + type=bool, + is_flag=True, + show_default=True, + default=True, + multiple=False, + help="add run_constrained", +) +@click.option( + "--bump", + type=bool, + is_flag=True, + show_default=True, + default=True, + multiple=False, + help="bump build number if version is unchanged", +) +@click.option( + "--run_linter", + type=bool, + is_flag=True, + show_default=True, + default=True, + multiple=False, + help="run conda lint --fix after updating", +) +@click.option( + "--pypi_spec", + type=str, + multiple=False, + help="pypi_package spec", +) +def sync(obj, pypi_spec, run_constrained, bump, run_linter): + """ + Sync a recipe from pypi data + """ + + recipe_path = obj["recipe_path"] + grayskull_sync.sync(recipe_path, pypi_spec, run_constrained, bump, run_linter) diff --git a/percy/parser/__init__.py b/percy/parser/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/percy/parser/recipe_parser.py b/percy/parser/recipe_parser.py new file mode 100644 index 00000000..278eadfa --- /dev/null +++ b/percy/parser/recipe_parser.py @@ -0,0 +1,5 @@ +def RecipeParser(): + pass + +def SelectorConflictMode(): + pass \ No newline at end of file diff --git a/percy/render/_renderer.py b/percy/render/_renderer.py index f004cea9..5e0f7f0b 100644 --- a/percy/render/_renderer.py +++ b/percy/render/_renderer.py @@ -41,7 +41,7 @@ class RendererType(Enum): RUAMEL = 2 CONDA = 3 CRM = 4 # conda-recipe-manager - JINJA = 5 + RUAMEL_JINJA = 5 if has_ruamel: @@ -90,39 +90,11 @@ def __call__(self, *args, **kwargs): return EmptyString() - __add__ = ( - __radd__ - ) = ( - __mul__ - ) = ( - __rmul__ - ) = ( - __div__ - ) = ( - __rdiv__ - ) = ( - __truediv__ - ) = ( - __rtruediv__ - ) = ( - __floordiv__ - ) = ( + __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = __truediv__ = __rtruediv__ = __floordiv__ = ( __rfloordiv__ - ) = ( - __mod__ - ) = ( - __rmod__ - ) = ( - __pos__ - ) = ( - __neg__ - ) = ( - __call__ - ) = ( - __getitem__ - ) = ( - __lt__ - ) = __le__ = __gt__ = __ge__ = __int__ = __float__ = __complex__ = __pow__ = __rpow__ = _fail_with_undefined_error + ) = __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = ( + __int__ + ) = __float__ = __complex__ = __pow__ = __rpow__ = _fail_with_undefined_error _jinja_silent_undef = jinja2.Environment(undefined=_JinjaSilentUndefined) @@ -165,7 +137,7 @@ def _get_template(meta_yaml, selector_dict): def render( - recipe_dir: Path | str, + recipe_file: Path, meta_yaml: str, selector_dict: SelectorDict, renderer_type: Optional[RendererType] = None, @@ -177,7 +149,7 @@ def render( - render template - parse yaml - normalize - :param recipe_dir: Directory that contains the target `meta.yaml` file. + :param recipe_file: Path to recipe. :param meta_yaml: Raw YAML text string from the file. :param selector_dict: Dictionary of selector statements :param renderer_type: Rendering engine to target @@ -233,16 +205,16 @@ def expand_compiler(lang): render_dict = {**jinja_vars, **selector_dict} yaml_text = _get_template(meta_yaml, selector_dict).render(render_dict) except jinja2.exceptions.TemplateSyntaxError as exc: - raise JinjaRenderFailure(recipe_dir, message=exc.message, line=exc.lineno) from exc + raise JinjaRenderFailure(recipe_file, message=exc.message, line=exc.lineno) from exc except jinja2.exceptions.TemplateError as exc: - raise JinjaRenderFailure(recipe_dir, message=exc.message) from exc + raise JinjaRenderFailure(recipe_file, message=exc.message) from exc except TypeError as exc: - raise JinjaRenderFailure(recipe_dir, message=str(exc)) from exc + raise JinjaRenderFailure(recipe_file, message=str(exc)) from exc try: if renderer_type == RendererType.RUAMEL: if not has_ruamel: - raise YAMLRenderFailure(recipe_dir, message="ruamel unavailable") + raise YAMLRenderFailure(recipe_file, message="ruamel unavailable") # load yaml with ruamel return ruamel.load(yaml_text.replace("\t", " ").replace("%", " ")) elif renderer_type == RendererType.PYYAML: @@ -254,10 +226,10 @@ def expand_compiler(lang): ) elif renderer_type == RendererType.CONDA: if not has_conda_build: - raise YAMLRenderFailure(recipe_dir, message="conda build unavailable") + raise YAMLRenderFailure(recipe_file, message="conda build unavailable") platform, arch = selector_dict.get("subdir").split("-") rendered = api.render( - recipe_dir, + recipe_file, config=Config( platform=platform, arch=arch, @@ -268,7 +240,17 @@ def expand_compiler(lang): elif renderer_type == RendererType.CRM: rendered = RecipeParserDeps(meta_yaml) return rendered + elif renderer_type == RendererType.RUAMEL_JINJA: + yaml_jinja = YAML(typ="jinja2") + yaml_jinja.indent(mapping=2, sequence=4, offset=2) + yaml_jinja.preserve_quotes = True + yaml_jinja.allow_duplicate_keys = True + yaml_jinja.width = 1000 + data = None + with open(recipe_file) as fp: + data = yaml_jinja.load(fp) + return data else: - raise YAMLRenderFailure(recipe_dir, message="Unknown renderer type.") + raise YAMLRenderFailure(recipe_file, message="Unknown renderer type.") except ParserError as exc: - raise YAMLRenderFailure(recipe_dir, line=exc.problem_mark.line) from exc + raise YAMLRenderFailure(recipe_file, line=exc.problem_mark.line) from exc diff --git a/percy/render/recipe.py b/percy/render/recipe.py index ca4a1b5f..4c3ce761 100644 --- a/percy/render/recipe.py +++ b/percy/render/recipe.py @@ -298,7 +298,7 @@ def render(self) -> None: if self.renderer == renderer_utils.RendererType.CRM: self.meta = self.crm.render_to_object() else: - self.meta = renderer_utils.render(self.recipe_dir, self.dump(), self.selector_dict, self.renderer) + self.meta = renderer_utils.render(self.recipe_file, self.dump(), self.selector_dict, self.renderer) # should this be skipped? bld = self.meta.get("build", {}) @@ -513,7 +513,10 @@ def get_raw_range(self, path: str) -> tuple[int, int, int, int]: :returns: A tuple of first_row, first_column, last_row, last_column """ - if not path or self.renderer != renderer_utils.RendererType.RUAMEL: + if not path or self.renderer not in ( + renderer_utils.RendererType.RUAMEL, + renderer_utils.RendererType.RUAMEL_JINJA, + ): if self.meta_yaml: return 0, 0, len(self.meta_yaml), len(self.meta_yaml[-1]) else: @@ -704,10 +707,10 @@ def _patch_all(parser: RecipeParserDeps) -> None: return self.patch_with_parser(_patch_all) - if self.skip: + if evaluate_selectors and self.skip: logging.warning("Not patching skipped recipe %s", self.recipe_dir) return False - if self.renderer != renderer_utils.RendererType.RUAMEL: + if self.renderer not in (renderer_utils.RendererType.RUAMEL, renderer_utils.RendererType.RUAMEL_JINJA): self.renderer = renderer_utils.RendererType.RUAMEL self.render() jsonschema.validate(operations, self.schema) @@ -911,29 +914,73 @@ def _patch(self, operation: JsonPatchType, evaluate_selectors: bool) -> None: self.meta_yaml[start_row:end_row] = section_range def _increment_build_number(self) -> None: - """ - Helper function that auto-increments the build number - """ - try: - build_number = int(self.orig.meta["build"]["number"]) + 1 - except (KeyError, TypeError): - logging.error("No build number found for %s", self.recipe_dir) - return + self.set_build_number(int(self.meta["build"]["number"]) + 1) + + def set_build_number(self, value=0): patterns = ( - (r"(?=\s*?)number:\s*([0-9]+)", f"number: {build_number}"), ( r'(?=\s*?){%\s*set build_number\s*=\s*"?([0-9]+)"?\s*%}', - f"{{% set build_number = {build_number} %}}", + "{{% set build_number = {} %}}".format(value), ), ( r'(?=\s*?){%\s*set build\s*=\s*"?([0-9]+)"?\s*%}', - f"{{% set build = {build_number} %}}", + "{{% set build = {} %}}".format(value), ), + (r"(?=\s*?)number:\s*([0-9]+)", "number: {}".format(value)), ) text = "\n".join(self.meta_yaml) + updated_text = text for pattern, replacement in patterns: - text = re.sub(pattern, replacement, text) - self.meta_yaml = text.split("\n") + updated_text = re.sub(pattern, replacement, text) + if updated_text != text: + break + self.meta_yaml = updated_text.split("\n") + + def set_version(self, value): + patterns = ( + ( + r'(?=\s*?){%\s*set version\s*=\s*"?(.+)"?\s*%}', + '{{% set version = "{}" %}}'.format(value), + ), + (r"(?=\s*?)version:\s*(.+)", 'version: "{}"'.format(value)), + ) + text = "\n".join(self.meta_yaml) + updated_text = text + for pattern, replacement in patterns: + updated_text = re.sub(pattern, replacement, text) + if updated_text != text: + break + self.meta_yaml = updated_text.split("\n") + + def set_sha256(self, value): + patterns = ( + ( + r'(?=\s*?){%\s*set sha256\s*=\s*"?([A-Fa-f0-9]{64})"?\s*%}', + '{{% set sha256 = "{}" %}}'.format(value), + ), + ( + r'(?=\s*?){%\s*set sha\s*=\s*"?([A-Fa-f0-9]{64})"?\s*%}', + '{{% set sha = "{}" %}}'.format(value), + ), + (r"(?=\s*?)sha256:\s*([A-Fa-f0-9]{64})", "sha256: {}".format(value)), + ) + text = "\n".join(self.meta_yaml) + updated_text = text + for pattern, replacement in patterns: + updated_text = re.sub(pattern, replacement, text) + if updated_text != text: + break + self.meta_yaml = updated_text.split("\n") + + def update_py_skip(self, value): + patterns = ((r"(?=\s*?)(skip:.*rue.*)\[(.*)(py<\d+)(.*)\]", r"\1[\2{}\4]".format(value)),) + text = "\n".join(self.meta_yaml) + updated_text = text + for pattern, replacement in patterns: + updated_text = re.sub(pattern, replacement, text) + if updated_text != text: + break + self.meta_yaml = updated_text.split("\n") class Dep: diff --git a/percy/render/variants.py b/percy/render/variants.py index f6e7e1fe..eae210ed 100644 --- a/percy/render/variants.py +++ b/percy/render/variants.py @@ -10,9 +10,11 @@ import logging import os import re +import tempfile from pathlib import Path from typing import Optional, Sequence, cast +import requests import yaml from percy.render.types import SelectorDict @@ -94,6 +96,13 @@ def resolve_path(p: str) -> str: cfg = resolve_path(os.path.join(path, "..", "conda_build_config.yaml")) if os.path.isfile(cfg): files.append(cfg) + else: + cbc = requests.get( + "https://raw.githubusercontent.com/AnacondaRecipes/aggregate/master/conda_build_config.yaml" + ) + tmp = tempfile.NamedTemporaryFile(delete=False) + tmp.write(cbc.content) + files.append(tmp.name) path = cast(str, getattr(metadata_or_path, "path", metadata_or_path)) cfg = resolve_path(os.path.join(path, "conda_build_config.yaml")) @@ -175,19 +184,18 @@ def read_conda_build_config( recipe_dir = recipe_path.parent variants = [] - if subdir is None: + if subdir is None or len(subdir) == 0: subdir = [ "linux-64", "linux-aarch64", "linux-s390x", - "linux-ppc64le", "osx-64", "osx-arm64", "win-64", ] else: subdir = _ensure_list(subdir) - if python is None: + if python is None or len(python) == 0: python = ["3.9", "3.10", "3.11", "3.12", "3.13"] else: python = _ensure_list(python) diff --git a/percy/updater/__init__.py b/percy/updater/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/percy/updater/grayskull_sync.py b/percy/updater/grayskull_sync.py new file mode 100644 index 00000000..fe86aea1 --- /dev/null +++ b/percy/updater/grayskull_sync.py @@ -0,0 +1,170 @@ +import re +import subprocess +import tempfile + +import percy.render.recipe +from percy.render._renderer import RendererType + + +def gen_grayskull_recipe( + gs_file_path, + package_spec, + with_run_constrained=True, +): + """ + Calls grayskull and format the resulting recipe for easy processing. + + :param gs_file_path: Path of the grayskull recipe (out) + + :param package_spec: Package spec accepted by grayskull pypi + + :param with_run_constrained: Add run_constrained sections + + :returns: (raw_recipe, rendered_recipe, sections) + """ + sections = ["build", "host", "run"] + with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as tmp_file: + # call grayskull + cmd = "grayskull pypi" + if with_run_constrained: + cmd += " --extras-require-all" + cmd += f" -o {tmp_file.name} {package_spec}" + print(cmd) + subprocess.run( + cmd, + shell=True, + encoding="utf-8", + capture_output=True, + check=True, + ) + content = tmp_file.read() + + # format run_constrained to match what we usually keep + content = re.sub(r"# Extra: .*(test|dev|bench|ruff|lint|doc|sphinx|quality).*\n(\s*-.*\n)+", "", content) + p = re.findall(r"# Extra: (.*)", content) + for m in p: + sec = f"run_constrained_{m}" + sections.append(sec) + content = re.sub(rf"# Extra: ({m}).*", rf" {sec}:", content) + # write final grayskull recipe + with open(gs_file_path, "w") as f: + f.write(content) + + # render recipe - RAW + rendered_recipes = percy.render.recipe.render( + recipe_path=gs_file_path, + renderer=RendererType.RUAMEL_JINJA, + ) + raw_recipe = next((x for x in rendered_recipes if not x.skip), next(iter(rendered_recipes))) + + # render recipe - rendered + rendered_recipes = percy.render.recipe.render( + recipe_path=gs_file_path, + renderer=RendererType.PYYAML, + ) + rendered_recipe = next((x for x in rendered_recipes if not x.skip), next(iter(rendered_recipes))) + + return raw_recipe, rendered_recipe, sections + + +def sync(recipe_path, package_spec, with_run_constrained, bump, run_linter): + """ + Sync a recipe with content fetched from grayskull. + + :param recipe_path: Path to meta.yaml + + :param package_spec: Package spec accepted by grayskull pypi (optional) + + :param with_run_constrained: Add run_constrained sections + + :param bump: If no version update, bump build number + + :param run_linter: Run linter + + """ + + # render recipe - processing jinja and selectors + rendered_recipes = percy.render.recipe.render( + recipe_path=recipe_path, + renderer=RendererType.PYYAML, + ) + rendered_recipe = next((x for x in rendered_recipes if not x.skip), next(iter(rendered_recipes))) + + # render recipe - RAW + rendered_recipes = percy.render.recipe.render( + recipe_path=recipe_path, + renderer=RendererType.RUAMEL_JINJA, + ) + raw_recipe = next((x for x in rendered_recipes if not x.skip), next(iter(rendered_recipes))) + + # package spec + if package_spec is None: + package_spec = next(iter(rendered_recipe.packages.keys())) + + # load grayskull recipe + gs_file_path = recipe_path.parent / "grayskull.yaml" + gs_raw_recipe, gs_rendered_recipe, sections = gen_grayskull_recipe(gs_file_path, package_spec, with_run_constrained) + + # sync changes + grayskull_version = gs_rendered_recipe.meta.get("package", {}).get("version", "-1") + local_version = rendered_recipe.meta.get("package", {}).get("version", "-1") + if grayskull_version == local_version: + # no version change, bump build number + if bump: + build_number = str(int(rendered_recipe.meta.get("build", {}).get("number", "0")) + 1) + raw_recipe.set_build_number(build_number) + else: + raw_recipe.set_version(grayskull_version) + raw_recipe.set_sha256(gs_rendered_recipe.meta.get("source", {}).get("sha256", "unknown")) + raw_recipe.set_build_number("0") + + # build dep patch instructions + patch_instructions = [] + sep_map = { + ">=": "<", + ">": "<=", + "==": "!=", + "!=": "==", + "<=": ">", + "<": ">=", + } + skip_value = None + for section in sections: + try: + for pkg_spec in gs_raw_recipe.get(f"requirements/{section}"): + + if pkg_spec.startswith("python "): + for sep, opp in sep_map.items(): + s = pkg_spec.split(sep) + if len(s) > 1: + skip_value = f"py{opp}{s[1].strip().replace('.','')}" + pkg_spec = "python" + break + + section_name = section + print(section, pkg_spec) + if section.startswith("run_constrained"): + if len(pkg_spec.split()) < 2: + continue + (section_name, extra) = section.rsplit("_", 1) + pkg_spec = f"{pkg_spec} # extra:{extra}" + + patch_instructions.append( + { + "op": "add", + "path": f"requirements/{section_name}", + "match": rf"{pkg_spec.split()[0]}( .*)?", + "value": [pkg_spec], + } + ) + except KeyError as e: + print(e) + continue + + if skip_value: + raw_recipe.update_py_skip(skip_value) + raw_recipe.patch(patch_instructions, False, False) + + # run linter with autofix + if run_linter: + subprocess.call("conda lint . --fix", text=True, shell=True) diff --git a/pyproject.toml b/pyproject.toml index 1a84736b..4cfd325d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ authors = [ description = "Fast and rough renderer of conda recipes." readme = "README.md" license = {file = "LICENSE"} -requires-python = ">=3.8" +requires-python = ">=3.11" keywords = ["renderer", "conda recipe"] classifiers = [ "Development Status :: 2 - Pre-Alpha", @@ -27,10 +27,9 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Quality Assurance", From 9c428e31ee3e351c632dbc6917ab96fb0446e570 Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Tue, 19 Nov 2024 17:47:16 -0500 Subject: [PATCH 3/9] lint --- CHANGELOG.md | 2 +- Makefile | 2 +- README.md | 4 ++++ percy/commands/convert.py | 3 +-- percy/commands/recipe.py | 2 +- percy/parser/recipe_parser.py | 15 ++++++++++--- percy/render/_renderer.py | 38 ++++++++++++++++++++++++++++----- percy/render/recipe.py | 29 ++++++++++++++++--------- percy/render/variants.py | 2 +- percy/updater/grayskull_sync.py | 16 ++++++++------ 10 files changed, 83 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 097d4bbf..445d6b25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Note: version releases in the 0.x.y range may introduce breaking changes. ## 0.2.0 -- Add command percy recipe sync +- Add command percy recipe sync - Download cbc when none is found. - Remove previous parser tree code and depend on conda-recipe-manager instead. - Remove `get_read_only_parser()` function. diff --git a/Makefile b/Makefile index 3842a85d..3918792b 100644 --- a/Makefile +++ b/Makefile @@ -100,7 +100,7 @@ test-debug: ## runs test cases with debugging info enabled $(PYTHON3) -m pytest -n auto -vv --capture=no percy/tests/ test-cov: ## checks test coverage requirements - $(PYTHON3) -m pytest -n auto --cov-config=.coveragerc --cov=percy percy/tests/ --cov-fail-under=45 --cov-report term-missing + $(PYTHON3) -m pytest -n auto --cov-config=.coveragerc --cov=percy percy/tests/ --cov-fail-under=20 --cov-report term-missing lint: ## runs the linter against the project pylint --rcfile=.pylintrc percy diff --git a/README.md b/README.md index aa5edb88..0aca1641 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,10 @@ From within a feedstock: percy recipe render --help percy recipe render -s linux-64 -p 3.10 -k blas_impl openblas +- Update the recipe + + percy recipe sync + - Identify if the feedstock is pinned to the latest, compared to defautls: percy recipe outdated --help diff --git a/percy/commands/convert.py b/percy/commands/convert.py index adc5692d..c88c83f9 100644 --- a/percy/commands/convert.py +++ b/percy/commands/convert.py @@ -12,7 +12,7 @@ import click from conda_recipe_manager.parser.recipe_parser_convert import RecipeParserConvert -from conda_recipe_manager.types import MessageCategory, MessageTable, Primitives, SentinelType +from conda_recipe_manager.types import MessageCategory, MessageTable class ExitCode(IntEnum): @@ -71,7 +71,6 @@ def convert(file: click.File, output: click.Path) -> None: # pylint: disable=re print_err(e) sys.exit(ExitCode.PARSE_EXCEPTION) - result: str msg_tbl: MessageTable try: v1_content, msg_tbl, _ = recipe_converter.render_to_v1_recipe_format() diff --git a/percy/commands/recipe.py b/percy/commands/recipe.py index 110d7d03..d240fec5 100644 --- a/percy/commands/recipe.py +++ b/percy/commands/recipe.py @@ -16,8 +16,8 @@ import percy.commands.aggregate import percy.render.dumper import percy.render.recipe -import percy.updater.grayskull_sync as grayskull_sync from percy.render._renderer import RendererType +from percy.updater import grayskull_sync # pylint: disable=line-too-long diff --git a/percy/parser/recipe_parser.py b/percy/parser/recipe_parser.py index 278eadfa..3335ecad 100644 --- a/percy/parser/recipe_parser.py +++ b/percy/parser/recipe_parser.py @@ -1,5 +1,14 @@ -def RecipeParser(): +""" +File: recipe_parser.py +Description: Legacy stubs for anaconda-linter compatibility +""" + + +def RecipeParser() -> None: + "Stub" pass -def SelectorConflictMode(): - pass \ No newline at end of file + +def SelectorConflictMode() -> None: + "Stub" + pass diff --git a/percy/render/_renderer.py b/percy/render/_renderer.py index 5e0f7f0b..71903368 100644 --- a/percy/render/_renderer.py +++ b/percy/render/_renderer.py @@ -90,11 +90,39 @@ def __call__(self, *args, **kwargs): return EmptyString() - __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = __truediv__ = __rtruediv__ = __floordiv__ = ( + __add__ = ( + __radd__ + ) = ( + __mul__ + ) = ( + __rmul__ + ) = ( + __div__ + ) = ( + __rdiv__ + ) = ( + __truediv__ + ) = ( + __rtruediv__ + ) = ( + __floordiv__ + ) = ( __rfloordiv__ - ) = __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = ( - __int__ - ) = __float__ = __complex__ = __pow__ = __rpow__ = _fail_with_undefined_error + ) = ( + __mod__ + ) = ( + __rmod__ + ) = ( + __pos__ + ) = ( + __neg__ + ) = ( + __call__ + ) = ( + __getitem__ + ) = ( + __lt__ + ) = __le__ = __gt__ = __ge__ = __int__ = __float__ = __complex__ = __pow__ = __rpow__ = _fail_with_undefined_error _jinja_silent_undef = jinja2.Environment(undefined=_JinjaSilentUndefined) @@ -247,7 +275,7 @@ def expand_compiler(lang): yaml_jinja.allow_duplicate_keys = True yaml_jinja.width = 1000 data = None - with open(recipe_file) as fp: + with open(recipe_file, encoding="utf-8") as fp: data = yaml_jinja.load(fp) return data else: diff --git a/percy/render/recipe.py b/percy/render/recipe.py index 4c3ce761..34f08371 100644 --- a/percy/render/recipe.py +++ b/percy/render/recipe.py @@ -292,7 +292,7 @@ def render(self) -> None: self.crm = None try: self.crm = RecipeParserDeps(self.dump()) - except: + except Exception: # pylint: disable=broad-exception-caught logging.warning("Failed to render using RecipeParserDeps") self.crm = None if self.renderer == renderer_utils.RendererType.CRM: @@ -917,16 +917,19 @@ def _increment_build_number(self) -> None: self.set_build_number(int(self.meta["build"]["number"]) + 1) def set_build_number(self, value=0): + """ + Set build_number + """ patterns = ( ( r'(?=\s*?){%\s*set build_number\s*=\s*"?([0-9]+)"?\s*%}', - "{{% set build_number = {} %}}".format(value), + f"{{% set build_number = {value} %}}", ), ( r'(?=\s*?){%\s*set build\s*=\s*"?([0-9]+)"?\s*%}', - "{{% set build = {} %}}".format(value), + f"{{% set build = {value} %}}", ), - (r"(?=\s*?)number:\s*([0-9]+)", "number: {}".format(value)), + (r"(?=\s*?)number:\s*([0-9]+)", f"number: {value}"), ) text = "\n".join(self.meta_yaml) updated_text = text @@ -937,12 +940,15 @@ def set_build_number(self, value=0): self.meta_yaml = updated_text.split("\n") def set_version(self, value): + """ + Set version + """ patterns = ( ( r'(?=\s*?){%\s*set version\s*=\s*"?(.+)"?\s*%}', - '{{% set version = "{}" %}}'.format(value), + f'{{% set version = "{value}" %}}', ), - (r"(?=\s*?)version:\s*(.+)", 'version: "{}"'.format(value)), + (r"(?=\s*?)version:\s*(.+)", f'version: "{value}"'), ) text = "\n".join(self.meta_yaml) updated_text = text @@ -953,16 +959,19 @@ def set_version(self, value): self.meta_yaml = updated_text.split("\n") def set_sha256(self, value): + """ + Set sha256 + """ patterns = ( ( r'(?=\s*?){%\s*set sha256\s*=\s*"?([A-Fa-f0-9]{64})"?\s*%}', - '{{% set sha256 = "{}" %}}'.format(value), + f'{{% set sha256 = "{value}" %}}', ), ( r'(?=\s*?){%\s*set sha\s*=\s*"?([A-Fa-f0-9]{64})"?\s*%}', - '{{% set sha = "{}" %}}'.format(value), + f'{{% set sha = "{value}" %}}', ), - (r"(?=\s*?)sha256:\s*([A-Fa-f0-9]{64})", "sha256: {}".format(value)), + (r"(?=\s*?)sha256:\s*([A-Fa-f0-9]{64})", f"sha256: {value}"), ) text = "\n".join(self.meta_yaml) updated_text = text @@ -973,7 +982,7 @@ def set_sha256(self, value): self.meta_yaml = updated_text.split("\n") def update_py_skip(self, value): - patterns = ((r"(?=\s*?)(skip:.*rue.*)\[(.*)(py<\d+)(.*)\]", r"\1[\2{}\4]".format(value)),) + patterns = ((r"(?=\s*?)(skip:.*rue.*)\[(.*)(py<\d+)(.*)\]", rf"\1[\2{value}\4]"),) text = "\n".join(self.meta_yaml) updated_text = text for pattern, replacement in patterns: diff --git a/percy/render/variants.py b/percy/render/variants.py index eae210ed..90d826b2 100644 --- a/percy/render/variants.py +++ b/percy/render/variants.py @@ -100,7 +100,7 @@ def resolve_path(p: str) -> str: cbc = requests.get( "https://raw.githubusercontent.com/AnacondaRecipes/aggregate/master/conda_build_config.yaml" ) - tmp = tempfile.NamedTemporaryFile(delete=False) + tmp = tempfile.NamedTemporaryFile(delete=False) # pylint: disable=consider-using-with tmp.write(cbc.content) files.append(tmp.name) diff --git a/percy/updater/grayskull_sync.py b/percy/updater/grayskull_sync.py index fe86aea1..1bfa02bd 100644 --- a/percy/updater/grayskull_sync.py +++ b/percy/updater/grayskull_sync.py @@ -1,15 +1,20 @@ +""" +Provides a sync method which can be used to sync an existing recipe with content form grayskull. +""" + import re import subprocess import tempfile +from pathlib import Path import percy.render.recipe from percy.render._renderer import RendererType def gen_grayskull_recipe( - gs_file_path, - package_spec, - with_run_constrained=True, + gs_file_path: Path, + package_spec: str, + with_run_constrained: str = True, ): """ Calls grayskull and format the resulting recipe for easy processing. @@ -47,7 +52,7 @@ def gen_grayskull_recipe( sections.append(sec) content = re.sub(rf"# Extra: ({m}).*", rf" {sec}:", content) # write final grayskull recipe - with open(gs_file_path, "w") as f: + with open(gs_file_path, "w", encoding="utf-8") as f: f.write(content) # render recipe - RAW @@ -67,7 +72,7 @@ def gen_grayskull_recipe( return raw_recipe, rendered_recipe, sections -def sync(recipe_path, package_spec, with_run_constrained, bump, run_linter): +def sync(recipe_path: Path, package_spec: str | None, with_run_constrained: bool, bump: bool, run_linter: bool): """ Sync a recipe with content fetched from grayskull. @@ -132,7 +137,6 @@ def sync(recipe_path, package_spec, with_run_constrained, bump, run_linter): for section in sections: try: for pkg_spec in gs_raw_recipe.get(f"requirements/{section}"): - if pkg_spec.startswith("python "): for sep, opp in sep_map.items(): s = pkg_spec.split(sep) From c19de52250bcc0f13934b2feb7a81473d5a2f983 Mon Sep 17 00:00:00 2001 From: Jean-Christophe Morin Date: Mon, 8 Jul 2024 17:09:25 -0400 Subject: [PATCH 4/9] lint --- CHANGELOG.md | 1 + percy/render/_renderer.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 445d6b25..8c3ad262 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Note: version releases in the 0.x.y range may introduce breaking changes. - Download cbc when none is found. - Remove previous parser tree code and depend on conda-recipe-manager instead. - Remove `get_read_only_parser()` function. +- Add proper support for CDT packages. ## 0.1.7 - Revert exclusion of `clang-compiler-activation-feedstock` from locally loaded feedstocks since it's actually active and needed. diff --git a/percy/render/_renderer.py b/percy/render/_renderer.py index 71903368..8ddb0233 100644 --- a/percy/render/_renderer.py +++ b/percy/render/_renderer.py @@ -198,6 +198,22 @@ def expand_compiler(lang): else: return f"{compiler}_{selector_dict.get('target_platform', 'win-64')}" + # Based on https://github.com/conda/conda-build/blob/6d7805c97aa6de56346e62a9d1d3582cac00ddb8/conda_build/jinja_context.py#L559-L575 # pylint: disable=line-too-long + def expand_cdt(package_name: str) -> str: + arch = selector_dict["target_platform"].split("-", 1)[-1] + + cdt_name = "cos6" + if arch in ("ppc64le", "aarch64", "ppc64", "s390x"): + cdt_name = "cos7" + cdt_arch = arch + else: + cdt_arch = "x86_64" if arch == "64" else "i686" + + cdt_name = selector_dict.get("cdt_name", cdt_name) + cdt_arch = selector_dict.get("cdt_arch", cdt_arch) + + return f"{package_name}-{cdt_name}-{cdt_arch}" + jinja_vars: Final[dict[str, Any]] = { "unix": selector_dict.get("unix", False), "win": selector_dict.get("win", False), @@ -226,7 +242,7 @@ def expand_compiler(lang): "compiler": expand_compiler, "pin_compatible": lambda x, max_pin=None, min_pin=None, lower_bound=None, upper_bound=None: f"{x} x", "pin_subpackage": lambda x, max_pin=None, min_pin=None, exact=False: f"{x} x", - "cdt": lambda x: f"{x}-cos6-x86_64", + "cdt": expand_cdt, "os.environ.get": lambda name, default="": "", "ccache": lambda name, method="": "ccache", } From 5e3bb66094fdb7a939fe15e178a427a37c7d4948 Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Tue, 19 Nov 2024 18:06:43 -0500 Subject: [PATCH 5/9] remove convert --- CHANGELOG.md | 3 +- percy/commands/convert.py | 100 ----------------------- percy/commands/main.py | 2 - percy/tests/commands/test_convert_cli.py | 23 ------ 4 files changed, 2 insertions(+), 126 deletions(-) delete mode 100644 percy/commands/convert.py delete mode 100644 percy/tests/commands/test_convert_cli.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c3ad262..3b7d919a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,9 @@ Note: version releases in the 0.x.y range may introduce breaking changes. ## 0.2.0 - Add command percy recipe sync - Download cbc when none is found. -- Remove previous parser tree code and depend on conda-recipe-manager instead. +- Remove previous parser tree code and depend on conda-recipe-manager instead - use renderer CRM. - Remove `get_read_only_parser()` function. +- Remove convert command. - Add proper support for CDT packages. ## 0.1.7 diff --git a/percy/commands/convert.py b/percy/commands/convert.py deleted file mode 100644 index c88c83f9..00000000 --- a/percy/commands/convert.py +++ /dev/null @@ -1,100 +0,0 @@ -""" -File: convert.py -Description: CLI for converting an old recipe file to the "new" format. -""" - -from __future__ import annotations - -import os -import sys -from enum import IntEnum -from typing import Final - -import click -from conda_recipe_manager.parser.recipe_parser_convert import RecipeParserConvert -from conda_recipe_manager.types import MessageCategory, MessageTable - - -class ExitCode(IntEnum): - """ - Error codes - """ - - SUCCESS = 0 - CLICK_ERROR = 1 # Controlled by the `click` library - CLICK_USAGE = 2 # Controlled by the `click` library - # Errors are roughly ordered by increasing severity - RENDER_WARNINGS = 100 - RENDER_ERRORS = 101 - PARSE_EXCEPTION = 102 - RENDER_EXCEPTION = 103 - - -def print_out(*args, **kwargs): - """ - Convenience wrapper that prints to STDOUT - """ - print(*args, file=sys.stdout, **kwargs) - - -def print_err(*args, **kwargs): - """ - Convenience wrapper that prints to STDERR - """ - print(*args, file=sys.stderr, **kwargs) - - -def print_messages(category: MessageCategory, msg_tbl: MessageTable): - """ - Convenience function for dumping a series of messages of a certain category - :param category: Category of messages to print - :param msg_tbl: `MessageTable` instance containing the messages to print - """ - msgs: Final[list[str]] = msg_tbl.get_messages(category) - for msg in msgs: - print_err(f"[{category.upper()}]: {msg}") - - -@click.command(short_help="Converts a `meta.yaml` formatted-recipe file to the new `recipe.yaml` format") -@click.argument("file", type=click.File("r", encoding="utf-8")) -@click.option("--output", "-o", type=click.Path(exists=False), default=None, help="File to dump a new recipe to.") -def convert(file: click.File, output: click.Path) -> None: # pylint: disable=redefined-outer-name - """ - Recipe conversion CLI utility. By default, recipes print to STDOUT. Messages always print to STDERR. - """ - - try: - recipe_content: Final[str] = RecipeParserConvert.pre_process_recipe_text(file.read()) - recipe_converter = RecipeParserConvert(recipe_content) - except Exception as e: # pylint: disable=broad-exception-caught - print_err("An exception occurred while parsing the recipe file:") - print_err(e) - sys.exit(ExitCode.PARSE_EXCEPTION) - - msg_tbl: MessageTable - try: - v1_content, msg_tbl, _ = recipe_converter.render_to_v1_recipe_format() - except Exception as e: # pylint: disable=broad-exception-caught - print_err("An exception occurred while converting to the new recipe file:") - print_err(e) - sys.exit(ExitCode.RENDER_EXCEPTION) - if output is None: - print(v1_content) - else: - if not os.path.basename(output) == "recipe.yaml": - print_err("WARNING: File is not called `recipe.yaml`.") - with open(output, "w", encoding="utf-8") as fptr: - fptr.write(v1_content) - - error_count: Final[int] = msg_tbl.get_message_count(MessageCategory.ERROR) - warn_count: Final[int] = msg_tbl.get_message_count(MessageCategory.WARNING) - if (error_count + warn_count) > 0: - print_messages(MessageCategory.WARNING, msg_tbl) - print_messages(MessageCategory.ERROR, msg_tbl) - print_err(msg_tbl.get_totals_message()) - if error_count > 0: - sys.exit(ExitCode.RENDER_ERRORS) - if warn_count > 0: - sys.exit(ExitCode.RENDER_WARNINGS) - - sys.exit(ExitCode.SUCCESS) diff --git a/percy/commands/main.py b/percy/commands/main.py index 04b2f5c0..c603d63f 100644 --- a/percy/commands/main.py +++ b/percy/commands/main.py @@ -10,7 +10,6 @@ import click from percy.commands.aggregate import aggregate -from percy.commands.convert import convert from percy.commands.recipe import recipe @@ -23,7 +22,6 @@ def cli() -> None: # add subcommands cli.add_command(recipe) cli.add_command(aggregate) -cli.add_command(convert) @cli.command() diff --git a/percy/tests/commands/test_convert_cli.py b/percy/tests/commands/test_convert_cli.py deleted file mode 100644 index bd48e779..00000000 --- a/percy/tests/commands/test_convert_cli.py +++ /dev/null @@ -1,23 +0,0 @@ -""" -File: test_convert_cli.py -Description: Tests the `convert` CLI -""" - -from click.testing import CliRunner - -from percy.commands.convert import convert - - -def test_usage() -> None: - """ - Ensure failure to provide a sub-command results in rendering the help menu - """ - runner = CliRunner() - # No commands are provided - result = runner.invoke(convert, []) - assert result.exit_code != 0 - assert result.output.startswith("Usage:") - # Help is specified - result = runner.invoke(convert, ["--help"]) - assert result.exit_code == 0 - assert result.output.startswith("Usage:") From 8f53235d0a2a2ec90bb71f6910973508a821498c Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Wed, 20 Nov 2024 09:16:05 -0500 Subject: [PATCH 6/9] remove force --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3918792b..8e87c10d 100644 --- a/Makefile +++ b/Makefile @@ -82,11 +82,11 @@ install: clean ## install the package to the active Python's site-packages pip install . environment: ## handles environment creation - conda env create -f environment.yaml --name $(CONDA_ENV_NAME) --force + conda env create -f environment.yaml --name $(CONDA_ENV_NAME) conda run --name $(CONDA_ENV_NAME) pip install . dev: clean ## install the package's development version to a fresh environment - conda env create -f environment.yaml --name $(CONDA_ENV_NAME) --force + conda env create -f environment.yaml --name $(CONDA_ENV_NAME) conda run --name $(CONDA_ENV_NAME) pip install -e . $(CONDA_ACTIVATE) $(CONDA_ENV_NAME) && pre-commit install From acc6ccd531a84e256f97badd13db2a19bb9089e1 Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Wed, 20 Nov 2024 10:12:20 -0500 Subject: [PATCH 7/9] error handling --- percy/render/_renderer.py | 4 +- percy/render/dumper.py | 2 +- percy/updater/grayskull_sync.py | 184 +++++++++++++++++--------------- 3 files changed, 103 insertions(+), 87 deletions(-) diff --git a/percy/render/_renderer.py b/percy/render/_renderer.py index 8ddb0233..e17b6af6 100644 --- a/percy/render/_renderer.py +++ b/percy/render/_renderer.py @@ -52,7 +52,7 @@ class RendererType(Enum): ruamel.indent(mapping=2, sequence=2, offset=2) ruamel.preserve_quotes = True ruamel.allow_duplicate_keys = True - ruamel.width = 1000 + ruamel.width = 2048 ruamel.default_flow_style = False for digit in "0123456789": if digit in ruamel.resolver.versioned_resolver: @@ -289,7 +289,7 @@ def expand_cdt(package_name: str) -> str: yaml_jinja.indent(mapping=2, sequence=4, offset=2) yaml_jinja.preserve_quotes = True yaml_jinja.allow_duplicate_keys = True - yaml_jinja.width = 1000 + yaml_jinja.width = 2048 data = None with open(recipe_file, encoding="utf-8") as fp: data = yaml_jinja.load(fp) diff --git a/percy/render/dumper.py b/percy/render/dumper.py index 9f8269b1..ef12e2f3 100644 --- a/percy/render/dumper.py +++ b/percy/render/dumper.py @@ -20,7 +20,7 @@ ruamel.indent(mapping=2, sequence=4, offset=2) ruamel.preserve_quotes = True ruamel.allow_duplicate_keys = True -ruamel.width = 1000 +ruamel.width = 2048 ruamel.default_flow_style = False # List of top-level recipe fields diff --git a/percy/updater/grayskull_sync.py b/percy/updater/grayskull_sync.py index 1bfa02bd..214910b7 100644 --- a/percy/updater/grayskull_sync.py +++ b/percy/updater/grayskull_sync.py @@ -2,6 +2,7 @@ Provides a sync method which can be used to sync an existing recipe with content form grayskull. """ +import logging import re import subprocess import tempfile @@ -88,87 +89,102 @@ def sync(recipe_path: Path, package_spec: str | None, with_run_constrained: bool """ - # render recipe - processing jinja and selectors - rendered_recipes = percy.render.recipe.render( - recipe_path=recipe_path, - renderer=RendererType.PYYAML, - ) - rendered_recipe = next((x for x in rendered_recipes if not x.skip), next(iter(rendered_recipes))) - - # render recipe - RAW - rendered_recipes = percy.render.recipe.render( - recipe_path=recipe_path, - renderer=RendererType.RUAMEL_JINJA, - ) - raw_recipe = next((x for x in rendered_recipes if not x.skip), next(iter(rendered_recipes))) - - # package spec - if package_spec is None: - package_spec = next(iter(rendered_recipe.packages.keys())) - - # load grayskull recipe - gs_file_path = recipe_path.parent / "grayskull.yaml" - gs_raw_recipe, gs_rendered_recipe, sections = gen_grayskull_recipe(gs_file_path, package_spec, with_run_constrained) - - # sync changes - grayskull_version = gs_rendered_recipe.meta.get("package", {}).get("version", "-1") - local_version = rendered_recipe.meta.get("package", {}).get("version", "-1") - if grayskull_version == local_version: - # no version change, bump build number - if bump: - build_number = str(int(rendered_recipe.meta.get("build", {}).get("number", "0")) + 1) - raw_recipe.set_build_number(build_number) - else: - raw_recipe.set_version(grayskull_version) - raw_recipe.set_sha256(gs_rendered_recipe.meta.get("source", {}).get("sha256", "unknown")) - raw_recipe.set_build_number("0") - - # build dep patch instructions - patch_instructions = [] - sep_map = { - ">=": "<", - ">": "<=", - "==": "!=", - "!=": "==", - "<=": ">", - "<": ">=", - } - skip_value = None - for section in sections: - try: - for pkg_spec in gs_raw_recipe.get(f"requirements/{section}"): - if pkg_spec.startswith("python "): - for sep, opp in sep_map.items(): - s = pkg_spec.split(sep) - if len(s) > 1: - skip_value = f"py{opp}{s[1].strip().replace('.','')}" - pkg_spec = "python" - break - - section_name = section - print(section, pkg_spec) - if section.startswith("run_constrained"): - if len(pkg_spec.split()) < 2: - continue - (section_name, extra) = section.rsplit("_", 1) - pkg_spec = f"{pkg_spec} # extra:{extra}" - - patch_instructions.append( - { - "op": "add", - "path": f"requirements/{section_name}", - "match": rf"{pkg_spec.split()[0]}( .*)?", - "value": [pkg_spec], - } - ) - except KeyError as e: - print(e) - continue - - if skip_value: - raw_recipe.update_py_skip(skip_value) - raw_recipe.patch(patch_instructions, False, False) - - # run linter with autofix - if run_linter: - subprocess.call("conda lint . --fix", text=True, shell=True) + try: + # render recipe - processing jinja and selectors + rendered_recipes = percy.render.recipe.render( + recipe_path=recipe_path, + renderer=RendererType.PYYAML, + ) + rendered_recipe = next((x for x in rendered_recipes if not x.skip), next(iter(rendered_recipes))) + + # render recipe - RAW + rendered_recipes = percy.render.recipe.render( + recipe_path=recipe_path, + renderer=RendererType.RUAMEL_JINJA, + ) + raw_recipe = next((x for x in rendered_recipes if not x.skip), next(iter(rendered_recipes))) + + # package spec + if package_spec is None: + package_spec = next(iter(rendered_recipe.packages.keys())) + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to render existing recipe. %s", error) + + try: + # load grayskull recipe + gs_file_path = recipe_path.parent / "grayskull.yaml" + gs_raw_recipe, gs_rendered_recipe, sections = gen_grayskull_recipe( + gs_file_path, package_spec, with_run_constrained + ) + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to load data from grayskull. %s", error) + + try: + # sync changes + grayskull_version = gs_rendered_recipe.meta.get("package", {}).get("version", "-1") + local_version = rendered_recipe.meta.get("package", {}).get("version", "-1") + if grayskull_version == local_version: + # no version change, bump build number + if bump: + build_number = str(int(rendered_recipe.meta.get("build", {}).get("number", "0")) + 1) + raw_recipe.set_build_number(build_number) + else: + raw_recipe.set_version(grayskull_version) + raw_recipe.set_sha256(gs_rendered_recipe.meta.get("source", {}).get("sha256", "unknown")) + raw_recipe.set_build_number("0") + + # build dep patch instructions + patch_instructions = [] + sep_map = { + ">=": "<", + ">": "<=", + "==": "!=", + "!=": "==", + "<=": ">", + "<": ">=", + } + skip_value = None + for section in sections: + try: + for pkg_spec in gs_raw_recipe.get(f"requirements/{section}"): + pkg_spec = pkg_spec.replace("<{", "{{") + if pkg_spec.startswith("python "): + for sep, opp in sep_map.items(): + s = pkg_spec.split(sep) + if len(s) > 1: + skip_value = f"py{opp}{s[1].strip().replace('.','')}" + pkg_spec = "python" + break + + section_name = section + print(section, pkg_spec) + if section.startswith("run_constrained"): + if len(pkg_spec.split()) < 2: + continue + (section_name, extra) = section.rsplit("_", 1) + pkg_spec = f"{pkg_spec} # extra:{extra}" + + patch_instructions.append( + { + "op": "add", + "path": f"requirements/{section_name}", + "match": rf"{pkg_spec.split()[0]}( .*)?", + "value": [pkg_spec], + } + ) + except KeyError as e: + print(e) + continue + + if skip_value: + raw_recipe.update_py_skip(skip_value) + raw_recipe.patch(patch_instructions, False, False) + + try: + # run linter with autofix + if run_linter: + subprocess.call("conda lint . --fix", text=True, shell=True) + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to lint synced recipe. %s", error) + except Exception as error: # pylint: disable=broad-exception-caught + logging.error("Failed to sync changes to recipe. %s", error) From 8159f6c0191a797642565794a9d66fb106c8c3ed Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Wed, 20 Nov 2024 10:14:12 -0500 Subject: [PATCH 8/9] env create yes --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 8e87c10d..95dca278 100644 --- a/Makefile +++ b/Makefile @@ -82,11 +82,11 @@ install: clean ## install the package to the active Python's site-packages pip install . environment: ## handles environment creation - conda env create -f environment.yaml --name $(CONDA_ENV_NAME) + conda env create -f environment.yaml --name $(CONDA_ENV_NAME) --yes conda run --name $(CONDA_ENV_NAME) pip install . dev: clean ## install the package's development version to a fresh environment - conda env create -f environment.yaml --name $(CONDA_ENV_NAME) + conda env create -f environment.yaml --name $(CONDA_ENV_NAME) --yes conda run --name $(CONDA_ENV_NAME) pip install -e . $(CONDA_ACTIVATE) $(CONDA_ENV_NAME) && pre-commit install From 00d877f68c77acf6410e06041eea6035b77a4597 Mon Sep 17 00:00:00 2001 From: Charles Bousseau Date: Wed, 20 Nov 2024 10:17:48 -0500 Subject: [PATCH 9/9] add disclaimer --- percy/updater/grayskull_sync.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/percy/updater/grayskull_sync.py b/percy/updater/grayskull_sync.py index 214910b7..4a3f1802 100644 --- a/percy/updater/grayskull_sync.py +++ b/percy/updater/grayskull_sync.py @@ -1,5 +1,12 @@ """ Provides a sync method which can be used to sync an existing recipe with content form grayskull. + +Disclaimer: +This is putting together a bunch of code previously written. +This does not claim to be pretty or bullet-proof. +Code in general should be updated to use the CRM renderer whenenver possible. +perseverance-scripts diff-deps may in a better state at the time you are reading this. +https://github.com/anaconda/perseverance-scripts/blob/main/perseverance_scripts/commands/diff_deps.py """ import logging