Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Preprocessor: Replace dot with bar functions #67

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
4010ae4
Adds some missing build section transforms
schuylermartin45 May 21, 2024
dd22227
Improves deprecated field system by moving the logic to one function
schuylermartin45 May 21, 2024
f668ab4
Modifies unit tests to handle messaging changes
schuylermartin45 May 21, 2024
805941c
Redacts license family from the conda recipe so the conversion build …
schuylermartin45 May 21, 2024
c3bc1ae
Conversion process will no longer emit an emtpy context object
schuylermartin45 May 22, 2024
682afaa
Ensures canonical ordering of the Python test element
schuylermartin45 May 22, 2024
47e13a0
Preserves insert-order for patch add ops
schuylermartin45 May 22, 2024
54f287b
Adds in missing import; caused by bad git parts command
schuylermartin45 May 22, 2024
dc2607f
Makes pre-commit happy
schuylermartin45 May 22, 2024
6b6712c
Adds support for upgrading _most_ `script_env` situations
schuylermartin45 May 23, 2024
81c2588
Adds unit tests for script-env work
schuylermartin45 May 23, 2024
8bd0ae3
Adds known edge case that is not currently supported into the unit te…
schuylermartin45 May 23, 2024
004dde7
Merge branch 'main' of github.com:conda-incubator/conda-recipe-manage…
schuylermartin45 May 24, 2024
4ac6b73
Adds a fix for recipes that include a hash_type JINJA variable as a k…
schuylermartin45 May 28, 2024
c95bd7d
Adds unit tests for hash_type preprocessor replacement
schuylermartin45 May 28, 2024
3113440
Adds some pre-processing support for converting functions invoked wit…
schuylermartin45 May 30, 2024
62a6495
Adds unit tests for dot function support
schuylermartin45 May 30, 2024
367057b
Merge branch 'main' of github.com:conda-incubator/conda-recipe-manage…
schuylermartin45 Jun 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion conda_recipe_manager/parser/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,16 +127,36 @@ class Regex:
PRE_PROCESS_JINJA_HASH_TYPE_KEY: Final[re.Pattern[str]] = re.compile(
r"'{0,1}\{\{ (hash_type|hash|hashtype) \}\}'{0,1}:"
)
# Finds set statements that use dot functions over piped functions (`foo.replace(...)` vs `foo | replace(...)`).
# Group 1 and Group 2 match the left and right sides of the period mark.
PRE_PROCESS_JINJA_DOT_FUNCTION_IN_ASSIGNMENT: Final[re.Pattern[str]] = re.compile(
r"(\{%\s*set.*=.*)\.(.*\(.*\)\s*%\})"
)
PRE_PROCESS_JINJA_DOT_FUNCTION_IN_SUBSTITUTION: Final[re.Pattern[str]] = re.compile(
r"(\{\{\s*[a-zA-Z0-9_]*.*)\.([a-zA-Z0-9_]*\(.*\)\s*\}\})"
)
# Strips empty parenthesis artifacts on functions like `| lower`
PRE_PROCESS_JINJA_DOT_FUNCTION_STRIP_EMPTY_PARENTHESIS = re.compile(r"(\|\s*(lower|upper))(\(\))")

## Jinja regular expressions ##
JINJA_SUB: Final[re.Pattern[str]] = re.compile(r"{{\s*" + _JINJA_VAR_FUNCTION_PATTERN + r"\s*}}")
JINJA_FUNCTION_LOWER: Final[re.Pattern[str]] = re.compile(r"\|\s*lower")
JINJA_LINE: Final[re.Pattern[str]] = re.compile(r"({%.*%}|{#.*#})\n")
JINJA_SET_LINE: Final[re.Pattern[str]] = re.compile(r"{%\s*set\s*" + _JINJA_VAR_FUNCTION_PATTERN + r"\s*=.*%}\s*\n")
# Useful for replacing the older `{{` JINJA substitution with the newer `${{` WITHOUT accidentally doubling-up the
# newer syntax when multiple replacements are possible.
JINJA_REPLACE_V0_STARTING_MARKER: Final[re.Pattern[str]] = re.compile(r"(?<!\$)\{\{")

# All recognized JINJA functions are kept in a set for the convenience of trying to match against all of them.
# Group 1 contains the function name, Group 2 contains the arguments, if any.
JINJA_FUNCTION_LOWER: Final[re.Pattern[str]] = re.compile(r"\|\s*(lower)")
JINJA_FUNCTION_UPPER: Final[re.Pattern[str]] = re.compile(r"\|\s*(upper)")
JINJA_FUNCTION_REPLACE: Final[re.Pattern[str]] = re.compile(r"\|\s*(replace)\((.*)\)")
JINJA_FUNCTIONS_SET: Final[set[re.Pattern[str]]] = {
JINJA_FUNCTION_LOWER,
JINJA_FUNCTION_UPPER,
JINJA_FUNCTION_REPLACE,
}

SELECTOR: Final[re.Pattern[str]] = re.compile(r"\[.*\]")
# Detects the 6 common variants (3 |'s, 3 >'s). See this guide for more info:
# https://stackoverflow.com/questions/3790454/how-do-i-break-a-string-in-yaml-over-multiple-lines/21699210
Expand Down
16 changes: 16 additions & 0 deletions conda_recipe_manager/parser/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from __future__ import annotations

import json
import re
from collections.abc import Iterable
from typing import Final, cast

from conda_recipe_manager.parser._types import (
Expand Down Expand Up @@ -200,3 +202,17 @@ def set_key_conditionally(dictionary: dict[str, JsonType], key: str, value: Json
"""
if value:
dictionary[key] = value


def search_any_regex(re_set: Iterable[re.Pattern[str]], s: str) -> bool:
"""
Convenience function that checks a string against many regular expressions
:param re_set: Set of regular expressions to check
:param s: Target string
:returns: True if any regex in the set matches. False otherwise.
"""
for r in re_set:
if r.search(s):
return True

return False
10 changes: 4 additions & 6 deletions conda_recipe_manager/parser/recipe_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,20 +241,18 @@ def _render_jinja_vars(self, s: str) -> JsonType:
"""
# Search the string, replacing all substitutions we can recognize
for match in cast(list[str], Regex.JINJA_SUB.findall(s)):
lower_case = False
# The regex guarantees the string starts and ends with double braces
key = match[2:-2].strip()
# A brief search through `aggregate` shows that `|lower` is a commonly used Jinja command. Few, if any,
# other commands are used, as of writing. If others are found, we might need to support them here.
# Check for and interpret common JINJA functions
# TODO add support for UPPER and REPLACE
lower_match = Regex.JINJA_FUNCTION_LOWER.search(key)
if lower_match is not None:
lower_case = True
if lower_match:
key = key.replace(lower_match.group(), "").strip()

if key in self._vars_tbl:
# Replace value as a string. Re-interpret the entire value before returning.
value = str(self._vars_tbl[key])
if lower_case:
if lower_match:
value = value.lower()
s = s.replace(match, value)
return cast(JsonType, yaml.safe_load(s))
Expand Down
18 changes: 17 additions & 1 deletion conda_recipe_manager/parser/recipe_parser_convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,12 @@
V1_SOURCE_SECTION_KEY_SORT_ORDER,
Regex,
)
from conda_recipe_manager.parser._utils import set_key_conditionally, stack_path_to_str, str_to_stack_path
from conda_recipe_manager.parser._utils import (
search_any_regex,
set_key_conditionally,
stack_path_to_str,
str_to_stack_path,
)
from conda_recipe_manager.parser.recipe_parser import RecipeParser
from conda_recipe_manager.parser.types import CURRENT_RECIPE_SCHEMA_FORMAT, MessageCategory, MessageTable
from conda_recipe_manager.types import JsonPatchType, JsonType, Primitives, SentinelType
Expand Down Expand Up @@ -153,6 +158,9 @@ def _upgrade_jinja_to_context_obj(self) -> None:
if not isinstance(value, (str, int, float, bool)):
self._msg_tbl.add_message(MessageCategory.WARNING, f"The variable `{name}` is an unsupported type.")
continue
# Function calls need to preserve JINJA escaping or else they turn into unevaluated strings.
if isinstance(value, str) and search_any_regex(Regex.JINJA_FUNCTIONS_SET, value):
value = "{{" + value + "}}"
context_obj[name] = value
# Ensure that we do not include an empty context object (which is forbidden by the schema).
if context_obj:
Expand Down Expand Up @@ -622,6 +630,14 @@ def pre_process_recipe_text(content: str) -> str:
:param content: Recipe file contents to pre-process
:returns: Pre-processed recipe file contents
"""
# Some recipes use `foo.<function()>` instead of `{{ foo | <function()> }}` in JINJA statements. This causes
# rattler-build to fail with `invalid operation: object has no method named <function()>`
# NOTE: This is currently done BEFORE converting to use `env.get()` to wipe-out those changes.
content = Regex.PRE_PROCESS_JINJA_DOT_FUNCTION_IN_ASSIGNMENT.sub(r"\1 | \2", content)
content = Regex.PRE_PROCESS_JINJA_DOT_FUNCTION_IN_SUBSTITUTION.sub(r"\1 | \2", content)
# Strip any problematic parenthesis that may be left over from the previous operations.
content = Regex.PRE_PROCESS_JINJA_DOT_FUNCTION_STRIP_EMPTY_PARENTHESIS.sub(r"\1", content)

# Convert the old JINJA `environ[""]` variable usage to the new `get.env("")` syntax.
# NOTE:
# - This is mostly used by Bioconda recipes and R-based-packages in the `license_file` field.
Expand Down
4 changes: 3 additions & 1 deletion tests/parser/test_recipe_parser_convert.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@
@pytest.mark.parametrize(
"input_file,expected_file",
[
#
# {{ hash_type }} as a sha256 key replacement
("hash_type_replacement.yaml", "pre_processor/pp_hash_type_replacement.yaml"),
# Environment syntax replacement
("simple-recipe_environ.yaml", "pre_processor/pp_simple-recipe_environ.yaml"),
# Dot-function for pipe equivalent replacement
("dot_function_replacement.yaml", "pre_processor/pp_dot_function_replacement.yaml"),
# Unchanged file
("simple-recipe.yaml", "simple-recipe.yaml"),
],
Expand Down
29 changes: 29 additions & 0 deletions tests/test_aux_files/dot_function_replacement.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{% set name = dot_function_replacement.replace("foo", "bar") %}
{% set version = foo.upper() %}

package:
name: {{ name.lower() }}
version: {{ version.replace("foo", "bar") }}

source:
url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/script-env-{{ version }}.tar.gz
sha256: 6d3ac79e36c9ee593c5d4fb33a50cca0e3adceb6ef5cff8b8e5aef67b4c4aaf2

build:
number: 0
script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation

requirements:
host:
- python
run:
- python

about:
home: https://github.com/python/typeshed
summary: Typing stubs for toml
description: Tests replacing dot functions with pipe counterparts
license: MIT
license_file: LICENSE
dev_url: https://github.com/python/typeshed
doc_url: https://pypi.org/project/types-toml/
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{% set name = dot_function_replacement | replace("foo", "bar") %}
{% set version = foo | upper %}

package:
name: {{ name | lower }}
version: {{ version | replace("foo", "bar") }}

source:
url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/script-env-{{ version }}.tar.gz
sha256: 6d3ac79e36c9ee593c5d4fb33a50cca0e3adceb6ef5cff8b8e5aef67b4c4aaf2

build:
number: 0
script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation

requirements:
host:
- python
run:
- python

about:
home: https://github.com/python/typeshed
summary: Typing stubs for toml
description: Tests replacing dot functions with pipe counterparts
license: MIT
license_file: LICENSE
dev_url: https://github.com/python/typeshed
doc_url: https://pypi.org/project/types-toml/