Merge branch 'main' of github.com:conda-incubator/conda-recipe-manage…

…r into 244-bump-recipe-should-exit-and-save-gracefully-on-some-failures

README.md

@@ -79,9 +79,11 @@ Usage: crm [OPTIONS] COMMAND [ARGS]...
   Command line interface for conda recipe management commands.
 
 Options:
-  --help  Show this message and exit.
+  -v, --verbose  Enables verbose logging (for commands that use the logger).
+  --help         Show this message and exit.
 
 Commands:
+  bump-recipe         Bumps a recipe file to a new version.
   convert             Converts a `meta.yaml` formatted-recipe file to the new
                       `recipe.yaml` format.
   graph               Interactive CLI for examining recipe dependency graphs.

conda_recipe_manager/commands/README.md

@@ -27,6 +27,50 @@ to unique error cases.
 
 # List of Tools
 
+## `bump-recipe`
+This tool takes an existing recipe file and "bumps" it to target a new software version.
+
+As of writing, this tool can automatically update:
+- The `/build/number` field
+- The package version number (either by canonical `version` variable or `/package/version` field)
+- Any applicable `sha-256` fields in the `/source` section
+
+If a network request is required to perform a modification, this script will use a retry-with-a-back-off mechanism to
+recover in the event of a network failure.
+
+If the recipe only needs to increment the `/build/number` field, `--build-num` flag can be used.
+
+### Usage
+```sh
+Usage: crm bump-recipe [OPTIONS] RECIPE_FILE_PATH
+
+  Bumps a recipe to a new version.
+
+  RECIPE_FILE_PATH: Path to the target recipe file
+
+Options:
+  -b, --build-num             Bump the build number by 1.
+  -d, --dry-run               Performs a dry-run operation that prints the
+                              recipe to STDOUT and does not save to the recipe
+                              file.
+  -t, --target-version TEXT   New project version to target. Required if
+                              `--build-num` is NOT specified.
+  -i, --retry-interval FLOAT  Retry interval (in seconds) for network
+                              requests. Scales with number of failed attempts.
+                              Defaults to 30 seconds
+  --help                      Show this message and exit.
+```
+
+### Example
+Upgrade the `types-toml` feedstock from version `0.10.8.6` to `0.10.8.20240310`.
+```sh
+crm bump-recipe -t 0.10.8.20240310 types-toml-feedstock/recipe/meta.yaml
+```
+Only increment the `/build/number` field.
+```sh
+crm bump-recipe --build-num types-toml-feedstock/recipe/meta.yaml
+```
+
 ## `convert`
 This tool converts one or more recipe files from the V0 recipe format to the
 [V1 recipe format](https://github.com/conda/ceps/blob/main/cep-14.md).

conda_recipe_manager/parser/_types.py

@@ -132,6 +132,13 @@ class Regex:
     # Finds `environ[]` used by a some recipe files. Requires a whitespace character to prevent matches with
     # `os.environ[]`, which is very rare.
     PRE_PROCESS_ENVIRON: Final[re.Pattern[str]] = re.compile(r"\s+environ\[(\"|')(.*)(\"|')\]")
+
+    # Finds `environ.get` which is used in a closed source community of which the author of this comment
+    # participates in. See Issue #271.
+    PRE_PROCESS_ENVIRON_GET: Final[re.Pattern[str]] = re.compile(
+        r"\s+environ \| get\((\"|')(.*)(\"|'), *(\"|')(.*)(\"|')\)"
+    )
+
     # Finds commonly used variants of `{{ hash_type }}:` which is a substitution for the `sha256` field
     PRE_PROCESS_JINJA_HASH_TYPE_KEY: Final[re.Pattern[str]] = re.compile(
         r"'{0,1}\{\{ (hash_type|hash|hashtype) \}\}'{0,1}:"

conda_recipe_manager/parser/recipe_parser_convert.py

@@ -132,7 +132,10 @@ def _upgrade_jinja_to_context_obj(self) -> None:
                 self._msg_tbl.add_message(MessageCategory.WARNING, f"The variable `{name}` is an unsupported type.")
                 continue
             # Function calls need to preserve JINJA escaping or else they turn into unevaluated strings.
-            if isinstance(value, str) and search_any_regex(Regex.JINJA_FUNCTIONS_SET, value):
+            # See issue #271 for details about env.get( string here.
+            if isinstance(value, str) and (
+                search_any_regex(Regex.JINJA_FUNCTIONS_SET, value) or value.startswith("env.get(")
+            ):
                 value = "{{" + value + "}}"
             context_obj[name] = value
         # Ensure that we do not include an empty context object (which is forbidden by the schema).
@@ -728,7 +731,7 @@ def pre_process_recipe_text(content: str) -> str:
         #   - This is mostly used by Bioconda recipes and R-based-packages in the `license_file` field.
         #   - From our search, it looks like we never deal with more than one set of outer quotes within the brackets
         replacements: list[tuple[str, str]] = []
-        for groups in cast(list[str], Regex.PRE_PROCESS_ENVIRON.findall(content)):
+        for groups in cast(list[tuple[str, ...]], Regex.PRE_PROCESS_ENVIRON.findall(content)):
             # Each match should return ["<quote char>", "<key>", "<quote_char>"]
             quote_char = groups[0]
             key = groups[1]
@@ -738,6 +741,18 @@ def pre_process_recipe_text(content: str) -> str:
                     f"env.get({quote_char}{key}{quote_char})",
                 )
             )
+
+        for groups in cast(list[tuple[str, ...]], Regex.PRE_PROCESS_ENVIRON_GET.findall(content)):
+            environ_key = f"{groups[0]}{groups[1]}{groups[2]}"
+            environ_default = f"{groups[3]}{groups[4]}{groups[5]}"
+
+            replacements.append(
+                (
+                    f"environ | get({environ_key}, {environ_default})",
+                    f"env.get({environ_key}, default={environ_default})",
+                )
+            )
+
         for old, new in replacements:
             content = content.replace(old, new, 1)
 

conda_recipe_manager/parser/recipe_reader.py

@@ -703,8 +703,8 @@ def render(self) -> str:
         # `/context`.
         if self._schema_version == SchemaVersion.V0:
             for key, val in self._vars_tbl.items():
-                # Double quote strings
-                if isinstance(val, str):
+                # Double quote strings, except for when we detect a env.get() expression. See issue #271.
+                if isinstance(val, str) and not val.startswith("env.get("):
                     val = f'"{val}"'
                 lines.append(f"{{% set {key} = {val} %}}")
             # Add spacing if variables have been set

tests/parser/test_recipe_parser_convert.py

@@ -22,6 +22,8 @@
         ("dot_function_replacement.yaml", "pre_processor/pp_dot_function_replacement.yaml"),
         # Upgrading multiline quoted strings
         ("quoted_multiline_str.yaml", "pre_processor/pp_quoted_multiline_str.yaml"),
+        # Issue #271 environ.get() conversions
+        ("unprocessed_environ_get.yaml", "pre_processor/pp_environ_get.yaml"),
         # Unchanged file
         ("simple-recipe.yaml", "simple-recipe.yaml"),
     ],
@@ -173,6 +175,12 @@ def test_pre_process_recipe_text(input_file: str, expected_file: str) -> None:
                 "Field at `/about/license_family` is no longer supported.",
             ],
         ),
+        # Issue #271 properly elevate environ.get() into context
+        (
+            "environ_get.yaml",
+            [],
+            [],
+        ),
         # TODO complete: The `rust.yaml` test contains many edge cases and selectors that aren't directly supported in
         # the V1 recipe format
         # (

tests/test_aux_files/environ_get.yaml

@@ -0,0 +1,6 @@
+{% set a = env.get("a") %}
+{% set b = env.get("b", default="0") %}
+
+package:
+  name: abc
+  version: 0

tests/test_aux_files/pre_processor/pp_environ_get.yaml

@@ -0,0 +1,6 @@
+{% set a = env.get("a") %}
+{% set b = env.get("b", default="0") %}
+
+package:
+  name: abc
+  version: 0

tests/test_aux_files/unprocessed_environ_get.yaml

@@ -0,0 +1,6 @@
+{% set a = environ["a"] %}
+{% set b = environ.get("b", "0") %}
+
+package:
+  name: abc
+  version: 0

tests/test_aux_files/v1_format/v1_environ_get.yaml

@@ -0,0 +1,9 @@
+schema_version: 1
+
+context:
+  a: ${{env.get("a")}}
+  b: ${{env.get("b", default="0")}}
+
+package:
+  name: abc
+  version: 0