Merge branch 'main' into main

.azure-pipelines/azure-pipelines-linux.yml

@@ -54,50 +54,6 @@ jobs:
   - publish: build_artifacts/noarch/
     artifact: conda_pkgs_noarch
 
-- job: linux_64_cuda_112
-  dependsOn: linux_64
-  condition: and(not(eq(variables['Build.SourceBranch'], 'refs/heads/main')), eq(dependencies.linux_64.outputs['linux_64_build.NEED_CUDA'], '1'))
-  pool:
-    vmImage: ubuntu-latest
-  timeoutInMinutes: 360
-  steps:
-  - script: |
-      sudo mkdir -p /opt/empty_dir
-      for d in \
-          /opt/ghc \
-          /opt/hostedtoolcache \
-          /usr/lib/jvm \
-          /usr/local/.ghcup \
-          /usr/local/android \
-          /usr/local/powershell \
-          /usr/share/dotnet \
-          /usr/share/swift \
-          ; do
-          sudo rsync --stats -a --delete /opt/empty_dir/ $d || true
-      done
-    displayName: Manage disk space
-
-  - script: |
-      # sudo pip install --upgrade pip
-      sudo pip install setuptools shyaml
-    displayName: Install dependencies
-
-  - script: |
-      set -e
-
-      # make sure there is a package directory so that artifact publishing works
-      mkdir -p build_artifacts/linux-64/
-
-      export CI=azure
-      export CONFIG=linux64_cuda112
-      export DOCKER_IMAGE=quay.io/condaforge/linux-anvil-cuda:11.2
-      export AZURE=True
-      .scripts/run_docker_build.sh
-
-    displayName: Run docker build for CUDA 11.2
-  - publish: build_artifacts/linux-64/
-    artifact: conda_pkgs_linux_64_cuda112
-
 - job: linux_64_cuda_118
   dependsOn: linux_64
   condition: and(not(eq(variables['Build.SourceBranch'], 'refs/heads/main')), eq(dependencies.linux_64.outputs['linux_64_build.NEED_CUDA'], '1'))

.azure-pipelines/azure-pipelines-osx.yml

@@ -6,7 +6,7 @@ jobs:
 - job: osx
   condition: not(eq(variables['Build.SourceBranch'], 'refs/heads/main'))
   pool:
-    vmImage: macOS-11
+    vmImage: macOS-13
   strategy:
     matrix:
       osx_64:

.ci_support/build_all.py

@@ -1,3 +1,5 @@
+from shutil import rmtree
+import tempfile
 import conda.base.context
 import conda.core.index
 import conda.resolve
@@ -6,6 +8,8 @@
 import networkx as nx
 from compute_build_graph import construct_graph
 import argparse
+import atexit
+import re
 import os
 from collections import OrderedDict
 import sys
@@ -18,6 +22,9 @@
     from yaml import BaseLoader, load
 
 
+EXAMPLE_RECIPE_FOLDERS = ["example", "example-v1"]
+
+
 def get_host_platform():
     from sys import platform
     if platform == "linux" or platform == "linux2":
@@ -43,21 +50,59 @@ def build_all(recipes_dir, arch):
     script_dir = os.path.dirname(os.path.realpath(__file__))
     variant_config_file = os.path.join(script_dir, "{}.yaml".format(get_config_name(arch)))
 
+    has_meta_yaml = False
+    has_recipe_yaml = False
+
     found_cuda = False
     found_centos7 = False
     for folder in folders:
         meta_yaml = os.path.join(recipes_dir, folder, "meta.yaml")
         if os.path.exists(meta_yaml):
+            has_meta_yaml = True
             with(open(meta_yaml, "r", encoding="utf-8")) as f:
                 text = ''.join(f.readlines())
                 if 'cuda' in text:
                     found_cuda = True
                 if 'sysroot_linux-64' in text:
                     found_centos7 = True
+
+        recipe_yaml = os.path.join(recipes_dir, folder, "recipe.yaml")
+        if os.path.exists(recipe_yaml):
+            has_recipe_yaml = True
+            with open(recipe_yaml, "r", encoding="utf-8") as f:
+                text = "".join(f.readlines())
+                if "cuda" in text:
+                    found_cuda = True
+                if "sysroot_linux-64" in text:
+                    found_centos7 = True
+
+        cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml")
+        if os.path.exists(cbc):
+            with open(cbc, "r") as f:
+                lines = f.readlines()
+            pat = re.compile(r"^([^\#]*?)\s+\#\s\[.*(not\s(linux|unix)|(?<!not\s)(osx|win)).*\]\s*$")
+            # remove lines with selectors that don't apply to linux, i.e. if they contain
+            # "not linux", "not unix", "osx" or "win"; this also removes trailing newlines
+            lines = [pat.sub("", x) for x in lines]
+            text = "\n".join(lines)
+            if platform == 'linux' and ('c_stdlib_version' in text):
+                config = load(text, Loader=BaseLoader)
+                if 'c_stdlib_version' in config:
+                    for version in config['c_stdlib_version']:
+                        version = tuple([int(x) for x in version.split('.')])
+                        print(f"Found c_stdlib_version for linux: {version=}")
+                        found_centos7 |= version == (2, 17)
+
+    if has_meta_yaml and has_recipe_yaml:
+        raise ValueError("Mixing meta.yaml and recipe.yaml recipes is not supported")
+    if not has_meta_yaml and not has_recipe_yaml:
+        raise ValueError("Neither a meta.yaml or a recipe.yaml recipes was found")
+
     if found_cuda:
         print('##vso[task.setvariable variable=NEED_CUDA;isOutput=true]1')
     if found_centos7:
         os.environ["DEFAULT_LINUX_VERSION"] = "cos7"
+        print("Overriding DEFAULT_LINUX_VERSION to be cos7")
 
     deployment_version = (0, 0)
     sdk_version = (0, 0)
@@ -66,16 +111,27 @@ def build_all(recipes_dir, arch):
         cbc = os.path.join(recipes_dir, folder, "conda_build_config.yaml")
         if os.path.exists(cbc):
             with open(cbc, "r") as f:
-                text = ''.join(f.readlines())
+                lines = f.readlines()
+            pat = re.compile(r"^([^\#]*?)\s+\#\s\[.*(not\s(osx|unix)|(?<!not\s)(linux|win)).*\]\s*$")
+            # remove lines with selectors that don't apply to osx, i.e. if they contain
+            # "not osx", "not unix", "linux" or "win"; this also removes trailing newlines
+            lines = [pat.sub("", x) for x in lines]
+            text = "\n".join(lines)
             if platform == 'osx' and (
                     'MACOSX_DEPLOYMENT_TARGET' in text or
-                    'MACOSX_SDK_VERSION' in text):
+                    'MACOSX_SDK_VERSION' in text or
+                    'c_stdlib_version' in text):
                 config = load(text, Loader=BaseLoader)
 
                 if 'MACOSX_DEPLOYMENT_TARGET' in config:
                     for version in config['MACOSX_DEPLOYMENT_TARGET']:
                         version = tuple([int(x) for x in version.split('.')])
                         deployment_version = max(deployment_version, version)
+                if 'c_stdlib_version' in config:
+                    for version in config['c_stdlib_version']:
+                        version = tuple([int(x) for x in version.split('.')])
+                        print(f"Found c_stdlib_version for osx: {version=}")
+                        deployment_version = max(deployment_version, version)
                 if 'MACOSX_SDK_VERSION' in config:
                     for version in config['MACOSX_SDK_VERSION']:
                         version = tuple([int(x) for x in version.split('.')])
@@ -117,8 +173,17 @@ def build_all(recipes_dir, arch):
 
     if 'conda-forge' not in channel_urls:
         raise ValueError('conda-forge needs to be part of channel_sources')
-    print("Building {} with {}".format(','.join(folders), ','.join(channel_urls)))
-    build_folders(recipes_dir, folders, arch, channel_urls)
+
+    if has_meta_yaml:
+        print("Building {} with {}".format(','.join(folders), ','.join(channel_urls)))
+        build_folders(recipes_dir, folders, arch, channel_urls)
+    elif has_recipe_yaml:
+        print(
+            "Building {} with {} using rattler-build".format(
+                ",".join(folders), ",".join(channel_urls)
+            )
+        )
+        build_folders_rattler_build(recipes_dir, platform, arch, channel_urls)
 
 
 def get_config(arch, channel_urls):
@@ -177,6 +242,50 @@ def build_folders(recipes_dir, folders, arch, channel_urls):
         conda_build.api.build([recipe], config=get_config(arch, channel_urls))
 
 
+def build_folders_rattler_build(
+    recipes_dir: str, platform, arch, channel_urls: list[str]
+):
+    config = get_config(arch, channel_urls)
+
+    # Remove the example recipes to ensure that they are not also build.
+    for example_recipe in EXAMPLE_RECIPE_FOLDERS:
+        rmtree(os.path.join(recipes_dir, example_recipe), ignore_errors=True)
+
+    # Determine the locations for the variant config files.
+    specs = OrderedDict()
+    for f in config.exclusive_config_files:
+        specs[f] = conda_build.variants.parse_config_file(
+            os.path.abspath(os.path.expanduser(os.path.expandvars(f))), config
+        )
+
+    # Combine all the variant config files together
+    combined_spec = conda_build.variants.combine_specs(specs, log_output=config.verbose)
+    variant_config = yaml.dump(combined_spec)
+
+    # Define the arguments for rattler-build
+    args = [
+        "rattler-build",
+        "build",
+        "--recipe-dir",
+        recipes_dir,
+        "--target-platform",
+        f"{platform}-{arch}",
+    ]
+    for channel_url in channel_urls:
+        # Local is automatically added by rattler-build so we just remove it.
+        if channel_url != "local":
+            args.extend(["-c", channel_url])
+
+    # Construct a temporary file where we write the combined variant config. We can then pass that
+    # to rattler-build.
+    with tempfile.NamedTemporaryFile(delete=False) as fp:
+        fp.write(variant_config.encode("utf-8"))   
+        atexit.register(os.unlink, fp.name)
+
+    # Execute rattler-build.
+    subprocess.run(args + ["--variant-config", fp.name], check=True)
+
+
 def check_recipes_in_correct_dir(root_dir, correct_dir):
     from pathlib import Path
     for path in Path(root_dir).rglob('meta.yaml'):
@@ -198,7 +307,7 @@ def read_mambabuild(recipes_dir):
     folders = os.listdir(recipes_dir)
     conda_build_tools = []
     for folder in folders:
-        if folder == "example":
+        if folder in EXAMPLE_RECIPE_FOLDERS:
             continue
         cf = os.path.join(recipes_dir, folder, "conda-forge.yml")
         if os.path.exists(cf):

.ci_support/compute_build_graph.py

@@ -39,15 +39,13 @@
 import functools
 from functools import lru_cache
 
-from frozendict import frozendict
+from frozendict import frozendict, deepfreeze
 import networkx as nx
 from conda.models.match_spec import MatchSpec
 from conda.models.records import PackageRecord
 from conda_build import api
 from conda_build.metadata import find_recipe, MetaData
 
-from conda_build.utils import HashableDict
-
 
 log = logging.getLogger(__file__)
 CONDA_BUILD_CACHE = os.environ.get("CONDA_BUILD_CACHE")
@@ -347,7 +345,7 @@ def collapse_subpackage_nodes(graph):
             if master_meta.name() == meta.name():
                 master = True
             group = node_groups.get(meta_path, {})
-            subgroup = group.get(HashableDict(meta.config.variant), {})
+            subgroup = group.get(deepfreeze(meta.config.variant), {})
             if master:
                 if 'master' in subgroup:
                     raise ValueError("tried to set more than one node in a group as master")
@@ -356,7 +354,7 @@ def collapse_subpackage_nodes(graph):
                 sps = subgroup.get('subpackages', [])
                 sps.append(node)
                 subgroup['subpackages'] = sps
-            group[HashableDict(meta.config.variant)] = subgroup
+            group[deepfreeze(meta.config.variant)] = subgroup
             node_groups[meta_path] = group
 
     for recipe_path, group in node_groups.items():

.ci_support/linux64.yaml

@@ -1,3 +1,7 @@
+c_stdlib_version:
+- 2.17
+c_stdlib:
+- sysroot
 cdt_name:
 - cos7
 c_compiler:

.ci_support/linux64_cuda118.yaml

@@ -14,6 +14,10 @@ cxx_compiler_version:
 - 11
 fortran_compiler_version:
 - 11
+c_stdlib:
+- sysroot
+c_stdlib_version:
+- 2.17
 cdt_name:
 - cos7
 target_platform:
@@ -27,4 +31,4 @@ cuda_compiler:
 cuda_compiler_version:
 - 11.8
 cuda_compiler_version_min:
-- 11.2
+- 11.8

.ci_support/linux64_cuda120.yaml

@@ -14,6 +14,10 @@ cxx_compiler_version:
 - 12
 fortran_compiler_version:
 - 12
+c_stdlib:
+- sysroot
+c_stdlib_version:
+- 2.17
 cdt_name:
 - cos7
 target_platform:
@@ -27,4 +31,4 @@ cuda_compiler:
 cuda_compiler_version:
 - 12.0
 cuda_compiler_version_min:
-- 11.2
+- 11.8

.ci_support/osx64.yaml

@@ -2,6 +2,12 @@ c_compiler:
   - clang
 cxx_compiler:
   - clangxx
+c_stdlib:
+  - macosx_deployment_target
+c_stdlib_version:
+  - 10.13
+MACOSX_DEPLOYMENT_TARGET:
+  - 10.13
 fortran_compiler:
   - gfortran
 go_compiler:

.ci_support/requirements.txt

@@ -1,8 +1,9 @@
 conda>=23.7.3
 conda-libmamba-solver>=23.7.0
-conda-build>=3.16,!=3.28.3
+conda-build>=24.3
 conda-index>=0.3.0
-conda-forge-ci-setup=3.*
+conda-forge-ci-setup>=4.9.3,<5.0
 conda-forge-pinning
 frozendict
 networkx=2.4
+rattler-build-conda-compat>=1.2.0,<2.0.0a0

.ci_support/win64.yaml

@@ -1,5 +1,7 @@
 go_compiler:
   - go-nocgo
+c_stdlib:
+  - vs
 cgo_compiler:
   - go-cgo
 channel_sources: