diff --git a/.azure-pipelines/azure-pipelines-linux.yml b/.azure-pipelines/azure-pipelines-linux.yml index 47db1f7..6710233 100755 --- a/.azure-pipelines/azure-pipelines-linux.yml +++ b/.azure-pipelines/azure-pipelines-linux.yml @@ -8,30 +8,36 @@ jobs: vmImage: ubuntu-latest strategy: matrix: - linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython: - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython + linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython: + CONFIG: linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 - linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython: - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython + SHORT_CONFIG: linux_64_c_compiler_version10cuda_c_hf86a599877 + linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython: + CONFIG: linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 - linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython: - CONFIG: linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython + SHORT_CONFIG: linux_64_c_compiler_version10cuda_c_hb8fd6f0b42 + linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython: + CONFIG: linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cuda:11.2 - linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython: - CONFIG: linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython + SHORT_CONFIG: linux_64_c_compiler_version10cuda_c_he98a3b1d0d + linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython: + CONFIG: linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 - linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython: - CONFIG: linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython + SHORT_CONFIG: linux_64_c_compiler_version12cuda_c_haa23569174 + linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython: + CONFIG: linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 - linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython: - CONFIG: linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython + SHORT_CONFIG: linux_64_c_compiler_version12cuda_c_h97d80f754f + linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython: + CONFIG: linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython UPLOAD_PACKAGES: 'True' DOCKER_IMAGE: quay.io/condaforge/linux-anvil-cos7-x86_64 + SHORT_CONFIG: linux_64_c_compiler_version12cuda_c_h6dd2ecf2e7 timeoutInMinutes: 360 steps: @@ -62,4 +68,33 @@ jobs: env: BINSTAR_TOKEN: $(BINSTAR_TOKEN) FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN) - STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) \ No newline at end of file + STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) + - script: | + export CI=azure + export CI_RUN_ID=$(build.BuildNumber).$(system.JobAttempt) + export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) + export CONDA_BLD_DIR=build_artifacts + export ARTIFACT_STAGING_DIR="$(Build.ArtifactStagingDirectory)" + # Archive everything in CONDA_BLD_DIR except environments + export BLD_ARTIFACT_PREFIX=conda_artifacts + if [[ "$AGENT_JOBSTATUS" == "Failed" ]]; then + # Archive the CONDA_BLD_DIR environments only when the job fails + export ENV_ARTIFACT_PREFIX=conda_envs + fi + ./.scripts/create_conda_build_artifacts.sh + displayName: Prepare conda build artifacts + condition: succeededOrFailed() + + - task: PublishPipelineArtifact@1 + displayName: Store conda build artifacts + condition: not(eq(variables.BLD_ARTIFACT_PATH, '')) + inputs: + targetPath: $(BLD_ARTIFACT_PATH) + artifactName: $(BLD_ARTIFACT_NAME) + + - task: PublishPipelineArtifact@1 + displayName: Store conda build environment artifacts + condition: not(eq(variables.ENV_ARTIFACT_PATH, '')) + inputs: + targetPath: $(ENV_ARTIFACT_PATH) + artifactName: $(ENV_ARTIFACT_NAME) \ No newline at end of file diff --git a/.azure-pipelines/azure-pipelines-osx.yml b/.azure-pipelines/azure-pipelines-osx.yml index 3c2bd11..23d0873 100755 --- a/.azure-pipelines/azure-pipelines-osx.yml +++ b/.azure-pipelines/azure-pipelines-osx.yml @@ -11,12 +11,27 @@ jobs: osx_64_python3.10.____cpython: CONFIG: osx_64_python3.10.____cpython UPLOAD_PACKAGES: 'True' + SHORT_CONFIG: osx_64_python3.10.____cpython osx_64_python3.8.____cpython: CONFIG: osx_64_python3.8.____cpython UPLOAD_PACKAGES: 'True' + SHORT_CONFIG: osx_64_python3.8.____cpython osx_64_python3.9.____cpython: CONFIG: osx_64_python3.9.____cpython UPLOAD_PACKAGES: 'True' + SHORT_CONFIG: osx_64_python3.9.____cpython + osx_arm64_python3.10.____cpython: + CONFIG: osx_arm64_python3.10.____cpython + UPLOAD_PACKAGES: 'True' + SHORT_CONFIG: osx_arm64_python3.10.____cpython + osx_arm64_python3.8.____cpython: + CONFIG: osx_arm64_python3.8.____cpython + UPLOAD_PACKAGES: 'True' + SHORT_CONFIG: osx_arm64_python3.8.____cpython + osx_arm64_python3.9.____cpython: + CONFIG: osx_arm64_python3.9.____cpython + UPLOAD_PACKAGES: 'True' + SHORT_CONFIG: osx_arm64_python3.9.____cpython timeoutInMinutes: 360 steps: @@ -36,4 +51,33 @@ jobs: env: BINSTAR_TOKEN: $(BINSTAR_TOKEN) FEEDSTOCK_TOKEN: $(FEEDSTOCK_TOKEN) - STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) \ No newline at end of file + STAGING_BINSTAR_TOKEN: $(STAGING_BINSTAR_TOKEN) + - script: | + export CI=azure + export CI_RUN_ID=$(build.BuildNumber).$(system.JobAttempt) + export FEEDSTOCK_NAME=$(basename ${BUILD_REPOSITORY_NAME}) + export CONDA_BLD_DIR=/Users/runner/miniforge3/conda-bld + export ARTIFACT_STAGING_DIR="$(Build.ArtifactStagingDirectory)" + # Archive everything in CONDA_BLD_DIR except environments + export BLD_ARTIFACT_PREFIX=conda_artifacts + if [[ "$AGENT_JOBSTATUS" == "Failed" ]]; then + # Archive the CONDA_BLD_DIR environments only when the job fails + export ENV_ARTIFACT_PREFIX=conda_envs + fi + ./.scripts/create_conda_build_artifacts.sh + displayName: Prepare conda build artifacts + condition: succeededOrFailed() + + - task: PublishPipelineArtifact@1 + displayName: Store conda build artifacts + condition: not(eq(variables.BLD_ARTIFACT_PATH, '')) + inputs: + targetPath: $(BLD_ARTIFACT_PATH) + artifactName: $(BLD_ARTIFACT_NAME) + + - task: PublishPipelineArtifact@1 + displayName: Store conda build environment artifacts + condition: not(eq(variables.ENV_ARTIFACT_PATH, '')) + inputs: + targetPath: $(ENV_ARTIFACT_PATH) + artifactName: $(ENV_ARTIFACT_NAME) \ No newline at end of file diff --git a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython.yaml b/.ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython.yaml similarity index 96% rename from .ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython.yaml index 84ef703..8d3a3ae 100644 --- a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython.yaml @@ -31,6 +31,7 @@ target_platform: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython.yaml b/.ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython.yaml similarity index 96% rename from .ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython.yaml index 4e95228..e91809c 100644 --- a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython.yaml @@ -31,6 +31,7 @@ target_platform: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython.yaml similarity index 96% rename from .ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython.yaml index 67fca76..ac6b72f 100644 --- a/.ci_support/linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython.yaml @@ -31,6 +31,7 @@ target_platform: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython.yaml similarity index 95% rename from .ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython.yaml index 03edc60..15bb7e6 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython.yaml @@ -9,7 +9,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- nvcc +- None cuda_compiler_version: - None cxx_compiler: @@ -31,6 +31,7 @@ target_platform: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython.yaml similarity index 95% rename from .ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython.yaml index 678466d..190799a 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython.yaml @@ -9,7 +9,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- nvcc +- None cuda_compiler_version: - None cxx_compiler: @@ -31,6 +31,7 @@ target_platform: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/.ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython.yaml b/.ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython.yaml similarity index 95% rename from .ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython.yaml rename to .ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython.yaml index 70f81a9..975b320 100644 --- a/.ci_support/linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython.yaml +++ b/.ci_support/linux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython.yaml @@ -9,7 +9,7 @@ channel_sources: channel_targets: - conda-forge main cuda_compiler: -- nvcc +- None cuda_compiler_version: - None cxx_compiler: @@ -31,6 +31,7 @@ target_platform: zip_keys: - - c_compiler_version - cxx_compiler_version + - cuda_compiler - cuda_compiler_version - cdt_name - docker_image diff --git a/.ci_support/osx_arm64_python3.10.____cpython.yaml b/.ci_support/osx_arm64_python3.10.____cpython.yaml new file mode 100644 index 0000000..f57eb00 --- /dev/null +++ b/.ci_support/osx_arm64_python3.10.____cpython.yaml @@ -0,0 +1,33 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '15' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '15' +llvm_openmp: +- '15' +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.10.* *_cpython +pytorch: +- '2.0' +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/.ci_support/osx_arm64_python3.8.____cpython.yaml b/.ci_support/osx_arm64_python3.8.____cpython.yaml new file mode 100644 index 0000000..6a822ff --- /dev/null +++ b/.ci_support/osx_arm64_python3.8.____cpython.yaml @@ -0,0 +1,33 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '15' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '15' +llvm_openmp: +- '15' +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.8.* *_cpython +pytorch: +- '2.0' +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/.ci_support/osx_arm64_python3.9.____cpython.yaml b/.ci_support/osx_arm64_python3.9.____cpython.yaml new file mode 100644 index 0000000..33db40f --- /dev/null +++ b/.ci_support/osx_arm64_python3.9.____cpython.yaml @@ -0,0 +1,33 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +c_compiler: +- clang +c_compiler_version: +- '15' +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '15' +llvm_openmp: +- '15' +macos_machine: +- arm64-apple-darwin20.0.0 +pin_run_as_build: + python: + min_pin: x.x + max_pin: x.x +python: +- 3.9.* *_cpython +pytorch: +- '2.0' +target_platform: +- osx-arm64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/.scripts/build_steps.sh b/.scripts/build_steps.sh index 595f8b5..a6b09f6 100755 --- a/.scripts/build_steps.sh +++ b/.scripts/build_steps.sh @@ -45,6 +45,9 @@ source run_conda_forge_build_setup # make the build number clobber make_build_number "${FEEDSTOCK_ROOT}" "${RECIPE_ROOT}" "${CONFIG_FILE}" +if [[ "${HOST_PLATFORM}" != "${BUILD_PLATFORM}" ]] && [[ "${HOST_PLATFORM}" != linux-* ]] && [[ "${BUILD_WITH_CONDA_DEBUG:-0}" != 1 ]]; then + EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" +fi ( endgroup "Configuring conda" ) 2> /dev/null diff --git a/.scripts/create_conda_build_artifacts.sh b/.scripts/create_conda_build_artifacts.sh new file mode 100755 index 0000000..17ec086 --- /dev/null +++ b/.scripts/create_conda_build_artifacts.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +# INPUTS (environment variables that need to be set before calling this script): +# +# CI (azure/github_actions/UNSET) +# CI_RUN_ID (unique identifier for the CI job run) +# FEEDSTOCK_NAME +# CONFIG (build matrix configuration string) +# SHORT_CONFIG (uniquely-shortened configuration string) +# CONDA_BLD_DIR (path to the conda-bld directory) +# ARTIFACT_STAGING_DIR (use working directory if unset) +# BLD_ARTIFACT_PREFIX (prefix for the conda build artifact name, skip if unset) +# ENV_ARTIFACT_PREFIX (prefix for the conda build environments artifact name, skip if unset) + +# OUTPUTS +# +# BLD_ARTIFACT_NAME +# BLD_ARTIFACT_PATH +# ENV_ARTIFACT_NAME +# ENV_ARTIFACT_PATH + +source .scripts/logging_utils.sh + +# DON'T do set -x, because it results in double echo-ing pipeline commands +# and that might end up inserting extraneous quotation marks in output variables +set -e + +# Check that the conda-build directory exists +if [ ! -d "$CONDA_BLD_DIR" ]; then + echo "conda-build directory does not exist" + exit 1 +fi + +# Set staging dir to the working dir, in Windows style if applicable +if [[ -z "${ARTIFACT_STAGING_DIR}" ]]; then + if pwd -W; then + ARTIFACT_STAGING_DIR=$(pwd -W) + else + ARTIFACT_STAGING_DIR=$PWD + fi +fi +echo "ARTIFACT_STAGING_DIR: $ARTIFACT_STAGING_DIR" + +FEEDSTOCK_ROOT=$(cd "$(dirname "$0")/.."; pwd;) +if [ -z ${FEEDSTOCK_NAME} ]; then + export FEEDSTOCK_NAME=$(basename ${FEEDSTOCK_ROOT}) +fi + +# Set a unique ID for the artifact(s), specialized for this particular job run +ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${CONFIG}" +if [[ ${#ARTIFACT_UNIQUE_ID} -gt 80 ]]; then + ARTIFACT_UNIQUE_ID="${CI_RUN_ID}_${SHORT_CONFIG}" +fi +echo "ARTIFACT_UNIQUE_ID: $ARTIFACT_UNIQUE_ID" + +# Set a descriptive ID for the archive(s), specialized for this particular job run +ARCHIVE_UNIQUE_ID="${CI_RUN_ID}_${CONFIG}" + +# Make the build artifact zip +if [[ ! -z "$BLD_ARTIFACT_PREFIX" ]]; then + export BLD_ARTIFACT_NAME="${BLD_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" + export BLD_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${BLD_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.zip" + + ( startgroup "Archive conda build directory" ) 2> /dev/null + + # Try 7z and fall back to zip if it fails (for cross-platform use) + if ! 7z a "$BLD_ARTIFACT_PATH" "$CONDA_BLD_DIR" '-xr!.git/' '-xr!_*_env*/' '-xr!*_cache/' -bb; then + pushd "$CONDA_BLD_DIR" + zip -r -y -T "$BLD_ARTIFACT_PATH" . -x '*.git/*' '*_*_env*/*' '*_cache/*' + popd + fi + + ( endgroup "Archive conda build directory" ) 2> /dev/null + + echo "BLD_ARTIFACT_NAME: $BLD_ARTIFACT_NAME" + echo "BLD_ARTIFACT_PATH: $BLD_ARTIFACT_PATH" + + if [[ "$CI" == "azure" ]]; then + echo "##vso[task.setVariable variable=BLD_ARTIFACT_NAME]$BLD_ARTIFACT_NAME" + echo "##vso[task.setVariable variable=BLD_ARTIFACT_PATH]$BLD_ARTIFACT_PATH" + elif [[ "$CI" == "github_actions" ]]; then + echo "BLD_ARTIFACT_NAME=$BLD_ARTIFACT_NAME" >> $GITHUB_OUTPUT + echo "BLD_ARTIFACT_PATH=$BLD_ARTIFACT_PATH" >> $GITHUB_OUTPUT + fi +fi + +# Make the environments artifact zip +if [[ ! -z "$ENV_ARTIFACT_PREFIX" ]]; then + export ENV_ARTIFACT_NAME="${ENV_ARTIFACT_PREFIX}_${ARTIFACT_UNIQUE_ID}" + export ENV_ARTIFACT_PATH="${ARTIFACT_STAGING_DIR}/${FEEDSTOCK_NAME}_${ENV_ARTIFACT_PREFIX}_${ARCHIVE_UNIQUE_ID}.zip" + + ( startgroup "Archive conda build environments" ) 2> /dev/null + + # Try 7z and fall back to zip if it fails (for cross-platform use) + if ! 7z a "$ENV_ARTIFACT_PATH" -r "$CONDA_BLD_DIR"/'_*_env*/' -bb; then + pushd "$CONDA_BLD_DIR" + zip -r -y -T "$ENV_ARTIFACT_PATH" . -i '*_*_env*/*' + popd + fi + + ( endgroup "Archive conda build environments" ) 2> /dev/null + + echo "ENV_ARTIFACT_NAME: $ENV_ARTIFACT_NAME" + echo "ENV_ARTIFACT_PATH: $ENV_ARTIFACT_PATH" + + if [[ "$CI" == "azure" ]]; then + echo "##vso[task.setVariable variable=ENV_ARTIFACT_NAME]$ENV_ARTIFACT_NAME" + echo "##vso[task.setVariable variable=ENV_ARTIFACT_PATH]$ENV_ARTIFACT_PATH" + elif [[ "$CI" == "github_actions" ]]; then + echo "ENV_ARTIFACT_NAME=$ENV_ARTIFACT_NAME" >> $GITHUB_OUTPUT + echo "ENV_ARTIFACT_PATH=$ENV_ARTIFACT_PATH" >> $GITHUB_OUTPUT + fi +fi \ No newline at end of file diff --git a/.scripts/run_osx_build.sh b/.scripts/run_osx_build.sh index f169576..bb00584 100755 --- a/.scripts/run_osx_build.sh +++ b/.scripts/run_osx_build.sh @@ -55,6 +55,10 @@ source run_conda_forge_build_setup echo -e "\n\nMaking the build clobber file" make_build_number ./ ./recipe ./.ci_support/${CONFIG}.yaml +if [[ "${HOST_PLATFORM}" != "${BUILD_PLATFORM}" ]]; then + EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" +fi + if [[ -f LICENSE.txt ]]; then cp LICENSE.txt "recipe/recipe-scripts-license.txt" diff --git a/README.md b/README.md index 7583b78..164389d 100644 --- a/README.md +++ b/README.md @@ -29,45 +29,45 @@ Current build status - + - + - + - + - + - + @@ -91,6 +91,27 @@ Current build status variant + + + + + + + + +
VariantStatus
linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.10.____cpythonlinux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.10.____cpython - variant + variant
linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.8.____cpythonlinux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.8.____cpython - variant + variant
linux_64_c_compiler_version10cuda_compiler_version11.2cxx_compiler_version10python3.9.____cpythonlinux_64_c_compiler_version10cuda_compilernvcccuda_compiler_version11.2cxx_compiler_version10python3.9.____cpython - variant + variant
linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.10.____cpythonlinux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.10.____cpython - variant + variant
linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.8.____cpythonlinux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.8.____cpython - variant + variant
linux_64_c_compiler_version12cuda_compiler_versionNonecxx_compiler_version12python3.9.____cpythonlinux_64_c_compiler_version12cuda_compilerNonecuda_compiler_versionNonecxx_compiler_version12python3.9.____cpython - variant + variant
osx_arm64_python3.10.____cpython + + variant + +
osx_arm64_python3.8.____cpython + + variant + +
osx_arm64_python3.9.____cpython + + variant + +
diff --git a/conda-forge.yml b/conda-forge.yml index 993de48..4ff9e86 100644 --- a/conda-forge.yml +++ b/conda-forge.yml @@ -1,6 +1,15 @@ +build_platform: + osx_arm64: osx_64 +conda_build: + pkg_format: '2' +conda_forge_output_validation: true github: branch_name: main tooling_branch_name: main -conda_forge_output_validation: true -conda_build: - pkg_format: '2' +test: native_and_emulated +azure: + # flag for forcing the building all supported providers + force: false + # toggle for storing the conda build_artifacts directory (including the + # built packages) as an Azure pipeline artifact that can be downloaded + store_build_artifacts: true diff --git a/recipe/build.sh b/recipe/build.sh index 0497131..75a48fa 100644 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -24,11 +24,38 @@ else export FORCE_CUDA=0 fi -# export USE_MKL_BLAS=1 # only used for >0.1.0 -export Torch_DIR=$(python -c 'import torch; print(torch.utils.cmake_prefix_path)') +# Dynamic libraries need to be lazily loaded so that torch +# can be imported on system without a GPU +export LDFLAGS="${LDFLAGS//-Wl,-z,now/-Wl,-z,lazy}" +# export USE_MKL_BLAS=1 # only used for >0.1.0 export FORCE_NINJA=1 export EXTERNAL_PHMAP_INCLUDE_DIR="${BUILD_PREFIX}/include/" export EXTERNAL_CUTLASS_INCLUDE_DIR="${BUILD_PREFIX}/include/" -python -m pip install . -vvv +export PYG_CMAKE_ARGS="${CMAKE_ARGS}" + +# get torch libraries for osx-arm64 +# from https://github.com/conda-forge/openmm-torch-feedstock/blob/f7b09cd93f69d7213acd88dca1b0b1770b0ac2bc/recipe/build.sh#L7 +LIBTORCH_DIR=${BUILD_PREFIX} +if [[ "$OSTYPE" == "darwin"* && $OSX_ARCH == "arm64" ]]; then + + LIBTORCH_DIR=${RECIPE_DIR}/libtorch + conda list -p ${BUILD_PREFIX} >packages.txt + cat packages.txt + PYTORCH_PACKAGE_VERSION=$(grep pytorch packages.txt | awk -F ' ' '{print $2}') + CONDA_SUBDIR=osx-arm64 conda create -y -p ${LIBTORCH_DIR} --no-deps pytorch=${PYTORCH_PACKAGE_VERSION} python=${PY_VER} + + export PYG_CMAKE_ARGS="${PYG_CMAKE_ARGS} -DTorch_DIR=${LIBTORCH_DIR}/lib/python${PY_VER}/site-packages/torch/share/cmake/Torch" +else + # For everything else than osx-arm64 + TORCH_PREFIX=$(${PYTHON} -c "import torch;print(torch.utils.cmake_prefix_path)") + export PYG_CMAKE_ARGS="${PYG_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${TORCH_PREFIX}" +fi + +${PYTHON} -m pip install . -vvv + +if [[ "$OSTYPE" == "darwin"* && $OSX_ARCH == "arm64" ]]; then + # clean up, otherwise, environment is stored in package + rm -fr ${LIBTORCH_DIR} +fi diff --git a/recipe/fix-cmake.patch b/recipe/fix-cmake.patch new file mode 100644 index 0000000..ccd19fb --- /dev/null +++ b/recipe/fix-cmake.patch @@ -0,0 +1,23 @@ +diff --git a/setup.py b/setup.py +index b9158f7..5a2f4aa 100644 +--- a/setup.py ++++ b/setup.py +@@ -61,7 +61,7 @@ class CMakeBuild(build_ext): + f'-DWITH_CUDA={"ON" if WITH_CUDA else "OFF"}', + f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={extdir}', + f'-DCMAKE_BUILD_TYPE={self.build_type}', +- f'-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}', ++ # f'-DCMAKE_PREFIX_PATH={torch.utils.cmake_prefix_path}', + ] + + if CMakeBuild.check_env_flag('USE_MKL_BLAS'): +@@ -77,6 +77,9 @@ class CMakeBuild(build_ext): + warnings.warn("Building times of 'pyg-lib' can be heavily improved" + " by installing 'ninja': `pip install ninja`") + ++ if os.getenv("PYG_CMAKE_ARGS") is not None: ++ cmake_args += os.getenv("PYG_CMAKE_ARGS").split(" ") ++ + build_args = [] + + subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, diff --git a/recipe/meta.yaml b/recipe/meta.yaml index 52e0255..f8c4238 100644 --- a/recipe/meta.yaml +++ b/recipe/meta.yaml @@ -10,8 +10,11 @@ source: url: https://github.com/pyg-team/pyg-lib/archive/refs/tags/{{ version }}.tar.gz sha256: 3c486a72a421b4769cd1ca114522eedf25d4115ca529091db06c10264a406ac4 + patches: + - fix-cmake.patch + build: - number: 2 + number: 10 string: cuda{{ cuda_compiler_version | replace('.', '') }}py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} # [cuda_compiler_version != "None"] string: cpu_py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} # [cuda_compiler_version == "None"] # Only build for CUDA >=11.2 @@ -21,14 +24,13 @@ requirements: build: - {{ compiler('c') }} - {{ compiler('cxx') }} - - {{ compiler('cuda') }} # [cuda_compiler_version not in (undefined, 'None')] - - sysroot_linux-64 ==2.17 # [linux64] - - python # [build_platform != target_platform] - - cross-python_{{ target_platform }} # [build_platform != target_platform] - - pytorch # [build_platform != target_platform] - - pytorch =*={{ torch_proc_type }}* # [build_platform != target_platform] + - {{ compiler('cuda') }} # [cuda_compiler_version not in (undefined, 'None')] + - sysroot_linux-64 ==2.17 # [linux64] + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - pytorch ={{ pytorch }}={{ torch_proc_type }}* # [build_platform != target_platform] - cmake - - ninja + - ninja # NOTE(hadim): here ninja is used for linux and osx - llvm-openmp # [osx] - libgomp # [linux] - parallel-hashmap