diff --git a/ci/build_docs.sh b/ci/build_docs.sh index 8c2ee8ae95..ea408b6940 100755 --- a/ci/build_docs.sh +++ b/ci/build_docs.sh @@ -6,28 +6,22 @@ set -euo pipefail rapids-logger "Create test conda environment" . /opt/conda/etc/profile.d/conda.sh -export RAPIDS_VERSION="$(rapids-version)" +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-dependency-file-generator \ --output conda \ --file-key docs \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n docs +rapids-mamba-retry env create -yq -f env.yaml -n docs conda activate docs rapids-print-env -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) - -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "kvikio=${RAPIDS_VERSION}" \ - "libkvikio=${RAPIDS_VERSION}" - export RAPIDS_DOCS_DIR="$(mktemp -d)" rapids-logger "Build CPP docs" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 6b9458b273..102beaa2ba 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -36,7 +36,10 @@ function sed_runner() { echo "${NEXT_FULL_TAG}" > VERSION DEPENDENCIES=( + kvikio libkvikio + libkvikio-tests + rapids-dask-dependency ) for DEP in "${DEPENDENCIES[@]}"; do for FILE in dependencies.yaml conda/environments/*.yaml; do diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh index 7dfde2187f..ef7933f150 100755 --- a/ci/test_cpp.sh +++ b/ci/test_cpp.sh @@ -5,33 +5,28 @@ set -euo pipefail . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) rapids-logger "Generate C++ testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_cpp \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \ + --prepend-channel "${CPP_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n test +rapids-mamba-retry env create -qy -f env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u conda activate test set -u -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/ mkdir -p "${RAPIDS_TESTS_DIR}" -SUITEERROR=0 rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - "libkvikio=${RAPIDS_VERSION}" \ - "libkvikio-tests=${RAPIDS_VERSION}" - rapids-logger "Check GPU usage" nvidia-smi diff --git a/ci/test_python.sh b/ci/test_python.sh index 1f4ff6d2f1..df16f20f9f 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -8,37 +8,31 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../ . /opt/conda/etc/profile.d/conda.sh -RAPIDS_VERSION="$(rapids-version)" +rapids-logger "Downloading artifacts from previous jobs" +CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) +PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) rapids-logger "Generate Python testing dependencies" rapids-dependency-file-generator \ --output conda \ --file-key test_python \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \ + --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \ + | tee env.yaml -rapids-mamba-retry env create --yes -f env.yaml -n test +rapids-mamba-retry env create -qy -f env.yaml -n test # Temporarily allow unbound variables for conda activation. set +u conda activate test set -u -rapids-logger "Downloading artifacts from previous jobs" -CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) -PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python) - RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"} RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"} mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}" rapids-print-env -rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "libkvikio=${RAPIDS_VERSION}" \ - "kvikio=${RAPIDS_VERSION}" - rapids-logger "Check GPU usage" nvidia-smi diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml index 2de4597657..7fee941a1d 100644 --- a/conda/environments/all_cuda-118_arch-aarch64.yaml +++ b/conda/environments/all_cuda-118_arch-aarch64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-aarch64=11.* - libcurl>=7.87.0 @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8b63a6726b..b89468e062 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-64=11.* - libcufile-dev=1.4.0.31 @@ -35,6 +34,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml index 87759faf53..e12ab092ef 100644 --- a/conda/environments/all_cuda-125_arch-aarch64.yaml +++ b/conda/environments/all_cuda-125_arch-aarch64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-aarch64=11.* - libcufile-dev @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index e0a10af655..131efa85d5 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - cython>=3.0.0 -- dask>=2022.05.2 - doxygen=1.9.1 - gcc_linux-64=11.* - libcufile-dev @@ -33,6 +32,7 @@ dependencies: - python>=3.10,<3.13 - rangehttpserver - rapids-build-backend>=0.3.0,<0.4.0.dev0 +- rapids-dask-dependency==24.12.*,>=0.0.0a0 - scikit-build-core>=0.10.0 - sphinx - sphinx-click diff --git a/conda/recipes/kvikio/conda_build_config.yaml b/conda/recipes/kvikio/conda_build_config.yaml index 8af5f6f9c2..639a56f509 100644 --- a/conda/recipes/kvikio/conda_build_config.yaml +++ b/conda/recipes/kvikio/conda_build_config.yaml @@ -4,6 +4,9 @@ c_compiler_version: cxx_compiler_version: - 11 +cmake_version: + - ">=3.26.4,!=3.30.0" + cuda_compiler: - cuda-nvcc @@ -16,8 +19,18 @@ c_stdlib: c_stdlib_version: - "2.17" -cmake_version: - - ">=3.26.4,!=3.30.0" +# The CTK libraries below are missing from the conda-forge::cudatoolkit package +# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages +# and the "*_run_*" version specifiers correspond to `11.x` packages. + +cuda11_libcufile_host_version: + - "1.4.0.31" + +cuda11_libcufile_run_version: + - ">=1.0.0.82,<=1.4.0.31" + +libcurl_version: + - "==7.87.0" nvcomp_version: - "=4.1.0.6" diff --git a/conda/recipes/kvikio/meta.yaml b/conda/recipes/kvikio/meta.yaml index 3c41af3310..271712ac51 100644 --- a/conda/recipes/kvikio/meta.yaml +++ b/conda/recipes/kvikio/meta.yaml @@ -37,6 +37,8 @@ build: - {{ compiler('cuda11') }} {% else %} - {{ compiler('cuda') }} + - cuda-cudart-dev + - libcufile-dev # [linux] {% endif %} requirements: @@ -58,13 +60,17 @@ requirements: - cython >=3.0.0 {% if cuda_major == "11" %} - cudatoolkit + - libcufile {{ cuda11_libcufile_run_version }} # [linux64] + {% else %} + - cuda-cudart-dev + - libcufile-dev # [linux] {% endif %} - cuda-version ={{ cuda_version }} - nvcomp {{ nvcomp_version }} - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 - libkvikio ={{ version }} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} run: - python - numpy >=1.23,<3.0a0 @@ -76,6 +82,10 @@ requirements: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} - cudatoolkit + - libcufile {{ cuda11_libcufile_run_version }} # [linux64] + {% else %} + - cuda-cudart + - libcufile # [linux] {% endif %} test: diff --git a/conda/recipes/libkvikio/conda_build_config.yaml b/conda/recipes/libkvikio/conda_build_config.yaml index 9cf2923599..b895b842f3 100644 --- a/conda/recipes/libkvikio/conda_build_config.yaml +++ b/conda/recipes/libkvikio/conda_build_config.yaml @@ -28,3 +28,6 @@ cuda11_libcufile_host_version: cuda11_libcufile_run_version: - ">=1.0.0.82,<=1.4.0.31" + +libcurl_version: + - "==7.87.0" diff --git a/conda/recipes/libkvikio/meta.yaml b/conda/recipes/libkvikio/meta.yaml index 999b9fc2c1..4019a55ec8 100644 --- a/conda/recipes/libkvikio/meta.yaml +++ b/conda/recipes/libkvikio/meta.yaml @@ -52,7 +52,7 @@ requirements: {% else %} - libcufile-dev # [linux] {% endif %} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} outputs: - name: libkvikio @@ -75,7 +75,7 @@ outputs: - cmake {{ cmake_version }} host: - cuda-version ={{ cuda_version }} - - libcurl==7.87.0 + - libcurl {{ libcurl_version }} run: - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} {% if cuda_major == "11" %} diff --git a/dependencies.yaml b/dependencies.yaml index fe85506ce5..5c35e5192c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -24,12 +24,14 @@ files: output: none includes: - cuda_version + - test_libkvikio - test_cpp test_python: output: none includes: - cuda_version - py_version + - test_kvikio - test_python checks: output: none @@ -42,6 +44,7 @@ files: - cuda_version - docs - py_version + - test_kvikio py_build_kvikio: output: pyproject pyproject_dir: python/kvikio @@ -358,6 +361,18 @@ dependencies: # See https://github.com/zarr-developers/numcodecs/pull/475 - numcodecs !=0.12.0 - packaging + test_libkvikio: + common: + - output_types: conda + packages: + - libkvikio==24.12.*,>=0.0.0a0 + - libkvikio-tests==24.12.*,>=0.0.0a0 + test_kvikio: + common: + - output_types: conda + packages: + - libkvikio==24.12.*,>=0.0.0a0 + - kvikio==24.12.*,>=0.0.0a0 test_cpp: common: - output_types: conda @@ -367,7 +382,7 @@ dependencies: common: - output_types: [conda, requirements, pyproject] packages: - - &dask dask>=2022.05.2 + - rapids-dask-dependency==24.12.*,>=0.0.0a0 - pytest - pytest-cov - rangehttpserver diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index cafde598bd..4869a63fab 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -42,11 +42,11 @@ classifiers = [ test = [ "boto3>=1.21.21", "cuda-python>=11.7.1,<12.0a0,<=11.8.3", - "dask>=2022.05.2", "moto[server]>=4.0.8", "pytest", "pytest-cov", "rangehttpserver", + "rapids-dask-dependency==24.12.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls]