Add CUDA libs in Python Conda, Consolidate Conda CI installs & use rapids-dask-dependency

ci/build_docs.sh

@@ -6,28 +6,22 @@ set -euo pipefail
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
-export RAPIDS_VERSION="$(rapids-version)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-dependency-file-generator \
   --output conda \
   --file-key docs \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \
+  | tee env.yaml
 
-rapids-mamba-retry env create --yes -f env.yaml -n docs
+rapids-mamba-retry env create -yq -f env.yaml -n docs
 conda activate docs
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "kvikio=${RAPIDS_VERSION}" \
-  "libkvikio=${RAPIDS_VERSION}"
-
 export RAPIDS_DOCS_DIR="$(mktemp -d)"
 
 rapids-logger "Build CPP docs"

ci/release/update-version.sh

@@ -36,7 +36,10 @@ function sed_runner() {
 echo "${NEXT_FULL_TAG}" > VERSION
 
 DEPENDENCIES=(
+  kvikio
   libkvikio
+  libkvikio-tests
+  rapids-dask-dependency
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml; do

ci/test_cpp.sh

@@ -5,33 +5,28 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION="$(rapids-version)"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_cpp \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \
+  --prepend-channel "${CPP_CHANNEL}" \
+  | tee env.yaml
 
-rapids-mamba-retry env create --yes -f env.yaml -n test
+rapids-mamba-retry env create -qy -f env.yaml -n test
 
 # Temporarily allow unbound variables for conda activation.
 set +u
 conda activate test
 set -u
 
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
-SUITEERROR=0
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  "libkvikio=${RAPIDS_VERSION}" \
-  "libkvikio-tests=${RAPIDS_VERSION}"
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

ci/test_python.sh

@@ -8,37 +8,31 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION="$(rapids-version)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_python \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \
+  | tee env.yaml
 
-rapids-mamba-retry env create --yes -f env.yaml -n test
+rapids-mamba-retry env create -qy -f env.yaml -n test
 
 # Temporarily allow unbound variables for conda activation.
 set +u
 conda activate test
 set -u
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
 RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
 mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "libkvikio=${RAPIDS_VERSION}" \
-  "kvikio=${RAPIDS_VERSION}"
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

conda/environments/all_cuda-118_arch-aarch64.yaml

@@ -15,7 +15,6 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-aarch64=11.*
 - libcurl>=7.87.0
@@ -33,6 +32,7 @@ dependencies:
 - python>=3.10,<3.13
 - rangehttpserver
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - sphinx
 - sphinx-click

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -15,7 +15,6 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - libcufile-dev=1.4.0.31
@@ -35,6 +34,7 @@ dependencies:
 - python>=3.10,<3.13
 - rangehttpserver
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - sphinx
 - sphinx-click

conda/environments/all_cuda-125_arch-aarch64.yaml

@@ -15,7 +15,6 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-aarch64=11.*
 - libcufile-dev
@@ -33,6 +32,7 @@ dependencies:
 - python>=3.10,<3.13
 - rangehttpserver
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - sphinx
 - sphinx-click

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -15,7 +15,6 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - libcufile-dev
@@ -33,6 +32,7 @@ dependencies:
 - python>=3.10,<3.13
 - rangehttpserver
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - sphinx
 - sphinx-click

conda/recipes/kvikio/conda_build_config.yaml

@@ -4,6 +4,9 @@ c_compiler_version:
 cxx_compiler_version:
   - 11
 
+cmake_version:
+  - ">=3.26.4,!=3.30.0"
+
 cuda_compiler:
   - cuda-nvcc
 
@@ -16,8 +19,18 @@ c_stdlib:
 c_stdlib_version:
   - "2.17"
 
-cmake_version:
-  - ">=3.26.4,!=3.30.0"
+# The CTK libraries below are missing from the conda-forge::cudatoolkit package
+# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages
+# and the "*_run_*" version specifiers correspond to `11.x` packages.
+
+cuda11_libcufile_host_version:
+  - "1.4.0.31"
+
+cuda11_libcufile_run_version:
+  - ">=1.0.0.82,<=1.4.0.31"
+
+libcurl_version:
+  - "==7.87.0"
 
 nvcomp_version:
   - "=4.1.0.6"

conda/recipes/kvikio/meta.yaml

@@ -37,6 +37,8 @@ build:
     - {{ compiler('cuda11') }}
     {% else %}
     - {{ compiler('cuda') }}
+    - cuda-cudart-dev
+    - libcufile-dev  # [linux]
     {% endif %}
 
 requirements:
@@ -58,13 +60,17 @@ requirements:
     - cython >=3.0.0
     {% if cuda_major == "11" %}
     - cudatoolkit
+    - libcufile {{ cuda11_libcufile_run_version }}  # [linux64]
+    {% else %}
+    - cuda-cudart-dev
+    - libcufile-dev  # [linux]
     {% endif %}
     - cuda-version ={{ cuda_version }}
     - nvcomp {{ nvcomp_version }}
     - rapids-build-backend >=0.3.0,<0.4.0.dev0
     - scikit-build-core >=0.10.0
     - libkvikio ={{ version }}
-    - libcurl==7.87.0
+    - libcurl {{ libcurl_version }}
   run:
     - python
     - numpy >=1.23,<3.0a0
@@ -76,6 +82,10 @@ requirements:
     - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
     {% if cuda_major == "11" %}
     - cudatoolkit
+    - libcufile {{ cuda11_libcufile_run_version }}  # [linux64]
+    {% else %}
+    - cuda-cudart
+    - libcufile  # [linux]
     {% endif %}
 
 test:

conda/recipes/libkvikio/conda_build_config.yaml

@@ -28,3 +28,6 @@ cuda11_libcufile_host_version:
 
 cuda11_libcufile_run_version:
   - ">=1.0.0.82,<=1.4.0.31"
+
+libcurl_version:
+  - "==7.87.0"

conda/recipes/libkvikio/meta.yaml

@@ -52,7 +52,7 @@ requirements:
     {% else %}
     - libcufile-dev  # [linux]
     {% endif %}
-    - libcurl==7.87.0
+    - libcurl {{ libcurl_version }}
 
 outputs:
   - name: libkvikio
@@ -75,7 +75,7 @@ outputs:
         - cmake {{ cmake_version }}
       host:
         - cuda-version ={{ cuda_version }}
-        - libcurl==7.87.0
+        - libcurl {{ libcurl_version }}
       run:
         - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
         {% if cuda_major == "11" %}

dependencies.yaml

@@ -24,12 +24,14 @@ files:
     output: none
     includes:
       - cuda_version
+      - test_libkvikio
       - test_cpp
   test_python:
     output: none
     includes:
       - cuda_version
       - py_version
+      - test_kvikio
       - test_python
   checks:
     output: none
@@ -42,6 +44,7 @@ files:
       - cuda_version
       - docs
       - py_version
+      - test_kvikio
   py_build_kvikio:
     output: pyproject
     pyproject_dir: python/kvikio
@@ -358,6 +361,18 @@ dependencies:
           # See https://github.com/zarr-developers/numcodecs/pull/475
           - numcodecs !=0.12.0
           - packaging
+  test_libkvikio:
+    common:
+      - output_types: conda
+        packages:
+          - libkvikio==24.12.*,>=0.0.0a0
+          - libkvikio-tests==24.12.*,>=0.0.0a0
+  test_kvikio:
+    common:
+      - output_types: conda
+        packages:
+          - libkvikio==24.12.*,>=0.0.0a0
+          - kvikio==24.12.*,>=0.0.0a0
   test_cpp:
     common:
       - output_types: conda
@@ -367,7 +382,7 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - &dask dask>=2022.05.2
+          - rapids-dask-dependency==24.12.*,>=0.0.0a0
           - pytest
           - pytest-cov
           - rangehttpserver

python/kvikio/pyproject.toml

@@ -42,11 +42,11 @@ classifiers = [
 test = [
     "boto3>=1.21.21",
     "cuda-python>=11.7.1,<12.0a0,<=11.8.3",
-    "dask>=2022.05.2",
     "moto[server]>=4.0.8",
     "pytest",
     "pytest-cov",
     "rangehttpserver",
+    "rapids-dask-dependency==24.12.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]