Add CUDA libs in Python Conda, Consolidate Conda CI installs & use `r…

…apids-dask-dependency` (#513)

Explicitly include dependencies on `libcufile*` and `*cudart*` in the Python Conda package. These seem to be pulled in implicitly by the C++ package. However as they are dependencies of the Python built shared objects, they should also be listed here.

Also consolidate the Conda environment creation and installation of nightly dependencies into one step. By doing this in one step, we guarantee that all of our constraints are taken into account during environment creation.

Lastly switch from using `dask` directly to `rapids-dask-dependency` to align Dask installation and pinning with the rest of RAPIDS.

xref: rapidsai/build-planning#22
xef: #378

Authors:
  - https://github.com/jakirkham
  - Mads R. B. Kristensen (https://github.com/madsbk)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Mads R. B. Kristensen (https://github.com/madsbk)

URL: #513

ci/build_docs.sh

@@ -6,28 +6,22 @@ set -euo pipefail
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
-export RAPIDS_VERSION="$(rapids-version)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-dependency-file-generator \
   --output conda \
   --file-key docs \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \
+  | tee env.yaml
 
-rapids-mamba-retry env create --yes -f env.yaml -n docs
+rapids-mamba-retry env create -yq -f env.yaml -n docs
 conda activate docs
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "kvikio=${RAPIDS_VERSION}" \
-  "libkvikio=${RAPIDS_VERSION}"
-
 export RAPIDS_DOCS_DIR="$(mktemp -d)"
 
 rapids-logger "Build CPP docs"

ci/release/update-version.sh

@@ -36,7 +36,10 @@ function sed_runner() {
 echo "${NEXT_FULL_TAG}" > VERSION
 
 DEPENDENCIES=(
+  kvikio
   libkvikio
+  libkvikio-tests
+  rapids-dask-dependency
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml; do

ci/test_cpp.sh

@@ -5,33 +5,28 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION="$(rapids-version)"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_cpp \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \
+  --prepend-channel "${CPP_CHANNEL}" \
+  | tee env.yaml
 
-rapids-mamba-retry env create --yes -f env.yaml -n test
+rapids-mamba-retry env create -qy -f env.yaml -n test
 
 # Temporarily allow unbound variables for conda activation.
 set +u
 conda activate test
 set -u
 
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
-SUITEERROR=0
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  "libkvikio=${RAPIDS_VERSION}" \
-  "libkvikio-tests=${RAPIDS_VERSION}"
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

ci/test_python.sh

@@ -8,37 +8,31 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION="$(rapids-version)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_python \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" --prepend-channel "${PYTHON_CHANNEL}" \
+  | tee env.yaml
 
-rapids-mamba-retry env create --yes -f env.yaml -n test
+rapids-mamba-retry env create -qy -f env.yaml -n test
 
 # Temporarily allow unbound variables for conda activation.
 set +u
 conda activate test
 set -u
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
 RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
 mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "libkvikio=${RAPIDS_VERSION}" \
-  "kvikio=${RAPIDS_VERSION}"
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

conda/environments/all_cuda-118_arch-aarch64.yaml

@@ -15,7 +15,6 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-aarch64=11.*
 - libcurl>=7.87.0
@@ -33,6 +32,7 @@ dependencies:
 - python>=3.10,<3.13
 - rangehttpserver
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - sphinx
 - sphinx-click

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -15,7 +15,6 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - libcufile-dev=1.4.0.31
@@ -35,6 +34,7 @@ dependencies:
 - python>=3.10,<3.13
 - rangehttpserver
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - sphinx
 - sphinx-click

conda/environments/all_cuda-125_arch-aarch64.yaml

@@ -15,7 +15,6 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-aarch64=11.*
 - libcufile-dev
@@ -33,6 +32,7 @@ dependencies:
 - python>=3.10,<3.13
 - rangehttpserver
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - sphinx
 - sphinx-click

conda/environments/all_cuda-125_arch-x86_64.yaml

@@ -15,7 +15,6 @@ dependencies:
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask>=2022.05.2
 - doxygen=1.9.1
 - gcc_linux-64=11.*
 - libcufile-dev
@@ -33,6 +32,7 @@ dependencies:
 - python>=3.10,<3.13
 - rangehttpserver
 - rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.12.*,>=0.0.0a0
 - scikit-build-core>=0.10.0
 - sphinx
 - sphinx-click

conda/recipes/kvikio/conda_build_config.yaml

@@ -4,6 +4,9 @@ c_compiler_version:
 cxx_compiler_version:
   - 11
 
+cmake_version:
+  - ">=3.26.4,!=3.30.0"
+
 cuda_compiler:
   - cuda-nvcc
 
@@ -16,8 +19,18 @@ c_stdlib:
 c_stdlib_version:
   - "2.17"
 
-cmake_version:
-  - ">=3.26.4,!=3.30.0"
+# The CTK libraries below are missing from the conda-forge::cudatoolkit package
+# for CUDA 11. The "*_host_*" version specifiers correspond to `11.8` packages
+# and the "*_run_*" version specifiers correspond to `11.x` packages.
+
+cuda11_libcufile_host_version:
+  - "1.4.0.31"
+
+cuda11_libcufile_run_version:
+  - ">=1.0.0.82,<=1.4.0.31"
+
+libcurl_version:
+  - "==7.87.0"
 
 nvcomp_version:
   - "=4.1.0.6"

conda/recipes/kvikio/meta.yaml

@@ -37,6 +37,8 @@ build:
     - {{ compiler('cuda11') }}
     {% else %}
     - {{ compiler('cuda') }}
+    - cuda-cudart-dev
+    - libcufile-dev  # [linux]
     {% endif %}
 
 requirements:
@@ -58,13 +60,17 @@ requirements:
     - cython >=3.0.0
     {% if cuda_major == "11" %}
     - cudatoolkit
+    - libcufile {{ cuda11_libcufile_run_version }}  # [linux64]
+    {% else %}
+    - cuda-cudart-dev
+    - libcufile-dev  # [linux]
     {% endif %}
     - cuda-version ={{ cuda_version }}
     - nvcomp {{ nvcomp_version }}
     - rapids-build-backend >=0.3.0,<0.4.0.dev0
     - scikit-build-core >=0.10.0
     - libkvikio ={{ version }}
-    - libcurl==7.87.0
+    - libcurl {{ libcurl_version }}
   run:
     - python
     - numpy >=1.23,<3.0a0
@@ -76,6 +82,10 @@ requirements:
     - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
     {% if cuda_major == "11" %}
     - cudatoolkit
+    - libcufile {{ cuda11_libcufile_run_version }}  # [linux64]
+    {% else %}
+    - cuda-cudart
+    - libcufile  # [linux]
     {% endif %}
 
 test:

conda/recipes/libkvikio/conda_build_config.yaml

@@ -28,3 +28,6 @@ cuda11_libcufile_host_version:
 
 cuda11_libcufile_run_version:
   - ">=1.0.0.82,<=1.4.0.31"
+
+libcurl_version:
+  - "==7.87.0"

conda/recipes/libkvikio/meta.yaml

@@ -52,7 +52,7 @@ requirements:
     {% else %}
     - libcufile-dev  # [linux]
     {% endif %}
-    - libcurl==7.87.0
+    - libcurl {{ libcurl_version }}
 
 outputs:
   - name: libkvikio
@@ -75,7 +75,7 @@ outputs:
         - cmake {{ cmake_version }}
       host:
         - cuda-version ={{ cuda_version }}
-        - libcurl==7.87.0
+        - libcurl {{ libcurl_version }}
       run:
         - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}
         {% if cuda_major == "11" %}

dependencies.yaml

@@ -24,12 +24,14 @@ files:
     output: none
     includes:
       - cuda_version
+      - test_libkvikio
       - test_cpp
   test_python:
     output: none
     includes:
       - cuda_version
       - py_version
+      - test_kvikio
       - test_python
   checks:
     output: none
@@ -42,6 +44,7 @@ files:
       - cuda_version
       - docs
       - py_version
+      - test_kvikio
   py_build_kvikio:
     output: pyproject
     pyproject_dir: python/kvikio
@@ -358,6 +361,18 @@ dependencies:
           # See https://github.com/zarr-developers/numcodecs/pull/475
           - numcodecs !=0.12.0
           - packaging
+  test_libkvikio:
+    common:
+      - output_types: conda
+        packages:
+          - libkvikio==24.12.*,>=0.0.0a0
+          - libkvikio-tests==24.12.*,>=0.0.0a0
+  test_kvikio:
+    common:
+      - output_types: conda
+        packages:
+          - libkvikio==24.12.*,>=0.0.0a0
+          - kvikio==24.12.*,>=0.0.0a0
   test_cpp:
     common:
       - output_types: conda
@@ -367,7 +382,7 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - &dask dask>=2022.05.2
+          - rapids-dask-dependency==24.12.*,>=0.0.0a0
           - pytest
           - pytest-cov
           - rangehttpserver

python/kvikio/pyproject.toml

@@ -42,11 +42,11 @@ classifiers = [
 test = [
     "boto3>=1.21.21",
     "cuda-python>=11.7.1,<12.0a0,<=11.8.3",
-    "dask>=2022.05.2",
     "moto[server]>=4.0.8",
     "pytest",
     "pytest-cov",
     "rangehttpserver",
+    "rapids-dask-dependency==24.12.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]