update abseil, grpc, and ucx versions for cuml compatibility (#177)

installing cuml 22.08 along side srf 22.09 does not work, as the abseil, grpc, and ucx versions mismatch. This PR resolves those issues. In doing so, a bug presented itself in how conda build was determining hashes. SRF was attempting to look for a version of libsrf that was not being generated, to fix this, I added abseil as a dependency of SRF, which is a workaround mentioned in conda-forge/arrow-cpp-feedstock#814.

Authors:
  - Christopher Harris (https://github.com/cwharris)
  - Michael Demoret (https://github.com/mdemoret-nv)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #177

Dockerfile

@@ -40,7 +40,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/
 RUN conda config --set ssl_verify false &&\
     conda config --add pkgs_dirs /opt/conda/pkgs &&\
     conda config --env --add channels conda-forge &&\
-    /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" python=${PYTHON_VER}
+    /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.12" python=${PYTHON_VER}
     # conda clean -afy
 
 # All code will be under /work

ci/conda/environments/ci_env.yml

@@ -18,6 +18,6 @@ name: srf
 channels:
     - conda-forge
 dependencies:
-    - boa>=0.1
+    - boa>=0.12
     - conda-pack=0.7
     - sccache=0.3

ci/conda/environments/dev_env.yml

@@ -21,7 +21,7 @@ dependencies:
   - glog=0.6
   - gmock=1.10
   - graphviz=3.0
-  - grpc-cpp=1.45
+  - grpc-cpp=1.46
   - gtest=1.10
   - gxx_linux-64=9.4
   - jinja2=3.0
@@ -42,7 +42,7 @@ dependencies:
   - scikit-build>=0.12
   - spdlog=1.8.5
   - sysroot_linux-64=2.17
-  - ucx=1.12
+  - ucx=1.13
   - pip:
     - cython
     - flake8

ci/conda/environments/dev_env_nogcc.yml

@@ -17,7 +17,7 @@ dependencies:
   - glog=0.6
   - gmock=1.10
   - graphviz=3.0
-  - grpc-cpp=1.45
+  - grpc-cpp=1.46
   - gtest=1.10
   - libhwloc=2.5
   - libprotobuf=3.20
@@ -28,7 +28,7 @@ dependencies:
   - python=3.8
   - scikit-build>=0.12
   - spdlog=1.8.5
-  - ucx=1.12
+  - ucx=1.13
   - pip:
     - cython
     - flake8

ci/conda/recipes/libsrf/build.sh

@@ -62,6 +62,7 @@ CMAKE_ARGS="-DSRF_BUILD_PYTHON=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DSRF_RAPIDS_VERSION=${rapids_version} ${CMAKE_ARGS}"
+CMAKE_ARGS="-DUCX_VERSION=${ucx} ${CMAKE_ARGS}"
 
 echo "CC          : ${CC}"
 echo "CXX         : ${CXX}"

ci/conda/recipes/libsrf/conda_build_config.yaml

@@ -30,19 +30,36 @@ python:
   - 3.9
 
 # Setup the dependencies to build with multiple versions of RAPIDS
-rapids_version:
-  - 22.04 # Keep around compatibility with current version -1
+rapids_version: # Keep around compatibility with current version -1
   - 22.06
   - 22.08
+  - 22.08
 
+# Multiple versions of abseil are required to satisfy the solver for some
+# environments. RAPIDS 22.06 only works with gRPC 1.45 and 22.08 only works with
+# 1.46. For each version of gRPC, support 2 abseil versions. Zip all of the keys
+# together to avoid impossible combinations
 abseil_cpp:
-  - 20211102.0
-  - 20210324.2
   - 20210324.2
+  - 20211102.0
+  - 20220623.0
+
+grpc_cpp:
+  - 1.45
+  - 1.46
+  - 1.46
+
+# UCX 1.12 is required for RAPIDS 22.06
+ucx:
+  - 1.12
+  - 1.13
+  - 1.13
 
 zip_keys:
   - rapids_version
   - abseil_cpp
+  - grpc_cpp
+  - ucx
 
 # The following mimic what is available in the pinning feedstock:
 # https://github.com/conda-forge/conda-forge-pinning-feedstock/blob/main/recipe/conda_build_config.yaml
@@ -54,12 +71,8 @@ gflags:
   - 2.2
 glog:
   - 0.6
-grpc_cpp:
-  - 1.45
 libprotobuf:
   - 3.20
-ucx:
-  - 1.12
 
 
 pin_run_as_build:

ci/conda/recipes/libsrf/meta.yaml

@@ -127,15 +127,16 @@ outputs:
       host:
         # Only should need libsrf and python. Keep sorted!
         - {{ pin_subpackage('libsrf', exact=True) }}
+        - abseil-cpp # srf does not require abseil at build time. See https://github.com/conda-forge/arrow-cpp-feedstock/issues/814
         - python {{ python }}
       run:
         - {{ pin_subpackage('libsrf', exact=True) }}
-        - rmm {{ rapids_version }} # This is not necessary but required until this is fixed: https://github.com/mamba-org/boa/issues/232
+        - rmm {{ rapids_version }}.* # This is not necessary but required until this is fixed: https://github.com/mamba-org/boa/issues/232
         - python
     test:
       imports:
         - srf
-      script: test_libsrf.sh
+      script: test_srf.sh
       source_files:
         # Copy the pytest source files
         - python/pytest.ini
@@ -145,6 +146,7 @@ outputs:
         - numpy
         - nvtx
         - pytest
+        - cuml {{ rapids_version }}.* # Ensure we can install cuml. This can cause issues solving abseil-cpp
 
 about:
   home: https://www.nvidia.com/

cmake/dependencies.cmake

@@ -64,7 +64,7 @@ include(deps/Configure_boost)
 
 # UCX
 # ===
-set(UCX_VERSION "1.12" CACHE STRING "Version of ucx to use")
+set(UCX_VERSION "1.13" CACHE STRING "Version of ucx to use")
 include(deps/Configure_ucx)
 
 # hwloc

cmake/deps/Configure_ucx.cmake

@@ -20,7 +20,7 @@ function(find_and_configure_ucx version)
   list(APPEND CMAKE_MESSAGE_CONTEXT "ucx")
 
   # Try to find UCX and download from source if not found
-  rapids_cpm_find(ucx 1.12
+  rapids_cpm_find(ucx ${version}
     GLOBAL_TARGETS
       ucx ucx::ucp ucx::uct ucx_ucx ucx::ucp ucx::uct ucx::ucx
     BUILD_EXPORT_SET