From a916825778fa5e0b6a713d664741e5f1d0195b62 Mon Sep 17 00:00:00 2001 From: Christopher Harris Date: Tue, 11 Oct 2022 16:35:05 -0500 Subject: [PATCH] update abseil, grpc, and ucx versions for cuml compatibility (#177) installing cuml 22.08 along side srf 22.09 does not work, as the abseil, grpc, and ucx versions mismatch. This PR resolves those issues. In doing so, a bug presented itself in how conda build was determining hashes. SRF was attempting to look for a version of libsrf that was not being generated, to fix this, I added abseil as a dependency of SRF, which is a workaround mentioned in https://github.com/conda-forge/arrow-cpp-feedstock/issues/814. Authors: - Christopher Harris (https://github.com/cwharris) - Michael Demoret (https://github.com/mdemoret-nv) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: https://github.com/nv-morpheus/SRF/pull/177 --- Dockerfile | 2 +- ci/conda/environments/ci_env.yml | 2 +- ci/conda/environments/dev_env.yml | 4 +- ci/conda/environments/dev_env_nogcc.yml | 4 +- ci/conda/recipes/libsrf/build.sh | 1 + .../recipes/libsrf/conda_build_config.yaml | 29 +++++++++---- ci/conda/recipes/libsrf/meta.yaml | 6 ++- cmake/dependencies.cmake | 2 +- cmake/deps/Configure_gRPC.cmake | 41 ------------------- cmake/deps/Configure_ucx.cmake | 2 +- 10 files changed, 34 insertions(+), 59 deletions(-) delete mode 100644 cmake/deps/Configure_gRPC.cmake diff --git a/Dockerfile b/Dockerfile index 89512e255..5f2bf6d99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/ RUN conda config --set ssl_verify false &&\ conda config --add pkgs_dirs /opt/conda/pkgs &&\ conda config --env --add channels conda-forge &&\ - /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.10" python=${PYTHON_VER} + /opt/conda/bin/conda install -y -n base -c conda-forge "mamba >=0.22" "boa >=0.12" python=${PYTHON_VER} # conda clean -afy # All code will be under /work diff --git a/ci/conda/environments/ci_env.yml b/ci/conda/environments/ci_env.yml index 993824428..34aa9fcbe 100644 --- a/ci/conda/environments/ci_env.yml +++ b/ci/conda/environments/ci_env.yml @@ -18,6 +18,6 @@ name: srf channels: - conda-forge dependencies: - - boa>=0.1 + - boa>=0.12 - conda-pack=0.7 - sccache=0.3 diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml index b6ccd973e..879ae15c8 100644 --- a/ci/conda/environments/dev_env.yml +++ b/ci/conda/environments/dev_env.yml @@ -21,7 +21,7 @@ dependencies: - glog=0.6 - gmock=1.10 - graphviz=3.0 - - grpc-cpp=1.45 + - grpc-cpp=1.46 - gtest=1.10 - gxx_linux-64=9.4 - jinja2=3.0 @@ -42,7 +42,7 @@ dependencies: - scikit-build>=0.12 - spdlog=1.8.5 - sysroot_linux-64=2.17 - - ucx=1.12 + - ucx=1.13 - pip: - cython - flake8 diff --git a/ci/conda/environments/dev_env_nogcc.yml b/ci/conda/environments/dev_env_nogcc.yml index f1b8ec610..5be1b1455 100644 --- a/ci/conda/environments/dev_env_nogcc.yml +++ b/ci/conda/environments/dev_env_nogcc.yml @@ -17,7 +17,7 @@ dependencies: - glog=0.6 - gmock=1.10 - graphviz=3.0 - - grpc-cpp=1.45 + - grpc-cpp=1.46 - gtest=1.10 - libhwloc=2.5 - libprotobuf=3.20 @@ -28,7 +28,7 @@ dependencies: - python=3.8 - scikit-build>=0.12 - spdlog=1.8.5 - - ucx=1.12 + - ucx=1.13 - pip: - cython - flake8 diff --git a/ci/conda/recipes/libsrf/build.sh b/ci/conda/recipes/libsrf/build.sh index 711277d0d..11a4254a0 100644 --- a/ci/conda/recipes/libsrf/build.sh +++ b/ci/conda/recipes/libsrf/build.sh @@ -62,6 +62,7 @@ CMAKE_ARGS="-DSRF_BUILD_PYTHON=ON ${CMAKE_ARGS}" CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"ALL"} ${CMAKE_ARGS}" CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" CMAKE_ARGS="-DSRF_RAPIDS_VERSION=${rapids_version} ${CMAKE_ARGS}" +CMAKE_ARGS="-DUCX_VERSION=${ucx} ${CMAKE_ARGS}" echo "CC : ${CC}" echo "CXX : ${CXX}" diff --git a/ci/conda/recipes/libsrf/conda_build_config.yaml b/ci/conda/recipes/libsrf/conda_build_config.yaml index d11d84f59..84db3f5eb 100644 --- a/ci/conda/recipes/libsrf/conda_build_config.yaml +++ b/ci/conda/recipes/libsrf/conda_build_config.yaml @@ -30,19 +30,36 @@ python: - 3.9 # Setup the dependencies to build with multiple versions of RAPIDS -rapids_version: - - 22.04 # Keep around compatibility with current version -1 +rapids_version: # Keep around compatibility with current version -1 - 22.06 - 22.08 + - 22.08 +# Multiple versions of abseil are required to satisfy the solver for some +# environments. RAPIDS 22.06 only works with gRPC 1.45 and 22.08 only works with +# 1.46. For each version of gRPC, support 2 abseil versions. Zip all of the keys +# together to avoid impossible combinations abseil_cpp: - - 20211102.0 - - 20210324.2 - 20210324.2 + - 20211102.0 + - 20220623.0 + +grpc_cpp: + - 1.45 + - 1.46 + - 1.46 + +# UCX 1.12 is required for RAPIDS 22.06 +ucx: + - 1.12 + - 1.13 + - 1.13 zip_keys: - rapids_version - abseil_cpp + - grpc_cpp + - ucx # The following mimic what is available in the pinning feedstock: # https://github.com/conda-forge/conda-forge-pinning-feedstock/blob/main/recipe/conda_build_config.yaml @@ -54,12 +71,8 @@ gflags: - 2.2 glog: - 0.6 -grpc_cpp: - - 1.45 libprotobuf: - 3.20 -ucx: - - 1.12 pin_run_as_build: diff --git a/ci/conda/recipes/libsrf/meta.yaml b/ci/conda/recipes/libsrf/meta.yaml index e39a5ee8a..2eb2ff035 100644 --- a/ci/conda/recipes/libsrf/meta.yaml +++ b/ci/conda/recipes/libsrf/meta.yaml @@ -127,15 +127,16 @@ outputs: host: # Only should need libsrf and python. Keep sorted! - {{ pin_subpackage('libsrf', exact=True) }} + - abseil-cpp # srf does not require abseil at build time. See https://github.com/conda-forge/arrow-cpp-feedstock/issues/814 - python {{ python }} run: - {{ pin_subpackage('libsrf', exact=True) }} - - rmm {{ rapids_version }} # This is not necessary but required until this is fixed: https://github.com/mamba-org/boa/issues/232 + - rmm {{ rapids_version }}.* # This is not necessary but required until this is fixed: https://github.com/mamba-org/boa/issues/232 - python test: imports: - srf - script: test_libsrf.sh + script: test_srf.sh source_files: # Copy the pytest source files - python/pytest.ini @@ -145,6 +146,7 @@ outputs: - numpy - nvtx - pytest + - cuml {{ rapids_version }}.* # Ensure we can install cuml. This can cause issues solving abseil-cpp about: home: https://www.nvidia.com/ diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 72262cca5..dc162a29c 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -64,7 +64,7 @@ include(deps/Configure_boost) # UCX # === -set(UCX_VERSION "1.12" CACHE STRING "Version of ucx to use") +set(UCX_VERSION "1.13" CACHE STRING "Version of ucx to use") include(deps/Configure_ucx) # hwloc diff --git a/cmake/deps/Configure_gRPC.cmake b/cmake/deps/Configure_gRPC.cmake deleted file mode 100644 index fddae0dd5..000000000 --- a/cmake/deps/Configure_gRPC.cmake +++ /dev/null @@ -1,41 +0,0 @@ -#============================================================================= -# SPDX-FileCopyrightText: Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -#============================================================================= - -function(find_and_configure_gRPC VERSION) - - list(APPEND CMAKE_MESSAGE_CONTEXT "gRPC") - - rapids_cpm_find(gRPC ${GRPC_VERSION} - GLOBAL_TARGETS - gRPC::grpc++_unsecure gRPC::grpc_cpp_plugin gRPC::gpr - BUILD_EXPORT_SET - ${PROJECT_NAME}-core-exports - INSTALL_EXPORT_SET - ${PROJECT_NAME}-core-exports - CPM_ARGS - GIT_REPOSITORY https://github.com/grpc/grpc.git - GIT_TAG v${GRPC_VERSION} - GIT_SHALLOW TRUE - OPTIONS "BUILD_TESTS OFF" - "BUILD_BENCHMARKS OFF" - "CUDA_STATIC_RUNTIME ON" - "DISABLE_DEPRECATION_WARNING ${DISABLE_DEPRECATION_WARNINGS}" - ) - -endfunction() - -find_and_configure_gRPC(${GRPC_VERSION}) diff --git a/cmake/deps/Configure_ucx.cmake b/cmake/deps/Configure_ucx.cmake index 2bcd5da9f..550999d30 100644 --- a/cmake/deps/Configure_ucx.cmake +++ b/cmake/deps/Configure_ucx.cmake @@ -20,7 +20,7 @@ function(find_and_configure_ucx version) list(APPEND CMAKE_MESSAGE_CONTEXT "ucx") # Try to find UCX and download from source if not found - rapids_cpm_find(ucx 1.12 + rapids_cpm_find(ucx ${version} GLOBAL_TARGETS ucx ucx::ucp ucx::uct ucx_ucx ucx::ucp ucx::uct ucx::ucx BUILD_EXPORT_SET