Combine conda recipes into one one meta.yaml generating multiple outputs

Signed-off-by: Anuradha Karuppiah <anuradhak@nvidia.com>
nv-morpheus · Sep 30, 2024 · 03ce1cf · 03ce1cf
1 parent dbb34a8
commit 03ce1cf
Show file tree

Hide file tree

Showing 15 changed files with 166 additions and 527 deletions.
diff --git a/ci/conda/recipes/morpheus-dfp/conda_build_config.yaml b/ci/conda/recipes/morpheus-dfp/conda_build_config.yaml
diff --git a/ci/conda/recipes/morpheus-dfp/meta.yaml b/ci/conda/recipes/morpheus-dfp/meta.yaml
diff --git a/...ecipes/morpheus-dfp/morpheus_dfp_build.sh → ...nda/recipes/morpheus-libs/cmake_common.sh b/...ecipes/morpheus-dfp/morpheus_dfp_build.sh → ...nda/recipes/morpheus-libs/cmake_common.sh
@@ -1,3 +1,4 @@
+
 # SPDX-FileCopyrightText: Copyright (c) 2022-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -18,28 +19,8 @@
 
 # This will store all of the cmake args. Make sure to prepend args to allow
 # incoming values to overwrite them
-CMAKE_ARGS=${CMAKE_ARGS:-""}
-
-export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
-export USE_SCCACHE=${USE_SCCACHE:-""}
-
-# Check for some mrc environment variables. Append to front of args to allow users to overwrite them
-if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
-   # Set the cache variable, then set the Staging prefix to allow for host searching
-   CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
-
-   # Double check that the cache dir has been created
-   mkdir -p ${MORPHEUS_CACHE_DIR}
-fi
-
-# Enable DFP
-CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=ON ${CMAKE_ARGS}"
-
-# Disable core, llm and doca
-CMAKE_ARGS="-DMORPHEUS_SUPPORT_DOCA=OFF ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_CORE=OFF ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF ${CMAKE_ARGS}"
 
+# CMake flags common across all libraries
 CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
 CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
@@ -55,32 +36,12 @@ CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
 CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11
 CMAKE_ARGS="--log-level=VERBOSE ${CMAKE_ARGS}"
 
-if [[ "${USE_SCCACHE}" == "1" ]]; then
-   CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}"
-fi
-
-echo "========Begin Env========"
-env
-echo "========End Env========"
-
-BUILD_DIR="build-conda"
+# Append to front of args to allow users to overwrite them
+if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
+   # Set the cache variable, then set the Staging prefix to allow for host searching
+   CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
 
-# Check if the build directory already exists. And if so, delete the
-# CMakeCache.txt and CMakeFiles to ensure a clean configuration
-if [[ -d "./${BUILD_DIR}" ]]; then
-   echo "Deleting old CMake files at ./${BUILD_DIR}"
-   rm -rf "./${BUILD_DIR}/CMakeCache.txt"
-   rm -rf "./${BUILD_DIR}/CMakeFiles"
+   # Double check that the cache dir has been created
+   mkdir -p ${MORPHEUS_CACHE_DIR}
 fi
 
-# Run configure
-cmake -B ${BUILD_DIR} \
-   ${CMAKE_ARGS} \
-   --log-level=verbose \
-   .
-
-# Build the components
-cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
-
-# Install the mprpheus dfp python wheel components
-${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus_dfp/dist/*.whl
diff --git a/ci/conda/recipes/morpheus-libs/meta.yaml b/ci/conda/recipes/morpheus-libs/meta.yaml
@@ -17,14 +17,14 @@
 {% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
 
 package:
-  name: morpheus-core
+  name: morpheus-libs
   version: {{ version }}
 
 source:
   git_url: ../../../..
 
 outputs:
-
+  ############################### morpheus-core #############################
   - name: morpheus-core
     build:
       number: {{ GIT_DESCRIBE_NUMBER }}
@@ -45,14 +45,12 @@ outputs:
         - {{ compiler("c") }}
         - {{ compiler("cuda") }}
         - {{ compiler("cxx") }}
-        - automake =1.16.5 # Needed for DOCA build
         - ccache
         - cmake =3.27
         - cuda-cudart-dev {{ cuda_compiler_version }}.* # Needed by CMake to compile a test application
         - cuda-version {{ cuda_compiler_version }}.*
-        - libtool # Needed for DOCA build
         - ninja =1.11
-        - pkg-config =0.29 # for mrc cmake
+        - pkg-config =0.29
         - sysroot_linux-64 =2.17
       host:
         # CUDA dependencies
@@ -98,7 +96,7 @@ outputs:
         - numpydoc =1.5.*
         - pydantic
         - pluggy =1.3.*
-        - python
+        - python {{ python }}
         - python-confluent-kafka >=1.9.2,<1.10.0a0
         - python-graphviz
         - pytorch-cuda
@@ -130,9 +128,78 @@ outputs:
       commands:
           - echo # make sure test requirements get installed
 
+  ############################### morpheus-dfp #############################
+  - name: morpheus-dfp
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+      script_env:
+        - CMAKE_CUDA_ARCHITECTURES
+        - MORPHEUS_CACHE_DIR
+        - MORPHEUS_PYTHON_BUILD_STUBS
+        - PARALLEL_LEVEL
+      run_exports:
+        - {{ pin_subpackage("morpheus-dfp", max_pin="x.x") }}
+    script: morpheus_dfp_build.sh
+
+    requirements:
+      build:
+        - ccache
+        - cmake =3.27
+      host:
+        - {{ pin_subpackage('morpheus-core', exact=True) }}
+        - pip
+        - scikit-build 0.17.6
+        - versioneer-518
+      run:
+        - {{ pin_subpackage('morpheus-core', exact=True) }}
+
+    #test: Tests will be added post test refactoring
+
+  ############################### morpheus-llm #############################
+  - name: morpheus-llm
+    build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: cuda_{{ cuda_compiler_version }}_py{{ python }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+      script_env:
+        - CMAKE_CUDA_ARCHITECTURES
+        - MORPHEUS_CACHE_DIR
+        - MORPHEUS_PYTHON_BUILD_STUBS
+        - PARALLEL_LEVEL
+      run_exports:
+        - {{ pin_subpackage("morpheus-llm", max_pin="x.x") }}
+    script: morpheus_llm_build.sh
+
+    requirements:
+      build:
+        - {{ compiler("c") }}
+        - {{ compiler("cuda") }}
+        - {{ compiler("cxx") }}
+        - ccache
+        - cmake =3.27
+        - ninja =1.11
+        - pkg-config =0.29
+      host:
+        - cudf {{ rapids_version }}
+        - cython 3.0.*
+        - glog 0.6.*
+        - libcudf {{ rapids_version }}
+        - librdkafka >=1.9.2,<1.10.0a0
+        - {{ pin_subpackage('morpheus-core', exact=True) }}
+        - pip
+        - pybind11-stubgen 0.10.5
+        - rapidjson 1.1.0
+        - scikit-build 0.17.6
+        - versioneer-518
+        - zlib 1.2.13 # required to build triton client
+      run:
+        - {{ pin_subpackage('morpheus-core', exact=True) }}
+
+    #test: Tests will be added post test refactoring
+
 about:
   home: https://github.com/nv-morpheus/Morpheus
   license: Apache-2.0
   license_family: Apache
   license_file: LICENSE
-  summary: Morpheus Cybersecurity Core Library
+  summary: Morpheus Cybersecurity Library
diff --git a/ci/conda/recipes/morpheus-libs/morpheus_core_build.sh b/ci/conda/recipes/morpheus-libs/morpheus_core_build.sh
@@ -18,12 +18,14 @@
 
 # This will store all of the cmake args. Make sure to prepend args to allow
 # incoming values to overwrite them
+
+source $RECIPE_DIR/cmake_common.sh
+
 CMAKE_ARGS=${CMAKE_ARGS:-""}
 
 export CCACHE_BASEDIR=$(realpath ${SRC_DIR}/..)
 export USE_SCCACHE=${USE_SCCACHE:-""}
 
-# Check for some mrc environment variables. Append to front of args to allow users to overwrite them
 if [[ -n "${MORPHEUS_CACHE_DIR}" ]]; then
    # Set the cache variable, then set the Staging prefix to allow for host searching
    CMAKE_ARGS="-DMORPHEUS_CACHE_DIR=${MORPHEUS_CACHE_DIR} ${CMAKE_ARGS}"
@@ -40,21 +42,6 @@ CMAKE_ARGS="-DMORPHEUS_SUPPORT_DOCA=OFF ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_DFP=OFF ${CMAKE_ARGS}"
 CMAKE_ARGS="-DMORPHEUS_BUILD_MORPHEUS_LLM=OFF ${CMAKE_ARGS}"
 
-CMAKE_ARGS="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DCMAKE_INSTALL_PREFIX=$PREFIX ${CMAKE_ARGS}"
-CMAKE_ARGS="-DCMAKE_INSTALL_LIBDIR=lib ${CMAKE_ARGS}"
-CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_USE_CONDA=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_USE_CCACHE=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_STUBS=${MORPHEUS_PYTHON_BUILD_STUBS=-"ON"} ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_PYTHON_INPLACE_BUILD=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DMORPHEUS_PYTHON_BUILD_WHEEL=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DCMAKE_BUILD_RPATH_USE_ORIGIN=ON ${CMAKE_ARGS}"
-CMAKE_ARGS="-DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES=-"RAPIDS"} ${CMAKE_ARGS}"
-CMAKE_ARGS="-DPython_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}"
-CMAKE_ARGS="-DPYTHON_EXECUTABLE=${PYTHON} ${CMAKE_ARGS}" # for pybind11
-CMAKE_ARGS="--log-level=VERBOSE ${CMAKE_ARGS}"
-
 if [[ "${USE_SCCACHE}" == "1" ]]; then
    CMAKE_ARGS="-DCCACHE_PROGRAM_PATH=$(which sccache) ${CMAKE_ARGS}"
 fi
@@ -69,7 +56,7 @@ echo "========Begin Env========"
 env
 echo "========End Env========"
 
-BUILD_DIR="build-conda"
+BUILD_DIR="build-conda-core"
 
 # Check if the build directory already exists. And if so, delete the
 # CMakeCache.txt and CMakeFiles to ensure a clean configuration
@@ -88,5 +75,5 @@ cmake -B ${BUILD_DIR} \
 # Build the components
 cmake --build ${BUILD_DIR} -j${PARALLEL_LEVEL:-$(nproc)} --target install
 
-# Install just the mprpheus core python wheel components
+# Install just the morpheus core python wheel components
 ${PYTHON} -m pip install -vv ${BUILD_DIR}/python/morpheus/dist/*.whl