Extraction of common scripts and utilities + cleanup and imrpovements

ci/runner/README.md

@@ -0,0 +1,43 @@
+<!--
+ SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ SPDX-License-Identifier: Apache-2.0
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+The `Dockerfile` in this directory defines the images used by the CI runner not for the repo itself.
+
+# Building CI images
+The `Dockerfile` defines two targets: `base` and `driver`. The `driver` target includes the Nvidia driver needed to build MRC on a machine without access to a GPU.
+
+To build the images from the root of the repo run:
+```bash
+SKIP_PUSH=1 external/utilities/ci/runner/build_and_push.sh
+```
+
+# Build and push CI images
+This will require being a member of the `Morpheus Early Access CI` group in [NGC](https://catalog.ngc.nvidia.com) and logging into the `nvcr.io` registry prior to running.
+
+From the root of the MRC repo run:
+```bash
+external/utilities/ci/runner/build_and_push.sh
+```
+
+If the images are already built, the build step can be skipped by setting `SKIP_BUILD=1`.
+
+# Updating CI to use the new images
+Update `.github/workflows/pull_request.yml` changing these two lines with the new image names:
+```yaml
+      container: nvcr.io/ea-nvidia-morpheus/morpheus:${repo_name}-ci-driver-221128
+      test_container: nvcr.io/ea-nvidia-morpheus/morpheus:${repo_name}-ci-base-221128
+```

ci/runner/build_and_push.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DOCKER_TARGET=${DOCKER_TARGET:-"base" "driver"}
+DOCKER_BUILDKIT=${DOCKER_BUILDKIT:-1}
+DOCKER_REGISTRY_SERVER=${DOCKER_REGISTRY_SERVER:-"nvcr.io"}
+DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH:-"/ea-nvidia-morpheus/morpheus"}
+DOCKER_TAG_PREFIX=${DOCKER_TAG_PREFIX:-"mrc-ci"}
+DOCKER_TAG_POSTFIX=${DOCKER_TAG_POSTFIX:-"$(date +'%y%m%d')"}
+DOCKER_EXTRA_ARGS=${DOCKER_EXTRA_ARGS:-""}
+
+SKIP_BUILD=${SKIP_BUILD:-""}
+SKIP_PUSH=${SKIP_PUSH:-""}
+
+set -e
+
+function get_image_full_name() {
+   echo "${DOCKER_REGISTRY_SERVER}${DOCKER_REGISTRY_PATH}:${DOCKER_TAG_PREFIX}-${build_target}-${DOCKER_TAG_POSTFIX}"
+}
+
+if [[ "${SKIP_BUILD}" == "" ]]; then
+    for build_target in ${DOCKER_TARGET[@]}; do
+        FULL_NAME=$(get_image_full_name)
+        echo "Building target \"${build_target}\" as ${FULL_NAME}";
+        docker buildx build --network=host ${DOCKER_EXTRA_ARGS} --target ${build_target} -t ${FULL_NAME} -f ./Dockerfile .
+    done
+fi
+
+if [[ "${SKIP_PUSH}" == "" ]]; then
+    for build_target in ${DOCKER_TARGET[@]}; do
+        FULL_NAME=$(get_image_full_name)
+        echo "Pushing ${FULL_NAME}";
+        docker push ${FULL_NAME}
+    done
+fi

cmake/morpheus_utils/environment_config/ccache/register_api.cmake

@@ -0,0 +1,131 @@
+# =============================================================================
+# SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+# ######################################################################################################################
+# * CMake properties ------------------------------------------------------------------------------
+
+include_guard(GLOBAL)
+
+#### No External Requirements ####
+
+function(morpheus_utils_initialize_ccache cache_dir_name)
+  list(APPEND CMAKE_MESSAGE_CONTEXT "ccache")
+
+  find_program(CCACHE_PROGRAM_PATH ccache DOC "Location of ccache executable")
+
+  if(NOT CCACHE_PROGRAM_PATH)
+    message(WARNING "CCache option, ${cache_dir_name}, is enabled but ccache was not found.
+      Check ccache installation.")
+    return()
+  endif()
+
+  message(STATUS "Using ccache: ${CCACHE_PROGRAM_PATH}")
+
+  set(LOCAL_MODULES_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+  set(CCACHE_DIR "${${cache_dir_name}}/ccache")
+
+  message(STATUS "Using ccache directory: ${CCACHE_DIR}")
+
+  # Write or update the ccache configuration file
+  configure_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/templates/ccache.conf.in" "${CCACHE_DIR}/ccache.conf")
+
+  # Set the ccache options we need
+  set(CCACHE_CONFIGPATH "${CCACHE_DIR}/ccache.conf")
+
+  # Because CMake doesnt allow settings variables `CCACHE_COMPILERTYPE=gcc
+  # ccache` in CMAKE_C_COMPILER_LAUNCHER, we need to put everything into a
+  # single script and use that for CMAKE_C_COMPILER_LAUNCHER. Also, since
+  # gxx_linux-64 sets the compiler to c++ instead of g++, we need to set the
+  # value of CCACHE_COMPILERTYPE otherwise caching doesnt work correctly. So
+  # we need to make separate runners for each language with specific ccache
+  # settings for each
+  if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+    set(CCACHE_REMOVE_ARGS "^--driver-mode=.*")
+  endif()
+
+  # Set the base dir for relative ccache
+  set(CCACHE_BASEDIR "${PROJECT_SOURCE_DIR}")
+
+  # Configure ccache for C
+  if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
+    set(CCACHE_COMPILERTYPE "gcc")
+  elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Clang")
+    set(CCACHE_COMPILERTYPE "clang")
+  else()
+    set(CCACHE_COMPILERTYPE "auto")
+  endif()
+
+  configure_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/templates/run_ccache.sh.in"
+      "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_c.sh" @ONLY)
+
+  # Configure ccache for CXX
+  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+    set(CCACHE_COMPILERTYPE "gcc")
+  elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+    set(CCACHE_COMPILERTYPE "clang")
+  else()
+    set(CCACHE_COMPILERTYPE "auto")
+  endif()
+
+  configure_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/templates/run_ccache.sh.in"
+      "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_cxx.sh" @ONLY)
+  configure_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/templates/run_ccache_prefix.sh.in"
+      "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_prefix.sh" @ONLY)
+
+  # Configure ccache for CUDA
+  set(CCACHE_COMPILERTYPE "nvcc")
+  configure_file("${CMAKE_CURRENT_FUNCTION_LIST_DIR}/templates/run_ccache.sh.in"
+      "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_cuda.sh" @ONLY)
+
+  # Finally, set the compiler option
+  set(CMAKE_C_COMPILER_LAUNCHER "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_c.sh" PARENT_SCOPE)
+  set(CMAKE_CXX_COMPILER_LAUNCHER "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_cxx.sh" PARENT_SCOPE)
+  set(CMAKE_CUDA_COMPILER_LAUNCHER "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_cuda.sh" PARENT_SCOPE)
+
+  # PARENT_SCOPE here so others can use this value
+  set(CCACHE_DIR "${CCACHE_DIR}" PARENT_SCOPE)
+  set(MORPHEUS_UTILS_CCACHE_INITIALIZED ON PARENT_SCOPE)
+endfunction()
+
+
+function(morpheus_utils_check_cache_path cache_dir_name)
+  # First, ensure that the current cache dir can be found by find_package/find_path/etc
+  if((NOT "${CMAKE_FIND_ROOT_PATH}" STREQUAL "") AND("${CMAKE_FIND_ROOT_PATH_MODE_INCLUDE}" STREQUAL "ONLY"))
+    set(is_contained FALSE)
+
+    # Now check if ${cache_dir_name} is under anything in CMAKE_FIND_ROOT_PATH
+    foreach(path_to_search ${CMAKE_FIND_ROOT_PATH})
+      # Check if we are contained by the find path
+      cmake_path(IS_PREFIX path_to_search ${${cache_dir_name}} is_relative)
+
+      if(is_relative)
+        set(is_contained TRUE)
+        break()
+      endif()
+    endforeach()
+
+    if(NOT is_contained)
+      message(WARNING "The value for ${cache_dir_name} (${${cache_dir_name}}) is not contained in any CMAKE_FIND_ROOT_PATH (${CMAKE_FIND_ROOT_PATH}). "
+        "This will result in cmake being unable to find any downloaded packages. The cache path has been appended to the back "
+        "of CMAKE_FIND_ROOT_PATH")
+
+      list(APPEND CMAKE_FIND_ROOT_PATH ${${cache_dir_name}})
+
+      set(CMAKE_FIND_ROOT_PATH ${CMAKE_FIND_ROOT_PATH} PARENT_SCOPE)
+    endif()
+  endif()
+endfunction()

cmake/morpheus_utils/environment_config/ccache/templates/ccache.conf.in

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2021-2022,NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+max_size = 10G
+# hash_dir = false
+# let ccache preserve C++ comments, because some of them may be meaningful to the compiler
+keep_comments_cpp = true
+cache_dir = @CCACHE_DIR@
+compiler_check = %compiler% --version
+# Uncomment to debug ccache preprocessor errors/cache misses
+log_file = @CCACHE_DIR@/ccache.log
+
+# Force absolute paths in error output for IDEs
+absolute_paths_in_stderr = true

cmake/morpheus_utils/environment_config/ccache/templates/run_ccache.sh.in

@@ -0,0 +1,41 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Optionally, remove some arguments. For example, when using IWYU and ccache
+# together, CMake passes `--driver-mode` which breaks gcc
+ARGS_TO_REMOVE="@CCACHE_REMOVE_ARGS@"
+
+if [[ -n "$ARGS_TO_REMOVE" ]]; then
+   for arg in "$@"; do
+   shift
+   [[ "$arg" =~ $ARGS_TO_REMOVE ]] && continue
+   set -- "$@" "$arg"
+   done
+fi
+
+# Set a default for CCACHE_BASEDIR. Allows overriding during build phase
+export CCACHE_BASEDIR=${CCACHE_BASEDIR:-"@CCACHE_BASEDIR@"}
+export CCACHE_CONFIGPATH=${CCACHE_CONFIGPATH:-"@CCACHE_CONFIGPATH@"}
+export CCACHE_SLOPPINESS="system_headers"
+export CCACHE_COMPILERTYPE="@CCACHE_COMPILERTYPE@"
+
+# Uncomment the following to enable debug logs
+# export CCACHE_DEBUG=1
+# export CCACHE_DEBUGDIR="@CMAKE_CURRENT_BINARY_DIR@/ccache_debug"
+
+# Allows running ccache with options inside of CMake. CMake does not work well
+# with setting variables before calling a command
+@CCACHE_PROGRAM_PATH@ "$@"

cmake/morpheus_utils/environment_config/ccache/templates/run_ccache_prefix.sh.in

@@ -0,0 +1,23 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The first argument is treated as the prefix command to run
+export CCACHE_PREFIX="$1"
+
+shift
+
+# Pass all remaining arguments to ccache
+${MRC_CCACHE_WRAPPER} "$@"