Backport/issue 6299 fix native mac os build (#6300) (#6468) (#877)

* Backport/issue 6299 fix native mac os build (#6300) (#6468)

* Merge pull request #51710 from velavokr/velavokr-warn-cpu-utilization

A cmake warning on job limits underutilizing CPU

* Merge pull request #52196 from rschu1ze/simplify-job-limiting

CMake: Simplify job limiting

* Merge pull request #56971 from ClickHouse/own-cmake-for-grpc

Own CMake for GRPC

* Merge pull request #70671 from ClickHouse/fix-native-macos

Fix native macOS build

---------

Co-authored-by: Alexey Milovidov <milovidov@clickhouse.com>
Co-authored-by: Robert Schulze <robert@clickhouse.com>
Co-authored-by: Konstantin Bogdanov <thevar1able@users.noreply.github.com>

* fix

---------

Co-authored-by: Lisen <38773813+yl-lisen@users.noreply.github.com>
Co-authored-by: Alexey Milovidov <milovidov@clickhouse.com>
Co-authored-by: Robert Schulze <robert@clickhouse.com>
Co-authored-by: Konstantin Bogdanov <thevar1able@users.noreply.github.com>
Co-authored-by: Qijun Niu <niuqijun0702@qq.com>

.github/workflows/manual_trigger_build.yml

@@ -209,7 +209,7 @@ jobs:
     runs-on: [self-hosted, macOS, ARM64]
     env:
       build_directory: ${{ github.workspace }}/build
-      build_type: Release
+      build_type: RelWithDebInfo
     if: github.event.inputs.enable_native_build_for_macOS == 'true' && github.event.inputs.arch == 'arm'
     steps:
       - name: Checkout

.github/workflows/release_build.yml

@@ -297,7 +297,7 @@ jobs:
     runs-on: [self-hosted, macOS, ARM64]
     env:
       build_directory: ${{ github.workspace }}/build
-      build_type: Release
+      build_type: RelWithDebInfo
     steps:
       - name: Checkout
         uses: actions/checkout@v4.1.7

cmake/limit_jobs.cmake

@@ -1,38 +1,39 @@
-# Usage:
-# set (MAX_COMPILER_MEMORY 2000 CACHE INTERNAL "") # In megabytes
-# set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "")
-# include (cmake/limit_jobs.cmake)
+# Limit compiler/linker job concurrency to avoid OOMs on subtrees where compilation/linking is memory-intensive.
+#
+# Usage from CMake:
+#    set (MAX_COMPILER_MEMORY 2000 CACHE INTERNAL "") # megabyte
+#    set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "") # megabyte
+#    include (cmake/limit_jobs.cmake)
+#
+# (bigger values mean fewer jobs)
 
-cmake_host_system_information(RESULT TOTAL_PHYSICAL_MEMORY QUERY TOTAL_PHYSICAL_MEMORY) # Not available under freebsd
+cmake_host_system_information(RESULT TOTAL_PHYSICAL_MEMORY QUERY TOTAL_PHYSICAL_MEMORY)
 cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)
 
-# 1 if not set
-option(PARALLEL_COMPILE_JOBS "Maximum number of concurrent compilation jobs" "")
-
-# 1 if not set
-option(PARALLEL_LINK_JOBS "Maximum number of concurrent link jobs" "")
+# Set to disable the automatic job-limiting
+option(PARALLEL_COMPILE_JOBS "Maximum number of concurrent compilation jobs" OFF)
+option(PARALLEL_LINK_JOBS "Maximum number of concurrent link jobs" OFF)
 
-if (NOT PARALLEL_COMPILE_JOBS AND TOTAL_PHYSICAL_MEMORY AND MAX_COMPILER_MEMORY)
+if (NOT PARALLEL_COMPILE_JOBS AND MAX_COMPILER_MEMORY)
     math(EXPR PARALLEL_COMPILE_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_COMPILER_MEMORY})
 
     if (NOT PARALLEL_COMPILE_JOBS)
         set (PARALLEL_COMPILE_JOBS 1)
     endif ()
+    if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES)
+        message("The auto-calculated compile jobs limit (${PARALLEL_COMPILE_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_COMPILE_JOBS to override.")
+    endif()
 endif ()
 
-if (PARALLEL_COMPILE_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES))
-    set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR})
-    string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE})
-    set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS})
-endif ()
-
-
-if (NOT PARALLEL_LINK_JOBS AND TOTAL_PHYSICAL_MEMORY AND MAX_LINKER_MEMORY)
+if (NOT PARALLEL_LINK_JOBS AND MAX_LINKER_MEMORY)
     math(EXPR PARALLEL_LINK_JOBS ${TOTAL_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY})
 
     if (NOT PARALLEL_LINK_JOBS)
         set (PARALLEL_LINK_JOBS 1)
     endif ()
+    if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES)
+        message("The auto-calculated link jobs limit (${PARALLEL_LINK_JOBS}) underutilizes CPU cores (${NUMBER_OF_LOGICAL_CORES}). Set PARALLEL_LINK_JOBS to override.")
+    endif()
 endif ()
 
 # ThinLTO provides its own parallel linking
@@ -46,14 +47,16 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLE
     set (PARALLEL_LINK_JOBS 2)
 endif()
 
-if (PARALLEL_LINK_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES))
+message(STATUS "Building sub-tree with ${PARALLEL_COMPILE_JOBS} compile jobs and ${PARALLEL_LINK_JOBS} linker jobs (system: ${NUMBER_OF_LOGICAL_CORES} cores, ${TOTAL_PHYSICAL_MEMORY} MB DRAM, 'OFF' means the native core count).")
+
+if (PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES)
+    set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR})
+    string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE})
+    set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS})
+endif ()
+
+if (PARALLEL_LINK_JOBS LESS NUMBER_OF_LOGICAL_CORES)
     set(CMAKE_JOB_POOL_LINK link_job_pool${CMAKE_CURRENT_SOURCE_DIR})
     string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_LINK ${CMAKE_JOB_POOL_LINK})
     set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_LINK}=${PARALLEL_LINK_JOBS})
 endif ()
-
-if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS)
-    message(STATUS
-        "${CMAKE_CURRENT_SOURCE_DIR}: Have ${TOTAL_PHYSICAL_MEMORY} megabytes of memory.
-        Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS} (system has ${NUMBER_OF_LOGICAL_CORES} logical cores)")
-endif ()

contrib/abseil-cpp-cmake/CMakeLists.txt

@@ -2,6 +2,10 @@ set(ABSL_ROOT_DIR "${proton_SOURCE_DIR}/contrib/abseil-cpp")
 if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt")
   message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive")
 endif()
+
+# To avoid errors "'X' does not refer to a value" while using `offsetof` function.
+set(CMAKE_CXX_STANDARD 17)
+
 set(BUILD_TESTING OFF)
 set(ABSL_PROPAGATE_CXX_STD ON)
 add_subdirectory("${ABSL_ROOT_DIR}" "${proton_BINARY_DIR}/contrib/abseil-cpp")

contrib/grpc-cmake/CMakeLists.txt

@@ -6,53 +6,19 @@ if(NOT ENABLE_GRPC)
   return()
 endif()
 
+set(CMAKE_CXX_STANDARD 17)
+
 set(_gRPC_SOURCE_DIR "${proton_SOURCE_DIR}/contrib/grpc")
 set(_gRPC_BINARY_DIR "${proton_BINARY_DIR}/contrib/grpc")
 
-# Use re2 from proton contrib, not from gRPC third_party.
-set(gRPC_RE2_PROVIDER "clickhouse" CACHE STRING "" FORCE)
-set(_gRPC_RE2_INCLUDE_DIR "")
-set(_gRPC_RE2_LIBRARIES ch_contrib::re2)
-
-# Use zlib from proton contrib, not from gRPC third_party.
-set(gRPC_ZLIB_PROVIDER "clickhouse" CACHE STRING "" FORCE)
-set(_gRPC_ZLIB_INCLUDE_DIR "")
-set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib)
-
-# Use protobuf from proton contrib, not from gRPC third_party.
-set(gRPC_PROTOBUF_PROVIDER "clickhouse" CACHE STRING "" FORCE)
-set(_gRPC_PROTOBUF_LIBRARIES ch_contrib::protobuf)
-set(_gRPC_PROTOBUF_PROTOC "protoc")
-set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $<TARGET_FILE:protoc>)
-set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ch_contrib::protoc)
-
 if(TARGET OpenSSL::SSL)
   set(gRPC_USE_UNSECURE_LIBRARIES FALSE)
 else()
   set(gRPC_USE_UNSECURE_LIBRARIES TRUE)
 endif()
 
-# Use OpenSSL from proton contrib, not from gRPC third_party.
-set(gRPC_SSL_PROVIDER "clickhouse" CACHE STRING "" FORCE)
-set(_gRPC_SSL_INCLUDE_DIR "")
-set(_gRPC_SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL)
-
-# Use abseil-cpp from proton contrib, not from gRPC third_party.
-set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE)
-
-# We don't want to build C# extensions.
-set(gRPC_BUILD_CSHARP_EXT OFF)
-
-# TODO: Remove this. We generally like to compile with C++23 but grpc isn't ready yet.
-set (CMAKE_CXX_STANDARD 20)
-
-set(_gRPC_CARES_LIBRARIES ch_contrib::c-ares)
-set(gRPC_CARES_PROVIDER "clickhouse" CACHE STRING "" FORCE)
-add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}")
-
-# The contrib/grpc/CMakeLists.txt redefined the PROTOBUF_GENERATE_GRPC_CPP() function for its own purposes,
-# so we need to redefine it back.
-include("${proton_SOURCE_DIR}/contrib/grpc-cmake/protobuf_generate_grpc.cmake")
+include(grpc.cmake)
+include(protobuf_generate_grpc.cmake)
 
 set(gRPC_CPP_PLUGIN $<TARGET_FILE:grpc_cpp_plugin>)
 set(gRPC_PYTHON_PLUGIN $<TARGET_FILE:grpc_python_plugin>)