[chore] Prolog of hermetic build with GCC 11 and Clang 13. (#7712)

Prepare to generate hermetic build using GCC 11 and Clang 13. The ideal toolchain would be ldb toolchain generated by [ldb_toolchain_gen.sh](https://github.com/amosbird/ldb_toolchain_gen/releases/download/v0.3/ldb_toolchain_gen.sh) To kick off a clang build, set `DORIS_TOOLCHAIN=clang` before running any build scripts.
apache · Jan 21, 2022 · 800a363 · 800a363
1 parent 0efef1b
commit 800a363
Show file tree

Hide file tree

Showing 53 changed files with 769 additions and 247 deletions.
diff --git a/CONTRIBUTING_CN.md b/CONTRIBUTING_CN.md
@@ -1,4 +1,4 @@
-<!-- 
+<!--
 Licensed to the Apache Software Foundation (ASF) under one
 or more contributor license agreements.  See the NOTICE file
 distributed with this work for additional information
@@ -32,7 +32,7 @@ under the License.
 初次来到 Doris 社区，您可以：
 
 * 关注 Doris [Github 代码库](https://github.com/apache/incubator-doris)
-* 订阅我们的 [邮件列表](./subscribe-mail-list.md)； 
+* 订阅我们的 [邮件列表](./subscribe-mail-list.md)；
 * 加入 Doris 微信群(加微信号：morningman-cmy, 备注：加入Doris群) 随时提问；
 * 进入 Doris 的 [Gitter](./gitter.md) 聊天室；
 

diff --git a/README.md b/README.md
@@ -44,7 +44,7 @@ The main advantages of Doris are the simplicity (of developing, deploying and us
 
 [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
 
-> **Note**   
+> **Note**
 > Some licenses of the third-party dependencies are not compatible with Apache 2.0 License. So you need to disable
 some Doris features to be complied with Apache 2.0 License. For details, refer to the `thirdparty/LICENSE.txt`
 

diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
@@ -17,15 +17,6 @@
 
 cmake_minimum_required(VERSION 3.19.2)
 
-# set CMAKE_C_COMPILER, this must set before project command
-if (DEFINED ENV{DORIS_GCC_HOME})
-    set(CMAKE_C_COMPILER "$ENV{DORIS_GCC_HOME}/bin/gcc")
-    set(CMAKE_CXX_COMPILER "$ENV{DORIS_GCC_HOME}/bin/g++")
-    set(GCC_HOME $ENV{DORIS_GCC_HOME})
-else()
-    message(FATAL_ERROR "DORIS_GCC_HOME environment variable is not set")
-endif()
-
 project(doris CXX C)
 
 # Write compile_commands.json
@@ -52,6 +43,12 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)")
     set (ARCH_PPC64LE 1)
 endif ()
 
+if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set (COMPILER_GCC 1)
+elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    set (COMPILER_CLANG 1)
+endif ()
+
 option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}")
 
@@ -95,8 +92,6 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         message(FATAL_ERROR "Need GCC version at least 7.3.0")
     endif()
     message(STATUS "GCC version is greater than 7.3.0, disable -Werror. Be careful with compile warnings.")
-elseif (NOT APPLE)
-    message(FATAL_ERROR "Compiler should be GNU")
 endif()
 
 set(PIC_LIB_PATH "${THIRDPARTY_DIR}")
@@ -299,6 +294,9 @@ set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
 add_library(minizip STATIC IMPORTED)
 set_target_properties(minizip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libminizip.a)
 
+add_library(idn STATIC IMPORTED)
+set_target_properties(idn PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libidn.a)
+
 add_library(gsasl STATIC IMPORTED)
 set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
 
@@ -344,6 +342,106 @@ set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFI
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED")
 # Enable the cpu and heap profile of brpc
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBRPC_ENABLE_CPU_PROFILER")
+
+if (COMPILER_GCC)
+    # Avoid GCC 11 false alarm
+    # https://stackoverflow.com/questions/67584073/gcc-11-false-array-subscript-is-partly-outside-array-bounds-warning
+    # https://stackoverflow.com/questions/69426070/gcc-11-order-of-arguments-triggers-false-positive-wstringop-overflow-is-this-bu
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-array-bounds -Wno-stringop-overread")
+endif ()
+
+if (COMPILER_CLANG)
+    # TODO: Fix all these warnings
+    add_compile_options(
+        -Wno-address-of-packed-member
+        -Wno-ambiguous-reversed-operator
+        -Wno-atomic-implicit-seq-cst
+        -Wno-cast-align
+        -Wno-cast-qual
+        -Wno-comma
+        -Wno-conditional-uninitialized
+        -Wno-constant-conversion
+        -Wno-covered-switch-default
+        -Wno-delete-abstract-non-virtual-dtor
+        -Wno-delete-non-abstract-non-virtual-dtor
+        -Wno-deprecated
+        -Wno-deprecated-copy-with-dtor
+        -Wno-deprecated-copy-with-user-provided-copy
+        -Wno-deprecated-copy-with-user-provided-dtor
+        -Wno-deprecated-declarations
+        -Wno-deprecated-this-capture
+        -Wno-documentation
+        -Wno-embedded-directive
+        -Wno-enum-compare-switch
+        -Wno-extra-semi
+        -Wno-extra-semi-stmt
+        -Wno-format-nonliteral
+        -Wno-frame-larger-than
+        -Wno-gnu-label-as-value
+        -Wno-gnu-redeclared-enum
+        -Wno-gnu-statement-expression
+        -Wno-gnu-zero-variadic-macro-arguments
+        -Wno-header-hygiene
+        -Wno-ignored-qualifiers
+        -Wno-implicit-const-int-float-conversion
+        -Wno-implicit-exception-spec-mismatch
+        -Wno-implicit-fallthrough
+        -Wno-inconsistent-missing-destructor-override
+        -Wno-inconsistent-missing-override
+        -Wno-instantiation-after-specialization
+        -Wno-macro-redefined
+        -Wno-mismatched-new-delete
+        -Wno-mismatched-tags
+        -Wno-missing-field-initializers
+        -Wno-missing-noreturn
+        -Wno-newline-eof
+        -Wno-non-virtual-dtor
+        -Wno-null-conversion
+        -Wno-null-dereference
+        -Wno-old-style-cast
+        -Wno-overloaded-virtual
+        -Wno-pedantic
+        -Wno-pessimizing-move
+        -Wno-pragma-once-outside-header
+        -Wno-reserved-identifier
+        -Wno-reserved-macro-identifier
+        -Wno-return-type
+        -Wno-shadow
+        -Wno-shadow-field
+        -Wno-shadow-field-in-constructor
+        -Wno-shadow-uncaptured-local
+        -Wno-shift-count-overflow
+        -Wno-sign-compare
+        -Wno-string-plus-int
+        -Wno-suggest-destructor-override
+        -Wno-suggest-override
+        -Wno-tautological-constant-out-of-range-compare
+        -Wno-tautological-unsigned-zero-compare
+        -Wno-undef
+        -Wno-uninitialized
+        -Wno-unknown-pragmas
+        -Wno-unknown-warning-option
+        -Wno-unreachable-code
+        -Wno-unreachable-code-break
+        -Wno-unreachable-code-return
+        -Wno-unused-but-set-variable
+        -Wno-unused-const-variable
+        -Wno-unused-exception-parameter
+        -Wno-unused-function
+        -Wno-unused-lambda-capture
+        -Wno-unused-local-typedef
+        -Wno-unused-macros
+        -Wno-unused-member-function
+        -Wno-unused-parameter
+        -Wno-unused-private-field
+        -Wno-unused-result
+        -Wno-unused-value
+        -Wno-unused-variable
+        -Wno-variadic-macros
+        -Wno-zero-as-null-pointer-constant
+    )
+endif ()
+
 # https://github.com/boostorg/uuid/issues/92
 # We need to avoid using SYS_getrandom system calls
 set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1")
@@ -436,6 +534,10 @@ add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H)
 include_directories(
     ${SRC_DIR}/
     ${TEST_DIR}/
+)
+
+include_directories(
+    SYSTEM
     ${GENSRC_DIR}/
     ${THIRDPARTY_DIR}/include
     ${GPERFTOOLS_HOME}/include
@@ -498,6 +600,7 @@ set(COMMON_THIRDPARTY
     lz4
     libevent
     libevent_pthreads
+    idn
     gsasl
     curl
     ${LIBZ}
@@ -532,13 +635,13 @@ set(COMMON_THIRDPARTY
 if (${MAKE_TEST} STREQUAL "ON")
     set(COMMON_THIRDPARTY
         ${COMMON_THIRDPARTY}
-        benchmark       
+        benchmark
     )
 endif()
 
 # thirdparties dependescies that can only run on X86 platform
 set(X86_DEPENDENCIES
-    ${COMMON_THIRDPARTY}   
+    ${COMMON_THIRDPARTY}
     hdfs3
     xml2
     lzma
@@ -590,31 +693,41 @@ if ("${CMAKE_BUILD_TYPE}" STREQUAL "BCC")
     )
 endif()
 
+if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+    set(ASAN_LIBS -static-libasan)
+    set(LSAN_LIBS -static-liblsan)
+    set(UBSAN_LIBS -static-libubsan tcmalloc)
+    set(TSAN_LIBS -static-libtsan)
+else ()
+    set(UBSAN_LIBS tcmalloc)
+endif ()
+
 # Add sanitize static link flags or tcmalloc
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE" OR "${CMAKE_BUILD_TYPE}" STREQUAL "BCC")
     set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} tcmalloc)
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ASAN")
-    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -static-libasan)
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${ASAN_LIBS})
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "LSAN")
-    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -static-liblsan)
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${LSAN_LIBS})
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
-    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -static-libubsan tcmalloc)
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${UBSAN_LIBS})
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
-    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -static-libtsan)
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${TSAN_LIBS})
     add_definitions("-DTHREAD_SANITIZER")
 else()
     message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
 endif()
 
-set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
-    -lrt -l:libbfd.a -liberty -lc -lm -ldl -pthread
-)
-
+# NOTE(amos): This should come before -lc -lm to interpose symbols correctly.
 if (GLIBC_COMPATIBILITY)
     add_subdirectory(${SRC_DIR}/glibc-compatibility)
-    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} glibc-compatibility)
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} glibc-compatibility-explicit glibc-compatibility)
 endif()
 
+set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
+    -lrt -l:libbfd.a -liberty -lc -lm -ldl -pthread
+)
+
 # Set libraries for test
 set (TEST_LINK_LIBS ${DORIS_LINK_LIBS}
     ${WL_START_GROUP}
@@ -735,8 +848,7 @@ install(FILES
     ${BASE_DIR}/../conf/odbcinst.ini
     DESTINATION ${OUTPUT_DIR}/conf)
 
-
 get_property(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES)
 foreach(dir ${dirs})
-  message(STATUS "dir='${dir}'")
-  endforeach()
+    message(STATUS "dir='${dir}'")
+endforeach()
diff --git a/be/src/agent/topic_subscriber.h b/be/src/agent/topic_subscriber.h
@@ -19,6 +19,7 @@
 #define DORIS_BE_SRC_AGENT_TOPIC_SUBSCRIBER_H
 
 #include <map>
+#include <mutex>
 #include <shared_mutex>
 #include <thread>
 

diff --git a/be/src/exec/olap_utils.h b/be/src/exec/olap_utils.h
@@ -76,6 +76,7 @@ inline CompareLargeFunc get_compare_func(PrimitiveType type) {
     default:
         DCHECK(false) << "Unsupported Compare type";
     }
+    __builtin_unreachable();
 }
 
 static const char* NEGATIVE_INFINITY = "-oo";

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
@@ -942,11 +942,11 @@ Status IRuntimeFilter::init_with_desc(const TRuntimeFilterDesc* desc, const TQue
 }
 
 Status IRuntimeFilter::serialize(PMergeFilterRequest* request, void** data, int* len) {
-    return _serialize(request, data, len);
+    return serialize_impl(request, data, len);
 }
 
 Status IRuntimeFilter::serialize(PPublishFilterRequest* request, void** data, int* len) {
-    return _serialize(request, data, len);
+    return serialize_impl(request, data, len);
 }
 
 Status IRuntimeFilter::create_wrapper(const MergeRuntimeFilterParams* param, MemTracker* tracker,
@@ -1034,7 +1034,7 @@ void batch_copy(PInFilter* filter, HybridSetBase::IteratorBase* it,
 }
 
 template <class T>
-Status IRuntimeFilter::_serialize(T* request, void** data, int* len) {
+Status IRuntimeFilter::serialize_impl(T* request, void** data, int* len) {
     request->set_filter_type(get_type(_runtime_filter_type));
 
     if (_runtime_filter_type == RuntimeFilterType::IN_FILTER) {

diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
@@ -221,7 +221,7 @@ class IRuntimeFilter {
     void to_protobuf(PMinMaxFilter* filter);
 
     template <class T>
-    Status _serialize(T* request, void** data, int* len);
+    Status serialize_impl(T* request, void** data, int* len);
 
     template <class T>
     static Status _create_wrapper(const T* param, MemTracker* tracker, ObjectPool* pool,

diff --git a/be/src/exprs/table_function/explode_split.cpp b/be/src/exprs/table_function/explode_split.cpp
@@ -40,7 +40,7 @@ Status ExplodeSplitTableFunction::open() {
     // check if the delimiter argument(the 2nd arg) is constant.
     // if yes, cache it
     if (fn_ctx->is_arg_constant(1)) {
-        _is_delimiter_constant = true; 
+        _is_delimiter_constant = true;
         StringVal* delimiter = reinterpret_cast<StringVal*>(fn_ctx->get_constant_arg(1));
         _const_delimter = StringPiece((char*) delimiter->ptr, delimiter->len);
     }
@@ -65,7 +65,7 @@ Status ExplodeSplitTableFunction::process(TupleRow* tuple_row) {
             StringVal delimiter = _expr_context->root()->get_child(1)->get_string_val(_expr_context, tuple_row);
             _backup = strings::Split(StringPiece((char*) text.ptr, text.len), StringPiece((char*) delimiter.ptr, delimiter.len));
         }
-        for (const std::string str : _backup) {
+        for (const std::string & str : _backup) {
             _data.emplace_back(str);
         }
         _cur_size = _backup.size();

diff --git a/be/src/glibc-compatibility/CMakeLists.txt b/be/src/glibc-compatibility/CMakeLists.txt
@@ -48,9 +48,19 @@ if (GLIBC_COMPATIBILITY)
         list(APPEND glibc_compatibility_sources musl/getentropy.c)
     endif()
 
+    list(REMOVE_ITEM glibc_compatibility_sources musl/getrandom.c memcpy_wrapper.c)
+
     # Need to omit frame pointers to match the performance of glibc
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer")
 
+    # NOTE(amos): getrandom is hard to interpose since we build thirdparty deps
+    # without glibc-compatibility. Also sanitizers might generate getrandom
+    # libcalls. Workaround: Use object file so that linker will always take a
+    # look at its symbol table.
+    # NOTE(amos): sanitizers might generate memcpy references that are too late to
+    # refer. Let's also extract memcpy definitions explicitly to avoid UNDEF GLIBC 2.14.
+    add_library(glibc-compatibility-explicit OBJECT musl/getrandom.c memcpy_wrapper.c)
+    target_compile_options(glibc-compatibility-explicit PRIVATE -fPIC)
     add_library(glibc-compatibility STATIC ${glibc_compatibility_sources})
 
     if (COMPILER_CLANG)
@@ -60,6 +70,7 @@ if (GLIBC_COMPATIBILITY)
     endif ()
 
     target_include_directories(glibc-compatibility PRIVATE ${musl_arch_include_dir})
+    target_include_directories(glibc-compatibility-explicit PRIVATE ${musl_arch_include_dir})
 
     message (STATUS "Some symbols from glibc will be replaced for compatibility")
 else()