dep: upgrade jemalloc

contrib/jemalloc-cmake/CMakeLists.txt

@@ -4,65 +4,136 @@ endif()
 
 set(JEMALLOC_SOURCE_DIR ${TiFlash_SOURCE_DIR}/contrib/jemalloc)
 
-set(SRCS
-${JEMALLOC_SOURCE_DIR}/src/arena.c
-${JEMALLOC_SOURCE_DIR}/src/background_thread.c
-${JEMALLOC_SOURCE_DIR}/src/base.c
-${JEMALLOC_SOURCE_DIR}/src/bin.c
-${JEMALLOC_SOURCE_DIR}/src/bitmap.c
-${JEMALLOC_SOURCE_DIR}/src/ckh.c
-${JEMALLOC_SOURCE_DIR}/src/ctl.c
-${JEMALLOC_SOURCE_DIR}/src/div.c
-${JEMALLOC_SOURCE_DIR}/src/extent.c
-${JEMALLOC_SOURCE_DIR}/src/extent_dss.c
-${JEMALLOC_SOURCE_DIR}/src/extent_mmap.c
-${JEMALLOC_SOURCE_DIR}/src/hash.c
-${JEMALLOC_SOURCE_DIR}/src/hook.c
-${JEMALLOC_SOURCE_DIR}/src/jemalloc.c
-${JEMALLOC_SOURCE_DIR}/src/jemalloc_cpp.cpp
-${JEMALLOC_SOURCE_DIR}/src/large.c
-${JEMALLOC_SOURCE_DIR}/src/log.c
-${JEMALLOC_SOURCE_DIR}/src/malloc_io.c
-${JEMALLOC_SOURCE_DIR}/src/mutex.c
-${JEMALLOC_SOURCE_DIR}/src/mutex_pool.c
-${JEMALLOC_SOURCE_DIR}/src/nstime.c
-${JEMALLOC_SOURCE_DIR}/src/pages.c
-${JEMALLOC_SOURCE_DIR}/src/prng.c
-${JEMALLOC_SOURCE_DIR}/src/prof.c
-${JEMALLOC_SOURCE_DIR}/src/rtree.c
-${JEMALLOC_SOURCE_DIR}/src/sc.c
-${JEMALLOC_SOURCE_DIR}/src/stats.c
-${JEMALLOC_SOURCE_DIR}/src/sz.c
-${JEMALLOC_SOURCE_DIR}/src/tcache.c
-${JEMALLOC_SOURCE_DIR}/src/test_hooks.c
-${JEMALLOC_SOURCE_DIR}/src/ticker.c
-${JEMALLOC_SOURCE_DIR}/src/tsd.c
-${JEMALLOC_SOURCE_DIR}/src/witness.c
-${JEMALLOC_SOURCE_DIR}/src/safety_check.c
+set (SRCS
+        "${JEMALLOC_SOURCE_DIR}/src/arena.c"
+        "${JEMALLOC_SOURCE_DIR}/src/background_thread.c"
+        "${JEMALLOC_SOURCE_DIR}/src/base.c"
+        "${JEMALLOC_SOURCE_DIR}/src/bin.c"
+        "${JEMALLOC_SOURCE_DIR}/src/bin_info.c"
+        "${JEMALLOC_SOURCE_DIR}/src/bitmap.c"
+        "${JEMALLOC_SOURCE_DIR}/src/buf_writer.c"
+        "${JEMALLOC_SOURCE_DIR}/src/cache_bin.c"
+        "${JEMALLOC_SOURCE_DIR}/src/ckh.c"
+        "${JEMALLOC_SOURCE_DIR}/src/counter.c"
+        "${JEMALLOC_SOURCE_DIR}/src/ctl.c"
+        "${JEMALLOC_SOURCE_DIR}/src/decay.c"
+        "${JEMALLOC_SOURCE_DIR}/src/div.c"
+        "${JEMALLOC_SOURCE_DIR}/src/ecache.c"
+        "${JEMALLOC_SOURCE_DIR}/src/edata.c"
+        "${JEMALLOC_SOURCE_DIR}/src/edata_cache.c"
+        "${JEMALLOC_SOURCE_DIR}/src/ehooks.c"
+        "${JEMALLOC_SOURCE_DIR}/src/emap.c"
+        "${JEMALLOC_SOURCE_DIR}/src/eset.c"
+        "${JEMALLOC_SOURCE_DIR}/src/exp_grow.c"
+        "${JEMALLOC_SOURCE_DIR}/src/extent.c"
+        "${JEMALLOC_SOURCE_DIR}/src/extent_dss.c"
+        "${JEMALLOC_SOURCE_DIR}/src/extent_mmap.c"
+        "${JEMALLOC_SOURCE_DIR}/src/fxp.c"
+        "${JEMALLOC_SOURCE_DIR}/src/hook.c"
+        "${JEMALLOC_SOURCE_DIR}/src/hpa.c"
+        "${JEMALLOC_SOURCE_DIR}/src/hpa_hooks.c"
+        "${JEMALLOC_SOURCE_DIR}/src/hpdata.c"
+        "${JEMALLOC_SOURCE_DIR}/src/inspect.c"
+        "${JEMALLOC_SOURCE_DIR}/src/jemalloc.c"
+        "${JEMALLOC_SOURCE_DIR}/src/large.c"
+        "${JEMALLOC_SOURCE_DIR}/src/log.c"
+        "${JEMALLOC_SOURCE_DIR}/src/malloc_io.c"
+        "${JEMALLOC_SOURCE_DIR}/src/mutex.c"
+        "${JEMALLOC_SOURCE_DIR}/src/nstime.c"
+        "${JEMALLOC_SOURCE_DIR}/src/pa.c"
+        "${JEMALLOC_SOURCE_DIR}/src/pac.c"
+        "${JEMALLOC_SOURCE_DIR}/src/pa_extra.c"
+        "${JEMALLOC_SOURCE_DIR}/src/pages.c"
+        "${JEMALLOC_SOURCE_DIR}/src/pai.c"
+        "${JEMALLOC_SOURCE_DIR}/src/peak_event.c"
+        "${JEMALLOC_SOURCE_DIR}/src/prof.c"
+        "${JEMALLOC_SOURCE_DIR}/src/prof_data.c"
+        "${JEMALLOC_SOURCE_DIR}/src/prof_log.c"
+        "${JEMALLOC_SOURCE_DIR}/src/prof_recent.c"
+        "${JEMALLOC_SOURCE_DIR}/src/prof_stats.c"
+        "${JEMALLOC_SOURCE_DIR}/src/prof_sys.c"
+        "${JEMALLOC_SOURCE_DIR}/src/psset.c"
+        "${JEMALLOC_SOURCE_DIR}/src/rtree.c"
+        "${JEMALLOC_SOURCE_DIR}/src/safety_check.c"
+        "${JEMALLOC_SOURCE_DIR}/src/san_bump.c"
+        "${JEMALLOC_SOURCE_DIR}/src/san.c"
+        "${JEMALLOC_SOURCE_DIR}/src/sc.c"
+        "${JEMALLOC_SOURCE_DIR}/src/sec.c"
+        "${JEMALLOC_SOURCE_DIR}/src/stats.c"
+        "${JEMALLOC_SOURCE_DIR}/src/sz.c"
+        "${JEMALLOC_SOURCE_DIR}/src/tcache.c"
+        "${JEMALLOC_SOURCE_DIR}/src/test_hooks.c"
+        "${JEMALLOC_SOURCE_DIR}/src/thread_event.c"
+        "${JEMALLOC_SOURCE_DIR}/src/ticker.c"
+        "${JEMALLOC_SOURCE_DIR}/src/tsd.c"
+        "${JEMALLOC_SOURCE_DIR}/src/witness.c"
 )
 
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
   list(APPEND SRCS ${JEMALLOC_SOURCE_DIR}/src/zone.c)
 endif()
 
+if (ARCH_LINUX)
+    # ThreadPool select job randomly, and there can be some threads that had been
+    # performed some memory heavy task before and will be inactive for some time,
+    # but until it will became active again, the memory will not be freed since by
+    # default each thread has it's own arena, but there should be not more then
+    # 4*CPU arenas (see opt.nareans description).
+    #
+    # By enabling percpu_arena number of arenas limited to number of CPUs and hence
+    # this problem should go away.
+    #
+    # muzzy_decay_ms -- use MADV_FREE when available on newer Linuxes, to
+    # avoid spurious latencies and additional work associated with
+    # MADV_DONTNEED. See
+    # https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
+    set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000")
+else()
+    set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000")
+endif()
+
+message (STATUS "jemalloc malloc_conf: ${JEMALLOC_CONFIG_MALLOC_CONF}")
+
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
 endif ()
 
 add_library(jemalloc STATIC ${SRCS})
+set (JEMALLOC_INCLUDE_PREFIX)
+
+if (ARCH_LINUX)
+    set (JEMALLOC_INCLUDE_PREFIX "include_linux")
+    target_compile_definitions(jemalloc PRIVATE JEMALLOC_MADV_FREE=8)
+elseif (ARCH_FREEBSD)
+    set (JEMALLOC_INCLUDE_PREFIX "include_freebsd")
+elseif (APPLE)
+    set (JEMALLOC_INCLUDE_PREFIX "include_darwin")
+else ()
+    message (FATAL_ERROR "internal jemalloc: This OS is not supported")
+endif ()
 
-if (ARCH_ARM)
-    target_include_directories(jemalloc PUBLIC
-        ${CMAKE_CURRENT_SOURCE_DIR}/include
-        ${CMAKE_CURRENT_SOURCE_DIR}/include_linux_aarch64)
+if (ARCH_AMD64)
+    if (USE_MUSL)
+        set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_x86_64_musl")
+    else()
+        set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_x86_64")
+    endif()
+elseif (ARCH_AARCH64)
+    set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_aarch64")
+elseif (ARCH_PPC64LE)
+    set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_ppc64le")
+elseif (ARCH_RISCV64)
+    set(JEMALLOC_INCLUDE_PREFIX "${JEMALLOC_INCLUDE_PREFIX}_riscv64")
 else ()
-    target_include_directories(jemalloc PUBLIC
-        ${CMAKE_CURRENT_SOURCE_DIR}/include
-        ${CMAKE_CURRENT_SOURCE_DIR}/include_linux_x86_64)
+    message (FATAL_ERROR "internal jemalloc: This arch is not supported")
 endif ()
 
-target_include_directories(jemalloc PRIVATE
-    ${JEMALLOC_SOURCE_DIR}/include)
+configure_file(${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h.in
+        ${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal/jemalloc_internal_defs.h)
+target_include_directories(jemalloc SYSTEM PRIVATE
+        "${CMAKE_CURRENT_BINARY_DIR}/${JEMALLOC_INCLUDE_PREFIX}/jemalloc/internal")
+
+target_include_directories(jemalloc PUBLIC ${JEMALLOC_SOURCE_DIR}/include ${TiFlash_SOURCE_DIR}/contrib/jemalloc-cmake/include)
 
 target_compile_definitions(jemalloc PRIVATE -DJEMALLOC_NO_PRIVATE_NAMESPACE)
 
@@ -80,3 +151,5 @@ if (ENABLE_JEMALLOC_PROF)
         target_link_libraries (jemalloc PRIVATE ${UNWIND_LIBRARY})
     endif ()
 endif ()
+
+target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE)

contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_preamble.h → contrib/jemalloc-cmake/include/jemalloc/internal/jemalloc_preamble.h

@@ -1,27 +1,33 @@
 #ifndef JEMALLOC_PREAMBLE_H
 #define JEMALLOC_PREAMBLE_H
 
-#include "jemalloc_internal_defs.h"
 #include "jemalloc/internal/jemalloc_internal_decls.h"
+#include "jemalloc_internal_defs.h"
 
-#ifdef JEMALLOC_UTRACE
+#if defined(JEMALLOC_UTRACE) || defined(JEMALLOC_UTRACE_LABEL)
 #include <sys/ktrace.h>
+#if defined(JEMALLOC_UTRACE)
+#define UTRACE_CALL(p, l) utrace(p, l)
+#else
+#define UTRACE_CALL(p, l) utrace("jemalloc_process", p, l)
+#define JEMALLOC_UTRACE
+#endif
 #endif
 
 #define JEMALLOC_NO_DEMANGLE
 #ifdef JEMALLOC_JET
-#  undef JEMALLOC_IS_MALLOC
-#  define JEMALLOC_N(n) jet_##n
-#  include "jemalloc/internal/public_namespace.h"
-#  define JEMALLOC_NO_RENAME
-#  include "jemalloc/jemalloc.h"
-#  undef JEMALLOC_NO_RENAME
+#undef JEMALLOC_IS_MALLOC
+#define JEMALLOC_N(n) jet_##n
+#include "jemalloc/internal/public_namespace.h"
+#define JEMALLOC_NO_RENAME
+#include "jemalloc/jemalloc.h"
+#undef JEMALLOC_NO_RENAME
 #else
-#  define JEMALLOC_N(n) je_##n
-#  include "jemalloc/jemalloc.h"
+#define JEMALLOC_N(n) je_##n
+#include "jemalloc/jemalloc.h"
 #endif
 
-#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
+#if defined(JEMALLOC_OSATOMIC)
 #include <libkern/OSAtomic.h>
 #endif
 
@@ -39,16 +45,16 @@
  * possible.
  */
 #ifndef JEMALLOC_NO_PRIVATE_NAMESPACE
-#  ifndef JEMALLOC_JET
-#    include "jemalloc/internal/private_namespace.h"
-#  else
-#    include "jemalloc/internal/private_namespace_jet.h"
-#  endif
+#ifndef JEMALLOC_JET
+#include "jemalloc/internal/private_namespace.h"
+#else
+#include "jemalloc/internal/private_namespace_jet.h"
+#endif
 #endif
 #include "jemalloc/internal/test_hooks.h"
 
 #ifdef JEMALLOC_DEFINE_MADVISE_FREE
-#  define JEMALLOC_MADV_FREE 8
+#define JEMALLOC_MADV_FREE 8
 #endif
 
 static const bool config_debug =
@@ -161,7 +167,55 @@ static const bool config_log =
     false
 #endif
     ;
-#ifdef JEMALLOC_HAVE_SCHED_GETCPU
+/*
+ * Are extra safety checks enabled; things like checking the size of sized
+ * deallocations, double-frees, etc.
+ */
+static const bool config_opt_safety_checks =
+#ifdef JEMALLOC_OPT_SAFETY_CHECKS
+    true
+#elif defined(JEMALLOC_DEBUG)
+    /*
+     * This lets us only guard safety checks by one flag instead of two; fast
+     * checks can guard solely by config_opt_safety_checks and run in debug mode
+     * too.
+     */
+    true
+#else
+    false
+#endif
+    ;
+
+/*
+ * Extra debugging of sized deallocations too onerous to be included in the
+ * general safety checks.
+ */
+static const bool config_opt_size_checks =
+#if defined(JEMALLOC_OPT_SIZE_CHECKS) || defined(JEMALLOC_DEBUG)
+    true
+#else
+    false
+#endif
+    ;
+
+static const bool config_uaf_detection =
+#if defined(JEMALLOC_UAF_DETECTION) || defined(JEMALLOC_DEBUG)
+    true
+#else
+    false
+#endif
+    ;
+
+/* Whether or not the C++ extensions are enabled. */
+static const bool config_enable_cxx =
+#ifdef JEMALLOC_ENABLE_CXX
+    true
+#else
+    false
+#endif
+    ;
+
+#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU)
 /* Currently percpu_arena depends on sched_getcpu. */
 #define JEMALLOC_PERCPU_ARENA
 #endif
@@ -190,23 +244,16 @@ static const bool have_background_thread =
     false
 #endif
     ;
-
-#define JEMALLOC_GCC_U8_ATOMIC_ATOMICS 1
-#define JEMALLOC_GCC_U8_SYNC_ATOMICS 1
-
-/*
- * Are extra safety checks enabled; things like checking the size of sized
- * deallocations, double-frees, etc.
- */
-static const bool config_opt_safety_checks =
-#ifdef JEMALLOC_OPT_SAFETY_CHECKS
+static const bool config_high_res_timer =
+#ifdef JEMALLOC_HAVE_CLOCK_REALTIME
     true
-#elif defined(JEMALLOC_DEBUG)
-    /*
-     * This lets us only guard safety checks by one flag instead of two; fast
-     * checks can guard solely by config_opt_safety_checks and run in debug mode
-     * too.
-     */
+#else
+    false
+#endif
+    ;
+
+static const bool have_memcntl =
+#ifdef JEMALLOC_HAVE_MEMCNTL
     true
 #else
     false

contrib/jemalloc-cmake/include/jemalloc/jemalloc.h

@@ -4,13 +4,21 @@
 extern "C" {
 #endif
 
+#if !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wredundant-decls"
+#endif
+
 #include <jemalloc/jemalloc_defs.h>
-#include <jemalloc/jemalloc_rename.h>
 #include <jemalloc/jemalloc_macros.h>
 #include <jemalloc/jemalloc_protos.h>
+#include <jemalloc/jemalloc_rename.h>
 #include <jemalloc/jemalloc_typedefs.h>
 
+#if !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
 #ifdef __cplusplus
 }
 #endif
-