Fixed #166: Promote jemalloc as the alternate memory allocator of choice

.github/workflows/build.yaml

@@ -77,6 +77,7 @@ jobs:
         -DRUNTIME_CHECK=${{matrix.runtimeCheck}}
         -DSANITIZE_PYTHON=OFF
         -DBUILD_BENCHMARKS=ON
+        -DUSE_JEMALLOC=ON
 
       CCACHE_BASEDIR: ${{github.workspace}}
       CCACHE_DIR: ${{github.workspace}}/.ccache
@@ -139,7 +140,7 @@ jobs:
 
       - name: Install Linux build dependencies
         run: |
-          sudo apt update; sudo apt install -y libdw-dev swig libpython3-dev libsasl2-dev libjsoncpp-dev libwebsockets-dev libnghttp2-dev ccache ninja-build pixz libbenchmark-dev nginx
+          sudo apt update; sudo apt install -y libdw-dev swig libpython3-dev libsasl2-dev libjsoncpp-dev libwebsockets-dev libnghttp2-dev libjemalloc-dev ccache ninja-build pixz libbenchmark-dev nginx
 
       - name: Zero ccache stats
         run: ccache -z
@@ -249,7 +250,7 @@ jobs:
 
       - name: Install Linux runtime/test dependencies
         run: |
-          sudo apt update; sudo apt install -y libdw1 libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp25 libwebsockets16 libbenchmark1 pixz bubblewrap curl ncat gdb elfutils findutils file python3-dbg
+          sudo apt update; sudo apt install -y libdw1 libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp25 libwebsockets16 libbenchmark1 libjemalloc2 pixz bubblewrap curl ncat gdb elfutils findutils file python3-dbg
 
       - name: Unpack archive
         run: tar -I pixz -xf archive.tar.xz

CMakeLists.txt

@@ -161,6 +161,13 @@ find_package(LibWebSockets 4.0.1 REQUIRED)
 find_library(dw_lib dw DOC "libdw used to symbolize QD_MEMORY_DEBUG backtraces")
 find_package(libunwind)
 
+# jemalloc allocator linking is disabled by default
+OPTION(USE_JEMALLOC "Enable building and running with the jemalloc memory allocator" OFF)
+if(USE_JEMALLOC)
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(jemalloc REQUIRED IMPORTED_TARGET jemalloc)
+endif()
+
 # google benchmark tests are disabled by default
 OPTION(BUILD_BENCHMARKS "Enable building and running benchmarks with Google Benchmark" OFF)
 

decisions/0003-integrate-with-the-jemalloc-allocator.md

@@ -0,0 +1,35 @@
+# 3. Integrate with the jemalloc allocator
+
+Date: 2022-03-12
+
+## Status
+
+Proposed
+
+## Context
+
+Memory pools are an important technique to improve application performance.
+The router already uses a custom memory pool (`src/alloc_pool.c`).
+Proton currently does not use any such mechanism.
+
+The `jemalloc` library is a thread-safe pooled memory allocator, able to seamlessly replace malloc/free calls with a more optimized, tunable, and observable implementation.
+It turns out that linking the router with `jemalloc` shared library (without any additional tuning) is sufficient to noticeably improve the router's performance.
+This improvement is larger than what is provided by linking either `tcmalloc` or `mimalloc`, two other similar well-known libraries.
+
+Integrating `jemalloc` can be accomplished by setting `LD_PRELOAD=/usr/lib64/libjemalloc.so.2`.
+However, this procedure is hard to discover, easy to get wrong (e.g. make a typo), and in general not user-friendly.
+
+## Decision
+
+Router shall offer a CMake configuration option `-DUSE_JEMALLOC=ON` to link with `jemalloc` as part of the regular build.
+Interested users are to be encouraged to experiment with this option, and to try various `jemalloc` tuning parameters, which are plentiful.
+
+## Consequences
+
+This change should make the router instantaneously slightly more performant.
+
+In the future, tighter integration with the library can be considered, (refer to `man 3 jemalloc` for description of its additional nonstandard allocation API).
+Thorough performance evaluation should be performed beforehand, to establish whether careful tuning can unseat `jemalloc` as the fastest allocation library choice among the considered alternatives (`tcmalloc` and `mimalloc`.
+
+The router could possibly get out of the business of doing its own memory pooling, or implement special additional pooling strategies only for the important types (such as `qd_message_t` and `qd_buffer_t`).
+This would require reimplementing the weak pointers so that they do not rely on the current properties of the router memory pool.

src/CMakeLists.txt

@@ -168,6 +168,10 @@ else()
   set(qpid_dispatch_SOURCES ${qpid_dispatch_SOURCES} posix/symbolization_none.c)
 endif()
 
+if(USE_JEMALLOC)
+  list(APPEND qpid_dispatch_LIBRARIES PkgConfig::jemalloc)
+endif()
+
 if ((DEFINED ASAN_LIBRARY) OR (DEFINED UBSAN_LIBRARY) OR (DEFINED TSAN_LIBRARY))
   set(USING_SANITIZERS TRUE)
 endif()