Fixed #xxxx: Promote jemalloc as the memory allocator of choice

.github/workflows/build.yaml

@@ -56,6 +56,7 @@ jobs:
         -DRUNTIME_CHECK=${{matrix.runtimeCheck}}
         -DSANITIZE_3RD_PARTY=ON
         -DBUILD_BENCHMARKS=ON
+        -DUSE_JEMALLOC=ON
 
       CCACHE_BASEDIR: ${{github.workspace}}
       CCACHE_DIR: ${{github.workspace}}/.ccache
@@ -110,7 +111,7 @@ jobs:
       - name: Install Linux build dependencies
         if: ${{ runner.os == 'Linux' }}
         run: |
-          sudo apt update; sudo apt install -y swig libpython3-dev libsasl2-dev libjsoncpp-dev libwebsockets-dev libnghttp2-dev ccache ninja-build pixz libbenchmark-dev
+          sudo apt update; sudo apt install -y swig libpython3-dev libsasl2-dev libjsoncpp-dev libwebsockets-dev libnghttp2-dev libjemalloc-dev ccache ninja-build pixz libbenchmark-dev
 
       - name: Zero ccache stats
         run: ccache -z
@@ -212,7 +213,7 @@ jobs:
       - name: Install Linux runtime/test dependencies
         if: ${{ runner.os == 'Linux' }}
         run: |
-          sudo apt update; sudo apt install -y libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 pixz bubblewrap curl gdb elfutils findutils file python3-dbg
+          sudo apt update; sudo apt install -y libsasl2-2 libsasl2-modules sasl2-bin libjsoncpp1 libwebsockets15 libbenchmark1 libjemalloc2 pixz bubblewrap curl gdb elfutils findutils file python3-dbg
 
       - name: Unpack archive
         run: tar -I pixz -xf archive.tar.xz

CMakeLists.txt

@@ -79,6 +79,13 @@ message(STATUS "Found Proton: ${Proton_LIBRARIES} (found version \"${Proton_VERS
 ## Optional dependencies
 ##
 
+# jemalloc allocator linking is disabled by default
+OPTION(USE_JEMALLOC "Enable building and running with the jemalloc memory allocator" OFF)
+if(USE_JEMALLOC)
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(jemalloc REQUIRED IMPORTED_TARGET jemalloc)
+endif()
+
 # google benchmark tests are disabled by default
 OPTION(BUILD_BENCHMARKS "Enable building and running benchmarks with Google Benchmark" OFF)
 

decisions/0003-integrate-with-the-jemalloc-allocator.md

@@ -0,0 +1,35 @@
+# 3. Integrate with the jemalloc allocator
+
+Date: 2022-03-12
+
+## Status
+
+Proposed
+
+## Context
+
+Memory pools are an important technique to improve application performance.
+The router already uses a custom memory pool (`src/alloc_pool.c`).
+Proton currently does not use any such mechanism.
+
+The `jemalloc` library is a thread-safe pooled memory allocator, able to seamlessly replace malloc/free calls with a more optimized, tunable, and observable implementation.
+It turns out that linking the router with `jemalloc` shared library (without any additional tuning) is sufficient to noticeably improve the router's performance.
+This improvement is larger than what is provided by linking either `tcmalloc` or `mimalloc`, two other similar well-known libraries.
+
+Integrating `jemalloc` can be accomplished by setting `LD_PRELOAD=/usr/lib64/libjemalloc.so.2`.
+However, this procedure is hard to discover, easy to get wrong (e.g. make a typo), and in general not user-friendly.
+
+## Decision
+
+Router shall offer a CMake configuration option `-DUSE_JEMALLOC=ON` to link with `jemalloc` as part of the regular build.
+Interested users are to be encouraged to experiment with this option, and to try various `jemalloc` tuning parameters, which are plentiful.
+
+## Consequences
+
+This change should make the router instantaneously slightly more performant.
+
+In the future, tighter integration with the library can be considered, (refer to `man 3 jemalloc` for description of its additional nonstandard allocation API).
+Thorough performance evaluation should be performed beforehand, to establish whether careful tuning can unseat `jemalloc` as the fastest allocation library choice among the considered alternatives (`tcmalloc` and `mimalloc`.
+
+The router could possibly get out of the business of doing its own memory pooling, or implement special additional pooling strategies only for the important types (such as `qd_message_t` and `qd_buffer_t`).
+This would require reimplementing the weak pointers so that they do not rely on the current properties of the router memory pool.

src/CMakeLists.txt

@@ -144,6 +144,10 @@ set(qpid_dispatch_LIBRARIES
   ${Python_LIBRARIES}
   )
 
+if(USE_JEMALLOC)
+  list(APPEND qpid_dispatch_LIBRARIES PkgConfig::jemalloc)
+endif()
+
 # USE_LIBWEBSOCKETS is true only if LIBWEBSOCKETS_FOUND
 if(USE_LIBWEBSOCKETS)
   list(APPEND qpid_dispatch_SOURCES http-libwebsockets.c)