Merge pull request redpanda-data#23142 from rockwotj/pgo

bazel: PGO for bazel clang toolchain

MODULE.bazel

@@ -119,16 +119,14 @@ CXX_STANDARD = "c++20"
 llvm.toolchain(
     name = "llvm_17_toolchain",
     cxx_standard = {"": CXX_STANDARD},
-    llvm_versions = {
-        "": "17.0.6",
-    },
+    llvm_version = "17.0.6",
     sha256 = {
-        "linux-aarch64": "259da3d332e2080ecfe5a059dedb8f485f0b20e6ed2e3880bc5a5b57084f4036",
-        "linux-x86_64": "2c06b2acb362a10b9e7ac3b0aebdccd710a5329d9c7ee46c10889fdf00a41b9f",
+        "linux-aarch64": "28edd46b9666d99c31aa255dfa5497dba95f095d731c6c398fb0e2c0402a958f",
+        "linux-x86_64": "d75de539f9791b35eb4b58bed6c7605a846857c1b203c160a6dad2d620f8b090",
     },
     urls = {
-        "linux-aarch64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/llvm-17.0.6-ubuntu-22.04-aarch64.tar.gz"],
-        "linux-x86_64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/llvm-17.0.6-ubuntu-22.04-x86_64.tar.gz"],
+        "linux-aarch64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/pgo/llvm-17.0.6-ubuntu-22.04-aarch64.tar.gz"],
+        "linux-x86_64": ["http://redpanda-core-toolchain.s3-website-us-east-1.amazonaws.com/pgo/llvm-17.0.6-ubuntu-22.04-x86_64.tar.gz"],
     },
 )
 use_repo(llvm, "llvm_17_toolchain", "llvm_17_toolchain_llvm")
@@ -138,9 +136,7 @@ register_toolchains("@llvm_17_toolchain//:all")
 llvm.toolchain(
     name = "llvm_18_toolchain",
     cxx_standard = {"": CXX_STANDARD},
-    llvm_versions = {
-        "": "18.1.8",
-    },
+    llvm_version = "18.1.8",
     sha256 = {
         "linux-aarch64": "95223a9fd35a2cfea2582b34519bbf65ff6bed498a378eb545a9d4fb8ee5a832",
         "linux-x86_64": "f79565afd6f5d270c10a11cbe4d4b59ecf807df29051af2b479daf4524aa9c73",

bazel/thirdparty/libpciaccess.BUILD

@@ -27,6 +27,7 @@ cc_library(
         "include/pciaccess.h",
     ],
     copts = [
+        "-Wno-unused-result",
         "-Wpointer-arith",
         "-Wmissing-declarations",
         "-Wformat=2",

bazel/toolchain/Dockerfile.llvm

@@ -0,0 +1,161 @@
+FROM ubuntu:jammy AS base
+
+RUN apt update
+RUN apt upgrade -y
+RUN apt install -y git curl python3 cmake ninja-build wget lsb-release software-properties-common gnupg
+ADD --chmod=755 https://apt.llvm.org/llvm.sh /
+ARG SYSTEM_CLANG_VERSION=19
+RUN /llvm.sh ${SYSTEM_CLANG_VERSION}
+
+ARG TARGETARCH
+ADD --chmod=755 https://github.com/bazelbuild/bazelisk/releases/download/v1.20.0/bazelisk-linux-${TARGETARCH} /usr/bin/bazel
+# For some reason, with high concurrency settings, clang will segfault for me sometimes. Use this automatic retry script so that's less annoying.
+ADD --chmod=755 https://github.com/kadwanev/retry/raw/20997c7712a4/retry /usr/bin/retry
+
+WORKDIR /opt
+
+ARG LLVM_VERSION=17.0.6
+
+RUN git clone --branch llvmorg-${LLVM_VERSION} --single-branch https://github.com/llvm/llvm-project.git
+
+ENV CC=clang-${SYSTEM_CLANG_VERSION}
+ENV CXX=clang++-${SYSTEM_CLANG_VERSION}
+
+ARG MAX_LINKER_JOBS=0
+ARG MAX_COMPILER_JOBS=0
+
+COPY --chmod=755 <<EOF /opt/build.sh
+#!/usr/bin/env bash
+
+set -ex
+
+BUILD_DIR="/tmp/$(tr -dc A-Za-z0-9 </dev/urandom | head -c 13; echo)"
+
+mkdir -p "\$BUILD_DIR"
+
+cmake -G Ninja -DLLVM_TARGETS_TO_BUILD='AArch64;X86' \\
+  -S /opt/llvm-project/llvm/ \\
+  -B "\$BUILD_DIR" \\
+  -DCMAKE_BUILD_TYPE=Release \\
+  -DLLVM_BUILD_STATIC=OFF \\
+  -DLLVM_ENABLE_PIC=ON \\
+  -DLLVM_INCLUDE_TESTS=OFF \\
+  -DLLVM_INCLUDE_EXAMPLES=OFF \\
+  -DLLVM_INCLUDE_UTILS=OFF \\
+  -DLLVM_INCLUDE_DOCS=OFF \\
+  -DLLVM_ENABLE_ZLIB=OFF \\
+  -DLLVM_ENABLE_ZSTD=OFF \\
+  -DLLVM_ENABLE_Z3_SOLVER=OFF \\
+  -DLLVM_ENABLE_DIA_SDK=OFF \\
+  -DLLVM_ENABLE_LIBEDIT=OFF \\
+  -DLLVM_ENABLE_TERMINFO=OFF \\
+  -DLLVM_ENABLE_LIBXML2=OFF \\
+  -DLLVM_ENABLE_BACKTRACES=OFF \\
+  -DLLVM_ENABLE_PROJECTS="clang;lld;lldb;clang-tools-extra;bolt" \\
+  -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;compiler-rt" \\
+  -DLLVM_USE_LINKER=lld \\
+  -DLLVM_BUILD_TOOLS=ON \\
+  -DLLVM_ENABLE_PLUGINS=OFF \\
+  -DLLDB_ENABLE_CURSES=OFF \\
+  -DLLDB_ENABLE_LUA=OFF \\
+  -DLLDB_ENABLE_LZMA=OFF \\
+  -DLLDB_ENABLE_LIBXML2=OFF \\
+  -DLLDB_ENABLE_LIBEDIT=OFF \\
+  -DLLVM_BINUTILS_INCDIR=ON \\
+  -DLIBCLANG_BUILD_STATIC=ON \\
+  -DCLANG_PLUGIN_SUPPORT=OFF \\
+  -DCOMPILER_RT_INCLUDE_TESTS=OFF \\
+  -DCOMPILER_RT_BUILD_SANITIZERS=ON \\
+  -DCOMPILER_RT_ENABLE_IOS=OFF \\
+  -DCOMPILER_RT_ENABLE_WATCHOS=OFF \\
+  -DCOMPILER_RT_ENABLE_TVOS=OFF \\
+  -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON \\
+  -DLIBCXX_CXX_ABI=libcxxabi \\
+  -DLIBCXXABI_USE_LLVM_UNWINDER=OFF \\
+  -DLLVM_PARALLEL_COMPILE_JOBS=${MAX_COMPILER_JOBS} \\
+  -DLLVM_PARALLEL_LINK_JOBS=${MAX_LINKER_JOBS} \\
+  -DLIBCXX_ABI_VERSION=2 \\
+  -DLLVM_ENABLE_LIBCXX=ON \\
+  -DLLVM_STATIC_LINK_CXX_STDLIB=ON \\
+  -DLLVM_ENABLE_LTO=On \\
+  "\$@"
+
+retry --sleep=0 --tries=100 -- ninja -C "\$BUILD_DIR"
+ninja -C "\$BUILD_DIR" install
+# Save space and cleanup the build directory
+rm -rf "\$BUILD_DIR"
+EOF
+
+FROM base AS lto-build
+
+RUN /opt/build.sh \
+  -DCMAKE_INSTALL_PREFIX=/opt/install
+
+WORKDIR /opt/install
+
+RUN bash -c 'tar -czvf /opt/llvm-${LLVM_VERSION}-$(source /etc/os-release && echo -n "$ID")-$(source /etc/os-release && echo -n "$VERSION_ID")-$(uname -m).tar.gz .'
+
+FROM scratch AS lto
+
+COPY --from=lto-build /opt/llvm-*.tar.gz .
+
+FROM base AS pgo-build
+
+RUN mkdir -p /tmp/clang-pgo/data/
+
+RUN /opt/build.sh \
+  -DCMAKE_INSTALL_PREFIX=/opt/stage-1 \
+  -DLLVM_BUILD_INSTRUMENTED=IR \
+  -DLLVM_PROFILE_DATA_DIR=/tmp/clang-pgo-data/ \
+  -DLLVM_ENABLE_PROJECTS="clang;lld;bolt" \
+  -DCMAKE_C_FLAGS="-Xclang -mllvm -Xclang -vp-counters-per-site=6" \
+  -DCMAKE_CXX_FLAGS="-Xclang -mllvm -Xclang -vp-counters-per-site=6" \
+  -DLLVM_BUILD_RUNTIME=No
+
+RUN git clone --branch dev --single-branch https://github.com/redpanda-data/redpanda.git
+
+WORKDIR /opt/redpanda
+
+RUN ./bazel/install-deps.sh
+
+# Append this snippet to out MODULE file
+COPY <<EOF pgo_compiler.bzl
+llvm.toolchain(
+    name = "pgo_toolchain",
+    cxx_standard = {"": CXX_STANDARD},
+    llvm_version = "${LLVM_VERSION}",
+)
+llvm.toolchain_root(
+    name = "pgo_toolchain",
+    path = "/opt/stage-1",
+)
+use_repo(llvm, "pgo_toolchain")
+EOF
+RUN cat pgo_compiler.bzl >> MODULE.bazel
+
+RUN retry --sleep=0 --tries=100 -- bazel build \
+  --config=release \
+  --copt=-g \
+  --copt=-Wno-everything \
+  --extra_toolchains=@pgo_toolchain//:all \
+  //...
+
+RUN bazel clean --expunge
+
+WORKDIR /opt
+
+# Merge profile data
+RUN llvm-profdata-${SYSTEM_CLANG_VERSION} merge -output=/tmp/profdata.prof /tmp/clang-pgo-data/*.profraw
+
+# Rebuild clang toolchain with profile data this time
+RUN /opt/build.sh \
+  -DCMAKE_INSTALL_PREFIX=/opt/stage-2 \
+  -DLLVM_PROFDATA_FILE=/tmp/profdata.prof
+
+WORKDIR /opt/stage-2
+
+RUN bash -c 'tar -czvf /opt/llvm-${LLVM_VERSION}-$(source /etc/os-release && echo -n "$ID")-$(source /etc/os-release && echo -n "$VERSION_ID")-$(uname -m).tar.gz .'
+
+FROM scratch AS pgo
+
+COPY --from=pgo-build /opt/llvm-*.tar.gz .

bazel/toolchain/README.md

@@ -7,7 +7,7 @@ hermetic and allows us to atomically upgrade the compiler as desired.
 To build a toolchain use the following command:
 
 ```
-docker build --build-arg "LLVM_VERSION=17.0.6" --file Dockerfile.oldlinux --output $PWD .
+docker build --build-arg "LLVM_VERSION=17.0.6" --file Dockerfile.llvm --output $PWD .
 ```
 
 The compiler output will be in a tarball in the current directory, this can be uploaded to S3, then bazel can pull
@@ -21,5 +21,10 @@ You can build an `aarch64` toolchain on a `x86_64` host by installing QEMU:
 Then build the docker image using buildx like so:
 
 ```
-docker buildx build --platform=linux/arm64 --build-arg "LLVM_VERSION=17.0.6" --file Dockerfile.oldlinux --output $PWD .
+docker buildx build --platform=linux/arm64 --build-arg "LLVM_VERSION=17.0.6" --file Dockerfile.llvm --output $PWD .
 ```
+
+### LTO Builds
+
+By default we build with PGO+LTO, but if PGO is causing issues (like on AArch64), we can choose a different build (resulting
+in a slower compiler) by adding the flag `--target=lto`. The current default target is `--target=pgo`

src/v/kafka/server/quota_manager.cc

@@ -557,7 +557,7 @@ ss::future<> quota_manager::do_global_gc() {
 
     for (auto it = _global_map->begin(); it != _global_map->end();) {
         auto& [key, value] = *it;
-        if (value.unique()) {
+        if (value.use_count() == 1) {
             // The pointer in the global map is effectively a weak pointer in
             // that we want to destroy the quota when only the global map holds
             // a reference to it. The reason why we have a std::shared_ptr<>

src/v/wasm/tests/wasm_cache_test.cc

@@ -81,9 +81,7 @@ class fake_engine : public engine {
     }
 
     ss::future<> transform(
-      model::record_batch batch,
-      transform_probe*,
-      transform_callback) override {
+      model::record_batch, transform_probe*, transform_callback) override {
         if (_state->engine_transform_should_throw) {
             throw std::runtime_error("test error");
         }