Update toolchain to Clang 17 (#8537)

close #7193, close #8890
pingcap · Apr 2, 2024 · 52fc334 · 52fc334
1 parent 93d7395
commit 52fc334
Show file tree

Hide file tree

Showing 11 changed files with 76 additions and 116 deletions.
diff --git a/.toolchain.yml b/.toolchain.yml
@@ -0,0 +1,18 @@
+# Copyright 2022 PingCAP, Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Hi, there!
+# This is passed via jenkins podTemplate argument.
+# Instead of trying injections, take a look at https://en.pingcap.com/careers/.
+image_tag_suffix: -llvm-17.0.6
diff --git a/README.md b/README.md
@@ -37,7 +37,7 @@ And the following operating systems:
 The following packages are required:
 
 - CMake 3.21.0+
-- Clang 14.0.0+
+- Clang 17.0.0+ under Linux or AppleClang 15.0.0+ under MacOS
 - Rust
 - Python 3.0+
 - Ninja-Build or GNU Make
@@ -56,10 +56,10 @@ curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain none
 source $HOME/.cargo/env
 
 # Install LLVM, see https://apt.llvm.org for details
-# Clang will be available as /usr/bin/clang++-15
+# Clang will be available as /usr/bin/clang++-17
 wget https://apt.llvm.org/llvm.sh
 chmod +x llvm.sh
-sudo ./llvm.sh 15 all
+sudo ./llvm.sh 17 all
 
 # Install other dependencies
 sudo apt install -y cmake ninja-build zlib1g-dev libcurl4-openssl-dev ccache
@@ -162,8 +162,8 @@ Note: In Linux, usually you need to explicitly specify to use LLVM.
 ```shell
 # In cmake-build-debug directory:
 cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG \
-  -DCMAKE_C_COMPILER=/usr/bin/clang-14 \
-  -DCMAKE_CXX_COMPILER=/usr/bin/clang++-14
+  -DCMAKE_C_COMPILER=/usr/bin/clang-17 \
+  -DCMAKE_CXX_COMPILER=/usr/bin/clang++-17
 ```
 
 In MacOS, if you install llvm clang, you need to explicitly specify to use llvm clang.
@@ -250,7 +250,7 @@ cmake .. -GNinja -DCMAKE_BUILD_TYPE=DEBUG -DFOO=BAR
 
   There is another option to append extra paths for CMake to find system libraries:
 
-  - `PREBUILT_LIBS_ROOT`: Default as empty, can be specified with multiple values, seperated by `;`
+  - `PREBUILT_LIBS_ROOT`: Default as empty, can be specified with multiple values, separated by `;`
 
   </details>
 

diff --git a/contrib/arm-optimized-routines b/contrib/arm-optimized-routines
diff --git a/contrib/arm-optimized-routines-cmake/src/aor.c b/contrib/arm-optimized-routines-cmake/src/aor.c
@@ -13,75 +13,9 @@
 // limitations under the License.
 
 #include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
 #include <string.h>
 #include <stringlib.h>
-#include <sys/auxv.h>
 
-// Provide default macro definitions in case that they are not defined on current linux distro.
-// For example, TiFlash compiled on older linux kernels may also be used in newer ones.
-// These values should be stable for Linux: only false negative is expected when running on
-// older kernels, but it is acceptable as `google/cpu_features` is also doing so.
-#ifndef HWCAP2_MTE
-#define HWCAP2_MTE (1 << 18)
-#endif
-
-#ifndef HWCAP_SVE
-#define HWCAP_SVE (1 << 22)
-#endif
-
-#ifndef AT_HWCAP2
-#define AT_HWCAP2 26
-#endif
-
-#ifndef AT_HWCAP
-#define AT_HWCAP 16
-#endif
-
-/// check if MTE is supported in current environment
-static inline bool mte_supported(void)
-{
-    return (getauxval(AT_HWCAP2) & HWCAP2_MTE) != 0;
-}
-
-/// check if SVE is supported in current environment
-static inline bool sve_supported(void)
-{
-    return (getauxval(AT_HWCAP) & HWCAP_SVE) != 0;
-}
-
-#define STRINGIFY_IMPL(X) #X
-#define STRINGIFY(X) STRINGIFY_IMPL(X)
-/**
- *  \brief
- *  Symbol is defined as hidden visibility. Therefore, implementations here are only to override routines with TiFlash
- *  binary itself. This is because dependencies like `ld.so`, `libgcc_s.so`, etc will need essential routines like
- *  `memcpy` to finish the early loading procedure. Therefore, declare such symbols as visible indirect function will
- *  create cyclic dependency. It shall be good enough to override symbols within TiFlash, as most heavy computation works
- *  are happening in the main binary.
- *  \param NAME: exported symbol name
- *  \param SVE: preferred implementation when SVE is available
- *  \param MTE: preferred implementation when MTE is available
- *  \param ASIMD: preferred implementation for generic aarch64 targets (ASIMD is required by default for Armv8 and above)
- */
-#define DISPATCH(NAME, SVE, MTE, ASIMD)                                                                                  \
-    extern typeof(ASIMD) __tiflash_##NAME __attribute__((ifunc(STRINGIFY(__tiflash_##NAME##_resolver))));                \
-    extern typeof(ASIMD) NAME __attribute__((visibility("hidden"), alias(STRINGIFY(__tiflash_##NAME))));                 \
-    _Pragma("GCC diagnostic push")                                                                                       \
-        _Pragma("GCC diagnostic ignored \"-Wunused-function\"") static typeof(ASIMD) * __tiflash_##NAME##_resolver(void) \
-    {                                                                                                                    \
-        if (sve_supported())                                                                                             \
-        {                                                                                                                \
-            return SVE;                                                                                                  \
-        }                                                                                                                \
-        if (mte_supported())                                                                                             \
-        {                                                                                                                \
-            return MTE;                                                                                                  \
-        }                                                                                                                \
-        return ASIMD;                                                                                                    \
-    }                                                                                                                    \
-    _Pragma("GCC diagnostic pop")
 #undef memcpy
 #undef memmove
 #undef memset
@@ -98,18 +32,18 @@ static inline bool sve_supported(void)
 #undef strnlen
 #undef strncmp
 
-DISPATCH(memcpy, __memcpy_aarch64_sve, __memcpy_aarch64_simd, __memcpy_aarch64_simd)
-DISPATCH(memmove, __memmove_aarch64_sve, __memmove_aarch64_simd, __memmove_aarch64_simd)
-DISPATCH(memset, __memset_aarch64, __memset_aarch64, __memset_aarch64)
-DISPATCH(memchr, __memchr_aarch64_sve, __memchr_aarch64_mte, __memchr_aarch64)
-DISPATCH(memrchr, __memrchr_aarch64, __memrchr_aarch64, __memrchr_aarch64)
-DISPATCH(memcmp, __memcmp_aarch64_sve, __memcmp_aarch64, __memcmp_aarch64)
-DISPATCH(strcpy, __strcpy_aarch64_sve, __strcpy_aarch64, __strcpy_aarch64)
-DISPATCH(stpcpy, __stpcpy_aarch64_sve, __stpcpy_aarch64, __stpcpy_aarch64)
-DISPATCH(strcmp, __strcmp_aarch64_sve, __strcmp_aarch64, __strcmp_aarch64)
-DISPATCH(strchr, __strchr_aarch64_sve, __strchr_aarch64_mte, __strchr_aarch64)
-DISPATCH(strrchr, __strrchr_aarch64_sve, __strrchr_aarch64_mte, __strrchr_aarch64)
-DISPATCH(strchrnul, __strchrnul_aarch64_sve, __strchrnul_aarch64_mte, __strchrnul_aarch64)
-DISPATCH(strlen, __strlen_aarch64_sve, __strlen_aarch64_mte, __strlen_aarch64)
-DISPATCH(strnlen, __strnlen_aarch64_sve, __strnlen_aarch64, __strnlen_aarch64)
-DISPATCH(strncmp, __strncmp_aarch64_sve, __strncmp_aarch64, __strncmp_aarch64)
+#define memcpy __memcpy_aarch64_simd
+#define memmove __memmove_aarch64_simd
+#define memset __memset_aarch64
+#define memchr __memchr_aarch64
+#define memrchr __memrchr_aarch64
+#define memcmp __memcmp_aarch64
+#define strcpy __strcpy_aarch64
+#define stpcpy __stpcpy_aarch64
+#define strcmp __strcmp_aarch64
+#define strchr __strchr_aarch64
+#define strrchr __strrchr_aarch64
+#define strchrnul __strchrnul_aarch64
+#define strlen __strlen_aarch64
+#define strnlen __strnlen_aarch64
+#define strncmp __strncmp_aarch64
diff --git a/dbms/src/Storages/KVStore/tests/gtest_kvstore.cpp b/dbms/src/Storages/KVStore/tests/gtest_kvstore.cpp
@@ -1542,9 +1542,7 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): not found region_id=1");
+            ASSERT_EQ(res, "void DB::RegionsRangeIndex::remove(const RegionRange &, RegionID): not found region_id=1");
         }
 
         region_index.add(makeRegion(2, RecordKVFormat::genKey(1, 3), RecordKVFormat::genKey(1, 5)));
@@ -1558,9 +1556,7 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): not found start key");
+            ASSERT_EQ(res, "void DB::RegionsRangeIndex::remove(const RegionRange &, RegionID): not found start key");
         }
 
         try
@@ -1573,9 +1569,7 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): not found end key");
+            ASSERT_EQ(res, "void DB::RegionsRangeIndex::remove(const RegionRange &, RegionID): not found end key");
         }
 
         try
@@ -1590,7 +1584,7 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
             const auto & res = e.message();
             ASSERT_EQ(
                 res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): range of region_id=2 is "
+                "void DB::RegionsRangeIndex::remove(const RegionRange &, RegionID): range of region_id=2 is "
                 "empty");
         }
 
@@ -1602,9 +1596,7 @@ TEST_F(RegionKVStoreOldTest, RegionRange)
         catch (Exception & e)
         {
             const auto & res = e.message();
-            ASSERT_EQ(
-                res,
-                "void DB::RegionsRangeIndex::remove(const DB::RegionRange &, DB::RegionID): not found region_id=2");
+            ASSERT_EQ(res, "void DB::RegionsRangeIndex::remove(const RegionRange &, RegionID): not found region_id=2");
         }
 
         region_index.clear();

diff --git a/release-centos7-llvm/README.md b/release-centos7-llvm/README.md
@@ -45,6 +45,17 @@ Raft Proxy
 
 At last, change the suffix in `${REPO_DIR}/.toolchain.yml` and check the built target in the CI.
 
+
+How to get into a build container:
+
+```
+SUFFIX=-llvm-17.0.6
+# x86_64
+docker run -it --rm -v /your/path/to/tiflash:/build/tics hub.pingcap.net/tiflash/tiflash-llvm-base:amd64${SUFFIX} /bin/bash
+# arm
+docker run -it --rm -v /your/path/to/tiflash:/build/tics hub.pingcap.net/tiflash/tiflash-llvm-base:aarch64${SUFFIX} /bin/bash
+```
+
 ## Local toolchain prerequisites
 
 Check `env/README.md`
diff --git a/release-centos7-llvm/dockerfiles/misc/bake_llvm_base_aarch64.sh b/release-centos7-llvm/dockerfiles/misc/bake_llvm_base_aarch64.sh
@@ -30,18 +30,18 @@ function bake_llvm_base_aarch64() {
 
     # CMake
     source $SCRIPTPATH/install_cmake.sh
-    install_cmake "3.22.1" "aarch64"
+    install_cmake "3.24.2" "aarch64"
 
     # LLVM
     source $SCRIPTPATH/bootstrap_llvm.sh
-    bootstrap_llvm "13.0.0"
+    bootstrap_llvm "17.0.6"
     export CC=clang
     export CXX=clang++
     export LD=ld.lld
 
     # OpenSSL
     source $SCRIPTPATH/install_openssl.sh
-    install_openssl "1_1_1t"
+    install_openssl "1_1_1w"
     export OPENSSL_ROOT_DIR="/usr/local/opt/openssl"
 
     # Git

diff --git a/release-centos7-llvm/dockerfiles/misc/bake_llvm_base_amd64.sh b/release-centos7-llvm/dockerfiles/misc/bake_llvm_base_amd64.sh
@@ -29,18 +29,18 @@ function bake_llvm_base_amd64() {
 
     # CMake
     source $SCRIPTPATH/install_cmake.sh
-    install_cmake "3.22.1" "x86_64"
+    install_cmake "3.24.2" "x86_64"
 
     # LLVM
     source $SCRIPTPATH/bootstrap_llvm.sh
-    bootstrap_llvm "13.0.0"
+    bootstrap_llvm "17.0.6"
     export CC=clang
     export CXX=clang++
     export LD=ld.lld
 
     # OpenSSL
     source $SCRIPTPATH/install_openssl.sh
-    install_openssl "1_1_1t"
+    install_openssl "1_1_1w"
     export OPENSSL_ROOT_DIR="/usr/local/opt/openssl"
 
     # Git

diff --git a/release-centos7-llvm/dockerfiles/misc/bootstrap_llvm.sh b/release-centos7-llvm/dockerfiles/misc/bootstrap_llvm.sh
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
 
 # Boostrap LLVM envriroment for CI/CD.
 # Require: git, ninja, cmake, compiler(devtoolset-10)
@@ -29,6 +29,7 @@ function bootstrap_llvm() {
 
     cmake -DCMAKE_BUILD_TYPE=Release \
         -GNinja \
+        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
         -DLLVM_ENABLE_PROJECTS="clang;lld" \
         -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi" \
         -DLLVM_TARGETS_TO_BUILD=Native \
@@ -44,7 +45,8 @@ function bootstrap_llvm() {
 
     cmake -DCMAKE_BUILD_TYPE=Release \
         -GNinja \
-        -DLLVM_ENABLE_PROJECTS="clang;lld;polly;clang-tools-extra" \
+        -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
+        -DLLVM_ENABLE_PROJECTS="clang;lld;polly;clang-tools-extra;bolt" \
         -DLLVM_ENABLE_RUNTIMES="libcxx;libcxxabi;openmp;compiler-rt" \
         -DLLVM_TARGETS_TO_BUILD=Native \
         -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON \

diff --git a/release-centos7-llvm/env/prepare-sysroot.sh b/release-centos7-llvm/env/prepare-sysroot.sh
@@ -16,15 +16,15 @@
 
 set -ueox pipefail
 
-CMAKE_VERSION="3.22.1"
-GO_VERSION="1.20"
+CMAKE_VERSION="3.24.2"
+GO_VERSION="1.21.5"
 ARCH=$(uname -m)
 GO_ARCH=$([[ "${ARCH}" == "aarch64" ]] && echo "arm64" || echo "amd64")
-LLVM_VERSION="13.0.0"
+LLVM_VERSION="17.0.6"
 CCACHE_VERSION="4.5.1"
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
 SYSROOT="${SCRIPTPATH}/sysroot"
-OPENSSL_VERSION="1_1_1l"
+OPENSSL_VERSION="1_1_1w"
 
 function install_cmake() {
     wget "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh"
@@ -38,10 +38,10 @@ function install_llvm() {
     mkdir -p llvm-project/build
     cd llvm-project/build
 
-    # TODO: enable `bolt` for >= 14.0.0. https://github.com/llvm/llvm-project/tree/main/bolt
+    # `bolt` is enabled; https://github.com/llvm/llvm-project/tree/main/bolt
     cmake -DCMAKE_BUILD_TYPE=Release \
         -GNinja \
-        -DLLVM_ENABLE_PROJECTS="clang;lld;polly;clang-tools-extra" \
+        -DLLVM_ENABLE_PROJECTS="clang;lld;polly;clang-tools-extra;bolt" \
         -DLLVM_ENABLE_RUNTIMES="compiler-rt;libcxx;libcxxabi;libunwind;openmp" \
         -DLLVM_TARGETS_TO_BUILD=Native \
         -DCOMPILER_RT_USE_BUILTINS_LIBRARY=ON \
@@ -86,7 +86,7 @@ function install_openssl() {
 }
 
 function install_go() {
-    wget "https://dl.google.com/go/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz"
+    wget "https://go.dev/dl/go${GO_VERSION}.linux-${GO_ARCH}.tar.gz"
     tar -C "${SYSROOT}" -xzvf go${GO_VERSION}.linux-${GO_ARCH}.tar.gz
     mv "${SYSROOT}/go" "${SYSROOT}/go${GO_VERSION}" && \
         pushd "${SYSROOT}" && ln -sv "go${GO_VERSION}" "go" && popd

diff --git a/release-centos7-llvm/scripts/build-tiflash-release.sh b/release-centos7-llvm/scripts/build-tiflash-release.sh
@@ -38,6 +38,9 @@ fi
 
 set -ueox pipefail
 
+# workaround git ownership check
+git config --global --add safe.directory "/build/tics"
+
 SCRIPTPATH="$(
   cd "$(dirname "$0")"
   pwd -P