fix build: fix build all CI Ubuntu checks

Tests: протестировано CI Pull Request resolved: #730 Co-authored-by: Fedor [fiosetrova@gmail.com](mailto:fiosetrova@gmail.com) Co-authored-by: fdr400 [fdr400@yandex-team.com](mailto:fdr400@yandex-team.com) commit_hash:4dde7685a9d47db49599171ae760bd6a259a00a6
userver-framework · Oct 28, 2024 · 11e71bc · 11e71bc
1 parent e752549
commit 11e71bc
Show file tree

Hide file tree

Showing 32 changed files with 235 additions and 101 deletions.
diff --git a/.github/workflows/ci-conan.yml b/.github/workflows/ci-conan.yml
@@ -39,6 +39,7 @@ jobs:
             run: |
                 sudo apt-get install -y gcc g++ cmake wget git python3 python3-pip python3-venv ccache
                 pip install "conan==2.8.0"
+                pip install numpy
                 conan profile detect
                 conan profile show
 

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -28,7 +28,6 @@ jobs:
                         -DUSERVER_NAMESPACE_BEGIN="namespace userver_ns { inline namespace
                         v1 {"
                         -DUSERVER_NAMESPACE_END="}}"
-                        -DUSERVER_FEATURE_KAFKA=0
                         -DUSERVER_PG_SERVER_INCLUDE_DIR=/usr/include/postgresql/16/server
                         -DUSERVER_PG_SERVER_LIBRARY_DIR=/usr/lib/postgresql/16/lib
                     os: ubuntu-24.04
@@ -55,6 +54,8 @@ jobs:
                         -DUSERVER_FEATURE_CRYPTOPP_BLAKE2=0
                         -DUSERVER_FEATURE_REDIS_HI_MALLOC=1
                         -DUSERVER_NO_WERROR=1
+                        -DUSERVER_FORCE_DOWNLOAD_ABSEIL=0
+                        -DUSERVER_DOWNLOAD_PACKAGE_ABSEIL=0
                         -DCMAKE_BUILD_TYPE=Release
                     os: ubuntu-20.04
                     info: g++-9 + release
@@ -67,6 +68,8 @@ jobs:
                         -DUSERVER_FEATURE_CRYPTOPP_BLAKE2=0
                         -DUSERVER_FEATURE_REDIS_HI_MALLOC=1
                         -DUSERVER_NO_WERROR=1
+                        -DUSERVER_FORCE_DOWNLOAD_ABSEIL=0
+                        -DUSERVER_DOWNLOAD_PACKAGE_ABSEIL=0
                         -DCMAKE_BUILD_TYPE=Debug
                     os: ubuntu-20.04
                     info: g++-8 + debug
@@ -100,11 +103,50 @@ jobs:
                 df -h
                 # See https://stackoverflow.com/questions/75536771/github-runner-out-of-disk-space-after-building-docker-image
                 sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/lib/php* /opt/ghc \
-                    /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup || true
+                    /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
+                    /opt/hostedtoolcache/CodeQL || true
+                sudo docker image prune --all --force
                 df -h
 
-          - name: Install packages
+          - name: Add confluentinc repositories
+            if: matrix.os != 'ubuntu-24.04'
             run: |
+                # Adding librdkafka confluent repositories as in https://docs.confluent.io/platform/current/installation/installing_cp/deb-ubuntu.html#get-the-software
+                gpg_retrieve_keyserver() {
+                  GNUPGHOME=$(mktemp -d)
+                  GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring --keyring "/usr/share/keyrings/$2.gpg" --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $1
+                  rm -rf "$GNUPGHOME"
+                  chmod a+r "/usr/share/keyrings/$2.gpg"
+                }
+                gpg_retrieve_keyserver 8B1DA6120C2BF624 confluent
+                printf "\
+                deb [arch=amd64 signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/deb/7.6 stable main\n\
+                deb [signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/clients/deb $(lsb_release -cs) main\n" \
+                  | sudo tee /etc/apt/sources.list.d/confluent.list
+
+          - name: Install clang-format
+            if: matrix.os == 'ubuntu-20.04'
+            run: |
+                sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+                sudo apt update
+                wget https://apt.llvm.org/llvm.sh
+                chmod +x llvm.sh
+                sudo ./llvm.sh 14
+                sudo apt install clang-format-14
+                sudo ln -sf /usr/bin/clang-format-14 /usr/bin/clang-format
+                clang-format --version
+
+          - name: Install grpc, protobuf and abseil
+            if: matrix.os == 'ubuntu-20.04'
+            run: |
+                git clone --recursive -b v1.45.0 https://github.com/grpc/grpc new_grpc
+                cmake -S./new_grpc -B./new_grpc/build -DABSL_PROPAGATE_CXX_STD=ON -DABSL_ENABLE_INSTALL=ON
+                cmake --build new_grpc/build -- -j$(nproc)
+                sudo cmake --install new_grpc/build
+
+          - name: Add missing repositories
+            run: |
+                sudo apt update
                 # Instructions from https://clickhouse.com/docs/en/getting-started/install/
                 sudo apt install -y apt-transport-https ca-certificates dirmngr
                 sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
@@ -118,50 +160,39 @@ jobs:
                 #    | tee /etc/apt/sources.list.d/mariadb.list
                 echo "deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/mariadb.gpg] https://mirror.kumi.systems/mariadb/repo/10.11/ubuntu $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/mariadb.list
 
-                if [ "${{matrix.os}}" != "ubuntu-24.04" ]; then
-                  # Adding librdkafka confluent repositories as in https://docs.confluent.io/platform/current/installation/installing_cp/deb-ubuntu.html#get-the-software
-                  gpg_retrieve_keyserver() {
-                    GNUPGHOME=$(mktemp -d)
-                    GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring --keyring "/usr/share/keyrings/$2.gpg" --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $1
-                    rm -rf "$GNUPGHOME"
-                    chmod a+r "/usr/share/keyrings/$2.gpg"
-                  }
-                  gpg_retrieve_keyserver 8B1DA6120C2BF624 confluent
-                  printf "\
-                  deb [arch=amd64 signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/deb/7.6 stable main\n\
-                  deb [signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/clients/deb $(lsb_release -cs) main\n" \
-                    | sudo tee /etc/apt/sources.list.d/confluent.list
-
-                  # Install Kafka
-                  sudo apt install -y default-jre
-                  curl https://dlcdn.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz -o kafka.tgz
-                  sudo mkdir -p /etc/kafka
-                  sudo tar xf kafka.tgz --directory=/etc/kafka
-                  sudo cp -r /etc/kafka/kafka_2.13-3.8.0/* /etc/kafka/
-                  sudo rm -rf /etc/kafka/kafka_2.13-3.8.0
-                  ls -lha /etc/kafka
-                fi
-
+                # Adding mongodb repositories
+                wget -qO- https://pgp.mongodb.com/server-7.0.asc | sudo gpg --dearmor | sudo tee /usr/share/keyrings/mongodb-server-7.0.gpg >/dev/null
+                echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" \
+                      | sudo tee -a /etc/apt/sources.list.d/mongodb-org-7.0.list
+                # Mongo is not officially supported on Ubuntu 24.04, so install it from jammy
+
+                # Install Kafka
+                sudo apt install -y default-jre
+                curl https://dlcdn.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz -o kafka.tgz
+                sudo mkdir -p /etc/kafka
+                sudo tar xf kafka.tgz --directory=/etc/kafka
+                sudo cp -r /etc/kafka/kafka_2.13-3.8.0/* /etc/kafka/
+                sudo rm -rf /etc/kafka/kafka_2.13-3.8.0
+                ls -lha /etc/kafka
+
+          - name: Install common deps
+            run: |
                 sudo apt update
                 sudo apt install --allow-downgrades -y clickhouse-common-static redis-server postgresql $(cat scripts/docs/en/deps/${{matrix.os}}.md | tr '\n' ' ')
-                if [ "${{matrix.os}}" = "ubuntu-22.04" ]; then
-                  # GH Ubuntu image has libpq version 15+ from pg's repo that is API incompatible with postgresql-14.
-                  sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7FCC7D46ACCC4CF8
-                  echo "deb https://apt-archive.postgresql.org/pub/repos/apt jammy-pgdg-archive main" | sudo tee /etc/apt/sources.list.d/pgdg.list
-                  sudo apt update
-                  sudo apt install -y postgresql-15 postgresql-client-15 postgresql-server-dev-15
-                fi
 
-                if [ "${{matrix.info}}" = "g++-8 + debug" ]; then
-                  sudo apt install --allow-downgrades -y g++-8
-                fi
+          - name: Install postgres 15
+            if: matrix.os == 'ubuntu-22.04'
+            run: |
+                # GH Ubuntu image has libpq version 15+ from pg's repo that is API incompatible with postgresql-14.
+                sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7FCC7D46ACCC4CF8
+                echo "deb https://apt-archive.postgresql.org/pub/repos/apt jammy-pgdg-archive main" | sudo tee /etc/apt/sources.list.d/pgdg.list
+                sudo apt update
+                sudo apt install -y postgresql-15 postgresql-client-15 postgresql-server-dev-15
 
-                if [ "${{matrix.os}}" = "ubuntu-20.04" ]; then
-                  git clone --recursive -b v1.60.0 https://github.com/grpc/grpc new_grpc
-                  cmake -S./new_grpc -B./new_grpc/build
-                  cmake --build new_grpc/build -- -j4
-                  sudo cmake --install new_grpc/build
-                fi
+          - name: Install g++-8
+            if: matrix.info == 'g++-8 + debug'
+            run: |
+              sudo apt install --allow-downgrades -y g++-8
 
           - name: Install RabbitMQ packages
             run: |
@@ -233,14 +264,6 @@ jobs:
                 cd rabbitmq
                 ctest -V
 
-          - name: Run tests (mysql)
-            run: |
-                sudo apt install mariadb-server
-                cd build_debug
-                mkdir mysql || :
-                cd mysql
-                ctest -V
-
           - name: Run tests (postgresql)
             run: |
                 cd build_debug
@@ -257,18 +280,29 @@ jobs:
             # libbson unnecessarily overaligns structures and GCC-9 does not support overaligned new
             if: matrix.info != 'g++-9 + debug + sanitize addr+ub'
             run: |
+                sudo apt install mongodb-mongosh
                 cd build_debug
                 mkdir mongo || :
                 cd mongo
                 if [ -f ./userver-mongo-unittest ]; then ./userver-mongo-unittest ; fi
                 # Mongo is not available on Ubuntu 22.04 from default repos
                 if [ "${{matrix.os}}" != "ubuntu-22.04" ]; then ctest -V -R userver-mongo-mongotest ; fi
 
+
           - name: Run tests (gRPC)
-            # TODO: Sanitizer detects bool with value other than 1 or 0
-            if: matrix.info != 'clang-14 + debug + sanitize addr+ub'
+            if: matrix.os != 'ubuntu-24.04'
             run: |
                 cd build_debug
                 mkdir grpc || :
                 cd grpc
+                ulimit -n 4096 && ctest -V
+
+          - name: Run tests (mysql)
+            # MySQL benchmarks do not work on new Ubuntu
+            if: matrix.os != 'ubuntu-24.04'
+            run: |
+                sudo apt install mariadb-server
+                cd build_debug
+                mkdir mysql || :
+                cd mysql
                 ctest -V
diff --git a/.mapping.json b/.mapping.json
@@ -359,7 +359,7 @@
   "cmake/SetupPostgresqlDeps.cmake":"taxi/uservices/userver/cmake/SetupPostgresqlDeps.cmake",
   "cmake/SetupProtobuf.cmake":"taxi/uservices/userver/cmake/SetupProtobuf.cmake",
   "cmake/SetupRdKafka.cmake":"taxi/uservices/userver/cmake/SetupRdKafka.cmake",
-  "cmake/SetupRocksDeps.cmake":"taxi/uservices/userver/cmake/SetupRocksDeps.cmake",
+  "cmake/SetupRocksDB.cmake":"taxi/uservices/userver/cmake/SetupRocksDB.cmake",
   "cmake/SetupYdbCppSDK.cmake":"taxi/uservices/userver/cmake/SetupYdbCppSDK.cmake",
   "cmake/Stacktrace.cmake":"taxi/uservices/userver/cmake/Stacktrace.cmake",
   "cmake/UserverCxxCompileOptionsIfSupported.cmake":"taxi/uservices/userver/cmake/UserverCxxCompileOptionsIfSupported.cmake",
@@ -393,11 +393,13 @@
   "cmake/modules/FindGssApi.cmake":"taxi/uservices/userver/cmake/modules/FindGssApi.cmake",
   "cmake/modules/FindHiredis.cmake":"taxi/uservices/userver/cmake/modules/FindHiredis.cmake",
   "cmake/modules/FindJemalloc.cmake":"taxi/uservices/userver/cmake/modules/FindJemalloc.cmake",
+  "cmake/modules/FindLZ4.cmake":"taxi/uservices/userver/cmake/modules/FindLZ4.cmake",
   "cmake/modules/FindLibEv.cmake":"taxi/uservices/userver/cmake/modules/FindLibEv.cmake",
   "cmake/modules/FindNghttp2.cmake":"taxi/uservices/userver/cmake/modules/FindNghttp2.cmake",
   "cmake/modules/FindPostgreSQLInternal.cmake":"taxi/uservices/userver/cmake/modules/FindPostgreSQLInternal.cmake",
   "cmake/modules/FindPythonDev.cmake":"taxi/uservices/userver/cmake/modules/FindPythonDev.cmake",
   "cmake/modules/FindRdKafka.cmake":"taxi/uservices/userver/cmake/modules/FindRdKafka.cmake",
+  "cmake/modules/FindRocksDB.cmake":"taxi/uservices/userver/cmake/modules/FindRocksDB.cmake",
   "cmake/modules/FindSSL.cmake":"taxi/uservices/userver/cmake/modules/FindSSL.cmake",
   "cmake/modules/FindUserverGBench.cmake":"taxi/uservices/userver/cmake/modules/FindUserverGBench.cmake",
   "cmake/modules/FindUserverGrpc.cmake":"taxi/uservices/userver/cmake/modules/FindUserverGrpc.cmake",

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -129,6 +129,7 @@ option(USERVER_FEATURE_RABBITMQ "Provide asynchronous driver for RabbitMQ" "${US
 option(USERVER_FEATURE_MYSQL "Provide asynchronous driver for MariaDB/MySQL" "${USERVER_LIB_ENABLED_DEFAULT}")
 option(USERVER_FEATURE_ROCKS "Provide asynchronous driver for Rocks" "${USERVER_LIB_ENABLED_DEFAULT}")
 option(USERVER_FEATURE_YDB "Provide asynchronous driver for YDB" "${USERVER_YDB_DEFAULT}")
+option(USERVER_FEATURE_OTLP "Provide asynchronous OTLP exporters" "${USERVER_LIB_ENABLED_DEFAULT}")
 
 set(CMAKE_DEBUG_POSTFIX d)
 
@@ -157,8 +158,6 @@ init_debian_depends()
 
 include(SetupGTest)
 
-option(USERVER_FEATURE_OTLP "Provide asynchronous OTLP exporters" "${USERVER_LIB_ENABLED_DEFAULT}")
-
 if (USERVER_FEATURE_GRPC)
   include(SetupProtobuf)
 endif()

diff --git a/cmake/AddGoogleTests.cmake b/cmake/AddGoogleTests.cmake
@@ -18,7 +18,16 @@ function(add_google_benchmark_tests target)
     # This is currently a smoke test. That is, we don't try to detect
     # performance regressions, but instead just run 1 loop iteration and check
     # that the executable doesn't crash or hang.
-    set(BENCHMARK_MIN_TIME 0)
+    if (USERVER_CONAN)
+      set(BENCHMARK_VERSION ${benchmark_VERSION})
+    else()
+      set(BENCHMARK_VERSION ${UserverGBench_VERSION})
+    endif()
+    if(${BENCHMARK_VERSION} VERSION_LESS "1.8.0")
+      set(BENCHMARK_MIN_TIME "0")
+    else()
+      set(BENCHMARK_MIN_TIME "0.0s")
+    endif()
 
     add_test(
         NAME "${target}"

diff --git a/cmake/SetupAbseil.cmake b/cmake/SetupAbseil.cmake
@@ -1,5 +1,22 @@
 # userver does not use Abseil directly, but some libraries need it.
 
+option(USERVER_DOWNLOAD_PACKAGE_ABSEIL "Download and setup Abseil if no Abseil matching version was found" ${USERVER_DOWNLOAD_PACKAGES})
+option(USERVER_FORCE_DOWNLOAD_ABSEIL "Download Abseil even if it exists in a system" ${USERVER_DOWNLOAD_PACKAGES})
+
+if(NOT USERVER_FORCE_DOWNLOAD_ABSEIL)
+  set(ABSL_PROPAGATE_CXX_STD ON)
+
+  if(USERVER_DOWNLOAD_PACKAGE_ABSEIL)
+    find_package(absl QUIET)
+  else()
+    find_package(absl REQUIRED)
+  endif()
+
+  if(absl_FOUND)
+    return()
+  endif()
+endif()
+
 include(DownloadUsingCPM)
 
 CPMAddPackage(

diff --git a/cmake/SetupClickhouseCPP.cmake b/cmake/SetupClickhouseCPP.cmake
@@ -22,6 +22,7 @@ CPMAddPackage(
     SYSTEM
     OPTIONS
     "WITH_SYSTEM_ABSEIL ON"
+    "WITH_SYSTEM_LZ4 ON"
 )
 
 add_library(clickhouse-cpp ALIAS clickhouse-cpp-lib)

diff --git a/cmake/SetupProtobuf.cmake b/cmake/SetupProtobuf.cmake
@@ -34,6 +34,7 @@ endif()
 
 if(NOT USERVER_FORCE_DOWNLOAD_PROTOBUF)
   # Use the builtin CMake FindProtobuf
+  set(Protobuf_USE_STATIC_LIBS ON)
   if(USERVER_DOWNLOAD_PACKAGE_PROTOBUF)
     find_package(Protobuf QUIET)
   else()

diff --git a/cmake/SetupRdKafka.cmake b/cmake/SetupRdKafka.cmake
@@ -2,12 +2,13 @@ include_guard(GLOBAL)
 
 option(USERVER_DOWNLOAD_PACKAGE_KAFKA "Download and setup librdkafka if no librdkafka matching version was found" ${USERVER_DOWNLOAD_PACKAGES})
 
-set(USERVER_KAFKA_VERSION "2.4.0")
+set(USERVER_KAFKA_VERSION "2.3.0")
 
 find_package(OpenSSL COMPONENTS SSL Crypto REQUIRED)
 find_package(CURL REQUIRED)
 find_package(libz REQUIRED)
 find_package(libzstd REQUIRED)
+find_package(LZ4 REQUIRED)
 
 if(NOT USERVER_FORCE_DOWNLOAD_PACKAGES)
   if(USERVER_DOWNLOAD_PACKAGE_KAFKA)
@@ -36,9 +37,9 @@ CPMAddPackage(
   "WITH_SASL ON"
   "WITH CURL ON"
   "WITH_ZLIB ON"
-  "WITH_ZSTD OFF"
+  "WITH_ZSTD ON"
   "WITH_LIBDL OFF"
-  "ENABLE_LZ4_EXT OFF"
+  "ENABLE_LZ4_EXT ON"
 )
 
 set(KAFKA_CPM TRUE)

diff --git a/cmake/SetupRocksDeps.cmake → cmake/SetupRocksDB.cmake b/cmake/SetupRocksDeps.cmake → cmake/SetupRocksDB.cmake
@@ -2,16 +2,14 @@ include_guard(GLOBAL)
 
 option(USERVER_DOWNLOAD_PACKAGE_ROCKS "Download and setup RocksDB if no RocksDB matching version was found" ${USERVER_DOWNLOAD_PACKAGES})
 
-set(USERVER_ROCKSDB_VERSION "8.11.3")
-
-if (NOT USERVER_FORCE_DOWNLOAD_PACKAGES)
-  if (USERVER_DOWNLOAD_PACKAGE_ROCKS)
+if(NOT USERVER_FORCE_DOWNLOAD_PACKAGES)
+  if(USERVER_DOWNLOAD_PACKAGE_ROCKS)
     find_package(RocksDB QUIET)
   else()
     find_package(RocksDB REQUIRED)
   endif()
 
-  if (TARGET RocksDB::rocksdb)
+  if(RocksDB_FOUND)
     return()
   endif()
 endif()
@@ -27,6 +25,14 @@ include(DownloadUsingCPM)
 CPMAddPackage(
   NAME rocksdb
   GITHUB_REPOSITORY facebook/rocksdb
-  GIT_TAG v${USERVER_ROCKSDB_VERSION}
+  GIT_TAG v8.11.3
+  OPTIONS
+  "ROCKSDB_BUILD_SHARED OFF"
+  "WITH_TESTS OFF"
+  "WITH_BENCHMARK_TOOLS OFF"
+  "WITH_TOOLS OFF"
+  "USE_RTTI ON"
 )
-add_library(RocksDB::rocksdb ALIAS rocksdb)
+
+mark_targets_as_system("${rocksdb_SOURCE_DIR}")
+write_package_stub(rocksdb)
diff --git a/cmake/UserverTestsuite.cmake b/cmake/UserverTestsuite.cmake
@@ -514,11 +514,23 @@ function(userver_add_ubench_test)
   cmake_parse_arguments(
       ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
 
+  if (USERVER_CONAN)
+    set(BENCHMARK_VERSION ${benchmark_VERSION})
+  else()
+    set(BENCHMARK_VERSION ${UserverGBench_VERSION})
+  endif()
+
+  if(${BENCHMARK_VERSION} VERSION_LESS "1.8.0")
+    set(BENCHMARK_MIN_TIME "0")
+  else()
+    set(BENCHMARK_MIN_TIME "0.0s")
+  endif()
+
   userver_add_utest(
       NAME "${ARG_NAME}"
       DATABASES ${ARG_DATABASES}
       TEST_ENV ${ARG_TEST_ENV}
-      TEST_ARGS --benchmark_min_time=0 --benchmark_color=no
+      TEST_ARGS --benchmark_min_time=${BENCHMARK_MIN_TIME} --benchmark_color=no
       DISABLE_GTEST_XML_OUTPUT ON
   )
 endfunction()
-Original file line number
+Diff line change
@@ Expand Up / @@ -22,6 +22,7 @@ CPMAddPackage( @@
         SYSTEM
         OPTIONS
         "WITH_SYSTEM_ABSEIL ON"
+        "WITH_SYSTEM_LZ4 ON"
     )
     add_library(clickhouse-cpp ALIAS clickhouse-cpp-lib)
@@ Expand Down @@