From 11e71bcdbaccbadf7cfd3cd347b78f7aa9f19f9e Mon Sep 17 00:00:00 2001 From: fdr400 Date: Mon, 28 Oct 2024 17:54:03 +0300 Subject: [PATCH] fix build: fix build all CI Ubuntu checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: протестировано CI Pull Request resolved: #730 Co-authored-by: Fedor [fiosetrova@gmail.com](mailto:fiosetrova@gmail.com) Co-authored-by: fdr400 [fdr400@yandex-team.com](mailto:fdr400@yandex-team.com) commit_hash:4dde7685a9d47db49599171ae760bd6a259a00a6 --- .github/workflows/ci-conan.yml | 1 + .github/workflows/ci.yml | 140 +++++++++++------- .mapping.json | 4 +- CMakeLists.txt | 3 +- cmake/AddGoogleTests.cmake | 11 +- cmake/SetupAbseil.cmake | 17 +++ cmake/SetupClickhouseCPP.cmake | 1 + cmake/SetupProtobuf.cmake | 1 + cmake/SetupRdKafka.cmake | 7 +- ...etupRocksDeps.cmake => SetupRocksDB.cmake} | 20 ++- cmake/UserverTestsuite.cmake | 14 +- cmake/install/userver-kafka-config.cmake | 3 +- cmake/modules/FindLZ4.cmake | 19 +++ cmake/modules/FindRdKafka.cmake | 13 +- cmake/modules/FindRocksDB.cmake | 18 +++ cmake/modules/Findc-ares.cmake | 2 +- conanfile.py | 2 +- grpc/benchmarks/base.cpp | 7 + grpc/include/userver/ugrpc/protobuf_visit.hpp | 1 + grpc/src/ugrpc/protobuf_visit.cpp | 2 - kafka/CMakeLists.txt | 28 ++-- .../src/storages/mongo/cdriver/pool_impl.cpp | 8 + rocks/CMakeLists.txt | 4 +- scripts/docker/setup-base-ubuntu-22.04-env.sh | 2 +- scripts/docs/en/deps/arch.md | 1 + scripts/docs/en/deps/debian-11.md | 1 + scripts/docs/en/deps/macos.md | 1 + scripts/docs/en/deps/ubuntu-18.04.md | 1 + scripts/docs/en/deps/ubuntu-20.04.md | 1 + scripts/docs/en/deps/ubuntu-21.10.md | 1 + scripts/docs/en/deps/ubuntu-22.04.md | 1 + scripts/docs/en/deps/ubuntu-24.04.md | 1 + 32 files changed, 235 insertions(+), 101 deletions(-) rename cmake/{SetupRocksDeps.cmake => SetupRocksDB.cmake} (63%) create mode 100644 cmake/modules/FindLZ4.cmake create mode 100644 cmake/modules/FindRocksDB.cmake diff --git a/.github/workflows/ci-conan.yml b/.github/workflows/ci-conan.yml index 5b1090745138..d828271ea0f8 100644 --- a/.github/workflows/ci-conan.yml +++ b/.github/workflows/ci-conan.yml @@ -39,6 +39,7 @@ jobs: run: | sudo apt-get install -y gcc g++ cmake wget git python3 python3-pip python3-venv ccache pip install "conan==2.8.0" + pip install numpy conan profile detect conan profile show diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63c00c115bf4..99d465b1e843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,6 @@ jobs: -DUSERVER_NAMESPACE_BEGIN="namespace userver_ns { inline namespace v1 {" -DUSERVER_NAMESPACE_END="}}" - -DUSERVER_FEATURE_KAFKA=0 -DUSERVER_PG_SERVER_INCLUDE_DIR=/usr/include/postgresql/16/server -DUSERVER_PG_SERVER_LIBRARY_DIR=/usr/lib/postgresql/16/lib os: ubuntu-24.04 @@ -55,6 +54,8 @@ jobs: -DUSERVER_FEATURE_CRYPTOPP_BLAKE2=0 -DUSERVER_FEATURE_REDIS_HI_MALLOC=1 -DUSERVER_NO_WERROR=1 + -DUSERVER_FORCE_DOWNLOAD_ABSEIL=0 + -DUSERVER_DOWNLOAD_PACKAGE_ABSEIL=0 -DCMAKE_BUILD_TYPE=Release os: ubuntu-20.04 info: g++-9 + release @@ -67,6 +68,8 @@ jobs: -DUSERVER_FEATURE_CRYPTOPP_BLAKE2=0 -DUSERVER_FEATURE_REDIS_HI_MALLOC=1 -DUSERVER_NO_WERROR=1 + -DUSERVER_FORCE_DOWNLOAD_ABSEIL=0 + -DUSERVER_DOWNLOAD_PACKAGE_ABSEIL=0 -DCMAKE_BUILD_TYPE=Debug os: ubuntu-20.04 info: g++-8 + debug @@ -100,11 +103,50 @@ jobs: df -h # See https://stackoverflow.com/questions/75536771/github-runner-out-of-disk-space-after-building-docker-image sudo rm -rf /usr/share/dotnet /usr/local/lib/android /usr/lib/php* /opt/ghc \ - /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup || true + /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \ + /opt/hostedtoolcache/CodeQL || true + sudo docker image prune --all --force df -h - - name: Install packages + - name: Add confluentinc repositories + if: matrix.os != 'ubuntu-24.04' run: | + # Adding librdkafka confluent repositories as in https://docs.confluent.io/platform/current/installation/installing_cp/deb-ubuntu.html#get-the-software + gpg_retrieve_keyserver() { + GNUPGHOME=$(mktemp -d) + GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring --keyring "/usr/share/keyrings/$2.gpg" --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $1 + rm -rf "$GNUPGHOME" + chmod a+r "/usr/share/keyrings/$2.gpg" + } + gpg_retrieve_keyserver 8B1DA6120C2BF624 confluent + printf "\ + deb [arch=amd64 signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/deb/7.6 stable main\n\ + deb [signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/clients/deb $(lsb_release -cs) main\n" \ + | sudo tee /etc/apt/sources.list.d/confluent.list + + - name: Install clang-format + if: matrix.os == 'ubuntu-20.04' + run: | + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + sudo apt update + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 14 + sudo apt install clang-format-14 + sudo ln -sf /usr/bin/clang-format-14 /usr/bin/clang-format + clang-format --version + + - name: Install grpc, protobuf and abseil + if: matrix.os == 'ubuntu-20.04' + run: | + git clone --recursive -b v1.45.0 https://github.com/grpc/grpc new_grpc + cmake -S./new_grpc -B./new_grpc/build -DABSL_PROPAGATE_CXX_STD=ON -DABSL_ENABLE_INSTALL=ON + cmake --build new_grpc/build -- -j$(nproc) + sudo cmake --install new_grpc/build + + - name: Add missing repositories + run: | + sudo apt update # Instructions from https://clickhouse.com/docs/en/getting-started/install/ sudo apt install -y apt-transport-https ca-certificates dirmngr sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 @@ -118,50 +160,39 @@ jobs: # | tee /etc/apt/sources.list.d/mariadb.list echo "deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/mariadb.gpg] https://mirror.kumi.systems/mariadb/repo/10.11/ubuntu $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/mariadb.list - if [ "${{matrix.os}}" != "ubuntu-24.04" ]; then - # Adding librdkafka confluent repositories as in https://docs.confluent.io/platform/current/installation/installing_cp/deb-ubuntu.html#get-the-software - gpg_retrieve_keyserver() { - GNUPGHOME=$(mktemp -d) - GNUPGHOME="$GNUPGHOME" gpg --no-default-keyring --keyring "/usr/share/keyrings/$2.gpg" --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $1 - rm -rf "$GNUPGHOME" - chmod a+r "/usr/share/keyrings/$2.gpg" - } - gpg_retrieve_keyserver 8B1DA6120C2BF624 confluent - printf "\ - deb [arch=amd64 signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/deb/7.6 stable main\n\ - deb [signed-by=/usr/share/keyrings/confluent.gpg] https://packages.confluent.io/clients/deb $(lsb_release -cs) main\n" \ - | sudo tee /etc/apt/sources.list.d/confluent.list - - # Install Kafka - sudo apt install -y default-jre - curl https://dlcdn.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz -o kafka.tgz - sudo mkdir -p /etc/kafka - sudo tar xf kafka.tgz --directory=/etc/kafka - sudo cp -r /etc/kafka/kafka_2.13-3.8.0/* /etc/kafka/ - sudo rm -rf /etc/kafka/kafka_2.13-3.8.0 - ls -lha /etc/kafka - fi - + # Adding mongodb repositories + wget -qO- https://pgp.mongodb.com/server-7.0.asc | sudo gpg --dearmor | sudo tee /usr/share/keyrings/mongodb-server-7.0.gpg >/dev/null + echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" \ + | sudo tee -a /etc/apt/sources.list.d/mongodb-org-7.0.list + # Mongo is not officially supported on Ubuntu 24.04, so install it from jammy + + # Install Kafka + sudo apt install -y default-jre + curl https://dlcdn.apache.org/kafka/3.8.0/kafka_2.13-3.8.0.tgz -o kafka.tgz + sudo mkdir -p /etc/kafka + sudo tar xf kafka.tgz --directory=/etc/kafka + sudo cp -r /etc/kafka/kafka_2.13-3.8.0/* /etc/kafka/ + sudo rm -rf /etc/kafka/kafka_2.13-3.8.0 + ls -lha /etc/kafka + + - name: Install common deps + run: | sudo apt update sudo apt install --allow-downgrades -y clickhouse-common-static redis-server postgresql $(cat scripts/docs/en/deps/${{matrix.os}}.md | tr '\n' ' ') - if [ "${{matrix.os}}" = "ubuntu-22.04" ]; then - # GH Ubuntu image has libpq version 15+ from pg's repo that is API incompatible with postgresql-14. - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7FCC7D46ACCC4CF8 - echo "deb https://apt-archive.postgresql.org/pub/repos/apt jammy-pgdg-archive main" | sudo tee /etc/apt/sources.list.d/pgdg.list - sudo apt update - sudo apt install -y postgresql-15 postgresql-client-15 postgresql-server-dev-15 - fi - if [ "${{matrix.info}}" = "g++-8 + debug" ]; then - sudo apt install --allow-downgrades -y g++-8 - fi + - name: Install postgres 15 + if: matrix.os == 'ubuntu-22.04' + run: | + # GH Ubuntu image has libpq version 15+ from pg's repo that is API incompatible with postgresql-14. + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7FCC7D46ACCC4CF8 + echo "deb https://apt-archive.postgresql.org/pub/repos/apt jammy-pgdg-archive main" | sudo tee /etc/apt/sources.list.d/pgdg.list + sudo apt update + sudo apt install -y postgresql-15 postgresql-client-15 postgresql-server-dev-15 - if [ "${{matrix.os}}" = "ubuntu-20.04" ]; then - git clone --recursive -b v1.60.0 https://github.com/grpc/grpc new_grpc - cmake -S./new_grpc -B./new_grpc/build - cmake --build new_grpc/build -- -j4 - sudo cmake --install new_grpc/build - fi + - name: Install g++-8 + if: matrix.info == 'g++-8 + debug' + run: | + sudo apt install --allow-downgrades -y g++-8 - name: Install RabbitMQ packages run: | @@ -233,14 +264,6 @@ jobs: cd rabbitmq ctest -V - - name: Run tests (mysql) - run: | - sudo apt install mariadb-server - cd build_debug - mkdir mysql || : - cd mysql - ctest -V - - name: Run tests (postgresql) run: | cd build_debug @@ -257,6 +280,7 @@ jobs: # libbson unnecessarily overaligns structures and GCC-9 does not support overaligned new if: matrix.info != 'g++-9 + debug + sanitize addr+ub' run: | + sudo apt install mongodb-mongosh cd build_debug mkdir mongo || : cd mongo @@ -264,11 +288,21 @@ jobs: # Mongo is not available on Ubuntu 22.04 from default repos if [ "${{matrix.os}}" != "ubuntu-22.04" ]; then ctest -V -R userver-mongo-mongotest ; fi + - name: Run tests (gRPC) - # TODO: Sanitizer detects bool with value other than 1 or 0 - if: matrix.info != 'clang-14 + debug + sanitize addr+ub' + if: matrix.os != 'ubuntu-24.04' run: | cd build_debug mkdir grpc || : cd grpc + ulimit -n 4096 && ctest -V + + - name: Run tests (mysql) + # MySQL benchmarks do not work on new Ubuntu + if: matrix.os != 'ubuntu-24.04' + run: | + sudo apt install mariadb-server + cd build_debug + mkdir mysql || : + cd mysql ctest -V diff --git a/.mapping.json b/.mapping.json index 9c3a58963845..0164b7d5ca8d 100644 --- a/.mapping.json +++ b/.mapping.json @@ -359,7 +359,7 @@ "cmake/SetupPostgresqlDeps.cmake":"taxi/uservices/userver/cmake/SetupPostgresqlDeps.cmake", "cmake/SetupProtobuf.cmake":"taxi/uservices/userver/cmake/SetupProtobuf.cmake", "cmake/SetupRdKafka.cmake":"taxi/uservices/userver/cmake/SetupRdKafka.cmake", - "cmake/SetupRocksDeps.cmake":"taxi/uservices/userver/cmake/SetupRocksDeps.cmake", + "cmake/SetupRocksDB.cmake":"taxi/uservices/userver/cmake/SetupRocksDB.cmake", "cmake/SetupYdbCppSDK.cmake":"taxi/uservices/userver/cmake/SetupYdbCppSDK.cmake", "cmake/Stacktrace.cmake":"taxi/uservices/userver/cmake/Stacktrace.cmake", "cmake/UserverCxxCompileOptionsIfSupported.cmake":"taxi/uservices/userver/cmake/UserverCxxCompileOptionsIfSupported.cmake", @@ -393,11 +393,13 @@ "cmake/modules/FindGssApi.cmake":"taxi/uservices/userver/cmake/modules/FindGssApi.cmake", "cmake/modules/FindHiredis.cmake":"taxi/uservices/userver/cmake/modules/FindHiredis.cmake", "cmake/modules/FindJemalloc.cmake":"taxi/uservices/userver/cmake/modules/FindJemalloc.cmake", + "cmake/modules/FindLZ4.cmake":"taxi/uservices/userver/cmake/modules/FindLZ4.cmake", "cmake/modules/FindLibEv.cmake":"taxi/uservices/userver/cmake/modules/FindLibEv.cmake", "cmake/modules/FindNghttp2.cmake":"taxi/uservices/userver/cmake/modules/FindNghttp2.cmake", "cmake/modules/FindPostgreSQLInternal.cmake":"taxi/uservices/userver/cmake/modules/FindPostgreSQLInternal.cmake", "cmake/modules/FindPythonDev.cmake":"taxi/uservices/userver/cmake/modules/FindPythonDev.cmake", "cmake/modules/FindRdKafka.cmake":"taxi/uservices/userver/cmake/modules/FindRdKafka.cmake", + "cmake/modules/FindRocksDB.cmake":"taxi/uservices/userver/cmake/modules/FindRocksDB.cmake", "cmake/modules/FindSSL.cmake":"taxi/uservices/userver/cmake/modules/FindSSL.cmake", "cmake/modules/FindUserverGBench.cmake":"taxi/uservices/userver/cmake/modules/FindUserverGBench.cmake", "cmake/modules/FindUserverGrpc.cmake":"taxi/uservices/userver/cmake/modules/FindUserverGrpc.cmake", diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a20cfa10dcc..1f3509949318 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,6 +129,7 @@ option(USERVER_FEATURE_RABBITMQ "Provide asynchronous driver for RabbitMQ" "${US option(USERVER_FEATURE_MYSQL "Provide asynchronous driver for MariaDB/MySQL" "${USERVER_LIB_ENABLED_DEFAULT}") option(USERVER_FEATURE_ROCKS "Provide asynchronous driver for Rocks" "${USERVER_LIB_ENABLED_DEFAULT}") option(USERVER_FEATURE_YDB "Provide asynchronous driver for YDB" "${USERVER_YDB_DEFAULT}") +option(USERVER_FEATURE_OTLP "Provide asynchronous OTLP exporters" "${USERVER_LIB_ENABLED_DEFAULT}") set(CMAKE_DEBUG_POSTFIX d) @@ -157,8 +158,6 @@ init_debian_depends() include(SetupGTest) -option(USERVER_FEATURE_OTLP "Provide asynchronous OTLP exporters" "${USERVER_LIB_ENABLED_DEFAULT}") - if (USERVER_FEATURE_GRPC) include(SetupProtobuf) endif() diff --git a/cmake/AddGoogleTests.cmake b/cmake/AddGoogleTests.cmake index d2189b16e7d6..1261ca2d671d 100644 --- a/cmake/AddGoogleTests.cmake +++ b/cmake/AddGoogleTests.cmake @@ -18,7 +18,16 @@ function(add_google_benchmark_tests target) # This is currently a smoke test. That is, we don't try to detect # performance regressions, but instead just run 1 loop iteration and check # that the executable doesn't crash or hang. - set(BENCHMARK_MIN_TIME 0) + if (USERVER_CONAN) + set(BENCHMARK_VERSION ${benchmark_VERSION}) + else() + set(BENCHMARK_VERSION ${UserverGBench_VERSION}) + endif() + if(${BENCHMARK_VERSION} VERSION_LESS "1.8.0") + set(BENCHMARK_MIN_TIME "0") + else() + set(BENCHMARK_MIN_TIME "0.0s") + endif() add_test( NAME "${target}" diff --git a/cmake/SetupAbseil.cmake b/cmake/SetupAbseil.cmake index 55c6fe7570c6..61a80443c927 100644 --- a/cmake/SetupAbseil.cmake +++ b/cmake/SetupAbseil.cmake @@ -1,5 +1,22 @@ # userver does not use Abseil directly, but some libraries need it. +option(USERVER_DOWNLOAD_PACKAGE_ABSEIL "Download and setup Abseil if no Abseil matching version was found" ${USERVER_DOWNLOAD_PACKAGES}) +option(USERVER_FORCE_DOWNLOAD_ABSEIL "Download Abseil even if it exists in a system" ${USERVER_DOWNLOAD_PACKAGES}) + +if(NOT USERVER_FORCE_DOWNLOAD_ABSEIL) + set(ABSL_PROPAGATE_CXX_STD ON) + + if(USERVER_DOWNLOAD_PACKAGE_ABSEIL) + find_package(absl QUIET) + else() + find_package(absl REQUIRED) + endif() + + if(absl_FOUND) + return() + endif() +endif() + include(DownloadUsingCPM) CPMAddPackage( diff --git a/cmake/SetupClickhouseCPP.cmake b/cmake/SetupClickhouseCPP.cmake index 94cf5b4fd3df..40cf4069beca 100644 --- a/cmake/SetupClickhouseCPP.cmake +++ b/cmake/SetupClickhouseCPP.cmake @@ -22,6 +22,7 @@ CPMAddPackage( SYSTEM OPTIONS "WITH_SYSTEM_ABSEIL ON" + "WITH_SYSTEM_LZ4 ON" ) add_library(clickhouse-cpp ALIAS clickhouse-cpp-lib) diff --git a/cmake/SetupProtobuf.cmake b/cmake/SetupProtobuf.cmake index 7369e987289a..84e3c923d352 100644 --- a/cmake/SetupProtobuf.cmake +++ b/cmake/SetupProtobuf.cmake @@ -34,6 +34,7 @@ endif() if(NOT USERVER_FORCE_DOWNLOAD_PROTOBUF) # Use the builtin CMake FindProtobuf + set(Protobuf_USE_STATIC_LIBS ON) if(USERVER_DOWNLOAD_PACKAGE_PROTOBUF) find_package(Protobuf QUIET) else() diff --git a/cmake/SetupRdKafka.cmake b/cmake/SetupRdKafka.cmake index 929581b721a2..66e752826b4d 100644 --- a/cmake/SetupRdKafka.cmake +++ b/cmake/SetupRdKafka.cmake @@ -2,12 +2,13 @@ include_guard(GLOBAL) option(USERVER_DOWNLOAD_PACKAGE_KAFKA "Download and setup librdkafka if no librdkafka matching version was found" ${USERVER_DOWNLOAD_PACKAGES}) -set(USERVER_KAFKA_VERSION "2.4.0") +set(USERVER_KAFKA_VERSION "2.3.0") find_package(OpenSSL COMPONENTS SSL Crypto REQUIRED) find_package(CURL REQUIRED) find_package(libz REQUIRED) find_package(libzstd REQUIRED) +find_package(LZ4 REQUIRED) if(NOT USERVER_FORCE_DOWNLOAD_PACKAGES) if(USERVER_DOWNLOAD_PACKAGE_KAFKA) @@ -36,9 +37,9 @@ CPMAddPackage( "WITH_SASL ON" "WITH CURL ON" "WITH_ZLIB ON" - "WITH_ZSTD OFF" + "WITH_ZSTD ON" "WITH_LIBDL OFF" - "ENABLE_LZ4_EXT OFF" + "ENABLE_LZ4_EXT ON" ) set(KAFKA_CPM TRUE) diff --git a/cmake/SetupRocksDeps.cmake b/cmake/SetupRocksDB.cmake similarity index 63% rename from cmake/SetupRocksDeps.cmake rename to cmake/SetupRocksDB.cmake index e1a7933a3df6..e0673205611b 100644 --- a/cmake/SetupRocksDeps.cmake +++ b/cmake/SetupRocksDB.cmake @@ -2,16 +2,14 @@ include_guard(GLOBAL) option(USERVER_DOWNLOAD_PACKAGE_ROCKS "Download and setup RocksDB if no RocksDB matching version was found" ${USERVER_DOWNLOAD_PACKAGES}) -set(USERVER_ROCKSDB_VERSION "8.11.3") - -if (NOT USERVER_FORCE_DOWNLOAD_PACKAGES) - if (USERVER_DOWNLOAD_PACKAGE_ROCKS) +if(NOT USERVER_FORCE_DOWNLOAD_PACKAGES) + if(USERVER_DOWNLOAD_PACKAGE_ROCKS) find_package(RocksDB QUIET) else() find_package(RocksDB REQUIRED) endif() - if (TARGET RocksDB::rocksdb) + if(RocksDB_FOUND) return() endif() endif() @@ -27,6 +25,14 @@ include(DownloadUsingCPM) CPMAddPackage( NAME rocksdb GITHUB_REPOSITORY facebook/rocksdb - GIT_TAG v${USERVER_ROCKSDB_VERSION} + GIT_TAG v8.11.3 + OPTIONS + "ROCKSDB_BUILD_SHARED OFF" + "WITH_TESTS OFF" + "WITH_BENCHMARK_TOOLS OFF" + "WITH_TOOLS OFF" + "USE_RTTI ON" ) -add_library(RocksDB::rocksdb ALIAS rocksdb) + +mark_targets_as_system("${rocksdb_SOURCE_DIR}") +write_package_stub(rocksdb) diff --git a/cmake/UserverTestsuite.cmake b/cmake/UserverTestsuite.cmake index 7a153d03559f..84a2bf206f9b 100644 --- a/cmake/UserverTestsuite.cmake +++ b/cmake/UserverTestsuite.cmake @@ -514,11 +514,23 @@ function(userver_add_ubench_test) cmake_parse_arguments( ARG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if (USERVER_CONAN) + set(BENCHMARK_VERSION ${benchmark_VERSION}) + else() + set(BENCHMARK_VERSION ${UserverGBench_VERSION}) + endif() + + if(${BENCHMARK_VERSION} VERSION_LESS "1.8.0") + set(BENCHMARK_MIN_TIME "0") + else() + set(BENCHMARK_MIN_TIME "0.0s") + endif() + userver_add_utest( NAME "${ARG_NAME}" DATABASES ${ARG_DATABASES} TEST_ENV ${ARG_TEST_ENV} - TEST_ARGS --benchmark_min_time=0 --benchmark_color=no + TEST_ARGS --benchmark_min_time=${BENCHMARK_MIN_TIME} --benchmark_color=no DISABLE_GTEST_XML_OUTPUT ON ) endfunction() diff --git a/cmake/install/userver-kafka-config.cmake b/cmake/install/userver-kafka-config.cmake index c1cca2b19fd2..55ce1bb4dfc5 100644 --- a/cmake/install/userver-kafka-config.cmake +++ b/cmake/install/userver-kafka-config.cmake @@ -5,9 +5,10 @@ if(userver_kafka_FOUND) endif() find_package(userver REQUIRED COMPONENTS - core + core ) +include("${USERVER_CMAKE_DIR}/FindLZ4.cmake") include("${USERVER_CMAKE_DIR}/FindRdKafka.cmake") set(userver_kafka_FOUND TRUE) diff --git a/cmake/modules/FindLZ4.cmake b/cmake/modules/FindLZ4.cmake new file mode 100644 index 000000000000..6a35df651963 --- /dev/null +++ b/cmake/modules/FindLZ4.cmake @@ -0,0 +1,19 @@ +_userver_module_begin( + NAME LZ4 + VERSION 1.9.2 + DEBIAN_NAMES liblz4-dev + FORMULA_NAMES lz4 + PKG_CONFIG_NAMES liblz4 +) + +_userver_module_find_include( + NAMES lz4.h +) + +_userver_module_find_library( + NAMES liblz4.a +) + +_userver_module_end() + +add_library(LZ4::LZ4 ALIAS LZ4) diff --git a/cmake/modules/FindRdKafka.cmake b/cmake/modules/FindRdKafka.cmake index d268ccf5825e..e39ceaf353ca 100644 --- a/cmake/modules/FindRdKafka.cmake +++ b/cmake/modules/FindRdKafka.cmake @@ -1,17 +1,6 @@ -set(RDKAFKA_BUILD_STATIC ON) -set(RDKAFKA_BUILD_EXAMPLES OFF) -set(RDKAFKA_BUILD_TESTS OFF) -set(WITH_SSL ON) -set(WITH_SASL ON) -set(WITH_CURL ON) -set(WITH_ZLIB ON) -set(WITH_ZSTD OFF) -set(WITH_LIBDL OFF) -set(ENABLE_LZ4_EXT OFF) - _userver_module_begin( NAME RdKafka - VERSION 2.4.0 + VERSION 2.3.0 DEBIAN_NAMES librdkafka-dev FORMULA_NAMES librdkafka PACMAN_NAMES librdkafka diff --git a/cmake/modules/FindRocksDB.cmake b/cmake/modules/FindRocksDB.cmake new file mode 100644 index 000000000000..0c3d575d3d7f --- /dev/null +++ b/cmake/modules/FindRocksDB.cmake @@ -0,0 +1,18 @@ +_userver_module_begin( + NAME RocksDB + VERSION 8.9.1 + DEBIAN_NAMES librocksdb-dev + FORMULA_NAMES rocksdb + PACMAN_NAMES rocksdb + PKG_CONFIG_NAMES rocksdb +) + +_userver_module_find_include( + NAMES rocksdb/db.h +) + +_userver_module_find_library( + NAMES librocksdb.a +) + +_userver_module_end() diff --git a/cmake/modules/Findc-ares.cmake b/cmake/modules/Findc-ares.cmake index 6bcb20437bc2..e98fe19228c1 100644 --- a/cmake/modules/Findc-ares.cmake +++ b/cmake/modules/Findc-ares.cmake @@ -13,7 +13,7 @@ _userver_module_find_include( ) _userver_module_find_library( - NAMES cares + NAMES libcares.a libcares_static.a libcares.so ) _userver_module_end() diff --git a/conanfile.py b/conanfile.py index e2f050872097..f8e79ef56383 100644 --- a/conanfile.py +++ b/conanfile.py @@ -128,7 +128,7 @@ def requirements(self): ) if self.options.with_grpc: self.requires( - 'grpc/1.48.4', transitive_headers=True, transitive_libs=True, + 'grpc/1.50.1', transitive_headers=True, transitive_libs=True, ) self.requires( 'googleapis/cci.20230501', diff --git a/grpc/benchmarks/base.cpp b/grpc/benchmarks/base.cpp index 38d7fa1b2384..7a30ffe56545 100644 --- a/grpc/benchmarks/base.cpp +++ b/grpc/benchmarks/base.cpp @@ -267,7 +267,14 @@ void BatchOfNewClient(benchmark::State& state) { ); } +#if defined(__has_feature) +#if __has_feature(address_sanitizer) +BENCHMARK(BatchOfNewClient)->DenseRange(1, 1)->Unit(benchmark::kMillisecond); +#endif BENCHMARK(BatchOfNewClient)->DenseRange(1, 8)->Unit(benchmark::kMillisecond); +#else +BENCHMARK(BatchOfNewClient)->DenseRange(1, 8)->Unit(benchmark::kMillisecond); +#endif } // namespace ugrpc diff --git a/grpc/include/userver/ugrpc/protobuf_visit.hpp b/grpc/include/userver/ugrpc/protobuf_visit.hpp index 4c3355e1c0a2..f1f3a70bea24 100644 --- a/grpc/include/userver/ugrpc/protobuf_visit.hpp +++ b/grpc/include/userver/ugrpc/protobuf_visit.hpp @@ -3,6 +3,7 @@ /// @file userver/ugrpc/protobuf_visit.hpp /// @brief Utilities for visiting the fields of protobufs +#include #include #include #include diff --git a/grpc/src/ugrpc/protobuf_visit.cpp b/grpc/src/ugrpc/protobuf_visit.cpp index 7e8c96da3339..0d601bc3653f 100644 --- a/grpc/src/ugrpc/protobuf_visit.cpp +++ b/grpc/src/ugrpc/protobuf_visit.cpp @@ -1,7 +1,5 @@ #include -#include - #include #include #include diff --git a/kafka/CMakeLists.txt b/kafka/CMakeLists.txt index 4ccc5327c9ad..6dddb448f02c 100644 --- a/kafka/CMakeLists.txt +++ b/kafka/CMakeLists.txt @@ -12,17 +12,17 @@ else() endif() userver_module(kafka - SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" - LINK_LIBRARIES_PRIVATE RdKafka sasl2 z libzstd CURL::libcurl OpenSSL::SSL OpenSSL::Crypto - DBTEST_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/tests" - DBTEST_LINK_LIBRARIES userver::kafka-utest - DBTEST_DATABASES kafka - DBTEST_ENV - "TESTSUITE_KAFKA_SERVER_START_TIMEOUT=120.0" - "TESTSUITE_KAFKA_SERVER_HOST=[::1]" - "TESTSUITE_KAFKA_SERVER_PORT=8099" - "TESTSUITE_KAFKA_CONTROLLER_PORT=8100" - "TESTSUITE_KAFKA_CUSTOM_TOPICS=lt-1:4,lt-2:4,tt-1:1,tt-2:1,tt-3:1,tt-4:1,tt-5:1,tt-6:1,tt-7:1,tt-8:1" + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" + LINK_LIBRARIES_PRIVATE RdKafka sasl2 z libzstd LZ4 CURL::libcurl OpenSSL::SSL OpenSSL::Crypto + DBTEST_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/tests" + DBTEST_LINK_LIBRARIES userver::kafka-utest + DBTEST_DATABASES kafka + DBTEST_ENV + "TESTSUITE_KAFKA_SERVER_START_TIMEOUT=120.0" + "TESTSUITE_KAFKA_SERVER_HOST=[::1]" + "TESTSUITE_KAFKA_SERVER_PORT=8099" + "TESTSUITE_KAFKA_CONTROLLER_PORT=8100" + "TESTSUITE_KAFKA_CUSTOM_TOPICS=lt-1:4,lt-2:4,tt-1:1,tt-2:1,tt-3:1,tt-4:1,tt-5:1,tt-6:1,tt-7:1,tt-8:1" ) target_compile_options(${PROJECT_NAME} PRIVATE "-Wno-ignored-qualifiers") @@ -30,12 +30,14 @@ target_compile_options(${PROJECT_NAME} PRIVATE "-Wno-ignored-qualifiers") if(KAFKA_CPM) message(STATUS "Including from wrapper ${CMAKE_CURRENT_SOURCE_DIR}/wrapper/include") target_include_directories(${PROJECT_NAME} - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/wrapper/include" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/wrapper/include" ) endif() _userver_directory_install(COMPONENT kafka - FILES "${USERVER_ROOT_DIR}/cmake/modules/FindRdKafka.cmake" + FILES + "${USERVER_ROOT_DIR}/cmake/modules/FindRdKafka.cmake" + "${USERVER_ROOT_DIR}/cmake/modules/FindLZ4.cmake" DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/userver" ) diff --git a/mongo/src/storages/mongo/cdriver/pool_impl.cpp b/mongo/src/storages/mongo/cdriver/pool_impl.cpp index 2206ce27f7c3..33dc875a609a 100644 --- a/mongo/src/storages/mongo/cdriver/pool_impl.cpp +++ b/mongo/src/storages/mongo/cdriver/pool_impl.cpp @@ -227,13 +227,21 @@ std::string CreateTopologyChangeMessage(const mongoc_apm_topology_changed_t* eve mongoc_read_prefs_t* prefs = mongoc_read_prefs_new(MONGOC_READ_SECONDARY); +#if MONGOC_CHECK_VERSION(1, 17, 0) if (mongoc_topology_description_has_readable_server(new_td, prefs)) { +#else + if (mongoc_topology_description_has_readable_server(const_cast(new_td), prefs)) { +#endif topology_msg.append("\nSecondary AVAILABLE\n"); } else { topology_msg.append("\nSecondary UNAVAILABLE\n"); } +#if MONGOC_CHECK_VERSION(1, 17, 0) if (mongoc_topology_description_has_writable_server(new_td)) { +#else + if (mongoc_topology_description_has_writable_server(const_cast(new_td))) { +#endif topology_msg.append("Primary AVAILABLE"); } else { topology_msg.append("Primary UNAVAILABLE"); diff --git a/rocks/CMakeLists.txt b/rocks/CMakeLists.txt index 624fbe73ce0b..1e88356abea9 100644 --- a/rocks/CMakeLists.txt +++ b/rocks/CMakeLists.txt @@ -1,9 +1,9 @@ project(userver-rocks CXX) -include(SetupRocksDeps) +include(SetupRocksDB) userver_module(rocks SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}" - LINK_LIBRARIES RocksDB::rocksdb + LINK_LIBRARIES rocksdb UTEST_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*_test.cpp" ) diff --git a/scripts/docker/setup-base-ubuntu-22.04-env.sh b/scripts/docker/setup-base-ubuntu-22.04-env.sh index e0c8ec41e070..267a9eb896b3 100755 --- a/scripts/docker/setup-base-ubuntu-22.04-env.sh +++ b/scripts/docker/setup-base-ubuntu-22.04-env.sh @@ -126,7 +126,7 @@ apt clean all # You could override those versions from command line AMQP_VERSION=${AMQP_VERSION:=v4.3.18} CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION:=v2.3.0} -ROCKSDB_VERSION=${ROCKSDB_VERSION:=v8.11.3} +ROCKSDB_VERSION=${ROCKSDB_VERSION:=v8.9.1} # Installing amqp/rabbitmq client libraries from sources git clone --depth 1 -b ${AMQP_VERSION} https://github.com/CopernicaMarketingSoftware/AMQP-CPP.git amqp-cpp diff --git a/scripts/docs/en/deps/arch.md b/scripts/docs/en/deps/arch.md index c02bff9d0a8f..b3590c6e36a8 100644 --- a/scripts/docs/en/deps/arch.md +++ b/scripts/docs/en/deps/arch.md @@ -25,6 +25,7 @@ python-jinja python-pip python-voluptuous python-yaml +rocksdb yaml-cpp zlib makepkg|cctz diff --git a/scripts/docs/en/deps/debian-11.md b/scripts/docs/en/deps/debian-11.md index 2a052e2f9175..0417ce40879c 100644 --- a/scripts/docs/en/deps/debian-11.md +++ b/scripts/docs/en/deps/debian-11.md @@ -26,6 +26,7 @@ libjemalloc-dev libkrb5-dev libldap2-dev librdkafka-dev +librocksdb-dev libmongoc-dev libnghttp2-dev libpq-dev diff --git a/scripts/docs/en/deps/macos.md b/scripts/docs/en/deps/macos.md index 87c11f2e3e2f..e0654d6a334d 100644 --- a/scripts/docs/en/deps/macos.md +++ b/scripts/docs/en/deps/macos.md @@ -20,6 +20,7 @@ pugixml openssl yaml-cpp librdkafka +rocksdb postgresql@16 redis zlib diff --git a/scripts/docs/en/deps/ubuntu-18.04.md b/scripts/docs/en/deps/ubuntu-18.04.md index 59d5cb74a67a..7dc4792e5712 100644 --- a/scripts/docs/en/deps/ubuntu-18.04.md +++ b/scripts/docs/en/deps/ubuntu-18.04.md @@ -25,6 +25,7 @@ libjemalloc-dev libkrb5-dev libldap2-dev librdkafka-dev +librocksdb-dev libmongoc-dev libnghttp2-dev libpq-dev=10.* diff --git a/scripts/docs/en/deps/ubuntu-20.04.md b/scripts/docs/en/deps/ubuntu-20.04.md index 9ad81f135c9f..7bc79d260dd5 100644 --- a/scripts/docs/en/deps/ubuntu-20.04.md +++ b/scripts/docs/en/deps/ubuntu-20.04.md @@ -28,6 +28,7 @@ libjemalloc-dev libkrb5-dev libldap2-dev librdkafka-dev +librocksdb-dev libmariadb-dev libmongoc-dev libnghttp2-dev diff --git a/scripts/docs/en/deps/ubuntu-21.10.md b/scripts/docs/en/deps/ubuntu-21.10.md index 8890b21c5cd1..e84d63cdfae6 100644 --- a/scripts/docs/en/deps/ubuntu-21.10.md +++ b/scripts/docs/en/deps/ubuntu-21.10.md @@ -29,6 +29,7 @@ libjemalloc-dev libkrb5-dev libldap2-dev librdkafka-dev +librocksdb-dev libmongoc-dev libnghttp2-dev libpq-dev diff --git a/scripts/docs/en/deps/ubuntu-22.04.md b/scripts/docs/en/deps/ubuntu-22.04.md index b58a1236b03e..ccd0a626f227 100644 --- a/scripts/docs/en/deps/ubuntu-22.04.md +++ b/scripts/docs/en/deps/ubuntu-22.04.md @@ -34,6 +34,7 @@ libjemalloc-dev libkrb5-dev libldap2-dev librdkafka-dev +librocksdb-dev liblz4-dev libmariadb-dev libmongoc-dev diff --git a/scripts/docs/en/deps/ubuntu-24.04.md b/scripts/docs/en/deps/ubuntu-24.04.md index ef28d0590fb6..56acaff818c9 100644 --- a/scripts/docs/en/deps/ubuntu-24.04.md +++ b/scripts/docs/en/deps/ubuntu-24.04.md @@ -34,6 +34,7 @@ libjemalloc-dev libkrb5-dev libldap2-dev librdkafka-dev +librocksdb-dev liblz4-dev libmariadb-dev libmongoc-dev