diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index d8abac2fac7..6fe83e0ce8b 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -266,7 +266,8 @@ if (ENABLE_TESTS) # attach all dbms gtest sources grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms dbms_gtest_sources) - add_executable(unit_tests_dbms ${dbms_gtest_sources}) - target_link_libraries(unit_tests_dbms gtest_main dbms) - add_check(unit_tests_dbms) + add_executable(gtests_dbms EXCLUDE_FROM_ALL ${dbms_gtest_sources}) + target_link_libraries(gtests_dbms gtest_main dbms clickhouse_functions) + target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas) + add_check(gtests_dbms) endif () diff --git a/dbms/src/Analyzers/CMakeLists.txt b/dbms/src/Analyzers/CMakeLists.txt index 36cab0b3590..b2aec21574d 100644 --- a/dbms/src/Analyzers/CMakeLists.txt +++ b/dbms/src/Analyzers/CMakeLists.txt @@ -1,3 +1,3 @@ if(ENABLE_TESTS) - add_subdirectory(tests) + add_subdirectory(tests EXCLUDE_FROM_ALL) endif() diff --git a/dbms/src/Common/CMakeLists.txt b/dbms/src/Common/CMakeLists.txt index 61d9b9771a4..8af30705b15 100644 --- a/dbms/src/Common/CMakeLists.txt +++ b/dbms/src/Common/CMakeLists.txt @@ -4,5 +4,5 @@ add_subdirectory(StringUtils) #add_subdirectory(ConfigProcessor) if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Common/ZooKeeper/CMakeLists.txt b/dbms/src/Common/ZooKeeper/CMakeLists.txt index 197b53da00d..493dae7a3dc 100644 --- a/dbms/src/Common/ZooKeeper/CMakeLists.txt +++ b/dbms/src/Common/ZooKeeper/CMakeLists.txt @@ -7,5 +7,5 @@ add_library(clickhouse_common_zookeeper ${SPLIT_SHARED} ${clickhouse_common_zook target_link_libraries (clickhouse_common_zookeeper clickhouse_common_io) if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp index 915b31d420b..0947e60cf8a 100644 --- a/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp +++ b/dbms/src/Common/ZooKeeper/tests/gtest_zkutil_test_multi_exception.cpp @@ -15,7 +15,9 @@ using namespace DB; -TEST(zkutil, zookeeper_connected) +/// Disable zookeeper tests since tiflash don't use zookeeper + +TEST(zkutil, DISABLED_zookeeper_connected) { auto zookeeper = std::make_unique("localhost:2181"); try @@ -29,7 +31,7 @@ TEST(zkutil, zookeeper_connected) } } -TEST(zkutil, multi_nice_exception_msg) +TEST(zkutil, DISABLED_multi_nice_exception_msg) { auto zookeeper = std::make_unique("localhost:2181"); @@ -67,7 +69,7 @@ TEST(zkutil, multi_nice_exception_msg) } -TEST(zkutil, multi_async) +TEST(zkutil, DISABLED_multi_async) { auto zookeeper = std::make_unique("localhost:2181"); zkutil::Requests ops; @@ -129,7 +131,7 @@ TEST(zkutil, multi_async) } } -TEST(zkutil, watch_get_children_with_chroot) +TEST(zkutil, DISABLED_watch_get_children_with_chroot) { try { @@ -160,7 +162,7 @@ TEST(zkutil, watch_get_children_with_chroot) } } -TEST(zkutil, multi_create_sequential) +TEST(zkutil, DISABLED_multi_create_sequential) { try { diff --git a/dbms/src/Core/CMakeLists.txt b/dbms/src/Core/CMakeLists.txt index 65172356645..433b18404b7 100644 --- a/dbms/src/Core/CMakeLists.txt +++ b/dbms/src/Core/CMakeLists.txt @@ -1,3 +1,3 @@ if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/DataStreams/CMakeLists.txt b/dbms/src/DataStreams/CMakeLists.txt index 65172356645..433b18404b7 100644 --- a/dbms/src/DataStreams/CMakeLists.txt +++ b/dbms/src/DataStreams/CMakeLists.txt @@ -1,3 +1,3 @@ if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/DataStreams/tests/CMakeLists.txt b/dbms/src/DataStreams/tests/CMakeLists.txt index 8c3acb1332e..00f21f56dc4 100644 --- a/dbms/src/DataStreams/tests/CMakeLists.txt +++ b/dbms/src/DataStreams/tests/CMakeLists.txt @@ -3,28 +3,28 @@ include_directories (${CMAKE_CURRENT_BINARY_DIR}) set(SRCS ) add_executable (tab_separated_streams tab_separated_streams.cpp ${SRCS}) -target_link_libraries (tab_separated_streams dbms) +target_link_libraries (tab_separated_streams dbms clickhouse_functions) add_executable (block_row_transforms block_row_transforms.cpp ${SRCS}) -target_link_libraries (block_row_transforms dbms) +target_link_libraries (block_row_transforms dbms clickhouse_functions) add_executable (expression_stream expression_stream.cpp ${SRCS}) -target_link_libraries (expression_stream dbms clickhouse_storages_system) +target_link_libraries (expression_stream dbms clickhouse_storages_system clickhouse_functions) add_executable (native_streams native_streams.cpp ${SRCS}) -target_link_libraries (native_streams dbms) +target_link_libraries (native_streams dbms clickhouse_functions) add_executable (filter_stream filter_stream.cpp ${SRCS}) -target_link_libraries (filter_stream dbms clickhouse_storages_system) +target_link_libraries (filter_stream dbms clickhouse_storages_system clickhouse_functions) add_executable (filter_stream_hitlog filter_stream_hitlog.cpp ${SRCS}) -target_link_libraries (filter_stream_hitlog dbms) +target_link_libraries (filter_stream_hitlog dbms clickhouse_functions) add_executable (sorting_stream sorting_stream.cpp ${SRCS}) -target_link_libraries (sorting_stream dbms) +target_link_libraries (sorting_stream dbms clickhouse_functions) add_executable (union_stream2 union_stream2.cpp ${SRCS}) -target_link_libraries (union_stream2 dbms) +target_link_libraries (union_stream2 dbms clickhouse_functions) add_executable (collapsing_sorted_stream collapsing_sorted_stream.cpp ${SRCS}) -target_link_libraries (collapsing_sorted_stream dbms) +target_link_libraries (collapsing_sorted_stream dbms clickhouse_functions) diff --git a/dbms/src/DataTypes/CMakeLists.txt b/dbms/src/DataTypes/CMakeLists.txt index 65172356645..433b18404b7 100644 --- a/dbms/src/DataTypes/CMakeLists.txt +++ b/dbms/src/DataTypes/CMakeLists.txt @@ -1,3 +1,3 @@ if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt index cbc5288eac5..a827ad64130 100644 --- a/dbms/src/Functions/CMakeLists.txt +++ b/dbms/src/Functions/CMakeLists.txt @@ -100,5 +100,5 @@ if (USE_VECTORCLASS) endif () if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Functions/tests/CMakeLists.txt b/dbms/src/Functions/tests/CMakeLists.txt index b8ee368bf43..e9c98d2e1f7 100644 --- a/dbms/src/Functions/tests/CMakeLists.txt +++ b/dbms/src/Functions/tests/CMakeLists.txt @@ -7,4 +7,4 @@ add_executable (strings_trim strings_trim.cpp) target_link_libraries (strings_trim dbms gtest_main clickhouse_functions) add_executable (strings_pad strings_pad.cpp) -target_link_libraries (strings_pad dbms gtest_main clickhouse_functions) \ No newline at end of file +target_link_libraries (strings_pad dbms gtest_main clickhouse_functions) diff --git a/dbms/src/IO/CMakeLists.txt b/dbms/src/IO/CMakeLists.txt index 65172356645..433b18404b7 100644 --- a/dbms/src/IO/CMakeLists.txt +++ b/dbms/src/IO/CMakeLists.txt @@ -1,3 +1,3 @@ if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Interpreters/CMakeLists.txt b/dbms/src/Interpreters/CMakeLists.txt index 4df38a7f25b..ce411588515 100644 --- a/dbms/src/Interpreters/CMakeLists.txt +++ b/dbms/src/Interpreters/CMakeLists.txt @@ -46,5 +46,5 @@ set (CONFIG_COMPILE ${ClickHouse_BINARY_DIR}/dbms/src/Interpreters/config_compil configure_file (${ClickHouse_SOURCE_DIR}/dbms/src/Interpreters/config_compile.h.in ${CONFIG_COMPILE}) if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Parsers/CMakeLists.txt b/dbms/src/Parsers/CMakeLists.txt index 491560b1856..1bfef8d629c 100644 --- a/dbms/src/Parsers/CMakeLists.txt +++ b/dbms/src/Parsers/CMakeLists.txt @@ -5,5 +5,5 @@ target_link_libraries (clickhouse_parsers clickhouse_common_io) target_include_directories (clickhouse_parsers PUBLIC ${DBMS_INCLUDE_DIR}) if (ENABLE_TESTS) - add_subdirectory (tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Storages/CMakeLists.txt b/dbms/src/Storages/CMakeLists.txt index 75aaf50d5a2..33353edcc10 100644 --- a/dbms/src/Storages/CMakeLists.txt +++ b/dbms/src/Storages/CMakeLists.txt @@ -1,7 +1,7 @@ add_subdirectory (System) if (ENABLE_TESTS) - add_subdirectory (tests) - add_subdirectory (Transaction/tests) - add_subdirectory (Page/tests) + add_subdirectory (tests EXCLUDE_FROM_ALL) + add_subdirectory (Transaction/tests EXCLUDE_FROM_ALL) + add_subdirectory (Page/tests EXCLUDE_FROM_ALL) endif () diff --git a/dbms/src/Storages/Page/tests/CMakeLists.txt b/dbms/src/Storages/Page/tests/CMakeLists.txt index 33d9262f22a..c31f170193c 100644 --- a/dbms/src/Storages/Page/tests/CMakeLists.txt +++ b/dbms/src/Storages/Page/tests/CMakeLists.txt @@ -1,15 +1,29 @@ +add_headers_and_sources(page_storage ../) +add_library(page_storage + ${page_storage_headers} ${page_storage_sources}) +target_link_libraries(page_storage clickhouse_common_io) -add_executable(page_test_page_storage gtest_page_storage.cpp) -target_link_libraries(page_test_page_storage gtest_main dbms) +# glob all unit tests of dm into gtests_page_storage +macro(grep_gtest_sources BASE_DIR DST_VAR) + # Cold match files that are not in tests/ directories + file(GLOB_RECURSE "${DST_VAR}" RELATIVE "${BASE_DIR}" "gtest*.cpp") +endmacro() -add_executable(page_test_page_file gtest_page_file.cpp) -target_link_libraries(page_test_page_file gtest_main dbms) +# attach all dm gtest sources +grep_gtest_sources(${ClickHouse_SOURCE_DIR}/dbms/src/Storages/Page/tests ps_gtest_sources) +add_executable(gtests_page_storage ${ps_gtest_sources}) +target_link_libraries(gtests_page_storage gtest_main page_storage) +target_compile_options(gtests_page_storage PRIVATE -Wno-unknown-pragmas) +add_check(gtests_page_storage) + +# non googletest add_executable(page_stress_page_storage stress_page_stroage.cpp) -target_link_libraries(page_stress_page_storage dbms) +target_link_libraries(page_stress_page_storage page_storage) +target_compile_options(page_stress_page_storage PRIVATE -Wno-format) # turn off printf format check add_executable(page_utils_get_valid_pages utils_get_valid_pages.cpp) -target_link_libraries(page_utils_get_valid_pages dbms) +target_link_libraries(page_utils_get_valid_pages page_storage) target_compile_options(page_utils_get_valid_pages PRIVATE -Wno-format) add_executable(test_page_storage_write_disk_full test_page_storage_write_disk_full.cpp) diff --git a/dbms/src/Storages/Transaction/SchemaBuilder.cpp b/dbms/src/Storages/Transaction/SchemaBuilder.cpp index b366029b9f8..0da1d2e6f8f 100644 --- a/dbms/src/Storages/Transaction/SchemaBuilder.cpp +++ b/dbms/src/Storages/Transaction/SchemaBuilder.cpp @@ -723,8 +723,8 @@ bool SchemaBuilder::isIgnoreDB(const String & name) return context.getTMTContext().getIgnoreDatabases().count(name) > 0; } -template class SchemaBuilder; -template class SchemaBuilder; +template struct SchemaBuilder; +template struct SchemaBuilder; // end namespace } // namespace DB diff --git a/docker/builder/build.sh b/docker/builder/build.sh index 8db723b63ba..85ab30e9786 100755 --- a/docker/builder/build.sh +++ b/docker/builder/build.sh @@ -4,10 +4,14 @@ SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" SRCPATH=${1:-$(cd $SCRIPTPATH/../..; pwd -P)} NPROC=${NPROC:-$(nproc || grep -c ^processor /proc/cpuinfo)} -ENABLE_TEST=${ENABLE_TEST:-0} -ENABLE_EMBEDDED_COMPILER=${DENABLE_EMBEDDED_COMPILER:-1} +ENABLE_TEST=${ENABLE_TEST:-1} +ENABLE_EMBEDDED_COMPILER=${ENABLE_EMBEDDED_COMPILER:-1} CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Debug} +if [[ "${CMAKE_BUILD_TYPE}" != "Debug" ]]; then + ENABLE_TEST=0 +fi + set -xe if [ -d "$SRCPATH/contrib/kvproto" ]; then @@ -19,9 +23,11 @@ fi build_dir="$SRCPATH/build_docker" mkdir -p $build_dir && cd $build_dir -cmake "$SRCPATH" -DENABLE_EMBEDDED_COMPILER=$ENABLE_EMBEDDED_COMPILER -DENABLE_TESTS=$ENABLE_TEST -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE +cmake "$SRCPATH" \ + -DENABLE_EMBEDDED_COMPILER=$ENABLE_EMBEDDED_COMPILER \ + -DENABLE_TESTS=$ENABLE_TEST \ + -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE make -j $NPROC -#ctest -V -j $(nproc || grep -c ^processor /proc/cpuinfo) install_dir="$SRCPATH/docker/builder/tics" if [ -d "$install_dir" ]; then rm -rf "$install_dir"/*; else mkdir -p "$install_dir"; fi @@ -31,3 +37,11 @@ ldd "$build_dir/dbms/src/Server/theflash" | grep '/' | grep '=>' | \ awk -F '=>' '{print $2}' | awk '{print $1}' | while read lib; do cp -f "$lib" "$install_dir" done + +# copy gtest binary under Debug mode +if [[ "${CMAKE_BUILD_TYPE}" = "Debug" && ${ENABLE_TEST} -ne 0 ]]; then + #ctest -V -j $(nproc || grep -c ^processor /proc/cpuinfo) + make -j ${NPROC} gtests_dbms gtests_libcommon + cp -f "$build_dir/dbms/gtests_dbms" "$install_dir/" + cp -f "$build_dir/libs/libcommon/src/tests/gtests_libcommon" "$install_dir/" +fi diff --git a/libs/libcommon/CMakeLists.txt b/libs/libcommon/CMakeLists.txt index c9de793e939..05810c44aa1 100644 --- a/libs/libcommon/CMakeLists.txt +++ b/libs/libcommon/CMakeLists.txt @@ -118,5 +118,5 @@ if (RT_LIBRARY) endif () if (ENABLE_TESTS) - add_subdirectory (src/tests) + add_subdirectory (src/tests EXCLUDE_FROM_ALL) endif () diff --git a/libs/libcommon/src/tests/CMakeLists.txt b/libs/libcommon/src/tests/CMakeLists.txt index c8aaf476842..f65f6222aed 100644 --- a/libs/libcommon/src/tests/CMakeLists.txt +++ b/libs/libcommon/src/tests/CMakeLists.txt @@ -17,9 +17,9 @@ target_link_libraries (date_lut_default_timezone common ${PLATFORM_LIBS}) target_link_libraries (multi_version common) add_check(multi_version) -add_executable (unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp) -target_link_libraries (unit_tests_libcommon gtest_main common) -add_check(unit_tests_libcommon) +add_executable (gtests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp) +target_link_libraries (gtests_libcommon gtest_main common) +add_check(gtests_libcommon) add_executable (dump_variable dump_variable.cpp) target_link_libraries (dump_variable clickhouse_common_io) diff --git a/libs/libglibc-compatibility/CMakeLists.txt b/libs/libglibc-compatibility/CMakeLists.txt index f9139d2ccde..0b60869cb1b 100644 --- a/libs/libglibc-compatibility/CMakeLists.txt +++ b/libs/libglibc-compatibility/CMakeLists.txt @@ -12,4 +12,4 @@ musl/syscall.s musl/syscall_ret.c musl/sched_cpucount.c) -add_subdirectory (tests) +add_subdirectory (tests EXCLUDE_FROM_ALL) diff --git a/libs/libmysqlxx/CMakeLists.txt b/libs/libmysqlxx/CMakeLists.txt index e3c9e836c22..ec38ebce8f0 100644 --- a/libs/libmysqlxx/CMakeLists.txt +++ b/libs/libmysqlxx/CMakeLists.txt @@ -52,5 +52,5 @@ target_link_libraries (mysqlxx common ${MYSQLCLIENT_LIBRARIES} ${OPENSSL_LIBRARI target_include_directories (mysqlxx PRIVATE ${OPENSSL_INCLUDE_DIR}) if (ENABLE_TESTS) - add_subdirectory (src/tests) + add_subdirectory (src/tests EXCLUDE_FROM_ALL) endif () diff --git a/tests/_env.sh b/tests/_env.sh index ad2591453a9..6a15a30fab6 100644 --- a/tests/_env.sh +++ b/tests/_env.sh @@ -3,11 +3,13 @@ # Executable path if [ `uname` == "Darwin" ]; then - export storage_bin="../../build_clang/dbms/src/Server/theflash" + export build_dir="../../build_clang" else - export storage_bin="../../build/dbms/src/Server/theflash" + export build_dir="../../build" fi +export storage_bin="$build_dir/dbms/src/Server/theflash" + # Serve config for launching export storage_server_config="../../running/config/config.xml" diff --git a/tests/docker/_env.sh b/tests/docker/_env.sh index ecf42b57626..ae6ce78f58c 100755 --- a/tests/docker/_env.sh +++ b/tests/docker/_env.sh @@ -1,7 +1,9 @@ #!/bin/bash +export build_dir="/tics" + # Executable path -export storage_bin="/tics/theflash" +export storage_bin="$build_dir/theflash" # Server address for connecting export storage_server="127.0.0.1" diff --git a/tests/docker/docker-compose.yaml b/tests/docker/docker-compose.yaml index 3d14a4568ff..2729a116178 100644 --- a/tests/docker/docker-compose.yaml +++ b/tests/docker/docker-compose.yaml @@ -19,6 +19,16 @@ services: command: - --config-file - /config.xml + # tics-gtest container is for gtest cases + tics-gtest: + image: hub.pingcap.net/tiflash/tics:${TAG:-master} + volumes: + - ./config/config.xml:/config.xml:ro + - ./config/users.xml:/users.xml:ro + - ./log:/log + - ..:/tests + - ./_env.sh:/tests/_env.sh + entrypoint: sleep infinity # just wait pd0: image: hub.pingcap.net/tiflash/pd:190508 ports: diff --git a/tests/docker/run.sh b/tests/docker/run.sh index ab2d9a6ccae..bb205746fca 100755 --- a/tests/docker/run.sh +++ b/tests/docker/run.sh @@ -8,24 +8,25 @@ rm -rf ./data ./log ./build_learner_config.sh -docker-compose up -d --scale tics0=0 --scale tiflash0=0 --scale tikv-learner0=0 +docker-compose up -d --scale tics0=0 --scale tics-gtest=0 --scale tiflash0=0 --scale tikv-learner0=0 sleep 60 -docker-compose up -d --scale tics0=0 --scale tikv-learner0=0 --build +docker-compose up -d --scale tics0=0 --scale tics-gtest=0 --scale tikv-learner0=0 --build sleep 5 -docker-compose up -d --scale tics0=0 - +docker-compose up -d --scale tics0=0 --scale tics-gtest=0 docker-compose exec -T tiflash0 bash -c 'cd /tests ; ./run-test.sh fullstack-test true' - docker-compose down -docker-compose up -d --scale tiflash0=0 --scale tikv-learner0=0 --scale tikv0=0 --scale tidb0=0 --scale pd0=0 - +# (only tics0 up) +docker-compose up -d --scale tics-gtest=0 --scale tiflash0=0 --scale tikv-learner0=0 --scale tikv0=0 --scale tidb0=0 --scale pd0=0 docker-compose exec -T tics0 bash -c 'cd /tests ; ./run-test.sh mutable-test' - docker-compose down +# run gtest cases. (only tics-gtest up) +docker-compose up -d --scale tics0=0 --scale tiflash0=0 --scale tikv-learner0=0 --scale tikv0=0 --scale tidb0=0 --scale pd0=0 +docker-compose exec -T tics-gtest bash -c 'cd /tests && ./run-gtest.sh' +docker-compose down diff --git a/tests/run-gtest.sh b/tests/run-gtest.sh new file mode 100755 index 00000000000..28191108e6e --- /dev/null +++ b/tests/run-gtest.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +function run_test() +{ + local name="$1" + local bin_path=$(find . -name "$name") + if [[ "$continue_on_error" -eq 1 ]]; then + ${bin_path} --gtest_catch_exceptions=1 + else + ${bin_path} --gtest_break_on_failure --gtest_catch_exceptions=0 + fi +} + +source ./_env.sh + +continue_on_error="${1:-1}" # default 1 +set -ex + +cd "$build_dir" + +tests=( + "gtests_dbms" + "gtests_libcommon" +) + +for test in ${tests[@]}; do + run_test "$test" +done