diff --git a/clickhouse/columns/date.cpp b/clickhouse/columns/date.cpp index 790a08cf..2ddd8974 100644 --- a/clickhouse/columns/date.cpp +++ b/clickhouse/columns/date.cpp @@ -119,7 +119,7 @@ ColumnRef ColumnDateTime::Slice(size_t begin, size_t len) const { } ColumnRef ColumnDateTime::CloneEmpty() const { - return std::make_shared(); + return std::make_shared(); } void ColumnDateTime::Swap(Column& other) { diff --git a/ut/CMakeLists.txt b/ut/CMakeLists.txt index e7bdee01..11f91208 100644 --- a/ut/CMakeLists.txt +++ b/ut/CMakeLists.txt @@ -14,10 +14,15 @@ SET ( clickhouse-cpp-ut-src performance_tests.cpp tcp_server.cpp - utils.cpp readonly_client_test.cpp connection_failed_client_test.cpp - array_of_low_cardinality_tests.cpp) + array_of_low_cardinality_tests.cpp + CreateColumnByType_ut.cpp + Column_ut.cpp + + utils.cpp + value_generators.cpp +) IF (WITH_OPENSSL) LIST (APPEND clickhouse-cpp-ut-src ssl_ut.cpp) diff --git a/ut/Column_ut.cpp b/ut/Column_ut.cpp new file mode 100644 index 00000000..dd1dcebf --- /dev/null +++ b/ut/Column_ut.cpp @@ -0,0 +1,213 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include // for ipv4-ipv6 platform-specific stuff + +#include + +#include "utils.h" +#include "value_generators.h" + +namespace { +using namespace clickhouse; +} + + +// Generic tests for a Column subclass against basic API: +// 1. Constructor: Create, ensure that it is empty +// 2. Append: Create, add some data one by one via Append, make sure that values inserted match extracted with At() and operator[] +// 3. Slice: Create, add some data via Append, do Slice() +// 4. CloneEmpty Create, invoke CloneEmplty, ensure that clone is Empty +// 5. Clear: Create, add some data, invoke Clear(), make sure column is empty +// 6. Swap: create two instances, populate one with data, swap with second, make sure has data was transferred +// 7. Load/Save: create, append some data, save to buffer, load from same buffer into new column, make sure columns match. + +template +class GenericColumnTest : public testing::Test { +public: + using ColumnType = std::decay_t; + + static auto MakeColumn() { + if constexpr (std::is_same_v) { + return std::make_shared(12); + } else if constexpr (std::is_same_v) { + return std::make_shared(3); + } else if constexpr (std::is_same_v) { + return std::make_shared(10, 5); + } else { + return std::make_shared(); + } + } + + static auto GenerateValues(size_t values_size) { + if constexpr (std::is_same_v) { + return GenerateVector(values_size, FooBarGenerator); + } else if constexpr (std::is_same_v) { + return GenerateVector(values_size, FromVectorGenerator{MakeFixedStrings(12)}); + } else if constexpr (std::is_same_v) { + return GenerateVector(values_size, FromVectorGenerator{MakeDates()}); + } else if constexpr (std::is_same_v) { + return GenerateVector(values_size, FromVectorGenerator{MakeDateTimes()}); + } else if constexpr (std::is_same_v) { + return MakeDateTime64s(3u, values_size); + } else if constexpr (std::is_same_v) { + return GenerateVector(values_size, FromVectorGenerator{MakeIPv4s()}); + } else if constexpr (std::is_same_v) { + return GenerateVector(values_size, FromVectorGenerator{MakeIPv6s()}); + } else if constexpr (std::is_same_v) { + return GenerateVector(values_size, FromVectorGenerator{MakeInt128s()}); + } else if constexpr (std::is_same_v) { + return GenerateVector(values_size, FromVectorGenerator{MakeDecimals(3, 10)}); + } else if constexpr (std::is_same_v) { + return GenerateVector(values_size, FromVectorGenerator{MakeUUIDs()}); + } else if constexpr (std::is_integral_v) { + // ColumnUIntX and ColumnIntX + return GenerateVector(values_size, RandomGenerator()); + } else if constexpr (std::is_floating_point_v) { + // OR ColumnFloatX + return GenerateVector(values_size, RandomGenerator()); + } + } + + template + static void AppendValues(std::shared_ptr column, const Values& values) { + for (const auto & v : values) { + column->Append(v); + } + } + + static auto MakeColumnWithValues(size_t values_size) { + auto column = MakeColumn(); + auto values = GenerateValues(values_size); + AppendValues(column, values); + + return std::tuple{column, values}; + } +}; + +using ValueColumns = ::testing::Types< + ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64 + , ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64 + , ColumnFloat32, ColumnFloat64 + , ColumnString, ColumnFixedString + , ColumnDate, ColumnDateTime, ColumnDateTime64 + , ColumnIPv4, ColumnIPv6 + , ColumnInt128 + , ColumnDecimal + , ColumnUUID +>; +TYPED_TEST_SUITE(GenericColumnTest, ValueColumns); + +TYPED_TEST(GenericColumnTest, Construct) { + auto column = this->MakeColumn(); + ASSERT_EQ(0u, column->Size()); +} + +TYPED_TEST(GenericColumnTest, EmptyColumn) { + auto column = this->MakeColumn(); + ASSERT_EQ(0u, column->Size()); + + // verify that Column methods work as expected on empty column: + // some throw exceptions, some return poper values (like CloneEmpty) + + // Shouldn't be able to get items on empty column. + ASSERT_ANY_THROW(column->At(0)); + + { + auto slice = column->Slice(0, 0); + ASSERT_NO_THROW(slice->template AsStrict()); + ASSERT_EQ(0u, slice->Size()); + } + + { + auto clone = column->CloneEmpty(); + ASSERT_NO_THROW(clone->template AsStrict()); + ASSERT_EQ(0u, clone->Size()); + } + + ASSERT_NO_THROW(column->Clear()); + ASSERT_NO_THROW(column->Swap(*this->MakeColumn())); +} + +TYPED_TEST(GenericColumnTest, Append) { + auto column = this->MakeColumn(); + const auto values = this->GenerateValues(100); + + for (const auto & v : values) { + EXPECT_NO_THROW(column->Append(v)); + } + + EXPECT_TRUE(CompareRecursive(values, *column)); +} + +TYPED_TEST(GenericColumnTest, Slice) { + auto [column, values] = this->MakeColumnWithValues(100); + + auto untyped_slice = column->Slice(0, column->Size()); + auto slice = untyped_slice->template AsStrict(); + EXPECT_EQ(column->GetType(), slice->GetType()); + + EXPECT_TRUE(CompareRecursive(values, *slice)); + + // TODO: slices of different sizes +} + +TYPED_TEST(GenericColumnTest, CloneEmpty) { + auto [column, values] = this->MakeColumnWithValues(100); + EXPECT_EQ(values.size(), column->Size()); + + auto clone_untyped = column->CloneEmpty(); + // Check that type matches + auto clone = clone_untyped->template AsStrict(); + EXPECT_EQ(0u, clone->Size()); + + EXPECT_EQ(column->GetType(), clone->GetType()); +} + +TYPED_TEST(GenericColumnTest, Clear) { + auto [column, values] = this->MakeColumnWithValues(100); + EXPECT_EQ(values.size(), column->Size()); + + column->Clear(); + EXPECT_EQ(0u, column->Size()); +} + +TYPED_TEST(GenericColumnTest, Swap) { + auto [column_A, values] = this->MakeColumnWithValues(100); + auto column_B = this->MakeColumn(); + + column_A->Swap(*column_B); + + EXPECT_EQ(0u, column_A->Size()); + EXPECT_TRUE(CompareRecursive(values, *column_B)); +} + +TYPED_TEST(GenericColumnTest, LoadAndSave) { + auto [column_A, values] = this->MakeColumnWithValues(100); + + char buffer[4096] = {'\0'}; + { + ArrayOutput output(buffer, sizeof(buffer)); + // Save + EXPECT_NO_THROW(column_A->Save(&output)); + } + + auto column_B = this->MakeColumn(); + { + ArrayInput input(buffer, sizeof(buffer)); + // Load + EXPECT_TRUE(column_B->Load(&input, values.size())); + } + + EXPECT_TRUE(CompareRecursive(*column_A, *column_B)); +} diff --git a/ut/CreateColumnByType_ut.cpp b/ut/CreateColumnByType_ut.cpp new file mode 100644 index 00000000..fb7ffd85 --- /dev/null +++ b/ut/CreateColumnByType_ut.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include + +#include + +namespace { +using namespace clickhouse; +} + +TEST(CreateColumnByType, CreateSimpleAggregateFunction) { + auto col = CreateColumnByType("SimpleAggregateFunction(funt, Int32)"); + + ASSERT_EQ("Int32", col->Type()->GetName()); + ASSERT_EQ(Type::Int32, col->Type()->GetCode()); + ASSERT_NE(nullptr, col->As()); +} + +TEST(CreateColumnByType, UnmatchedBrackets) { + // When type string has unmatched brackets, CreateColumnByType must return nullptr. + ASSERT_EQ(nullptr, CreateColumnByType("FixedString(10")); + ASSERT_EQ(nullptr, CreateColumnByType("Nullable(FixedString(10000")); + ASSERT_EQ(nullptr, CreateColumnByType("Nullable(FixedString(10000)")); + ASSERT_EQ(nullptr, CreateColumnByType("LowCardinality(Nullable(FixedString(10000")); + ASSERT_EQ(nullptr, CreateColumnByType("LowCardinality(Nullable(FixedString(10000)")); + ASSERT_EQ(nullptr, CreateColumnByType("LowCardinality(Nullable(FixedString(10000))")); + ASSERT_EQ(nullptr, CreateColumnByType("Array(LowCardinality(Nullable(FixedString(10000")); + ASSERT_EQ(nullptr, CreateColumnByType("Array(LowCardinality(Nullable(FixedString(10000)")); + ASSERT_EQ(nullptr, CreateColumnByType("Array(LowCardinality(Nullable(FixedString(10000))")); + ASSERT_EQ(nullptr, CreateColumnByType("Array(LowCardinality(Nullable(FixedString(10000)))")); +} + +TEST(CreateColumnByType, LowCardinalityAsWrappedColumn) { + CreateColumnByTypeSettings create_column_settings; + create_column_settings.low_cardinality_as_wrapped_column = true; + + ASSERT_EQ(Type::String, CreateColumnByType("LowCardinality(String)", create_column_settings)->GetType().GetCode()); + ASSERT_EQ(Type::String, CreateColumnByType("LowCardinality(String)", create_column_settings)->As()->GetType().GetCode()); + + ASSERT_EQ(Type::FixedString, CreateColumnByType("LowCardinality(FixedString(10000))", create_column_settings)->GetType().GetCode()); + ASSERT_EQ(Type::FixedString, CreateColumnByType("LowCardinality(FixedString(10000))", create_column_settings)->As()->GetType().GetCode()); +} + +TEST(CreateColumnByType, DateTime) { + ASSERT_NE(nullptr, CreateColumnByType("DateTime")); + ASSERT_NE(nullptr, CreateColumnByType("DateTime('Europe/Moscow')")); + + ASSERT_EQ(CreateColumnByType("DateTime('UTC')")->As()->Timezone(), "UTC"); + ASSERT_EQ(CreateColumnByType("DateTime64(3, 'UTC')")->As()->Timezone(), "UTC"); +} + +class CreateColumnByTypeWithName : public ::testing::TestWithParam +{}; + +TEST_P(CreateColumnByTypeWithName, CreateColumnByType) +{ + const auto col = CreateColumnByType(GetParam()); + ASSERT_NE(nullptr, col); + EXPECT_EQ(col->GetType().GetName(), GetParam()); +} + +INSTANTIATE_TEST_SUITE_P(Basic, CreateColumnByTypeWithName, ::testing::Values( + "Int8", "Int16", "Int32", "Int64", + "UInt8", "UInt16", "UInt32", "UInt64", + "String", "Date", "DateTime", + "UUID", "Int128" +)); + +INSTANTIATE_TEST_SUITE_P(Parametrized, CreateColumnByTypeWithName, ::testing::Values( + "FixedString(0)", "FixedString(10000)", + "DateTime('UTC')", "DateTime64(3, 'UTC')", + "Decimal(9,3)", "Decimal(18,3)", + "Enum8('ONE' = 1, 'TWO' = 2)", + "Enum16('ONE' = 1, 'TWO' = 2, 'THREE' = 3, 'FOUR' = 4)" +)); + + +INSTANTIATE_TEST_SUITE_P(Nested, CreateColumnByTypeWithName, ::testing::Values( + "Nullable(FixedString(10000))", + "Nullable(LowCardinality(FixedString(10000)))", + "Array(Nullable(LowCardinality(FixedString(10000))))", + "Array(Enum8('ONE' = 1, 'TWO' = 2))" +)); diff --git a/ut/client_ut.cpp b/ut/client_ut.cpp index 288d0e0e..82032377 100644 --- a/ut/client_ut.cpp +++ b/ut/client_ut.cpp @@ -12,17 +12,6 @@ using namespace clickhouse; -namespace clickhouse { -std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info) { - return ostr << server_info.name << "/" << server_info.display_name - << " ver " - << server_info.version_major << "." - << server_info.version_minor << "." - << server_info.version_patch - << " (" << server_info.revision << ")"; -} -} - namespace { uint64_t versionNumber( @@ -1013,7 +1002,7 @@ ColumnRef RoundtripColumnValues(Client& client, ColumnRef expected) { result->Append(b[0]); }); - EXPECT_EQ(expected->Type(), result->Type()); + EXPECT_EQ(expected->GetType(), result->GetType()); EXPECT_EQ(expected->Size(), result->Size()); return result; } diff --git a/ut/columns_ut.cpp b/ut/columns_ut.cpp index 2355acf9..9a806e39 100644 --- a/ut/columns_ut.cpp +++ b/ut/columns_ut.cpp @@ -16,149 +16,24 @@ #include #include "utils.h" +#include "value_generators.h" #include #include #include - -// only compare PODs of equal size this way -template , std::is_pod>>> -bool operator==(const L & left, const R& right) { - return memcmp(&left, &right, sizeof(left)) == 0; -} - -bool operator==(const in6_addr & left, const std::string_view & right) { - return right.size() == sizeof(left) && memcmp(&left, right.data(), sizeof(left)) == 0; -} - -bool operator==(const std::string_view & left, const in6_addr & right) { - return left.size() == sizeof(right) && memcmp(left.data(), &right, sizeof(right)) == 0; -} +#include namespace { using namespace clickhouse; using namespace std::literals::string_view_literals; -in_addr MakeIPv4(uint32_t ip) { - static_assert(sizeof(in_addr) == sizeof(ip)); - in_addr result; - memcpy(&result, &ip, sizeof(ip)); - - return result; -} - -in6_addr MakeIPv6(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, - uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, - uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) { - return in6_addr{{{v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}}}; -} - -in6_addr MakeIPv6(uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) { - return in6_addr{{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, v10, v11, v12, v13, v14, v15}}}; -} - -static std::vector MakeNumbers() { - return std::vector - {1, 2, 3, 7, 11, 13, 17, 19, 23, 29, 31}; -} - -static std::vector MakeBools() { - return std::vector - {1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0}; -} - -static std::vector MakeFixedStrings() { - return std::vector - {"aaa", "bbb", "ccc", "ddd"}; -} - -static std::vector MakeStrings() { - return std::vector - {"a", "ab", "abc", "abcd"}; -} - -static std::vector MakeUUIDs() { - return std::vector - {0xbb6a8c699ab2414cllu, 0x86697b7fd27f0825llu, - 0x84b9f24bc26b49c6llu, 0xa03b4ab723341951llu, - 0x3507213c178649f9llu, 0x9faf035d662f60aellu}; -} - static const auto LOWCARDINALITY_STRING_FOOBAR_10_ITEMS_BINARY = "\x01\x00\x00\x00\x00\x00\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00" "\x09\x00\x00\x00\x00\x00\x00\x00\x00\x06\x46\x6f\x6f\x42\x61\x72" "\x01\x31\x01\x32\x03\x46\x6f\x6f\x01\x34\x03\x42\x61\x72\x01\x37" "\x01\x38\x0a\x00\x00\x00\x00\x00\x00\x00\x01\x02\x03\x04\x05\x06" "\x04\x07\x08\x04"sv; - -template -auto GenerateVector(size_t items, Generator && gen) { - std::vector> result; - result.reserve(items); - for (size_t i = 0; i < items; ++i) { - result.push_back(std::move(gen(i))); - } - - return result; -} - -std::string FooBarSeq(size_t i) { - std::string result; - if (i % 3 == 0) - result += "Foo"; - if (i % 5 == 0) - result += "Bar"; - if (result.empty()) - result = std::to_string(i); - - return result; -} - -template -auto SameValueSeq(const U & value) { - return [&value](size_t) -> T { - return value; - }; -} - -template -auto AlternateGenerators(Generator1 && gen1, Generator2 && gen2) { - return [&gen1, &gen2](size_t i) -> ResultType { - if (i % 2 == 0) - return gen1(i/2); - else - return gen2(i/2); - }; -} - -template -std::vector ConcatSequences(std::vector && vec1, std::vector && vec2) { - std::vector result(vec1); - - result.reserve(vec1.size() + vec2.size()); - result.insert(result.end(), vec2.begin(), vec2.end()); - - return result; -} - -static std::vector MakeDateTime64s() { - static const auto seconds_multiplier = 1'000'000; - static const auto year = 86400ull * 365 * seconds_multiplier; // ~approx, but this doesn't matter here. - - // Approximatelly +/- 200 years around epoch (and value of epoch itself) - // with non zero seconds and sub-seconds. - // Please note there are values outside of DateTime (32-bit) range that might - // not have correct string representation in CH yet, - // but still are supported as Int64 values. - return GenerateVector(200, - [] (size_t i )-> Int64 { - return (i - 100) * year * 2 + (i * 10) * seconds_multiplier + i; - }); -} - } // TODO: add tests for ColumnDecimal. @@ -184,7 +59,7 @@ TEST(ColumnsCase, NumericSlice) { TEST(ColumnsCase, FixedStringInit) { - const auto column_data = MakeFixedStrings(); + const auto column_data = MakeFixedStrings(3); auto col = std::make_shared(3, column_data); ASSERT_EQ(col->Size(), column_data.size()); @@ -201,7 +76,7 @@ TEST(ColumnsCase, FixedString_Append_SmallStrings) { // are padded with zeroes on insertion. const size_t string_size = 7; - const auto column_data = MakeFixedStrings(); + const auto column_data = MakeFixedStrings(3); auto col = std::make_shared(string_size); size_t i = 0; @@ -304,7 +179,7 @@ TEST(ColumnsCase, DateTime64_6) { TEST(ColumnsCase, DateTime64_Append_At) { auto column = std::make_shared(6ul); - const auto data = MakeDateTime64s(); + const auto data = MakeDateTime64s(6ul); for (const auto & v : data) { column->Append(v); } @@ -322,7 +197,7 @@ TEST(ColumnsCase, DateTime64_Clear) { ASSERT_NO_THROW(column->Clear()); ASSERT_EQ(0u, column->Size()); - const auto data = MakeDateTime64s(); + const auto data = MakeDateTime64s(6ul); for (const auto & v : data) { column->Append(v); } @@ -334,7 +209,7 @@ TEST(ColumnsCase, DateTime64_Clear) { TEST(ColumnsCase, DateTime64_Swap) { auto column = std::make_shared(6ul); - const auto data = MakeDateTime64s(); + const auto data = MakeDateTime64s(6ul); for (const auto & v : data) { column->Append(v); } @@ -364,7 +239,7 @@ TEST(ColumnsCase, DateTime64_Slice) { ASSERT_EQ(column->GetPrecision(), slice->GetPrecision()); } - const auto data = MakeDateTime64s(); + const auto data = MakeDateTime64s(6ul); const size_t size = data.size(); ASSERT_GT(size, 4u); // so the partial slice below has half of the elements of the column @@ -413,7 +288,7 @@ TEST(ColumnsCase, DateTime64_Slice_OUTOFBAND) { // Non-Empty slice on empty column EXPECT_EQ(0u, column->Slice(0, 10)->Size()); - const auto data = MakeDateTime64s(); + const auto data = MakeDateTime64s(6ul); for (const auto & v : data) { column->Append(v); } @@ -439,14 +314,6 @@ TEST(ColumnsCase, Date2038) { ASSERT_EQ(largeDate, col1->At(0)); } -TEST(ColumnsCase, DateTime) { - ASSERT_NE(nullptr, CreateColumnByType("DateTime")); - ASSERT_NE(nullptr, CreateColumnByType("DateTime('Europe/Moscow')")); - - ASSERT_EQ(CreateColumnByType("DateTime('UTC')")->As()->Timezone(), "UTC"); - ASSERT_EQ(CreateColumnByType("DateTime64(3, 'UTC')")->As()->Timezone(), "UTC"); -} - TEST(ColumnsCase, EnumTest) { std::vector enum_items = {{"Hi", 1}, {"Hello", 2}}; @@ -484,8 +351,17 @@ TEST(ColumnsCase, NullableSlice) { ASSERT_EQ(subData->At(3), 17u); } +// internal representation of UUID data in ColumnUUID +std::vector MakeUUID_data() { + return { + 0xbb6a8c699ab2414cllu, 0x86697b7fd27f0825llu, + 0x84b9f24bc26b49c6llu, 0xa03b4ab723341951llu, + 0x3507213c178649f9llu, 0x9faf035d662f60aellu + }; +} + TEST(ColumnsCase, UUIDInit) { - auto col = std::make_shared(std::make_shared(MakeUUIDs())); + auto col = std::make_shared(std::make_shared(MakeUUID_data())); ASSERT_EQ(col->Size(), 3u); ASSERT_EQ(col->At(0), UInt128(0xbb6a8c699ab2414cllu, 0x86697b7fd27f0825llu)); @@ -493,7 +369,7 @@ TEST(ColumnsCase, UUIDInit) { } TEST(ColumnsCase, UUIDSlice) { - auto col = std::make_shared(std::make_shared(MakeUUIDs())); + auto col = std::make_shared(std::make_shared(MakeUUID_data())); auto sub = col->Slice(1, 2)->As(); ASSERT_EQ(sub->Size(), 2u); @@ -745,7 +621,7 @@ TEST(ColumnsCase, ColumnDecimal128_from_string_overflow) { TEST(ColumnsCase, ColumnLowCardinalityString_Append_and_Read) { const size_t items_count = 11; ColumnLowCardinalityT col; - for (const auto & item : GenerateVector(items_count, &FooBarSeq)) { + for (const auto & item : GenerateVector(items_count, &FooBarGenerator)) { col.Append(item); } @@ -753,15 +629,15 @@ TEST(ColumnsCase, ColumnLowCardinalityString_Append_and_Read) { ASSERT_EQ(col.GetDictionarySize(), 8u + 1); // 8 unique items from sequence + 1 null-item for (size_t i = 0; i < items_count; ++i) { - ASSERT_EQ(col.At(i), FooBarSeq(i)) << " at pos: " << i; - ASSERT_EQ(col[i], FooBarSeq(i)) << " at pos: " << i; + ASSERT_EQ(col.At(i), FooBarGenerator(i)) << " at pos: " << i; + ASSERT_EQ(col[i], FooBarGenerator(i)) << " at pos: " << i; } } TEST(ColumnsCase, ColumnLowCardinalityString_Clear_and_Append) { const size_t items_count = 11; ColumnLowCardinalityT col; - for (const auto & item : GenerateVector(items_count, &FooBarSeq)) + for (const auto & item : GenerateVector(items_count, &FooBarGenerator)) { col.Append(item); } @@ -770,7 +646,7 @@ TEST(ColumnsCase, ColumnLowCardinalityString_Clear_and_Append) { ASSERT_EQ(col.Size(), 0u); ASSERT_EQ(col.GetDictionarySize(), 1u); // null-item - for (const auto & item : GenerateVector(items_count, &FooBarSeq)) + for (const auto & item : GenerateVector(items_count, &FooBarGenerator)) { col.Append(item); } @@ -789,7 +665,7 @@ TEST(ColumnsCase, ColumnLowCardinalityString_Load) { ASSERT_TRUE(col.Load(&buffer, items_count)); for (size_t i = 0; i < items_count; ++i) { - EXPECT_EQ(col.At(i), FooBarSeq(i)) << " at pos: " << i; + EXPECT_EQ(col.At(i), FooBarGenerator(i)) << " at pos: " << i; } } @@ -798,7 +674,7 @@ TEST(ColumnsCase, ColumnLowCardinalityString_Load) { TEST(ColumnsCase, DISABLED_ColumnLowCardinalityString_Save) { const size_t items_count = 10; ColumnLowCardinalityT col; - for (const auto & item : GenerateVector(items_count, &FooBarSeq)) { + for (const auto & item : GenerateVector(items_count, &FooBarGenerator)) { col.Append(item); } @@ -835,7 +711,7 @@ TEST(ColumnsCase, ColumnLowCardinalityString_SaveAndLoad) { // Verify that we can load binary representation back ColumnLowCardinalityT col; - const auto items = GenerateVector(10, &FooBarSeq); + const auto items = GenerateVector(10, &FooBarGenerator); for (const auto & item : items) { col.Append(item); } @@ -862,7 +738,7 @@ TEST(ColumnsCase, ColumnLowCardinalityString_SaveAndLoad) { TEST(ColumnsCase, ColumnLowCardinalityString_WithEmptyString_1) { // Verify that when empty string is added to a LC column it can be retrieved back as empty string. ColumnLowCardinalityT col; - const auto values = GenerateVector(10, AlternateGenerators(SameValueSeq(""), FooBarSeq)); + const auto values = GenerateVector(10, AlternateGenerators(SameValueGenerator(""), FooBarGenerator)); for (const auto & item : values) { col.Append(item); } @@ -876,7 +752,7 @@ TEST(ColumnsCase, ColumnLowCardinalityString_WithEmptyString_2) { // Verify that when empty string is added to a LC column it can be retrieved back as empty string. // (Ver2): Make sure that outcome doesn't depend if empty values are on odd positions ColumnLowCardinalityT col; - const auto values = GenerateVector(10, AlternateGenerators(FooBarSeq, SameValueSeq(""))); + const auto values = GenerateVector(10, AlternateGenerators(FooBarGenerator, SameValueGenerator(""))); for (const auto & item : values) { col.Append(item); } @@ -889,7 +765,7 @@ TEST(ColumnsCase, ColumnLowCardinalityString_WithEmptyString_2) { TEST(ColumnsCase, ColumnLowCardinalityString_WithEmptyString_3) { // When we have many leading empty strings and some non-empty values. ColumnLowCardinalityT col; - const auto values = ConcatSequences(GenerateVector(100, SameValueSeq("")), GenerateVector(5, FooBarSeq)); + const auto values = ConcatSequences(GenerateVector(100, SameValueGenerator("")), GenerateVector(5, FooBarGenerator)); for (const auto & item : values) { col.Append(item); } @@ -899,69 +775,3 @@ TEST(ColumnsCase, ColumnLowCardinalityString_WithEmptyString_3) { } } -TEST(ColumnsCase, CreateSimpleAggregateFunction) { - auto col = CreateColumnByType("SimpleAggregateFunction(funt, Int32)"); - - ASSERT_EQ("Int32", col->Type()->GetName()); - ASSERT_EQ(Type::Int32, col->Type()->GetCode()); - ASSERT_NE(nullptr, col->As()); -} - - -TEST(ColumnsCase, UnmatchedBrackets) { - // When type string has unmatched brackets, CreateColumnByType must return nullptr. - ASSERT_EQ(nullptr, CreateColumnByType("FixedString(10")); - ASSERT_EQ(nullptr, CreateColumnByType("Nullable(FixedString(10000")); - ASSERT_EQ(nullptr, CreateColumnByType("Nullable(FixedString(10000)")); - ASSERT_EQ(nullptr, CreateColumnByType("LowCardinality(Nullable(FixedString(10000")); - ASSERT_EQ(nullptr, CreateColumnByType("LowCardinality(Nullable(FixedString(10000)")); - ASSERT_EQ(nullptr, CreateColumnByType("LowCardinality(Nullable(FixedString(10000))")); - ASSERT_EQ(nullptr, CreateColumnByType("Array(LowCardinality(Nullable(FixedString(10000")); - ASSERT_EQ(nullptr, CreateColumnByType("Array(LowCardinality(Nullable(FixedString(10000)")); - ASSERT_EQ(nullptr, CreateColumnByType("Array(LowCardinality(Nullable(FixedString(10000))")); - ASSERT_EQ(nullptr, CreateColumnByType("Array(LowCardinality(Nullable(FixedString(10000)))")); -} - -TEST(ColumnsCase, LowCardinalityAsWrappedColumn) { - CreateColumnByTypeSettings create_column_settings; - create_column_settings.low_cardinality_as_wrapped_column = true; - - ASSERT_EQ(Type::String, CreateColumnByType("LowCardinality(String)", create_column_settings)->GetType().GetCode()); - ASSERT_EQ(Type::String, CreateColumnByType("LowCardinality(String)", create_column_settings)->As()->GetType().GetCode()); - - ASSERT_EQ(Type::FixedString, CreateColumnByType("LowCardinality(FixedString(10000))", create_column_settings)->GetType().GetCode()); - ASSERT_EQ(Type::FixedString, CreateColumnByType("LowCardinality(FixedString(10000))", create_column_settings)->As()->GetType().GetCode()); -} - -class ColumnsCaseWithName : public ::testing::TestWithParam -{}; - -TEST_P(ColumnsCaseWithName, CreateColumnByType) -{ - const auto col = CreateColumnByType(GetParam()); - ASSERT_NE(nullptr, col); - EXPECT_EQ(col->GetType().GetName(), GetParam()); -} - -INSTANTIATE_TEST_SUITE_P(Basic, ColumnsCaseWithName, ::testing::Values( - "Int8", "Int16", "Int32", "Int64", - "UInt8", "UInt16", "UInt32", "UInt64", - "String", "Date", "DateTime", - "UUID", "Int128" -)); - -INSTANTIATE_TEST_SUITE_P(Parametrized, ColumnsCaseWithName, ::testing::Values( - "FixedString(0)", "FixedString(10000)", - "DateTime('UTC')", "DateTime64(3, 'UTC')", - "Decimal(9,3)", "Decimal(18,3)", - "Enum8('ONE' = 1, 'TWO' = 2)", - "Enum16('ONE' = 1, 'TWO' = 2, 'THREE' = 3, 'FOUR' = 4)" -)); - - -INSTANTIATE_TEST_SUITE_P(Nested, ColumnsCaseWithName, ::testing::Values( - "Nullable(FixedString(10000))", - "Nullable(LowCardinality(FixedString(10000)))", - "Array(Nullable(LowCardinality(FixedString(10000))))", - "Array(Enum8('ONE' = 1, 'TWO' = 2))" -)); diff --git a/ut/performance_tests.cpp b/ut/performance_tests.cpp index d305e545..bafa07dd 100644 --- a/ut/performance_tests.cpp +++ b/ut/performance_tests.cpp @@ -15,6 +15,7 @@ #include #include "utils.h" +#include "utils_performance.h" using namespace clickhouse; diff --git a/ut/utils.cpp b/ut/utils.cpp index dc0cb876..a5ef5cfd 100644 --- a/ut/utils.cpp +++ b/ut/utils.cpp @@ -1,6 +1,7 @@ #include "utils.h" #include +#include #include #include #include @@ -154,28 +155,6 @@ std::ostream & operator<<(std::ostream & ostr, const ColumnValue& v) { } -std::ostream& operator<<(std::ostream & ostr, const Block & block) { - if (block.GetRowCount() == 0 || block.GetColumnCount() == 0) - return ostr; - - for (size_t col = 0; col < block.GetColumnCount(); ++col) { - const auto & c = block[col]; - ostr << c->GetType().GetName() << " ["; - - for (size_t row = 0; row < block.GetRowCount(); ++row) { - printColumnValue(c, row, ostr); - if (row != block.GetRowCount() - 1) - ostr << ", "; - } - ostr << "]"; - - if (col != block.GetColumnCount() - 1) - ostr << "\n"; - } - - return ostr; -} - std::ostream& operator<<(std::ostream & ostr, const PrettyPrintBlock & pretty_print_block) { // Pretty-print block: // - names of each column @@ -248,3 +227,42 @@ std::ostream& operator<<(std::ostream& ostr, const in6_addr& addr) { return ostr << ip_str; } + +namespace clickhouse { + +std::ostream& operator<<(std::ostream & ostr, const Block & block) { + if (block.GetRowCount() == 0 || block.GetColumnCount() == 0) + return ostr; + + for (size_t col = 0; col < block.GetColumnCount(); ++col) { + const auto & c = block[col]; + ostr << c->GetType().GetName() << " ["; + + for (size_t row = 0; row < block.GetRowCount(); ++row) { + printColumnValue(c, row, ostr); + if (row != block.GetRowCount() - 1) + ostr << ", "; + } + ostr << "]"; + + if (col != block.GetColumnCount() - 1) + ostr << "\n"; + } + + return ostr; +} + +std::ostream& operator<<(std::ostream & ostr, const Type & type) { + return ostr << type.GetName(); +} + +std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info) { + return ostr << server_info.name << "/" << server_info.display_name + << " ver " + << server_info.version_major << "." + << server_info.version_minor << "." + << server_info.version_patch + << " (" << server_info.revision << ")"; +} + +} diff --git a/ut/utils.h b/ut/utils.h index 626b1192..80348f76 100644 --- a/ut/utils.h +++ b/ut/utils.h @@ -1,14 +1,17 @@ #pragma once #include +#include + +#include "utils_meta.h" +#include "utils_comparison.h" -#include -#include #include #include +#include #include -#include #include +#include #include @@ -16,37 +19,38 @@ namespace clickhouse { class Block; - class Column; + class Type; + struct ServerInfo; } -template -struct Timer { - using DurationType = ChronoDurationType; - - Timer() - : started_at(Now()) - {} - - void Restart() { - started_at = Now(); - } - - void Start() { - Restart(); - } +template +auto getEnvOrDefault(const std::string& env, const char * default_val) { + const char* v = std::getenv(env.c_str()); + if (!v && !default_val) + throw std::runtime_error("Environment var '" + env + "' is not set."); - auto Elapsed() const { - return std::chrono::duration_cast(Now() - started_at); - } + const std::string value = v ? v : default_val; -private: - static auto Now() { - return std::chrono::high_resolution_clock::now().time_since_epoch(); + if constexpr (std::is_same_v) { + return value; + } else if constexpr (std::is_integral_v) { + // since std::from_chars is not available on GCC-7 on linux + if constexpr (std::is_signed_v) { + if constexpr (sizeof(ResultType) <= sizeof(int)) + return std::stoi(value); + else if constexpr (sizeof(ResultType) <= sizeof(long)) + return std::stol(value); + else if constexpr (sizeof(ResultType) <= sizeof(long long)) + return std::stoll(value); + } else if constexpr (std::is_unsigned_v) { + if constexpr (sizeof(ResultType) <= sizeof(unsigned long)) + return std::stoul(value); + else if constexpr (sizeof(ResultType) <= sizeof(unsigned long long)) + return std::stoull(value); + } } +} -private: - std::chrono::nanoseconds started_at; -}; template inline const char * getPrefix() { @@ -75,47 +79,14 @@ template inline ostream & operator<<(ostream & ostr, const chrono::duration & d) { return ostr << d.count() << ::getPrefix

() << "s"; } -} - -// Since result_of is deprecated in C++17, and invoke_result_of is unavailable until C++20... -template -using my_result_of_t = -#if __cplusplus >= 201703L - std::invoke_result_t; -#else - std::result_of_t; -#endif - -template -class MeasuresCollector { -public: - using Result = my_result_of_t; - - explicit MeasuresCollector(MeasureFunc && measurment_func, const size_t preallocate_results = 10) - : measurment_func_(std::move(measurment_func)) - { - results_.reserve(preallocate_results); - } - - template - void Add(NameType && name) { - results_.emplace_back(name, measurment_func_()); - } - - const auto & GetResults() const { - return results_; - } - -private: - MeasureFunc measurment_func_; - std::vector> results_; -}; -template -MeasuresCollector collect(MeasureFunc && f) { - return MeasuresCollector(std::forward(f)); +template +inline ostream & operator<<(ostream & ostr, const pair & t) { + return ostr << "{ " << t.first << ", " << t.second << " }"; +} } + struct in_addr; struct in6_addr; // Helper for pretty-printing of the Block @@ -123,118 +94,17 @@ struct PrettyPrintBlock { const clickhouse::Block & block; }; -std::ostream& operator<<(std::ostream & ostr, const clickhouse::Block & block); +namespace clickhouse { +std::ostream& operator<<(std::ostream & ostr, const Block & block); +std::ostream& operator<<(std::ostream & ostr, const Type & type); +std::ostream & operator<<(std::ostream & ostr, const ServerInfo & server_info); +} + std::ostream& operator<<(std::ostream & ostr, const PrettyPrintBlock & block); std::ostream& operator<<(std::ostream& ostr, const in_addr& addr); std::ostream& operator<<(std::ostream& ostr, const in6_addr& addr); -template -auto getEnvOrDefault(const std::string& env, const char * default_val) { - const char* v = std::getenv(env.c_str()); - if (!v && !default_val) - throw std::runtime_error("Environment var '" + env + "' is not set."); - - const std::string value = v ? v : default_val; - - if constexpr (std::is_same_v) { - return value; - } else if constexpr (std::is_integral_v) { - // since std::from_chars is not available on GCC-7 on linux - if constexpr (std::is_signed_v) { - if constexpr (sizeof(ResultType) <= sizeof(int)) - return std::stoi(value); - else if constexpr (sizeof(ResultType) <= sizeof(long)) - return std::stol(value); - else if constexpr (sizeof(ResultType) <= sizeof(long long)) - return std::stoll(value); - } else if constexpr (std::is_unsigned_v) { - if constexpr (sizeof(ResultType) <= sizeof(unsigned long)) - return std::stoul(value); - else if constexpr (sizeof(ResultType) <= sizeof(unsigned long long)) - return std::stoull(value); - } - } -} - - -// based on https://stackoverflow.com/a/31207079 -template -struct is_container : std::false_type {}; - -namespace details { -template -struct is_container_helper {}; - -// Make a column a RO stl-like container -template -struct ColumnAsContainerWrapper { - const NestedColumnType& nested_col; - - struct Iterator { - const NestedColumnType& nested_col; - size_t i = 0; - - auto& operator++() { - ++i; - return *this; - } - - auto operator*() const { - return nested_col[i]; - } - - bool operator==(const Iterator & other) const { - return &other.nested_col == &this->nested_col && other.i == this->i; - } - - bool operator!=(const Iterator & other) const { - return !(other == *this); - } - }; - - size_t size() const { - return nested_col.Size(); - } - - auto begin() const { - return Iterator{nested_col, 0}; - } - - auto end() const { - return Iterator{nested_col, nested_col.Size()}; - } -}; - -} - -template -auto maybeWrapColumnAsContainer(const T & t) { - if constexpr (std::is_base_of_v) { - return ::details::ColumnAsContainerWrapper{t}; - } else { - return t; - } -} - -// A very loose definition of container, nerfed to fit both C-array and ColumnArrayT::ArrayWrapper -template -struct is_container< - T, - std::conditional_t< - false, - ::details::is_container_helper< - decltype(std::declval().size()), - decltype(std::begin(std::declval())), - decltype(std::end(std::declval())) - >, - void - > - > : public std::true_type {}; - -template -inline constexpr bool is_container_v = is_container::value; - template struct PrintContainer { const Container & container_; @@ -266,48 +136,4 @@ std::ostream& operator<<(std::ostream & ostr, const PrintContainer& print_con return ostr << "]"; } -// Compare values to each other, if values are container-ish, then recursively deep compare those containers. -template -::testing::AssertionResult CompareRecursive(const Left & left, const Right & right); - -// Compare containers element-wise, if elements are containers themselves - compare recursevely -template -::testing::AssertionResult CompareCotainersRecursive(const LeftContainer& left, const RightContainer& right) { - if (left.size() != right.size()) - return ::testing::AssertionFailure() << "\nMismatching containers size, expected: " << left.size() << " actual: " << right.size(); - - auto l_i = std::begin(left); - auto r_i = std::begin(right); - - for (size_t i = 0; i < left.size(); ++i, ++l_i, ++r_i) { - auto result = CompareRecursive(*l_i, *r_i); - if (!result) - return result << "\n\nMismatch at pos: " << i + 1; - } - - return ::testing::AssertionSuccess(); -} - -template -::testing::AssertionResult CompareRecursive(const Left & left, const Right & right) { - if constexpr ((is_container_v || std::is_base_of_v>) - && (is_container_v || std::is_base_of_v>) ) { - - const auto & l = maybeWrapColumnAsContainer(left); - const auto & r = maybeWrapColumnAsContainer(right); - - if (auto result = CompareCotainersRecursive(l, r)) - return result; - else - return result << "\nExpected container: " << PrintContainer{l} - << "\nActual container : " << PrintContainer{r}; - } else { - if (left != right) - return ::testing::AssertionFailure() - << "\nExpected value: " << left - << "\nActual value : " << right; - - return ::testing::AssertionSuccess(); - } -} diff --git a/ut/utils_comparison.h b/ut/utils_comparison.h new file mode 100644 index 00000000..c40033b4 --- /dev/null +++ b/ut/utils_comparison.h @@ -0,0 +1,163 @@ +#pragma once + +#include "utils_meta.h" + +#include // for ipv4-ipv6 platform-specific stuff + +#include + +#include +#include + +namespace clickhouse { + class Block; + class Column; +} + +inline bool operator==(const in6_addr& left, const in6_addr& right) { + return memcmp(&left, &right, sizeof(left)) == 0; +} + +inline bool operator==(const in_addr& left, const in_addr& right) { + return memcmp(&left, &right, sizeof(left)) == 0; +} + +inline bool operator==(const in_addr & left, const uint32_t& right) { + return memcmp(&left, &right, sizeof(left)) == 0; +} + +inline bool operator==(const uint32_t& left, const in_addr& right) { + return memcmp(&left, &right, sizeof(left)) == 0; +} + +inline bool operator==(const in6_addr & left, const std::string_view & right) { + return right.size() == sizeof(left) && memcmp(&left, right.data(), sizeof(left)) == 0; +} + +inline bool operator==(const std::string_view & left, const in6_addr & right) { + return left.size() == sizeof(right) && memcmp(left.data(), &right, sizeof(right)) == 0; +} + +inline bool operator!=(const in6_addr& left, const in6_addr& right) { + return !(left == right); +} + +inline bool operator!=(const in_addr& left, const in_addr& right) { + return !(left == right); +} + +inline bool operator!=(const in_addr & left, const uint32_t& right) { + return !(left == right); +} + +inline bool operator!=(const uint32_t& left, const in_addr& right) { + return !(left == right); +} + +inline bool operator!=(const in6_addr & left, const std::string_view & right) { + return !(left == right); +} + +inline bool operator!=(const std::string_view & left, const in6_addr & right) { + return !(left == right); +} + +namespace details { +// Make a column a RO stl-like container +template +struct ColumnAsContainerWrapper { + const NestedColumnType& nested_col; + + struct Iterator { + const NestedColumnType& nested_col; + size_t i = 0; + + auto& operator++() { + ++i; + return *this; + } + + auto operator*() const { + return nested_col.At(i); + } + + bool operator==(const Iterator & other) const { + return &other.nested_col == &this->nested_col && other.i == this->i; + } + + bool operator!=(const Iterator & other) const { + return !(other == *this); + } + }; + + size_t size() const { + return nested_col.Size(); + } + + auto begin() const { + return Iterator{nested_col, 0}; + } + + auto end() const { + return Iterator{nested_col, nested_col.Size()}; + } +}; +} + +template +auto maybeWrapColumnAsContainer(const T & t) { + if constexpr (std::is_base_of_v) { + return ::details::ColumnAsContainerWrapper{t}; + } else { + return t; + } +} + + +// Compare values to each other, if values are container-ish, then recursively deep compare those containers. +template +::testing::AssertionResult CompareRecursive(const Left & left, const Right & right); + +// Compare containers element-wise, if elements are containers themselves - compare recursevely +template +::testing::AssertionResult CompareCotainersRecursive(const LeftContainer& left, const RightContainer& right) { + if (left.size() != right.size()) + return ::testing::AssertionFailure() << "\nMismatching containers size, expected: " << left.size() << " actual: " << right.size(); + + auto l_i = std::begin(left); + auto r_i = std::begin(right); + + for (size_t i = 0; i < left.size(); ++i, ++l_i, ++r_i) { + auto result = CompareRecursive(*l_i, *r_i); + if (!result) + return result << "\n\nMismatch at pos: " << i + 1; + } + + return ::testing::AssertionSuccess(); +} + +template +struct PrintContainer; + +template +::testing::AssertionResult CompareRecursive(const Left & left, const Right & right) { + if constexpr ((is_container_v || std::is_base_of_v>) + && (is_container_v || std::is_base_of_v>) ) { + + const auto & l = maybeWrapColumnAsContainer(left); + const auto & r = maybeWrapColumnAsContainer(right); + + if (auto result = CompareCotainersRecursive(l, r)) + return result; + else + return result << "\nExpected container: " << PrintContainer{l} + << "\nActual container : " << PrintContainer{r}; + } else { + if (left != right) + return ::testing::AssertionFailure() + << "\nExpected value: " << left + << "\nActual value : " << right; + + return ::testing::AssertionSuccess(); + } +} diff --git a/ut/utils_meta.h b/ut/utils_meta.h new file mode 100644 index 00000000..707f9aca --- /dev/null +++ b/ut/utils_meta.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include // for std::begin + +// based on https://stackoverflow.com/a/31207079 +template +struct is_container : std::false_type {}; + +namespace details { +template +struct is_container_helper {}; +} + +// A very loose definition of container, nerfed to fit both C-array and ColumnArrayT::ArrayWrapper +template +struct is_container< + T, + std::conditional_t< + false, + ::details::is_container_helper< + decltype(std::declval().size()), + decltype(std::begin(std::declval())), + decltype(std::end(std::declval())) + >, + void + > + > : public std::true_type {}; + +template +inline constexpr bool is_container_v = is_container::value; + +// Since result_of is deprecated in C++17, and invoke_result_of is unavailable until C++20... +template +using my_result_of_t = +#if __cplusplus >= 201703L + std::invoke_result_t; +#else + std::result_of_t; +#endif + diff --git a/ut/utils_performance.h b/ut/utils_performance.h new file mode 100644 index 00000000..b6123589 --- /dev/null +++ b/ut/utils_performance.h @@ -0,0 +1,67 @@ +#pragma once + +#include "utils_meta.h" + +#include +#include +#include +#include + +template +struct Timer { + using DurationType = ChronoDurationType; + + Timer() + : started_at(Now()) + {} + + void Restart() { + started_at = Now(); + } + + void Start() { + Restart(); + } + + auto Elapsed() const { + return std::chrono::duration_cast(Now() - started_at); + } + +private: + static auto Now() { + return std::chrono::high_resolution_clock::now().time_since_epoch(); + } + +private: + std::chrono::nanoseconds started_at; +}; + +template +class MeasuresCollector { +public: + using Result = my_result_of_t; + + explicit MeasuresCollector(MeasureFunc && measurment_func, const size_t preallocate_results = 10) + : measurment_func_(std::move(measurment_func)) + { + results_.reserve(preallocate_results); + } + + template + void Add(NameType && name) { + results_.emplace_back(name, measurment_func_()); + } + + const auto & GetResults() const { + return results_; + } + +private: + MeasureFunc measurment_func_; + std::vector> results_; +}; + +template +MeasuresCollector collect(MeasureFunc && f) { + return MeasuresCollector(std::forward(f)); +} diff --git a/ut/value_generators.cpp b/ut/value_generators.cpp new file mode 100644 index 00000000..668a5ca2 --- /dev/null +++ b/ut/value_generators.cpp @@ -0,0 +1,132 @@ +#include "value_generators.h" + +#include +#include + +namespace { +using namespace clickhouse; +} + +std::vector MakeNumbers() { + return std::vector {1, 2, 3, 7, 11, 13, 17, 19, 23, 29, 31}; +} + +std::vector MakeBools() { + return std::vector {1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0}; +} + +std::vector MakeFixedStrings(size_t string_size) { + std::vector result {"aaa", "bbb", "ccc", "ddd"}; + + std::for_each(result.begin(), result.end(), [string_size](auto& value) { + value.resize(string_size, '\0'); + }); + + return result; +} + +std::vector MakeStrings() { + return {"a", "ab", "abc", "abcd"}; +} + +std::vector MakeUUIDs() { + return { + UInt128(0llu, 0llu), + UInt128(0xbb6a8c699ab2414cllu, 0x86697b7fd27f0825llu), + UInt128(0x84b9f24bc26b49c6llu, 0xa03b4ab723341951llu), + UInt128(0x3507213c178649f9llu, 0x9faf035d662f60aellu) + }; +} + +std::vector MakeDateTime64s(size_t scale, size_t values_size) { + const auto seconds_multiplier = static_cast(std::pow(10, scale)); + const auto year = 86400ull * 365 * seconds_multiplier; // ~approx, but this doesn't matter here. + + // Approximatelly +/- 200 years around epoch (and value of epoch itself) + // with non zero seconds and sub-seconds. + // Please note there are values outside of DateTime (32-bit) range that might + // not have correct string representation in CH yet, + // but still are supported as Int64 values. + return GenerateVector(values_size, + [seconds_multiplier, year] (size_t i )-> Int64 { + return (i - 100) * year * 2 + (i * 10) * seconds_multiplier + i; + }); +} + +std::vector MakeDates() { + // in CH Date internally a UInt16 and stores a day number + // ColumnDate expects values to be seconds, which is then + // converted to day number internally, hence the `* 86400`. + std::vector result {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; + std::for_each(result.begin(), result.end(), [](auto& value) { + value *= 86400; + }); + + return result; +} + +std::vector MakeDateTimes() { + // in CH DateTime internally a UInt32 + return { + 0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, + 131072, 262144, 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, + 134217728, 268435456, 536870912, 1073741824, 2147483648, 4294967296 - 1 + }; +} + +std::vector MakeInt128s() { + return { + absl::MakeInt128(0xffffffffffffffffll, 0xffffffffffffffffll), // -1 + absl::MakeInt128(0, 0xffffffffffffffffll), // 2^64 + absl::MakeInt128(0xffffffffffffffffll, 0), + absl::MakeInt128(0x8000000000000000ll, 0), + Int128(0) + }; +} + +std::vector MakeDecimals(size_t /*precision*/, size_t scale) { + const auto scale_multiplier = static_cast(std::pow(10, scale)); + const auto rhs_value = 12345678910; + + const std::vector vals {0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536 - 1}; + + std::vector result; + result.reserve(vals.size()); + + std::transform(vals.begin(), vals.end(), std::back_inserter(result), [scale_multiplier, rhs_value](const auto& value) { + return value * scale_multiplier + rhs_value % scale_multiplier; + }); + + return result; +} + +std::string FooBarGenerator(size_t i) { + std::string result; + if (i % 3 == 0) + result += "Foo"; + if (i % 5 == 0) + result += "Bar"; + if (result.empty()) + result = std::to_string(i); + + return result; +} + +std::vector MakeIPv4s() { + return { + MakeIPv4(0x12345678), // 255.255.255.255 + MakeIPv4(0x0100007f), // 127.0.0.1 + MakeIPv4(3585395774), + MakeIPv4(0), + MakeIPv4(0x12345678), + }; +} + +std::vector MakeIPv6s() { + return { + MakeIPv6(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), // 1:203:405:607:809:a0b:c0d:e0f + MakeIPv6(0, 0, 0, 0, 0, 1), // ::1 + MakeIPv6(0, 0, 0, 0, 0, 0), // :: + MakeIPv6(0xff, 0xff, 204, 152, 189, 116), // ::ffff:204.152.189.116 + }; +} diff --git a/ut/value_generators.h b/ut/value_generators.h new file mode 100644 index 00000000..89a872a1 --- /dev/null +++ b/ut/value_generators.h @@ -0,0 +1,122 @@ +#pragma once + +#include // for ipv4-ipv6 platform-specific stuff +#include + +#include "utils.h" + +#include +#include + +inline in_addr MakeIPv4(uint32_t ip) { + static_assert(sizeof(in_addr) == sizeof(ip)); + in_addr result; + memcpy(&result, &ip, sizeof(ip)); + + return result; +} + +inline in6_addr MakeIPv6(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, + uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, + uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) { + return in6_addr{{{v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15}}}; +} + +inline in6_addr MakeIPv6(uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) { + return in6_addr{{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, v10, v11, v12, v13, v14, v15}}}; +} + +std::vector MakeNumbers(); +std::vector MakeBools(); +std::vector MakeFixedStrings(size_t string_size); +std::vector MakeStrings(); +std::vector MakeDateTime64s(size_t scale, size_t values_size = 200); +std::vector MakeDates(); +std::vector MakeDateTimes(); +std::vector MakeIPv4s(); +std::vector MakeIPv6s(); +std::vector MakeUUIDs(); +std::vector MakeInt128s(); +std::vector MakeDecimals(size_t precision, size_t scale); + +std::string FooBarGenerator(size_t i); + +template +auto GenerateVector(size_t items, Generator && gen) { + using ActualValueType = std::conditional_t, my_result_of_t, ValueType>; + std::vector result; + result.reserve(items); + for (size_t i = 0; i < items; ++i) { + result.push_back(std::move(gen(i))); + } + + return result; +} + +template +auto SameValueGenerator(const U & value) { + return [&value](size_t) -> T { + return value; + }; +} + +template +auto AlternateGenerators(Generator1 && gen1, Generator2 && gen2) { + return [&gen1, &gen2](size_t i) -> ResultType { + if (i % 2 == 0) + return gen1(i/2); + else + return gen2(i/2); + }; +} + +template +struct RandomGenerator { + using uniform_distribution = + typename std::conditional_t, std::uniform_real_distribution, + typename std::conditional_t, std::uniform_int_distribution, void>>; + + explicit RandomGenerator(T seed = 0, T value_min = std::numeric_limits::min(), T value_max = std::numeric_limits::max()) + : random_engine(seed) + , distribution(value_min, value_max) + { + } + + template + T operator()(U) { + return distribution(random_engine); + } + +private: + std::default_random_engine random_engine; + uniform_distribution distribution; +}; + +template +std::vector ConcatSequences(std::vector && vec1, std::vector && vec2) { + std::vector result(vec1); + + result.reserve(vec1.size() + vec2.size()); + result.insert(result.end(), vec2.begin(), vec2.end()); + + return result; +} + +template +struct FromVectorGenerator { + const std::vector data; + RandomGenerator random_generator; + + FromVectorGenerator(std::vector data_) + : data(std::move(data_)), + random_generator(0, 0, data.size() - 1) + { + if (data.size() == 0) + throw std::runtime_error("can't generate values from empty vector"); + } + + auto operator()(size_t pos) { + return data[random_generator(pos)]; + } +};