Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhancement/issue 874 backport for field #875

Merged
merged 7 commits into from
Dec 16, 2024
Merged

Enhancement/issue 874 backport for field #875

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
79f4dfd
fix read/write DecimalField (#6719)
yl-lisen Dec 5, 2024
c83ae93
backport for field (#6629)
yl-lisen Nov 28, 2024
559e33b
Merge branch 'develop' into enhancement/issue-874-backport-for-field
yl-lisen Dec 9, 2024
9e41c05
Merge branch 'develop' into enhancement/issue-874-backport-for-field
yl-lisen Dec 9, 2024
48d63c2
Merge branch 'develop' into enhancement/issue-874-backport-for-field
yl-lisen Dec 12, 2024
1674d20
Merge branch 'develop' into enhancement/issue-874-backport-for-field
yl-lisen Dec 16, 2024
c2e3a23
fix flaky test
yl-lisen Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 20 additions & 4 deletions src/Common/FieldVisitorWriteBinary.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,26 @@ void FieldVisitorWriteBinary::operator() (const Int256 & x, WriteBuffer & buf) c
void FieldVisitorWriteBinary::operator() (const UUID & x, WriteBuffer & buf) const { writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const IPv4 & x, WriteBuffer & buf) const { writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const IPv6 & x, WriteBuffer & buf) const { writeBinary(x, buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal32> & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal64> & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal256> & x, WriteBuffer & buf) const { writeBinary(x.getValue(), buf); }
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal32> & x, WriteBuffer & buf) const
{
writeBinary(x.getValue(), buf);
writeVarUInt(x.getScale(), buf);
}
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal64> & x, WriteBuffer & buf) const
{
writeBinary(x.getValue(), buf);
writeVarUInt(x.getScale(), buf);
}
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal128> & x, WriteBuffer & buf) const
{
writeBinary(x.getValue(), buf);
writeVarUInt(x.getScale(), buf);
}
void FieldVisitorWriteBinary::operator() (const DecimalField<Decimal256> & x, WriteBuffer & buf) const
{
writeBinary(x.getValue(), buf);
writeVarUInt(x.getScale(), buf);
}
void FieldVisitorWriteBinary::operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const
{
writeStringBinary(x.name, buf);
Expand Down
133 changes: 75 additions & 58 deletions src/Core/Field.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,27 @@
#include <Common/FieldVisitorWriteBinary.h>


using namespace std::literals;

namespace DB
{

namespace ErrorCodes
{
extern const int CANNOT_RESTORE_FROM_FIELD_DUMP;
extern const int DECIMAL_OVERFLOW;
extern const int INCORRECT_DATA;
}

inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
template <typename T>
T DecimalField<T>::getScaleMultiplier() const
{
switch (type)
return DecimalUtils::scaleMultiplier<T>(scale);
}

Field getBinaryValue(UInt8 type, ReadBuffer & buf)
{
switch (static_cast<Field::Types::Which>(type))
{
case Field::Types::Null:
{
Expand Down Expand Up @@ -96,7 +106,7 @@ inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
case Field::Types::Array:
{
Array value;
readBinary(value, buf);
readBinaryArray(value, buf);
return value;
}
case Field::Types::Tuple:
Expand Down Expand Up @@ -130,32 +140,31 @@ inline Field getBinaryValue(UInt8 type, ReadBuffer & buf)
readBinary(value, buf);
return bool(value);
}
case Field::Types::Decimal32:
case Field::Types::Decimal64:
case Field::Types::Decimal128:
case Field::Types::Decimal256:
return Field();
}
return Field();
throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown field type {}", std::to_string(type));
}

void readBinary(Array & x, ReadBuffer & buf)
void readBinaryArray(Array & x, ReadBuffer & buf)
{
size_t size;
UInt8 type;
readBinary(type, buf);
readBinary(size, buf);

for (size_t index = 0; index < size; ++index)
x.push_back(getBinaryValue(type, buf));
x.push_back(readFieldBinary(buf));
}

void writeBinary(const Array & x, WriteBuffer & buf)
void writeBinaryArray(const Array & x, WriteBuffer & buf)
{
UInt8 type = Field::Types::Null;
size_t size = x.size();
if (size)
type = x.front().getType();
writeBinary(type, buf);
writeBinary(size, buf);

for (const auto & elem : x)
Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem);
writeFieldBinary(elem, buf);
}

void writeText(const Array & x, WriteBuffer & buf)
Expand All @@ -170,11 +179,7 @@ void readBinary(Tuple & x, ReadBuffer & buf)
readBinary(size, buf);

for (size_t index = 0; index < size; ++index)
{
UInt8 type;
readBinary(type, buf);
x.push_back(getBinaryValue(type, buf));
}
x.push_back(readFieldBinary(buf));
}

void writeBinary(const Tuple & x, WriteBuffer & buf)
Expand All @@ -183,11 +188,7 @@ void writeBinary(const Tuple & x, WriteBuffer & buf)
writeBinary(size, buf);

for (const auto & elem : x)
{
const UInt8 type = elem.getType();
writeBinary(type, buf);
Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem);
}
writeFieldBinary(elem, buf);
}

void writeText(const Tuple & x, WriteBuffer & buf)
Expand All @@ -201,11 +202,7 @@ void readBinary(Map & x, ReadBuffer & buf)
readBinary(size, buf);

for (size_t index = 0; index < size; ++index)
{
UInt8 type;
readBinary(type, buf);
x.push_back(getBinaryValue(type, buf));
}
x.push_back(readFieldBinary(buf));
}

void writeBinary(const Map & x, WriteBuffer & buf)
Expand All @@ -214,11 +211,7 @@ void writeBinary(const Map & x, WriteBuffer & buf)
writeBinary(size, buf);

for (const auto & elem : x)
{
const UInt8 type = elem.getType();
writeBinary(type, buf);
Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem);
}
writeFieldBinary(elem, buf);
}

void writeText(const Map & x, WriteBuffer & buf)
Expand Down Expand Up @@ -292,6 +285,19 @@ void writeFieldText(const Field & x, WriteBuffer & buf)
buf.write(res.data(), res.size());
}

void writeFieldBinary(const Field & x, WriteBuffer & buf)
{
const UInt8 type = x.getType();
writeBinary(type, buf);
Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, x);
}

Field readFieldBinary(ReadBuffer & buf)
{
UInt8 type;
readBinary(type, buf);
return getBinaryValue(type, buf);
}

String Field::dump() const
{
Expand Down Expand Up @@ -549,7 +555,7 @@ template bool decimalLessOrEqual<Decimal256>(Decimal256 x, Decimal256 y, UInt32
template bool decimalLessOrEqual<DateTime64>(DateTime64 x, DateTime64 y, UInt32 x_scale, UInt32 y_scale);


inline void writeText(const Null & x, WriteBuffer & buf)
void writeText(const Null & x, WriteBuffer & buf)
{
if (x.isNegativeInfinity())
writeText("-inf", buf);
Expand All @@ -571,33 +577,44 @@ String toString(const Field & x)
x);
}

String fieldTypeToString(Field::Types::Which type)
std::string_view fieldTypeToString(Field::Types::Which type)
{
switch (type)
{
case Field::Types::Which::Null: return "null";
case Field::Types::Which::Array: return "array";
case Field::Types::Which::Tuple: return "tuple";
case Field::Types::Which::Map: return "map";
case Field::Types::Which::Object: return "json";
case Field::Types::Which::AggregateFunctionState: return "aggregate_function_state";
case Field::Types::Which::Bool: return "bool";
case Field::Types::Which::String: return "string";
case Field::Types::Which::Decimal32: return "decimal32";
case Field::Types::Which::Decimal64: return "decimal64";
case Field::Types::Which::Decimal128: return "decimal128";
case Field::Types::Which::Decimal256: return "decimal256";
case Field::Types::Which::Float64: return "float64";
case Field::Types::Which::Int64: return "int64";
case Field::Types::Which::Int128: return "int128";
case Field::Types::Which::Int256: return "int256";
case Field::Types::Which::UInt64: return "uint64";
case Field::Types::Which::UInt128: return "uint128";
case Field::Types::Which::UInt256: return "uint256";
case Field::Types::Which::UUID: return "uuid";
case Field::Types::Which::IPv4: return "ipv4";
case Field::Types::Which::IPv6: return "ipv6";
case Field::Types::Which::Null: return "null"sv;
case Field::Types::Which::Array: return "array"sv;
case Field::Types::Which::Tuple: return "tuple"sv;
case Field::Types::Which::Map: return "map"sv;
case Field::Types::Which::Object: return "json"sv;
case Field::Types::Which::AggregateFunctionState: return "aggregate_function_state"sv;
case Field::Types::Which::Bool: return "bool"sv;
case Field::Types::Which::String: return "string"sv;
case Field::Types::Which::Decimal32: return "decimal32"sv;
case Field::Types::Which::Decimal64: return "decimal64"sv;
case Field::Types::Which::Decimal128: return "decimal128"sv;
case Field::Types::Which::Decimal256: return "decimal256"sv;
case Field::Types::Which::Float64: return "float64"sv;
case Field::Types::Which::Int64: return "int64"sv;
case Field::Types::Which::Int128: return "int128"sv;
case Field::Types::Which::Int256: return "int256"sv;
case Field::Types::Which::UInt64: return "uint64"sv;
case Field::Types::Which::UInt128: return "uint128"sv;
case Field::Types::Which::UInt256: return "uint256"sv;
case Field::Types::Which::UUID: return "uuid"sv;
case Field::Types::Which::IPv4: return "ipv4"sv;
case Field::Types::Which::IPv6: return "ipv6"sv;
}
}

/// Keep in mind, that "magic_enum" is very expensive for compiler, that's why we don't use it.
std::string_view Field::getTypeName() const
{
return fieldTypeToString(which);
}

template class DecimalField<Decimal32>;
template class DecimalField<Decimal64>;
template class DecimalField<Decimal128>;
template class DecimalField<Decimal256>;
template class DecimalField<DateTime64>;
}
28 changes: 17 additions & 11 deletions src/Core/Field.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
#include <Common/AllocatorWithMemoryTracking.h>
#include <Core/Types.h>
#include <Core/Defines.h>
#include <Core/DecimalFunctions.h>
#include <Core/UUID.h>
#include <base/IPv4andIPv6.h>
#include <base/DayNum.h>
Expand Down Expand Up @@ -117,7 +116,7 @@ class DecimalField

operator T() const { return dec; } /// NOLINT
T getValue() const { return dec; }
T getScaleMultiplier() const { return DecimalUtils::scaleMultiplier<T>(scale); }
T getScaleMultiplier() const;
UInt32 getScale() const { return scale; }

template <typename U>
Expand Down Expand Up @@ -166,6 +165,12 @@ class DecimalField
UInt32 scale;
};

extern template class DecimalField<Decimal32>;
extern template class DecimalField<Decimal64>;
extern template class DecimalField<Decimal128>;
extern template class DecimalField<Decimal256>;
extern template class DecimalField<DateTime64>;

template <typename T> constexpr bool is_decimal_field = false;
template <> constexpr inline bool is_decimal_field<DecimalField<Decimal32>> = true;
template <> constexpr inline bool is_decimal_field<DecimalField<Decimal64>> = true;
Expand Down Expand Up @@ -414,7 +419,7 @@ class Field

Types::Which getType() const { return which; }

constexpr std::string_view getTypeName() const { return magic_enum::enum_name(which); }
std::string_view getTypeName() const;

bool isNull() const { return which == Types::Null; }
template <typename T>
Expand Down Expand Up @@ -902,14 +907,15 @@ inline Field & Field::operator=(String && str)
class ReadBuffer;
class WriteBuffer;

/// It is assumed that all elements of the array have the same type.
void readBinary(Array & x, ReadBuffer & buf);
[[noreturn]] inline void readText(Array &, ReadBuffer &) { throw Exception("Cannot read array.", ErrorCodes::NOT_IMPLEMENTED); }
[[noreturn]] inline void readQuoted(Array &, ReadBuffer &) { throw Exception("Cannot read array.", ErrorCodes::NOT_IMPLEMENTED); }
/// Binary serialization of generic field.
void writeFieldBinary(const Field & x, WriteBuffer & buf);
Field readFieldBinary(ReadBuffer & buf);

void readBinaryArray(Array & x, ReadBuffer & buf);
[[noreturn]] inline void readText(Array &, ReadBuffer &) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot read Array."); }
[[noreturn]] inline void readQuoted(Array &, ReadBuffer &) { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot read Array."); }

/// It is assumed that all elements of the array have the same type.
/// Also write size and type into buf. UInt64 and Int64 is written in variadic size form
void writeBinary(const Array & x, WriteBuffer & buf);
void writeBinaryArray(const Array & x, WriteBuffer & buf);
void writeText(const Array & x, WriteBuffer & buf);
[[noreturn]] inline void writeQuoted(const Array &, WriteBuffer &) { throw Exception("Cannot write array quoted.", ErrorCodes::NOT_IMPLEMENTED); }

Expand Down Expand Up @@ -957,7 +963,7 @@ void writeFieldText(const Field & x, WriteBuffer & buf);

String toString(const Field & x);

String fieldTypeToString(Field::Types::Which type);
std::string_view fieldTypeToString(Field::Types::Which type);

}

Expand Down
4 changes: 2 additions & 2 deletions src/Interpreters/ConvertStringsToEnumVisitor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ namespace
/// @note We place strings in ascending order here under the assumption it colud speed up String to Enum conversion.
String makeStringsEnum(const std::set<String> & values)
{
String enum_string = "Enum8(";
String enum_string = "enum8(";
if (values.size() >= 255)
enum_string = "Enum16(";
enum_string = "enum16(";

size_t number = 1;
for (const auto & item : values)
Expand Down
2 changes: 1 addition & 1 deletion tests/queries_ported/0_stateless/01324_if_transform_strings_to_enum.reference
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ other
other
other
SELECT
transform(number, [2, 4, 6], [\'google\', \'yandex\', \'yahoo\'], \'other\')
transform(number, [2, 4, 6], _cast([\'google\', \'yandex\', \'yahoo\'], \'array(enum8(\\\'google\\\' = 1, \\\'other\\\' = 2, \\\'yahoo\\\' = 3, \\\'yandex\\\' = 4))\'), _cast(\'other\', \'enum8(\\\'google\\\' = 1, \\\'other\\\' = 2, \\\'yahoo\\\' = 3, \\\'yandex\\\' = 4)\'))
FROM
system.numbers
LIMIT 10
Expand Down
Loading
Loading