Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FLASH-617] Make cast when applying DDL #328

Merged
merged 4 commits into from
Nov 25, 2019
Merged

[FLASH-617] Make cast when applying DDL #328

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
35cd227
make cast when applying DDL
leiysky Nov 22, 2019
f2e5ee6
handle nullable
leiysky Nov 25, 2019
eca13ee
enhance handleing nullable
leiysky Nov 25, 2019
eec992f
fix
leiysky Nov 25, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 0 additions & 101 deletions dbms/src/Storages/DeltaMerge/Chunk.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,107 +247,6 @@ void readChunkData(MutableColumns & columns,
{
tmp_col = define.type->createColumnConstWithDefaultValue(rows_limit);
}
// TODO: `define.default_value` may be not matched with `define.type`
// For example: ... ADD COLUMN f32 Float32 '1.23'
// After parsing and interpreting, we will get a `type` Float32,
// while `default_value` will be inferred as String.
// So we should do some process before reading value or during applying alter to guarantee the correctness.
else if (define.type->equals(*DataTypeFactory::instance().get("Float32"))
|| define.type->equals(*DataTypeFactory::instance().get("Float64")))
{
Float64 real;
auto dec32 = DecimalField(Decimal32(), 0);
auto dec64 = DecimalField(Decimal64(), 0);
auto dec128 = DecimalField(Decimal128(), 0);
auto dec256 = DecimalField(Decimal256(), 0);
if (define.default_value.tryGet(dec32))
{
real = static_cast<Float64>(dec32);
}
else if (define.default_value.tryGet(dec64))
{
real = static_cast<Float64>(dec64);
}
else if (define.default_value.tryGet(dec128))
{
real = static_cast<Float64>(dec128);
}
else if (define.default_value.tryGet(dec256))
{
real = static_cast<Float64>(dec256);
}
else
{
throw Exception("Unsupported literal for default value", ErrorCodes::NOT_IMPLEMENTED);
}
tmp_col = define.type->createColumnConst(rows_limit, Field(real));
}
else if (define.type->equals(*DataTypeFactory::instance().get("DateTime")))
{
auto date = safeGet<String>(define.default_value);
time_t time = 0;
ReadBufferFromMemory buf(date.data(), date.size());
readDateTimeText(time, buf);
tmp_col = define.type->createColumnConst(rows_limit, Field(UInt64(time)));
}
else if (std::strcmp(define.type->getFamilyName(), "Decimal") == 0)
{
Int64 value = 0;
Int128 value128 = 0;
Int256 value256 = 0;
UInt32 scale = 0;
{
auto dec32 = DecimalField(Decimal32(), 0);
auto dec64 = DecimalField(Decimal64(), 0);
auto dec128 = DecimalField(Decimal128(), 0);
auto dec256 = DecimalField(Decimal256(), 0);
if (define.default_value.tryGet(dec32))
{
value = dec32.getValue().value;
scale = dec32.getScale();
}
else if (define.default_value.tryGet(dec64))
{
value = dec64.getValue().value;
scale = dec64.getScale();
}
else if (define.default_value.tryGet(dec128))
{
value128 = dec128.getValue().value;
scale = dec128.getScale();
}
else if (define.default_value.tryGet(dec256))
{
value256 = dec256.getValue().value;
scale = dec256.getScale();
}
}

if (define.type->getTypeId() == TypeIndex::Decimal32)
{
auto dec = DecimalField<Decimal32>(value, scale);
tmp_col = define.type->createColumnConst(rows_limit, toField(dec));
}
else if (define.type->getTypeId() == TypeIndex::Decimal64)
{
auto dec = DecimalField<Decimal64>(value, scale);
tmp_col = define.type->createColumnConst(rows_limit, toField(dec));
}
else if (define.type->getTypeId() == TypeIndex::Decimal128)
{
auto dec = DecimalField<Decimal128>(value128, scale);
tmp_col = define.type->createColumnConst(rows_limit, toField(dec));
}
else if (define.type->getTypeId() == TypeIndex::Decimal256)
{
auto dec = DecimalField<Decimal256>(value256, scale);
tmp_col = define.type->createColumnConst(rows_limit, toField(dec));
}
else
{
throw Exception("Unsupported literal for default value", ErrorCodes::NOT_IMPLEMENTED);
}
}
else
{
tmp_col = define.type->createColumnConst(rows_limit, define.default_value);
Expand Down
87 changes: 86 additions & 1 deletion dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -939,6 +939,90 @@ namespace
// TODO maybe move to -internal.h ?
inline void setColumnDefineDefaultValue(const AlterCommand & command, ColumnDefine & define)
{
std::function<Field(Field, DataTypePtr)> castDefaultValue; // for lazy bind
castDefaultValue = [&](Field value, DataTypePtr type) -> Field {
if (type->equals(*DataTypeFactory::instance().get("Float32")) || type->equals(*DataTypeFactory::instance().get("Float64")))
leiysky marked this conversation as resolved.
Show resolved Hide resolved
{
if (value.getType() == Field::Types::Float64)
{
Float64 res = applyVisitor(FieldVisitorConvertToNumber<Float64>(), value);
return toField(res);
}
else if (value.getType() == Field::Types::Decimal32)
{
DecimalField<Decimal32> dec = safeGet<DecimalField<Decimal32>>(value);
Float64 res = dec.getValue().toFloat<Float64>(dec.getScale());
return toField(res);
}
else if (value.getType() == Field::Types::Decimal64)
{
DecimalField<Decimal64> dec = safeGet<DecimalField<Decimal64>>(value);
Float64 res = dec.getValue().toFloat<Float64>(dec.getScale());
return toField(res);
}
leiysky marked this conversation as resolved.
Show resolved Hide resolved
}
else if (std::strcmp(type->getFamilyName(), "FixedString") == 0)
{
String res = get<String>(value);
return toField(res);
}
else if (type->equals(*DataTypeFactory::instance().get("Int8")) || type->equals(*DataTypeFactory::instance().get("Int16"))
|| type->equals(*DataTypeFactory::instance().get("Int32")) || type->equals(*DataTypeFactory::instance().get("Int64")))
{
Int64 res = applyVisitor(FieldVisitorConvertToNumber<Int64>(), value);
return toField(res);
}
else if (type->equals(*DataTypeFactory::instance().get("UInt8")) || type->equals(*DataTypeFactory::instance().get("UInt16"))
|| type->equals(*DataTypeFactory::instance().get("UInt32")) || type->equals(*DataTypeFactory::instance().get("UInt64")))
{
UInt64 res = applyVisitor(FieldVisitorConvertToNumber<UInt64>(), value);
return toField(res);
}
else if (type->equals(*DataTypeFactory::instance().get("DateTime")))
{
auto date = safeGet<String>(value);
time_t time = 0;
ReadBufferFromMemory buf(date.data(), date.size());
readDateTimeText(time, buf);
return toField(time);
}
else if (std::strcmp(type->getFamilyName(), "Decimal") == 0)
{
if (auto dec = std::dynamic_pointer_cast<const DataTypeDecimal32>(type); dec)
{
Int64 v = applyVisitor(FieldVisitorConvertToNumber<Int64>(), value);
ScaleType scale = dec->getScale();
return DecimalField(Decimal32(v), scale);
}
else if (auto dec = std::dynamic_pointer_cast<const DataTypeDecimal64>(type); dec)
{
Int64 v = applyVisitor(FieldVisitorConvertToNumber<Int64>(), value);
ScaleType scale = dec->getScale();
return DecimalField(Decimal64(v), scale);
}
else if (auto dec = std::dynamic_pointer_cast<const DataTypeDecimal128>(type); dec)
{
Int64 v = applyVisitor(FieldVisitorConvertToNumber<Int64>(), value);
ScaleType scale = dec->getScale();
return DecimalField(Decimal128(v), scale);
}
else if (auto dec = std::dynamic_pointer_cast<const DataTypeDecimal256>(type); dec)
{
Int64 v = applyVisitor(FieldVisitorConvertToNumber<Int64>(), value);
ScaleType scale = dec->getScale();
return DecimalField(Decimal256(v), scale);
}
}
else if (std::strcmp(type->getFamilyName(), "Nullable") == 0)
{
auto nullable = std::dynamic_pointer_cast<const DataTypeNullable>(type);
DataTypePtr nested_type = nullable->getNestedType();
return castDefaultValue(value, nested_type);
}

leiysky marked this conversation as resolved.
Show resolved Hide resolved
return type->getDefault();
};

if (command.default_expression)
{
// a cast function
Expand All @@ -961,7 +1045,8 @@ inline void setColumnDefineDefaultValue(const AlterCommand & command, ColumnDefi
auto default_literal_in_cast = typeid_cast<const ASTLiteral *>(default_cast_expr->arguments->children[0].get());
if (default_literal_in_cast)
{
define.default_value = default_literal_in_cast->value;
Field default_value = castDefaultValue(default_literal_in_cast->value, define.type);
define.default_value = default_value;
}
else
{
Expand Down
7 changes: 4 additions & 3 deletions dbms/src/Storages/DeltaMerge/tests/gtest_dm_delta_merge_store.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1112,11 +1112,11 @@ try
com.column_name = col_name_to_add;

// mock default value
// actual ddl is like: ADD COLUMN `f32` Float32 DEFAULT 1.23
// actual ddl is like: ADD COLUMN `f32` Float32 DEFAULT 1.125
auto cast = std::make_shared<ASTFunction>();
{
cast->name = "CAST";
ASTPtr arg = std::make_shared<ASTLiteral>(toField(DecimalField(Decimal32(123), 2)));
ASTPtr arg = std::make_shared<ASTLiteral>(toField(DecimalField(Decimal32(1125), 3)));
cast->arguments = std::make_shared<ASTExpressionList>();
cast->children.push_back(cast->arguments);
cast->arguments->children.push_back(arg);
Expand Down Expand Up @@ -1153,7 +1153,8 @@ try
c->get(i, tmp);
// There is some loss of precision during the convertion, so we just do a rough comparison
Float64 epsilon = 0.00001;
EXPECT_TRUE(std::abs(tmp.get<Float64>() - 1.23) < epsilon);
Float64 v = std::abs(tmp.get<Float64>());
EXPECT_TRUE(v - 1.125 < epsilon);
}
}
}
Expand Down