[FLASH-453]Support default value of CAST expression (pingcap#286)

* refactor default_value * Apply suggestions from code review * enable alter default value test * add default_value to make sure read can fill value with it * fix issue with reading Float default value * add test for read Float default value * resolve suggestions * add abs * fix issue with DateTime default value * add test for DateTime default value * modify test case * handle Decimal64 * modify expect values * format * fix issue with Decimal default_value * modify test case * resolve suggestions * fix bug: column in StorageDeltaMerge but not in TiDB::TableInfo::columns
JaySon-Huang · Nov 1, 2019 · 4fab794 · 4fab794
1 parent 3db1e16
commit 4fab794
Show file tree

Hide file tree

Showing 11 changed files with 334 additions and 50 deletions.
diff --git a/dbms/src/Core/ColumnWithTypeAndName.h b/dbms/src/Core/ColumnWithTypeAndName.h
@@ -25,10 +25,11 @@ struct ColumnWithTypeAndName
 
     /// TODO Handle column_id properly after we support DDL.
     Int64 column_id;
+    Field default_value;
 
     ColumnWithTypeAndName(): ColumnWithTypeAndName(nullptr, nullptr, "") {}
-    ColumnWithTypeAndName(ColumnPtr column_, const DataTypePtr & type_, const String & name_, Int64 column_id_ = 0)
-        : column(std::move(column_)), type(type_), name(name_), column_id(column_id_) {}
+    ColumnWithTypeAndName(ColumnPtr column_, const DataTypePtr & type_, const String & name_, Int64 column_id_ = 0, const Field & default_value_ = Field())
+        : column(std::move(column_)), type(type_), name(name_), column_id(column_id_), default_value(default_value_) {}
 
     /// Uses type->createColumn() to create column
     ColumnWithTypeAndName(const DataTypePtr & type_, const String & name_)

diff --git a/dbms/src/Storages/DeltaMerge/Chunk.cpp b/dbms/src/Storages/DeltaMerge/Chunk.cpp
@@ -1,9 +1,12 @@
+#include <cstring>
+
 #include <Storages/DeltaMerge/Chunk.h>
 
 #include <DataTypes/isSupportedDataTypeCast.h>
 #include <Functions/FunctionHelpers.h>
 #include <IO/CompressedReadBuffer.h>
 #include <IO/CompressedWriteBuffer.h>
+#include <IO/ReadHelpers.h>
 
 namespace DB
 {
@@ -238,20 +241,120 @@ void readChunkData(MutableColumns &      columns,
             // New column after ddl is not exist in chunk's meta, fill with default value
             IColumn & col = *columns[index];
 
-            if (define.default_value.empty())
+            // Read default value from `define.default_value`
+            ColumnPtr tmp_col;
+            if (define.default_value.isNull())
+            {
+                tmp_col = define.type->createColumnConstWithDefaultValue(rows_limit);
+            }
+            // TODO: `define.default_value` may be not matched with `define.type`
+            // For example: ... ADD COLUMN f32 Float32 '1.23'
+            // After parsing and interpreting, we will get a `type` Float32,
+            // while `default_value` will be inferred as String.
+            // So we should do some process before reading value or during applying alter to guarantee the correctness.
+            else if (define.type->equals(*DataTypeFactory::instance().get("Float32"))
+                     || define.type->equals(*DataTypeFactory::instance().get("Float64")))
             {
-                ColumnPtr tmp_col = define.type->createColumnConstWithDefaultValue(rows_limit)->convertToFullColumnIfConst();
-                col.insertRangeFrom(*tmp_col, 0, rows_limit);
+                Float64 real;
+                auto    dec32  = DecimalField(Decimal32(), 0);
+                auto    dec64  = DecimalField(Decimal64(), 0);
+                auto    dec128 = DecimalField(Decimal128(), 0);
+                auto    dec256 = DecimalField(Decimal256(), 0);
+                if (define.default_value.tryGet(dec32))
+                {
+                    real = static_cast<Float64>(dec32);
+                }
+                else if (define.default_value.tryGet(dec64))
+                {
+                    real = static_cast<Float64>(dec64);
+                }
+                else if (define.default_value.tryGet(dec128))
+                {
+                    real = static_cast<Float64>(dec128);
+                }
+                else if (define.default_value.tryGet(dec256))
+                {
+                    real = static_cast<Float64>(dec256);
+                }
+                else
+                {
+                    throw Exception("Unsupported literal for default value", ErrorCodes::NOT_IMPLEMENTED);
+                }
+                tmp_col = define.type->createColumnConst(rows_limit, Field(real));
+            }
+            else if (define.type->equals(*DataTypeFactory::instance().get("DateTime")))
+            {
+                auto                 date = safeGet<String>(define.default_value);
+                time_t               time;
+                ReadBufferFromMemory buf(date.data(), date.size());
+                readDateTimeText(time, buf);
+                tmp_col = define.type->createColumnConst(rows_limit, Field(UInt64(time)));
+            }
+            else if (std::strcmp(define.type->getFamilyName(), "Decimal") == 0)
+            {
+                Int64  value;
+                Int128 value128;
+                Int256 value256;
+                UInt32 scale;
+                {
+                    auto dec32  = DecimalField(Decimal32(), 0);
+                    auto dec64  = DecimalField(Decimal64(), 0);
+                    auto dec128 = DecimalField(Decimal128(), 0);
+                    auto dec256 = DecimalField(Decimal256(), 0);
+                    if (define.default_value.tryGet(dec32))
+                    {
+                        value = dec32.getValue().value;
+                        scale = dec32.getScale();
+                    }
+                    else if (define.default_value.tryGet(dec64))
+                    {
+                        value = dec64.getValue().value;
+                        scale = dec64.getScale();
+                    }
+                    else if (define.default_value.tryGet(dec128))
+                    {
+                        value128 = dec128.getValue().value;
+                        scale = dec128.getScale();
+                    }
+                    else if (define.default_value.tryGet(dec256))
+                    {
+                        value256 = dec256.getValue().value;
+                        scale = dec256.getScale();
+                    }
+                }
+
+                if (define.type->getTypeId() == TypeIndex::Decimal32)
+                {
+                    auto dec = DecimalField<Decimal32>(value, scale);
+                    tmp_col  = define.type->createColumnConst(rows_limit, toField(dec));
+                }
+                else if (define.type->getTypeId() == TypeIndex::Decimal64)
+                {
+                    auto dec = DecimalField<Decimal64>(value, scale);
+                    tmp_col  = define.type->createColumnConst(rows_limit, toField(dec));
+                }
+                else if (define.type->getTypeId() == TypeIndex::Decimal128)
+                {
+                    auto dec = DecimalField<Decimal128>(value128, scale);
+                    tmp_col  = define.type->createColumnConst(rows_limit, toField(dec));
+                }
+                else if (define.type->getTypeId() == TypeIndex::Decimal256)
+                {
+                    auto dec = DecimalField<Decimal256>(value256, scale);
+                    tmp_col  = define.type->createColumnConst(rows_limit, toField(dec));
+                }
+                else
+                {
+                    throw Exception("Unsupported literal for default value", ErrorCodes::NOT_IMPLEMENTED);
+                }
             }
             else
             {
-                // Read default value from `define.default_value`
-                MutableColumnPtr     tmp_col = define.type->createColumn();
-                ReadBufferFromMemory buff(define.default_value.c_str(), define.default_value.size());
-                define.type->deserializeTextEscaped(*tmp_col, buff);
-                ColumnPtr tmp_full_col = tmp_col->replicate(IColumn::Offsets(1, rows_limit));
-                col.insertRangeFrom(*tmp_full_col, 0, rows_limit);
+                tmp_col = define.type->createColumnConst(rows_limit, define.default_value);
             }
+            tmp_col = tmp_col->convertToFullColumnIfConst();
+
+            col.insertRangeFrom(*tmp_col, 0, rows_limit);
         }
     }
 
@@ -643,11 +746,21 @@ void insertRangeFromWithNumericTypeCast(const ColumnPtr &    from_col, //
         /// We are applying cast from nullable to not null, scan to fill "NULL" with default value
 
         TypeTo default_value = 0; // if read_define.default_value is empty, fill with 0
-        if (!read_define.default_value.empty())
+        if (read_define.default_value.isNull())
+        {
+            // Do nothing
+        }
+        else if (read_define.default_value.getType() == Field::Types::Int64)
+        {
+            default_value = read_define.default_value.safeGet<Int64>();
+        }
+        else if (read_define.default_value.getType() == Field::Types::UInt64)
+        {
+            default_value = read_define.default_value.safeGet<UInt64>();
+        }
+        else
         {
-            // parse from text
-            ReadBufferFromMemory buff(read_define.default_value.c_str(), read_define.default_value.size());
-            readIntTextUnsafe(default_value, buff);
+            throw Exception("Invalid column value type", ErrorCodes::BAD_ARGUMENTS);
         }
 
         const size_t to_offset_before_inserted = to_array_ptr->size() - rows_limit;

diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h b/dbms/src/Storages/DeltaMerge/DeltaMergeDefines.h
@@ -72,7 +72,7 @@ struct ColumnDefine
     ColId       id;
     String      name;
     DataTypePtr type;
-    String      default_value;
+    Field       default_value;
 
     explicit ColumnDefine(ColId id_ = 0, String name_ = "", DataTypePtr type_ = nullptr)
         : id(id_), name(std::move(name_)), type(std::move(type_))

diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h b/dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h
@@ -123,17 +123,17 @@ inline PaddedPODArray<T> const * getColumnVectorDataPtr(const Block & block, siz
     return toColumnVectorDataPtr<T>(block.getByPosition(pos).column);
 }
 
-inline void addColumnToBlock(Block & block, ColId col_id, const String & col_name, const DataTypePtr & col_type, const ColumnPtr & col)
+inline void addColumnToBlock(Block & block, ColId col_id, const String & col_name, const DataTypePtr & col_type, const ColumnPtr & col, const Field & default_value = Field())
 {
-    ColumnWithTypeAndName column(col, col_type, col_name, col_id);
+    ColumnWithTypeAndName column(col, col_type, col_name, col_id, default_value);
     block.insert(std::move(column));
 }
 
 inline Block toEmptyBlock(const ColumnDefines & columns)
 {
     Block block;
     for (auto & c : columns)
-        addColumnToBlock(block, c.id, c.name, c.type, c.type->createColumn());
+        addColumnToBlock(block, c.id, c.name, c.type, c.type->createColumn(), c.default_value);
     return block;
 }
 

diff --git a/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp b/dbms/src/Storages/DeltaMerge/DeltaMergeStore.cpp
@@ -820,8 +820,7 @@ inline void setColumnDefineDefaultValue(const AlterCommand & command, ColumnDefi
         if (auto default_literal = typeid_cast<const ASTLiteral *>(command.default_expression.get());
             default_literal && default_literal->value.getType() == Field::Types::String)
         {
-            const auto default_val = safeGet<String>(default_literal->value);
-            define.default_value   = default_val;
+            define.default_value = default_literal->value;
         }
         else if (auto default_cast_expr = typeid_cast<const ASTFunction *>(command.default_expression.get());
                  default_cast_expr && default_cast_expr->name == "CAST" /* ParserCastExpression::name */)
@@ -833,14 +832,13 @@ inline void setColumnDefineDefaultValue(const AlterCommand & command, ColumnDefi
             }
 
             auto default_literal_in_cast = typeid_cast<const ASTLiteral *>(default_cast_expr->arguments->children[0].get());
-            if (default_literal_in_cast && default_literal_in_cast->value.getType() == Field::Types::String)
+            if (default_literal_in_cast)
             {
-                const auto default_value = safeGet<String>(default_literal_in_cast->value);
-                define.default_value     = default_value;
+                define.default_value = default_literal_in_cast->value;
             }
             else
             {
-                throw Exception("First argument in CAST expression must be a string", ErrorCodes::NOT_IMPLEMENTED);
+                throw Exception("Invalid CAST expression", ErrorCodes::BAD_ARGUMENTS);
             }
         }
         else

diff --git a/dbms/src/Storages/DeltaMerge/tests/gtest_dm_chunk.cpp b/dbms/src/Storages/DeltaMerge/tests/gtest_dm_chunk.cpp
@@ -295,7 +295,7 @@ TEST(ChunkColumnCast_test, DISABLED_CastNullableToNotNullWithNonZeroDefaultValue
     {
         DataTypePtr  read_data_type = typeFromString(to_type);
         ColumnDefine read_define(0, "c", read_data_type);
-        read_define.default_value      = "5";
+        read_define.default_value      = Field(String("5"));
         MutableColumnPtr memory_column = read_data_type->createColumn();
         memory_column->reserve(3);