TiFlash crash because meeting null on non-nullable column

[JaySon-Huang]

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

According to the user, they adding a uniq index with column denomination recently

2. What did you expect to see? (Required)

3. What did you see instead (Required)

posted at https://asktug.com/t/topic/1017862

[2023/11/24 10:10:37.234 +08:00] [ERROR] [Exception.cpp:89] [“Code: 49, e.displayText() = DB::Exception: Detected invalid null when decoding data of column denomination with column type Decimal64: physical_table_id=3668: (while preHandleSnapshot region_id=2680177673, index=847, term=21), e.what() = DB::Exception, Stack trace:
 0x1718afe  DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits, std::__1::allocator > const&, int) [tiflash+24218366]
    dbms/src/Common/Exception.h:46
 0x6b1536a  bool DB::appendRowV2ToBlockImpl(std::__1::basic_string<char, std::__1::char_traits, std::__1::allocator > const&, std::__1::__map_const_iterator<std::__1::__tree_const_iterator<std::__1::__value_type<long, unsigned long>, std::__1::__tree_node<std::__1::__value_type<long, unsigned long>, void*>, long> >, std::__1::__map_const_iterator<std::__1::__tree_const_iterator<std::__1::__value_type<long, unsigned long>, std::__1::__tree_node<std::__1::__value_type<long, unsigned long>, void>, long> >, DB::Block&, unsigned long, std::__1::vector<TiDB::ColumnInfo, std::__1::allocatorTiDB::ColumnInfo > const&, long, bool, bool) [tiflash+112284522]
    dbms/src/Storages/Transaction/RowCodec.cpp:487
 0x6b13824  DB::appendRowToBlock(std::__1::basic_string<char, std::__1::char_traits, std::__1::allocator > const&, std::__1::__map_const_iterator<std::__1::__tree_const_iterator<std::__1::__value_type<long, unsigned long>, std::__1::__tree_node<std::__1::__value_type<long, unsigned long>, void>, long> >, std::__1::__map_const_iterator<std::__1::__tree_const_iterator<std::__1::__value_type<long, unsigned long>, std::__1::__tree_node<std::__1::__value_type<long, unsigned long>, void>, long> >, DB::Block&, unsigned long, std::__1::shared_ptr<DB::DecodingStorageSchemaSnapshot const> const&, bool) [tiflash+112277540]
    dbms/src/Storages/Transaction/RowCodec.cpp:349
 0x6ae0e53  bool DB::RegionBlockReader::readImpl<(DB::TMTPKType)0>(DB::Block&, std::__1::vector<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject const> >, std::__1::allocator<std::__1::tuple<DB::RawTiDBPK, unsigned char, unsigned long, std::__1::shared_ptr<DB::StringObject const> > > > const&, bool) [tiflash+112070227]
    dbms/src/Storages/Transaction/RegionBlockReader.cpp:146
 0x6abac58  DB::GenRegionBlockDataWithSchema(std::__1::shared_ptrDB::Region const&, std::__1::shared_ptr<DB::DecodingStorageSchemaSnapshot const> const&, unsigned long, bool, DB::TMTContext&) [tiflash+111914072]
    dbms/src/Storages/Transaction/PartitionStreams.cpp:598
 0x6a7089a  DB::DM::SSTFilesToBlockInputStream::readCommitedBlock() [tiflash+111610010]
    dbms/src/Storages/DeltaMerge/SSTFilesToBlockInputStream.cpp:255
 0x6a6f30e  DB::DM::SSTFilesToBlockInputStream::read() [tiflash+111604494]
    dbms/src/Storages/DeltaMerge/SSTFilesToBlockInputStream.cpp:154
 0x6946ea5  DB::DM::readNextBlock(std::__1::shared_ptrDB::IBlockInputStream const&) [tiflash+110390949]
    dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h:253
 0x6a71dec  DB::DM::PKSquashingBlockInputStream::read() [tiflash+111615468]
    dbms/src/Storages/DeltaMerge/PKSquashingBlockInputStream.h:78
 0x6946ea5  DB::DM::readNextBlock(std::__1::shared_ptrDB::IBlockInputStream const&) [tiflash+110390949]
    dbms/src/Storages/DeltaMerge/DeltaMergeHelpers.h:253
 0x16cbd35  DB::DM::DMVersionFilterBlockInputStream<1>::initNextBlock() [tiflash+23903541]
    dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.h:137
 0x16cb56b  DB::DM::DMVersionFilterBlockInputStream<1>::read(DB::PODArray<unsigned char, 4096ul, Allocator, 15ul, 16ul>&, bool) [tiflash+23901547]
    dbms/src/Storages/DeltaMerge/DMVersionFilterBlockInputStream.cpp:323
 0x6a71018  DB::DM::BoundedSSTFilesToBlockInputStream::read() [tiflash+111611928]
    dbms/src/Storages/DeltaMerge/SSTFilesToBlockInputStream.cpp:307
 0x16cf574  DB::DM::SSTFilesToDTFilesOutputStream<std::__1::shared_ptrDB::DM::BoundedSSTFilesToBlockInputStream >::write() [tiflash+23917940]
    dbms/src/Storages/DeltaMerge/SSTFilesToDTFilesOutputStream.cpp:200
 0x6a67b3f  DB::KVStore::preHandleSSTsToDTFiles(std::__1::shared_ptrDB::Region, DB::SSTViewVec, unsigned long, unsigned long, DB::DM::FileConvertJobType, DB::TMTContext&) [tiflash+111573823]
    dbms/src/Storages/Transaction/ApplySnapshot.cpp:360
 0x6a67214  DB::KVStore::preHandleSnapshotToFiles(std::__1::shared_ptrDB::Region, DB::SSTViewVec, unsigned long, unsigned long, DB::TMTContext&) [tiflash+111571476]
    dbms/src/Storages/Transaction/ApplySnapshot.cpp:275
 0x6ac2516  PreHandleSnapshot [tiflash+111944982]
    dbms/src/Storages/Transaction/ProxyFFI.cpp:388
 0x7fd8813cc228 engine_store_ffi::$LT$impl$u20$engine_store_ffi…interfaces…root…DB…EngineStoreServerHelper$GT$::pre_handle_snapshot::hec57f9b0ef29a0bb [libtiflash_proxy.so+17646120]
 0x7fd8813c3d09 engine_store_ffi::observer::pre_handle_snapshot_impl::h0b40090f59175b24 [libtiflash_proxy.so+17612041]
 0x7fd8813b6b86 yatp::task::future::RawTask$LT$F$GT$::poll::hd3296fb5cae316b9 [libtiflash_proxy.so+17558406]
 0x7fd883242f13 $LT$yatp…task…future…Runner$u20$as$u20$yatp…pool…runner…Runner$GT$::handle::h0056e31c4da70e35 [libtiflash_proxy.so+49590035]
 0x7fd8832357fc std::sys_common::backtrace::__rust_begin_short_backtrace::h747afb2668c16dcb [libtiflash_proxy.so+49534972]
 0x7fd88323631c core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h83ec6721ad8db87f [libtiflash_proxy.so+49537820]
 0x7fd8829a36a5 std::sys::unix::thread::thread::new::thread_start::hd2791a9cabec1fda [libtiflash_proxy.so+40548005]
    /rustc/96ddd32c4bfb1d78f0cd03eb068b1710a8cebeef/library/std/src/sys/unix/thread.rs:108
 0x7fd8800e3e25 start_thread [libpthread.so.0+32293]
 0x7fd87f4e9bad clone [libc.so.6+1043373]”] [source=“DB::RawCppPtr DB::PreHandleSnapshot(DB::EngineStoreServerWrap *, DB::BaseBuffView, uint64_t, DB::SSTViewVec, uint64_t, uint64_t)”] [thread_id=197]

4. What is your TiFlash version? (Required)

v6.5.5

[JaySon-Huang]

Minimal reproduce step:

create table t (a int, b DECIMAL(20) NULL default 1.0, c int NOT NULL);
insert into t(a,b,c) values (1,NULL,1),(2,NULL,2),(3,NULL,3),(4,NULL,4);
insert into t(a,b,c) values (11,0.1,11),(12,0.2,12),(13,0.3,13),(14,0.4,14);
select * from t;
update t set b = 0 where b is null;
select * from t;
alter table t modify column b DECIMAL(20) NOT NULL;
select * from t;

alter table t set tiflash replica 1;

• Create a table with at least one nullable column, note that the nullable column must have a default value
• Filling all NULL values in the nullable column by a value
• Turning the nullable column type to a non-nullable type
• Adding a tiflash replica before TiKV GC the rows with NULL in the currently non-nullable column

The main reason is that in the Snapshot TiKV sends to TiFlash, there exist old version of rows that contains NULL in the old schema but not valid according to the latest schema.