Ingesting DTFile to DeltaTree storage make it unable to read data #1787
Description
# Prepare a table big enough like the "tpch_5.partsupp" table (4,000,000 rows)
# Add TiFlash replica so that TiFlash apply data
mysql> alter table partsupp set tiflash replica 1;
# Alter TiFlash replica num to be 0
mysql> alter table partsupp set tiflash replica 0;
# Alter TiFlash replica num to be 1 again, TiFlash will crash while applying snapshot
mysql> alter table partsupp set tiflash replica 1;
The error message in logging
[2021/04/16 03:21:48.448 +08:00] [ERROR] [<unknown>] ["void DB::ApplyPreHandledSnapshot(DB::EngineStoreServerWrap*, PreHandledSnapshot*) [with PreHandledSnapshot = DB::PreHandledSnapshotWithFiles]: Code: 49, e.displayText() = DB::Exception: Not enough delta data to read [offset=462174] [limit=8192] [read_offset=462174]: while reading DTFile /data1/jaysonhuang/tiup-deploy/data/tiflash-5014/data/t_175/stable/dmf_29, e.what() = DB::Exception, Stack trace:\n\n0. bin/tiflash/tiflash(StackTrace::StackTrace()+0x15) [0x3648525]\n1. bin/tiflash/tiflash(DB::Exception::Exception(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int)+0x25) [0x363edf5]\n2. bin/tiflash/tiflash(DB::DM::DPFileReader::readRowsOnce(std::vector<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn>, std::allocator<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn> > >&, unsigned long, unsigned long, DB::DM::RowKeyRange const*)+0x554) [0x71bf6d4]\n3. bin/tiflash/tiflash(DB::DM::DPFileReader::readRows(std::vector<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn>, std::allocator<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn> > >&, unsigned long, unsigned long, DB::DM::RowKeyRange const*)+0x54) [0x71bfde4]\n4. bin/tiflash/tiflash(DB::DM::DeltaValueReader::readRows(std::vector<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn>, std::allocator<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn> > >&, unsigned long, unsigned long, DB::DM::RowKeyRange const*)+0x18a) [0x71d2bea]\n5. bin/tiflash/tiflash(DB::DM::DeltaMergeBlockInputStream<DB::DM::DeltaValueReader, DB::DM::DTCompactedEntries<55ul, 20ul, 3ul>::Iterator, false>::writeInsertFromDelta(std::vector<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn>, std::allocator<COWPtr<DB::IColumn>::mutable_ptr<DB::IColumn> > >&, unsigned long&)+0x93) [0x7166d23]\n6. bin/tiflash/tiflash(DB::DM::DeltaMergeBlockInputStream<DB::DM::DeltaValueReader, DB::DM::DTCompactedEntries<55ul, 20ul, 3ul>::Iterator, false>::read()+0x86f) [0x71697bf]\n7. bin/tiflash/tiflash(DB::DM::DMRowKeyFilterBlockInputStream<true>::read()+0x5a) [0x7157baa]\n8. bin/tiflash/tiflash(DB::DM::ReorganizeBlockInputStream::read()+0x883) [0x71599c3]\n9. bin/tiflash/tiflash(DB::DM::DMVersionFilterBlockInputStream<1>::initNextBlock()+0x30) [0x7697fb0]\n10. bin/tiflash/tiflash(DB::DM::DMVersionFilterBlockInputStream<1>::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool)+0x14f7) [0x7699fd7]\n11. bin/tiflash/tiflash(DB::DM::DMVersionFilterBlockInputStream<1>::read()+0x17) [0x714aee7]\n12. bin/tiflash/tiflash(DB::DM::writeIntoNewDMFile(DB::DM::DMContext&, std::shared_ptr<std::vector<DB::DM::ColumnDefine, std::allocator<DB::DM::ColumnDefine> > > const&, std::shared_ptr<DB::IBlockInputStream> const&, unsigned long, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x1db) [0x7136e4b]\n13. bin/tiflash/tiflash(DB::DM::createNewStable(DB::DM::DMContext&, std::shared_ptr<std::vector<DB::DM::ColumnDefine, std::allocator<DB::DM::ColumnDefine> > > const&, std::shared_ptr<DB::IBlockInputStream> const&, unsigned long, DB::DM::WriteBatches&, bool)+0xab) [0x713704b]\n14. bin/tiflash/tiflash(DB::DM::Segment::prepareMergeDelta(DB::DM::DMContext&, std::shared_ptr<std::vector<DB::DM::ColumnDefine, std::allocator<DB::DM::ColumnDefine> > > const&, std::shared_ptr<DB::DM::SegmentSnapshot> const&, DB::DM::WriteBatches&, bool) const+0xeb) [0x714273b]\n15. bin/tiflash/tiflash(DB::DM::DeltaMergeStore::segmentMergeDelta(DB::DM::DMContext&, std::shared_ptr<DB::DM::Segment> const&, bool)+0x41a) [0x710fb5a]\n16. bin/tiflash/tiflash(DB::DM::DeltaMergeStore::checkSegmentUpdate(std::shared_ptr<DB::DM::DMContext> const&, std::shared_ptr<DB::DM::Segment> const&, DB::DM::DeltaMergeStore::ThreadType)+0xb88) [0x7111528]\n17. bin/tiflash/tiflash(DB::DM::DeltaMergeStore::ingestFiles(std::shared_ptr<DB::DM::DMContext> const&, DB::DM::RowKeyRange const&, std::vector<unsigned long, std::allocator<unsigned long> >, bool)+0x1ed6) [0x71172d6]\n18. bin/tiflash/tiflash(DB::StorageDeltaMerge::ingestFiles(DB::DM::RowKeyRange const&, std::vector<unsigned long, std::allocator<unsigned long> > const&, bool, DB::Settings const&)+0x15e) [0x708ae6e]\n19. bin/tiflash/tiflash(void DB::KVStore::onSnapshot<DB::RegionPtrWithSnapshotFiles>(DB::RegionPtrWithSnapshotFiles const&, std::shared_ptr<DB::Region>, unsigned long, DB::TMTContext&)+0x81a) [0x76cc45a]\n20. bin/tiflash/tiflash(void DB::KVStore::checkAndApplySnapshot<DB::RegionPtrWithSnapshotFiles>(DB::RegionPtrWithSnapshotFiles const&, DB::TMTContext&)+0x434) [0x76ccdd4]\n21. bin/tiflash/tiflash(void DB::KVStore::handlePreApplySnapshot<DB::RegionPtrWithSnapshotFiles>(DB::RegionPtrWithSnapshotFiles const&, DB::TMTContext&)+0x162) [0x76cdac2]\n22. bin/tiflash/tiflash(DB::ApplyPreHandledSnapshot(DB::EngineStoreServerWrap*, void*, unsigned int)+0x91) [0x730dbe1]\n23. /data1/jaysonhuang/tiup-deploy/deploy/tiflash-5014/bin/tiflash/libtiflash_proxy.so(+0xc9b149) [0x7fb9ec602149]\n24. /data1/jaysonhuang/tiup-deploy/deploy/tiflash-5014/bin/tiflash/libtiflash_proxy.so(+0xa53f0c) [0x7fb9ec3baf0c]\n25. /data1/jaysonhuang/tiup-deploy/deploy/tiflash-5014/bin/tiflash/libtiflash_proxy.so(+0x1dbd0a0) [0x7fb9ed7240a0]\n26. /data1/jaysonhuang/tiup-deploy/deploy/tiflash-5014/bin/tiflash/libtiflash_proxy.so(+0x56a146) [0x7fb9ebed1146]\n27. /data1/jaysonhuang/tiup-deploy/deploy/tiflash-5014/bin/tiflash/libtiflash_proxy.so(+0x5a839f) [0x7fb9ebf0f39f]\n28. /data1/jaysonhuang/tiup-deploy/deploy/tiflash-5014/bin/tiflash/libtiflash_proxy.so(+0x157ff7f) [0x7fb9ecee6f7f]\n29. /lib64/libpthread.so.0(+0x7ea4) [0x7fb9ea7edea4]\n30. /lib64/libc.so.6(clone+0x6c) [0x7fb9e9da48dc]\n"] [thread_id=16]
The delta index is placed with wrong status:
[2021/04/16 13:20:46.889 +08:00] [ERROR] [<unknown>] ["executeQuery: Code: 0, e.displayText() = DB::Exception: Placed status not match! Expected place rows:478898, deletes:2, but actually placed rows:36530, deletes:2, e.what() = DB::Exception (from 172.16.5.82:46046) (in query: select count(*) from t_175), Stack trace:\n\n0. bin/tiflash/tiflash(StackTrace::StackTrace()+0x15) [0x3648525]\n1. bin/tiflash/tiflash() [0x71359b7]\n2. bin/tiflash/tiflash(DB::DM::Segment::ensurePlace(DB::DM::DMContext const&, std::shared_ptr<DB::DM::StableValueSpace::Snapshot> const&, std::shared_ptr<DB::DM::DeltaValueReader> const&, std::vector<DB::DM::RowKeyRange, std::allocator<DB::DM::RowKeyRange> > const&, unsigned long) const+0x1377) [0x713f767]\n3. bin/tiflash/tiflash(DB::DM::Segment::getReadInfo(DB::DM::DMContext const&, std::vector<DB::DM::ColumnDefine, std::allocator<DB::DM::ColumnDefine> > const&, std::shared_ptr<DB::DM::SegmentSnapshot> const&, std::vector<DB::DM::RowKeyRange, std::allocator<DB::DM::RowKeyRange> > const&, unsigned long) const+0x236) [0x7140066]\n4. bin/tiflash/tiflash(DB::DM::Segment::getInputStream(DB::DM::DMContext const&, std::vector<DB::DM::ColumnDefine, std::allocator<DB::DM::ColumnDefine> > const&, std::shared_ptr<DB::DM::SegmentSnapshot> const&, std::vector<DB::DM::RowKeyRange, std::allocator<DB::DM::RowKeyRange> > const&, std::shared_ptr<DB::DM::RSOperator> const&, unsigned long, unsigned long)+0x132) [0x7140942]\n5. bin/tiflash/tiflash(DB::DM::DMSegmentThreadInputStream::readImpl(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool)+0x605) [0x7127a95]\n6. bin/tiflash/tiflash(DB::DM::DMSegmentThreadInputStream::readImpl()+0x1a) [0x711a75a]\n7. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool)+0x3bc) [0x6832c9c]\n8. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read()+0x16) [0x6832e86]\n9. bin/tiflash/tiflash(DB::ExpressionBlockInputStream::readImpl()+0x1a) [0x744e55a]\n10. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read(DB::PODArray<unsigned char, 4096ul, Allocator<false>, 15ul, 16ul>*&, bool)+0x3bc) [0x6832c9c]\n11. bin/tiflash/tiflash(DB::IProfilingBlockInputStream::read()+0x16) [0x6832e86]\n12. bin/tiflash/tiflash(DB::ParallelInputsProcessor<DB::ParallelAggregatingBlockInputStream::Handler, (DB::StreamUnionMode)0>::loop(unsigned long)+0xea) [0x774cbfa]\n13. bin/tiflash/tiflash(DB::ParallelInputsProcessor<DB::ParallelAggregatingBlockInputStream::Handler, (DB::StreamUnionMode)0>::thread(MemoryTracker*, unsigned long)+0x20c) [0x774d14c]\n14. bin/tiflash/tiflash() [0x84b6bee]\n15. /lib64/libpthread.so.0(+0x7ea4) [0x7f3362b59ea4]\n16. /lib64/libc.so.6(clone+0x6c) [0x7f33621108dc]\n"] [thread_id=15]