Bug Report

What version of TiKV are you using?

sh-5.1# ./tikv-server -V
TiKV
Release Version: 8.5.0-alpha
Edition: Community
Git Commit Hash: d01bd74
Git Commit Branch: heads/refs/tags/v8.5.0-alpha
UTC Build Time: 2024-11-03 11:37:49
Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27)
Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored
Profile: dist_release

What operating system and CPU are you using?

8c/32g

Steps to reproduce

[in-memory-engine]
enable = true
gc-run-interval = "2m"
capacity = "12GB"
evict-threshold = "10GB"
stop-load-threshold = "9GB"
# Aggressive load. Default 100.
mvcc-amplification-threshold = 5
cross-check-interval = "30s"

1、run sysbench databricks
./prepare.sh -h tc-tidb.glh-knhqn -u root -r 4000 -t 64 -w 2000 -d 0 -j 10000 -m true
./many_delete_mvcc.sh -h tc-tidb.glh-knhqn -u root -r 4000 -t 64 -w 2000 -j 10000 -s 10 -e 7200

What did you expect?

no panic

What did happened?

three tikv panic
tikv-2024-11-04T20-08-05.954.log.zip

[2024/11/04 20:07:55.194 +08:00] [FATAL] [lib.rs:479] ["assertion failed: !region.get_peers().is_empty()"] [backtrace=" 0: tikv_util::set_panic_hook::{{closure}}\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/tikv_util/src/lib.rs:478:18\n 1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2029:9\n std::panicking::rust_panic_with_hook\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:783:13\n 2: std::panicking::begin_panic_handler::{{closure}}\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:649:13\n 3: std::sys_common::backtrace::__rust_end_short_backtrace\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:171:18\n 4: rust_begin_unwind\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:645:5\n 5: core::panicking::panic_fmt\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:72:14\n 6: core::panicking::panic\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:144:5\n 7: keys::enc_start_key\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/keys/src/lib.rs:232:5\n engine_traits::region_cache_engine::CacheRegion::from_region\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/engine_traits/src/region_cache_engine.rs:154:20\n 8: <hybrid_engine::observer::load_eviction::LoadEvictionObserver as raftstore::coprocessor::DestroyPeerObserver>::on_destroy_peer\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/hybrid_engine/src/observer/load_eviction.rs:278:21\n 9: raftstore::coprocessor::dispatcher::CoprocessorHost<E>::on_destroy_peer\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/raftstore/src/coprocessor/dispatcher.rs:998:13\n raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::destroy_peer\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/peer.rs:3966:9\n 10: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_destroy_peer\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/peer.rs:3865:13\n raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::check_msg\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/peer.rs:3258:28\n 11: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::on_raft_message\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/peer.rs:2744:12\n raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/peer.rs:653:37\n 12: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm<EK>>>::handle_normal\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/raftstore/src/store/fsm/store.rs:1098:9\n 13: batch_system::batch::Poller<N,C,Handler>::poll\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/batch-system/src/batch.rs:416:27\n 14: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/batch-system/src/batch.rs:589:17\n <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n at home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tikv/components/tikv_util/src/sys/thread.rs:438:13\n std::sys_common::backtrace::__rust_begin_short_backtrace\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n 15: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n std::panicking::try::do_call\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n std::panicking::try\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n std::panic::catch_unwind\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n std::thread::Builder::spawn_unchecked_::{{closure}}\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n core::ops::function::FnOnce::call_once{{vtable.shim}}\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n 16: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n std::sys::unix::thread::Thread::new::thread_start\n at rust/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n 17: start_thread\n 18: __GI___clone\n"] [location=components/keys/src/lib.rs:232] [thread_name=raftstore-1005-0] [thread_id=109] [2024/11/04 20:07:56.984 +08:00] [INFO] [lib.rs:91] ["Welcome to TiKV"] [thread_id=1]