Bug Report

What version of TiKV are you using?

./tikv-server -V
TiKV
Release Version: 8.3.0
Edition: Community
Git Commit Hash: 72fc9f1
Git Commit Branch: HEAD
UTC Build Time: 2024-08-13 09:27:19
Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27)
Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored
Profile: dist_release
2024-08-15T03:26:47.879+0800

What operating system and CPU are you using?

8c/32g

Steps to reproduce

1、run sysbench prepare

What did you expect?

no panic

What did happened?

tikv panic
{"pod":"tc-tikv-1","log":"[lib.rs:478] ["unexpected error: Store(Other(Other(\"[components/raft_log_engine/src/engine.rs:807]: Corruption: file seqno out of range\"))), raft_id: 68764, region_id: 68763"] [backtrace=" 0: tikv_util::set_panic_hook::{{closure}}\n at workspace/source/tikv/components/tikv_util/src/lib.rs:477:18\n 1: <alloc::boxed::Box<F,A> as core::ops::function::Fn>::call\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2029:9\n std::panicking::rust_panic_with_hook\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:783:13\n 2: std::panicking::begin_panic_handler::{{closure}}\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:657:13\n 3: std::sys_common::backtrace::__rust_end_short_backtrace\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:171:18\n 4: rust_begin_unwind\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:645:5\n 5: core::panicking::panic_fmt\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:72:14\n 6: raft::raft_log::RaftLog::slice\n at workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raft_log.rs:666:26\n raft::raft_log::RaftLog::scan\n at workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raft_log.rs:621:24\n raft::raft::Raft::has_unapplied_conf_changes\n at workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raft.rs:1582:27\n 7: raft::raft::Raft::hup\n at workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raft.rs:1548:12\n 8: raft::raft::Raft::step\n at workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raft.rs:1471:36\n 9: raft::raft::Raft::tick_election\n at workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raft.rs:1108:17\n raft::raft::Raft::tick\n at workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raft.rs:1090:17\n 10: raft::raw_node::RawNode::tick\n at workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raw_node.rs:343:9\n raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::on_raft_base_tick\n at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:2340:37\n raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::on_tick\n at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:1308:31\n 11: raftstore::store::fsm::peer::PeerFsmDelegate<EK,ER,T>::handle_msgs\n at workspace/source/tikv/components/raftstore/src/store/fsm/peer.rs:699:40\n 12: <raftstore::store::fsm::store::RaftPoller<EK,ER,T> as batch_system::batch::PollHandler<raftstore::store::fsm::peer::PeerFsm<EK,ER>,raftstore::store::fsm::store::StoreFsm>>::handle_normal\n at workspace/source/tikv/components/raftstore/src/store/fsm/store.rs:1096:9\n 13: batch_system::batch::Poller<N,C,Handler>::poll\n at workspace/source/tikv/components/batch-system/src/batch.rs:422:27\n 14: batch_system::batch::BatchSystem<N,C>::start_poller::{{closure}}\n at workspace/source/tikv/components/batch-system/src/batch.rs:551:17\n <std::thread::Builder as tikv_util::sys::thread::StdThreadBuildWrapper>::spawn_wrapper::{{closure}}\n at workspace/source/tikv/components/tikv_util/src/sys/thread.rs:438:13\n std::sys_common::backtrace::rust_begin_short_backtrace\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\n 15: std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}}\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\n <core::panic::unwind_safe::AssertUnwindSafe as core::ops::function::FnOnce<()>>::call_once\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\n std::panicking::try::do_call\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\n std::panicking::try\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\n std::panic::catch_unwind\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\n std::thread::Builder::spawn_unchecked::{{closure}}\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\n core::ops::function::FnOnce::call_once{{vtable.shim}}\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\n 16: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\n std::sys::unix::thread::Thread::new::thread_start\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n 17: start_thread\n 18: clone3\n"] [location=/workspace/.cargo/git/checkouts/raft-rs-42b8049ef2e3af07/a76fb6e/src/raft_log.rs:666] [thread_name=raftstore-4-0] [thread_id=100]","container":"tikv","level":"FATAL","namespace":"e2e-diskfull-stab-fun-tps-7550994-1-164"}