Bug Report

What version of TiKV are you using?

./tikv-server -V
TiKV
Release Version: 8.5.0-alpha
Edition: Community
Git Commit Hash: 3bd8c24
Git Commit Branch: HEAD
UTC Build Time: 2024-11-20 10:00:25
Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27)
Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored
Profile: dist_release
2024-11-22T02:09:21.600+0800

What operating system and CPU are you using?

8c/32g

Steps to reproduce

1、set gc lifetime to 2h
2、create changefeed
3、run databricks workload
4、injection network partition between one of tikv and one of tidb

What did you expect?

no panic

What did happened?

multiple tikv panic at the same time when injection network partition between one of tikv and one of tidb

{"namespace":"test-databricks-tps-7685449-1-269","level":"FATAL","log":"[lib.rs:480] [\"called Option::unwrap()on aNone value\"] [backtrace=\" 0: tikv_util::set_panic_hook::{{closure}}\\n at workspace/source/tikv/components/tikv_util/src/lib.rs:479:18\\n 1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2029:9\\n std::panicking::rust_panic_with_hook\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:783:13\\n 2: std::panicking::begin_panic_handler::{{closure}}\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:649:13\\n 3: std::sys_common::backtrace::__rust_end_short_backtrace\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:171:18\\n 4: rust_begin_unwind\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:645:5\\n 5: core::panicking::panic_fmt\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:72:14\\n 6: core::panicking::panic\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:144:5\\n 7: core::option::Option<T>::unwrap\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/option.rs:931:21\\n cdc::delegate::Delegate::sink_delete\\n at workspace/source/tikv/components/cdc/src/delegate.rs:1075:62\\n cdc::delegate::Delegate::sink_data\\n at workspace/source/tikv/components/cdc/src/delegate.rs:875:36\\n 8: cdc::delegate::Delegate::on_batch\\n at workspace/source/tikv/components/cdc/src/delegate.rs:735:17\\n 9: cdc::endpoint::Endpoint<T,E,S>::on_multi_batch\\n at workspace/source/tikv/components/cdc/src/endpoint.rs:999:33\\n <cdc::endpoint::Endpoint<T,E,S> as tikv_util::worker::pool::Runnable>::run\\n at workspace/source/tikv/components/cdc/src/endpoint.rs:1225:18\\n 10: tikv_util::worker::pool::Worker::start_with_timer_impl::{{closure}}\\n at workspace/source/tikv/components/tikv_util/src/worker/pool.rs:532:25\\n <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll::{{closure}}\\n at workspace/source/tikv/components/tracker/src/tls.rs:64:23\\n std::thread::local::LocalKey<T>::try_with\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:270:16\\n std::thread::local::LocalKey<T>::with\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:246:9\\n <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll\\n at workspace/source/tikv/components/tracker/src/tls.rs:62:27\\n <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll\\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/future/future/map.rs:55:37\\n <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll\\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.31/src/lib.rs:86:13\\n yatp::task::future::RawTask<F>::poll\\n at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:59:9\\n 11: yatp::task::future::TaskCell::poll\\n at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:103:9\\n <yatp::task::future::Runner as yatp::pool::runner::Runner>::handle\\n at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:387:20\\n 12: <tikv_util::yatp_pool::YatpPoolRunner<T> as yatp::pool::runner::Runner>::handle\\n at workspace/source/tikv/components/tikv_util/src/yatp_pool/mod.rs:199:24\\n yatp::pool::worker::WorkerThread<T,R>::run\\n at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool/worker.rs:48:13\\n yatp::pool::builder::LazyBuilder<T>::build::{{closure}}\\n at workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool/builder.rs:114:25\\n std::sys_common::backtrace::__rust_begin_short_backtrace\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18\\n 13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17\\n <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9\\n std::panicking::try::do_call\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40\\n std::panicking::try\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19\\n std::panic::catch_unwind\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14\\n std::thread::Builder::spawn_unchecked_::{{closure}}\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30\\n core::ops::function::FnOnce::call_once{{vtable.shim}}\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5\\n 14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\\n <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9\\n std::sys::unix::thread::Thread::new::thread_start\\n at root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\\n 15: start_thread\\n 16: clone3\\n\"] [location=components/cdc/src/delegate.rs:1075] [thread_name=cdc-0] [thread_id=48]","container":"tikv","pod":"upstream-tikv-4"}