Bug Report

What version of TiKV are you using?

v7.1.6

What operating system and CPU are you using?

Steps to reproduce

In the process of adding multi index, inject pd failure fault

What did you expect?

no tikv panic

What did happened?

{"log":"[lib.rs:510] ["Resolved future is not supposed to be polled again."] [backtrace=" 0: tikv_util::set_panic_hook::{{closure}}\n at workspace/source/tikv/components/tikv_util/src/lib.rs:509:18\n 1: <alloc::boxed::Box<F,A> as core::ops::function::Fn>::call\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2032:9\n std::panicking::rust_panic_with_hook\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:692:13\n 2: std::panicking::begin_panic::{{closure}}\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:608:9\n 3: std::sys_common::backtrace::__rust_end_short_backtrace\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:137:18\n 4: std::panicking::begin_panic\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:607:12\n 5: <grpcio::task::CqFuture as core::future::future::Future>::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/grpcio-0.10.4/src/task/mod.rs:90:13\n 6: grpcio::call::SinkBase::poll_ready\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/grpcio-0.10.4/src/call/mod.rs:796:24\n 7: <grpcio::call::server::ServerStreamingSink as futures_sink::Sink<(T,grpcio::call::WriteFlags)>>::poll_close\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/grpcio-0.10.4/src/call/server.rs:562:28\n <&mut S as futures_sink::Sink>::poll_close\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-sink-0.3.15/src/lib.rs:130:9\n <futures_util::sink::close::Close<Si,Item> as core::future::future::Future>::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/futures-util-0.3.15/src/sink/close.rs:30:9\n 8: <tikv::import::sst_service::ImportSstService as kvproto::protos::import_sstpb_grpc::ImportSst>::duplicate_detect::{{closure}}\n at workspace/source/tikv/src/import/sst_service.rs:1300:33\n <core::future::from_generator::GenFuture as core::future::future::Future>::poll\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/mod.rs:91:19\n tokio::runtime::task::core::CoreStage::poll::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/core.rs:184:17\n tokio::loom::std::unsafe_cell::UnsafeCell::with_mut\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/loom/std/unsafe_cell.rs:14:9\n tokio::runtime::task::core::CoreStage::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/core.rs:174:13\n 9: tokio::runtime::task::harness::poll_future::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:480:19\n <core::panic::unwind_safe::AssertUnwindSafe as core::ops::function::FnOnce<()>>::call_once\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:271:9\n std::panicking::try::do_call\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:483:40\n std::panicking::try\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:447:19\n std::panic::catch_unwind\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:137:14\n tokio::runtime::task::harness::poll_future\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:468:18\n tokio::runtime::task::harness::Harness<T,S>::poll_inner\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:104:27\n tokio::runtime::task::harness::Harness<T,S>::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:57:15\n tokio::runtime::task::raw::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/raw.rs:194:5\n 10: tokio::runtime::task::raw::RawTask::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/raw.rs:134:18\n tokio::runtime::task::LocalNotified::run\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/mod.rs:385:9\n tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:421:13\n tokio::coop::with_budget::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/coop.rs:102:9\n std::thread::local::LocalKey::try_with\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:446:16\n std::thread::local::LocalKey::with\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:422:9\n tokio::coop::with_budget\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/coop.rs:95:5\n tokio::coop::budget\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/coop.rs:72:5\n tokio::runtime::scheduler::multi_thread::worker::Context::run_task\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:420:9\n 11: tokio::runtime::scheduler::multi_thread::worker::Context::run\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:387:24\n tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:372:17\n tokio::macros::scoped_tls::ScopedKey::set\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/macros/scoped_tls.rs:61:9\n tokio::runtime::scheduler::multi_thread::worker::run\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:369:5\n tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/scheduler/multi_thread/worker.rs:348:45\n <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/blocking/task.rs:42:21\n tokio::runtime::task::core::CoreStage::poll::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/core.rs:184:17\n tokio::loom::std::unsafe_cell::UnsafeCell::with_mut\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/loom/std/unsafe_cell.rs:14:9\n tokio::runtime::task::core::CoreStage::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/core.rs:174:13\n tokio::runtime::task::harness::poll_future::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:480:19\n <core::panic::unwind_safe::AssertUnwindSafe as core::ops::function::FnOnce<()>>::call_once\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:271:9\n std::panicking::try::do_call\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:483:40\n std::panicking::try\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:447:19\n std::panic::catch_unwind\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:137:14\n tokio::runtime::task::harness::poll_future\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:468:18\n tokio::runtime::task::harness::Harness<T,S>::poll_inner\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:104:27\n tokio::runtime::task::harness::Harness<T,S>::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/harness.rs:57:15\n tokio::runtime::task::raw::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/raw.rs:194:5\n 12: tokio::runtime::task::raw::RawTask::poll\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/raw.rs:134:18\n tokio::runtime::task::UnownedTask::run\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/task/mod.rs:422:9\n tokio::runtime::blocking::pool::Task::run\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/blocking/pool.rs:111:9\n tokio::runtime::blocking::pool::Inner::run\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/blocking/pool.rs:346:17\n tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n at workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/tokio-1.21.2/src/runtime/blocking/pool.rs:321:13\n std::sys_common::backtrace::rust_begin_short_backtrace\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:121:18\n 13: std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}}\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:551:17\n <core::panic::unwind_safe::AssertUnwindSafe as core::ops::function::FnOnce<()>>::call_once\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:271:9\n std::panicking::try::do_call\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:483:40\n std::panicking::try\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:447:19\n std::panic::catch_unwind\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:137:14\n std::thread::Builder::spawn_unchecked::{{closure}}\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:550:30\n core::ops::function::FnOnce::call_once{{vtable.shim}}\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:513:5\n 14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2000:9\n <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2000:9\n std::sys::unix::thread::Thread::new::thread_start\n at root/.rustup/toolchains/nightly-2022-11-15-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17\n 15: start_thread\n 16: clone3\n"] [location=/workspace/.cargo/registry/src/mirrors.tuna.tsinghua.edu.cn-df7c3c540f42cdbd/grpcio-0.10.4/src/task/mod.rs:90] [thread_name=sst-importer]","container":"tikv","pod":"tc-tikv-0","level":"FATAL","namespace":"endless-ha-test-tps-7682088-1-507"}