Ceph » CephFS

While running libcephfs client aio tests with object cacher disabled after fixing the issue https://tracker.ceph.com/issues/68146, the client crashes
with the assert "caps_ref[c] > 0"

Reproducer:
1. Included the fix for https://tracker.ceph.com/issues/68146
2. Ran libcephfs client aio tests on vstart cluster as below
- bin/ceph_test_client --client_oc=false

Please find the traceback below

----
#5  0x00007f649c4a8664 in __pthread_kill_implementation () from /lib64/libc.so.6
#6  0x00007f649c44fc4e in raise () from /lib64/libc.so.6
#7  0x00007f649c437902 in abort () from /lib64/libc.so.6
#8  0x00007f649d80ac45 in ceph::__ceph_assert_fail (assertion=0x5619d0aef0ad "cap_refs[c] > 0", file=<optimized out>, 
    line=207, func=0x5619d0aef0bd "int Inode::put_cap_ref(int)")
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/common/assert.cc:75
#9  0x00007f649d80ad20 in ceph::__ceph_assert_fail (ctx=...)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/common/assert.cc:80
#10 0x00005619d0a3b5a3 in Inode::put_cap_ref (this=this@entry=0x7f647000a4b0, cap=1, cap@entry=2048)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/client/Inode.cc:207
#11 0x00005619d0999cd0 in Client::put_cap_ref (this=0x5619ea31ba10, in=0x7f647000a4b0, cap=cap@entry=2048)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/client/Client.cc:3619
#12 0x00005619d09a7782 in Client::C_Read_Finisher::finish_io (this=0x5619ea31ea70, r=0)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/client/Client.cc:10736
#13 0x00005619d0a35a87 in Client::CRF_iofinish::finish (this=<optimized out>, r=<optimized out>)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/client/Client.h:1330
#14 0x00005619d09f3b8d in Client::CRF_iofinish::complete (this=0x5619ea2fa2f0, r=<optimized out>)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/client/Client.h:1336
#15 0x00005619d09df552 in Client::C_Read_Sync_NonBlocking::finish (this=0x5619ea2fa040, r=0)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/client/Client.cc:10827
#16 0x00005619d09f3bab in Client::C_Read_Sync_NonBlocking::complete (this=0x5619ea2fa040, r=<optimized out>)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/client/Client.h:1389
#17 0x00005619d0ad280d in Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}::operator()<Context*>(Context*&&) const (__closure=0x7f6495bfd140, 
    arg=@0x7f6495bfd360: 0x5619ea2fa040) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/osdc/Objecter.h:2075
#18 std::__invoke_impl<void, Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}, Context*>(std::__invoke_other, Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}&&, Context*&&) (__f=...) at /usr/include/c++/14/bits/invoke.h:61
#19 std::__invoke<Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}, Context*>(Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}&&, Context*&&) (__fn=...)
    at /usr/include/c++/14/bits/invoke.h:96
#20 std::__detail::__variant::__gen_vtable_impl<std::__detail::__variant::_Multi_array<std::__detail::__variant::__deduce_visit_result<void> (*)(Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}&&, std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&)>, std::integer_sequence<unsigned long, 2ul> >::__visit_invoke(Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda--Type <RET> for more, q to quit, c to continue without paging--
etail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&) (__visitor=..., __vars#0=...) at /usr/include/c++/14/variant:1060
#21 std::__do_visit<std::__detail::__variant::__deduce_visit_result<void>, Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}, std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*> >(Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}&&, std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&) (__visitor=...) at /usr/include/c++/14/variant:1821
#22 0x00005619d0ad282f in std::visit<Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}, std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*> >(Objecter::Op::complete(std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&, boost::system::error_code, int, boost::asio::io_context::basic_executor_type<std::allocator<void>, 0ul>)::{lambda(auto:1&&)#1}&&, std::variant<boost::asio::any_completion_handler<void (boost::system::error_code)>, fu2::abi_310::detail::function<fu2::abi_310::detail::config<true, false, 16ul>, fu2::abi_310::detail::property<true, false, void (boost::system::error_code)> >, Context*>&&) (__visitor=...) at /usr/include/c++/14/variant:1882
#23 0x00005619d0ad287f in Objecter::Op::complete (f=..., ec=..., r=r@entry=0, e=...) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/osdc/Objecter.h:2072
#24 0x00005619d0aa96a6 in Objecter::handle_osd_op_reply (this=this@entry=0x5619ea2f6690, m=m@entry=0x7f6480008410) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/osdc/Objecter.cc:3643
#25 0x00005619d0aaa0ad in Objecter::ms_dispatch (this=0x5619ea2f6690, m=0x7f6480008410) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/osdc/Objecter.cc:990
#26 0x00005619d0aacab8 in Objecter::ms_fast_dispatch (this=<optimized out>, m=0x7f6480008410) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/osdc/Objecter.h:2772
#27 0x00005619d0957fa9 in Dispatcher::ms_fast_dispatch2 (this=0x5619ea2f6698, m=...) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/Dispatcher.h:90
#28 0x00007f649d936e56 in Messenger::ms_fast_dispatch (this=<optimized out>, m=...) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/Messenger.h:713
#29 0x00007f649d93614c in DispatchQueue::fast_dispatch (this=0x5619ea1d0ce8, m=...) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/DispatchQueue.cc:74
#30 0x00007f649d9de389 in DispatchQueue::fast_dispatch (this=<optimized out>, m=m@entry=0x7f6480008410) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/DispatchQueue.h:203
#31 0x00007f649da0f0e3 in ProtocolV2::handle_message (this=this@entry=0x5619ea2642c0) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/ProtocolV2.cc:1513
#32 0x00007f649da1b91f in ProtocolV2::handle_read_frame_dispatch (this=this@entry=0x5619ea2642c0) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/ProtocolV2.cc:1170
#33 0x00007f649da1bab4 in ProtocolV2::_handle_read_frame_epilogue_main (this=this@entry=0x5619ea2642c0) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/ProtocolV2.cc:1360
#34 0x00007f649da1d258 in ProtocolV2::handle_read_frame_epilogue_main (this=0x5619ea2642c0, buffer=..., r=0) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/ProtocolV2.cc:1337
#35 0x00007f649da1d350 in CtRxNode<ProtocolV2>::call (this=<optimized out>, foo=<optimized out>) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/Protocol.h:67
#36 0x00007f649da0b191 in ProtocolV2::run_continuation (this=0x5619ea2642c0, continuation=...) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/ProtocolV2.cc:49
#37 0x00007f649da0b6ba in operator() (__closure=0x5619ea31e8e8, buffer=<optimized out>, r=0) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/ProtocolV2.cc:782
#38 0x00007f649da0b717 in std::__invoke_impl<void, ProtocolV2::read(CONTINUATION_RXBPTR_TYPE<ProtocolV2>&, rx_buffer_t&&)::<lambda(char*, int)>&, char*, long int> (__f=...) at /usr/include/c++/14/bits/invoke.h:61
#39 std::__invoke_r<void, ProtocolV2::read(CONTINUATION_RXBPTR_TYPE<ProtocolV2>&, rx_buffer_t&&)::<lambda(char*, int)>&, char*, long int> (__fn=...) at /usr/include/c++/14/bits/invoke.h:111
#40 std::_Function_handler<void(char*, long int), ProtocolV2::read(CONTINUATION_RXBPTR_TYPE<ProtocolV2>&, rx_buffer_t&&)::<lambda(char*, int)> >::_M_invoke(const std::_Any_data &, char *&&, long &&) (__functor=..., __args#0=<optimized out>, 
    __args#1=<optimized out>) at /usr/include/c++/14/bits/std_function.h:290
#41 0x00007f649d9de8cf in std::function<void(char*, long)>::operator() (this=this@entry=0x5619ea31e8e8, __args#0=<optimized out>, __args#1=<optimized out>, __args#1@entry=0) at /usr/include/c++/14/bits/std_function.h:591
#42 0x00007f649d9dc801 in AsyncConnection::process (this=0x5619ea31e510) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/AsyncConnection.cc:478
#43 0x00007f649d9dfa3f in C_handle_read::do_request (this=<optimized out>, fd_or_id=<optimized out>) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/AsyncConnection.cc:71
#44 0x00007f649da28a71 in EventCenter::process_events (this=0x5619ea0ffca8, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000, working_dur=working_dur@entry=0x7f6495bfdb78)
    at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/Event.cc:436
#45 0x00007f649da2dc55 in operator() (__closure=0x5619ea1ce5d8) at /home/kotresh/sandbox/upstream/kotresh-ceph2/ceph/src/msg/async/Stack.cc:50
#46 0x00007f649da2ddc9 in std::__invoke_impl<void, NetworkStack::add_thread(Worker*)::<lambda()>&> (__f=...) at /usr/include/c++/14/bits/invoke.h:60
#47 std::__invoke_r<void, NetworkStack::add_thread(Worker*)::<lambda()>&> (__fn=...) at /usr/include/c++/14/bits/invoke.h:111
#48 std::_Function_handler<void(), NetworkStack::add_thread(Worker*)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/14/bits/std_function.h:290
#49 0x00007f649da2d674 in std::function<void()>::operator() (this=<optimized out>) at /usr/include/c++/14/bits/std_function.h:591
#50 0x00007f649da2d68b in std::__invoke_impl<void, std::function<void ()>>(std::__invoke_other, std::function<void ()>&&) (__f=...) at /usr/include/c++/14/bits/invoke.h:60
#51 std::__invoke<std::function<void ()>>(std::function<void ()>&&) (__fn=...) at /usr/include/c++/14/bits/invoke.h:96
#52 std::thread::_Invoker<std::tuple<std::function<void()> > >::_M_invoke<0ul> (this=<optimized out>) at /usr/include/c++/14/bits/std_thread.h:301
#53 std::thread::_Invoker<std::tuple<std::function<void()> > >::operator() (this=<optimized out>) at /usr/include/c++/14/bits/std_thread.h:308
#54 std::thread::_State_impl<std::thread::_Invoker<std::tuple<std::function<void()> > > >::_M_run (this=<optimized out>) at /usr/include/c++/14/bits/std_thread.h:253
#55 0x00007f649c6e7564 in execute_native_thread_routine () from /lib64/libstdc++.so.6
#56 0x00007f649c4a66d7 in start_thread () from /lib64/libc.so.6
#57 0x00007f649c52a60c in clone3 () from /lib64/libc.so.6
----