Ceph » crimson

It's not clear from the logs what the actual event that caused the transition to Crashed is, will add logging

https://pulpito.ceph.com/sjust-2025-01-28_06:26:29-crimson-rados-wip-sjust-crimson-testing-2025-01-27-distro-default-smithi/8098656/

DEBUG 2025-01-28 08:02:23,647 [shard 0:main] osd - peering_event(id=5163, detail=PeeringEvent(from=1 pgid=6.5 sent=156 requested=156 evt=epoch_sent: 156 epoch_requested: 156 RemoteBackfillReserved)): exit
DEBUG 2025-01-28 08:02:23,647 [shard 0:main] osd - dispatch_backfill_event
DEBUG 2025-01-28 08:02:23,647 [shard 0:main] osd - BackfillState::StateHelper: exit crimson::osd::BackfillState::Done
ceph-osd: /home/jenkins-build/build/workspace/ceph-dev-new-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/19.3.0-7026-g11fb6e8c/rpm/el9/BUILD/ceph-19.3.0-7026-g11fb6e8c/redhat-linux-build/boost/include/boost/statechart/simple_state.hpp:682: static OtherContext& boost::statechart::simple_state<MostDerived, Context, InnerInitial, historyMode>::context_impl_other_cont
ext::context_impl(State&) [with OtherContext = crimson::osd::BackfillState::BackfillMachine; State = boost::statechart::simple_state<crimson::osd::BackfillState::Crashed, crimson::osd::BackfillState::BackfillMachine>; MostDerived = crimson::osd::BackfillState::Crashed; Context = crimson::osd::BackfillState::BackfillMachine; InnerInitial = boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na
, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>; boost::statechart::history_mode historyMode = boost::statechart::has_no_history]: Assertion `get_pointer( stt.pContext_ ) != 0' failed.
Aborting on shard 0.
Backtrace:
 0# 0x00007F25D1C8BFDC in /lib64/libc.so.6
 1# raise in /lib64/libc.so.6
 2# abort in /lib64/libc.so.6
 3# 0x00007F25D1C2875B in /lib64/libc.so.6
 4# 0x00007F25D1C375C6 in /lib64/libc.so.6
 5# crimson::osd::BackfillState::BackfillMachine& boost::statechart::simple_state<crimson::osd::BackfillState::Crashed, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::context_impl_other_context::context_impl<crimson::osd::BackfillState::BackfillMachine, boost::statechart::simple_state<crimson::osd::BackfillState::Crashed, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<crimson::osd::BackfillState::Crashed, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&) in ceph-osd
 6# crimson::osd::BackfillState::StateHelper<crimson::osd::BackfillState::Crashed>::pg() in ceph-osd
 7# crimson::osd::BackfillState::StateHelper<crimson::osd::BackfillState::Crashed>::StateHelper() in ceph-osd
 8# crimson::osd::BackfillState::Crashed::Crashed() in ceph-osd
 9# boost::statechart::simple_state<crimson::osd::BackfillState::Crashed, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::shallow_construct(boost::statechart::state_machine<crimson::osd::BackfillState::BackfillMachine, crimson::osd::BackfillState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>* const&, boost::statechart::state_machine<crimson::osd::BackfillState::BackfillMachine, crimson::osd::BackfillState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>&) in ceph-osd
10# boost::statechart::simple_state<crimson::osd::BackfillState::Crashed, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::deep_construct(boost::statechart::state_machine<crimson::osd::BackfillState::BackfillMachine, crimson::osd::BackfillState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>* const&, boost::statechart::state_machine<crimson::osd::BackfillState::BackfillMachine, crimson::osd::BackfillState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>&) in ceph-osd
11# boost::statechart::detail::safe_reaction_result boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::transit_impl<crimson::osd::BackfillState::Crashed, crimson::osd::BackfillState::BackfillMachine, boost::statechart::detail::no_transition_function>(boost::statechart::detail::no_transition_function const&) in ceph-osd
12# boost::statechart::detail::safe_reaction_result boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::transit<crimson::osd::BackfillState::Crashed>() in ceph-osd
13# boost::statechart::transition<boost::statechart::event_base, crimson::osd::BackfillState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function>::reactions<crimson::osd::BackfillState::Done>::react_without_action(crimson::osd::BackfillState::Done&) in ceph-osd
14# boost::statechart::detail::reaction_dispatcher<boost::statechart::transition<boost::statechart::event_base, crimson::osd::BackfillState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function>::reactions<crimson::osd::BackfillState::Done>, crimson::osd::BackfillState::Done, boost::statechart::event_base, boost::statechart::event_base, boost::statechart::detail::no_context<boost::statechart::event_base>, void const*>::without_action::react(crimson::osd::BackfillState::Done&, boost::statechart::event_base const&) in ceph-osd
15# boost::statechart::detail::reaction_dispatcher<boost::statechart::transition<boost::statechart::event_base, crimson::osd::BackfillState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function>::reactions<crimson::osd::BackfillState::Done>, crimson::osd::BackfillState::Done, boost::statechart::event_base, boost::statechart::event_base, boost::statechart::detail::no_context<boost::statechart::event_base>, void const*>::base::react(crimson::osd::BackfillState::Done&, boost::statechart::event_base const&, void const* const&) in ceph-osd
16# boost::statechart::detail::reaction_dispatcher<boost::statechart::transition<boost::statechart::event_base, crimson::osd::BackfillState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function>::reactions<crimson::osd::BackfillState::Done>, crimson::osd::BackfillState::Done, boost::statechart::event_base, boost::statechart::event_base, boost::statechart::detail::no_context<boost::statechart::event_base>, void const*>::react(crimson::osd::BackfillState::Done&, boost::statechart::event_base const&, void const* const&) in ceph-osd
17# boost::statechart::detail::reaction_result boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list1<boost::statechart::transition<boost::statechart::event_base, crimson::osd::BackfillState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function> >, boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*) in ceph-osd
18# boost::statechart::detail::reaction_result boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list<boost::statechart::custom_reaction<crimson::osd::BackfillState::CancelBackfill>, boost::statechart::transition<boost::statechart::event_base, crimson::osd::BackfillState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &boost::statechart::detail::no_context<boost::statechart::event_base>::no_function>, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*) in ceph-osd
19# boost::statechart::simple_state<crimson::osd::BackfillState::Done, crimson::osd::BackfillState::BackfillMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*) in ceph-osd
20# boost::statechart::detail::send_function<boost::statechart::detail::state_base<std::allocator<boost::statechart::none>, boost::statechart::detail::rtti_policy>, boost::statechart::event_base, void const*>::operator()() in ceph-osd
21# boost::statechart::state_machine<crimson::osd::BackfillState::BackfillMachine, crimson::osd::BackfillState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&) in ceph-osd
22# boost::statechart::state_machine<crimson::osd::BackfillState::BackfillMachine, crimson::osd::BackfillState::Initial, std::allocator<boost::statechart::none>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&) in ceph-osd
23# crimson::osd::BackfillState::process_event(boost::intrusive_ptr<boost::statechart::event_base const>) in ceph-osd
24# PGRecovery::dispatch_backfill_event(boost::intrusive_ptr<boost::statechart::event_base const>) in ceph-osd