ROS Version

noetic-devel
tag: 1.15.8

Thread Stack Trace

Thread 1

#0 __lll_lock_wait (futex=futex@entry=0x7f2f4802d248, private=0) at lowlevellock.c:52 #1 0x00007f2f68a6b131 in __GI___pthread_mutex_lock (mutex=mutex@entry=0x7f2f4802d248) at ../nptl/pthread_mutex_lock.c:115 #2 0x00007f2f68d35600 in boost::posix::pthread_mutex_lock (m=<optimized out>) at /usr/include/boost/thread/pthread/pthread_helpers.hpp:79 #3 boost::recursive_mutex::lock (this=<optimized out>) at /usr/include/boost/thread/pthread/recursive_mutex.hpp:108 #4 boost::unique_lock<boost::recursive_mutex>::lock (this=0x7f2f5dff9ad0) at /usr/include/boost/thread/lock_types.hpp:346 #5 0x00007f2f68b42bf0 in boost::unique_lock<boost::recursive_mutex>::unique_lock (m_=..., this=0x7f2f5dff9ad0) at /usr/include/boost/thread/lock_types.hpp:121 #6 ros::Connection::drop (this=0x7f2f4802d0c0, reason=reason@entry=ros::Connection::Destructing) at ../../../src/roscpp/src/libros/connection.cpp:330 #7 0x00007f2f68b9ac66 in ros::ServiceServerLink::processNextCall (this=0x7f2f4802b2b0) at /usr/include/boost/smart_ptr/shared_ptr.hpp:732 #8 0x00007f2f68b9b56d in ros::ServiceServerLink::callFinished (this=<optimized out>) at ../../../src/roscpp/src/libros/service_server_link.cpp:277 #9 0x00007f2f68b9babd in ros::ServiceServerLink::onResponse (this=0x7f2f4802b2b0, conn=..., buffer=..., size=<optimized out>, success=<optimized out>) at ../../../src/roscpp/src/libros/service_server_link.cpp:250 #10 0x00007f2f68b9bfc0 in ros::ServiceServerLink::onResponseOkAndLength (this=0x7f2f4802b2b0, conn=..., buffer=..., size=<optimized out>, success=<optimized out>) at /usr/include/boost/smart_ptr/detail/shared_count.hpp:122 #11 0x00007f2f68b3f974 in boost::function4<void, boost::shared_ptr<ros::Connection> const&, boost::shared_array<unsigned char> const&, unsigned int, bool>::operator() (a3=true, a2=5, a1=..., a0=..., this=0x7f2f5dff9eb0) at /usr/include/boost/function/function_template.hpp:677 #12 ros::Connection::readTransport (this=0x7f2f4802d0c0) at ../../../src/roscpp/src/libros/connection.cpp:176 #13 0x00007f2f68b40812 in ros::Connection::read(unsigned int, boost::function<void (boost::shared_ptr<ros::Connection> const&, boost::shared_array<unsigned char> const&, unsigned int, bool)> const&) (this=this@entry=0x7f2f4802d0c0, size=size@entry=5, callback=...) at ../../../src/roscpp/src/libros/connection.cpp:286 #14 0x00007f2f68b9a7ce in ros::ServiceServerLink::onRequestWritten (this=0x7f2f4802b2b0, conn=...) at /usr/include/boost/function/function_template.hpp:912 #15 0x00007f2f68b41609 in boost::function1<void, boost::shared_ptr<ros::Connection> const&>::operator() (a0=..., this=0x7f2f5dffa090) at /usr/include/boost/function/function_template.hpp:677 #16 ros::Connection::writeTransport (this=0x7f2f4802d0c0) at ../../../src/roscpp/src/libros/connection.cpp:241 #17 0x00007f2f68b41e4b in ros::Connection::write(boost::shared_array<unsigned char> const&, unsigned int, boost::function<void (boost::shared_ptr<ros::Connection> const&)> const&, bool) (this=this@entry=0x7f2f4802d0c0, buffer=..., size=5, callback=..., immediate=immediate@entry=true) at ../../../src/roscpp/src/libros/connection.cpp:313 #18 0x00007f2f68b9ab92 in ros::ServiceServerLink::processNextCall (this=0x7f2f4802b2b0) at /usr/include/c++/9/new:174 #19 0x00007f2f68b9c88c in ros::ServiceServerLink::call (this=0x7f2f4802b2b0, req=..., resp=...) at ../../../src/roscpp/src/libros/service_server_link.cpp:362 #20 0x00007f2f68ba020e in ros::ServiceClient::call (this=this@entry=0x558278bafcc8, req=..., resp=..., service_md5sum="a24c062c736be593254f72e3ac2e6bc2") at ../../../src/roscpp/src/libros/service_client.cpp:143 #21 0x00007f2f64332b78 in ros::ServiceClient::call<autonomy_msgs::SetLidarRangeRequest_<std::allocator<void> >, autonomy_msgs::SetLidarRangeResponse_<std::allocator<void> > > (service_md5sum="a24c062c736be593254f72e3ac2e6bc2", resp=<synthetic pointer>..., req=<synthetic pointer>..., this=0x558278bafcc8) at /usr/include/boost/smart_ptr/detail/shared_count.hpp:122 ...

Thread 2

#0 0x00007f2f6871189b in sched_yield () at ../sysdeps/unix/syscall-template.S:78 #1 0x00007f2f68b9cf2d in ros::ServiceServerLink::cancelCall (this=<optimized out>, info=...) at ../../../src/roscpp/src/libros/service_server_link.cpp:83 #2 0x00007f2f68b9d0e6 in ros::ServiceServerLink::clearCalls (this=0x7f2f4802b2b0) at ../../../src/roscpp/src/libros/service_server_link.cpp:99 #3 0x00007f2f68b9d466 in ros::ServiceServerLink::onConnectionDropped (this=0x7f2f4802b2b0, conn=...) at ../../../src/roscpp/src/libros/service_server_link.cpp:175 #4 0x00007f2f68b4b962 in boost::function2<void, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>::operator() (a1=<optimized out>, a0=..., this=<optimized out>) at /usr/include/boost/function/function_template.hpp:677 #5 boost::signals2::detail::call_with_tuple_args<boost::signals2::detail::void_type>::m_invoke<boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>, 0u, 1u, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason&>(boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>&, boost::signals2::detail::unsigned_meta_array<0u, 1u>, std::tuple<boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason&> const&, boost::enable_if<boost::is_void<boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>::result_type>, void>::type*) const (this=<optimized out>, args=std::tuple containing = {...}, func=...) at /usr/include/boost/signals2/detail/variadic_slot_invoker.hpp:105 #6 boost::signals2::detail::call_with_tuple_args<boost::signals2::detail::void_type>::operator()<boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason&, 2ul>(boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>&, std::tuple<boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason&> const&, mpl_::size_t<2ul>) const (this=<optimized out>, args=std::tuple containing = {...}, func=...) at /usr/include/boost/signals2/detail/variadic_slot_invoker.hpp:90 #7 boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>::operator()<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >(boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > const&) const (connectionBody=..., this=0x7f2f653ee860) at /usr/include/boost/signals2/detail/variadic_slot_invoker.hpp:133 #8 boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> >::dereference() const (this=0x7f2f653ee5b0) at /usr/include/boost/signals2/detail/slot_call_iterator.hpp:110 #9 boost::iterators::iterator_core_access::dereference<boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >(boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > const&) (f=...) at /usr/include/boost/iterator/iterator_facade.hpp:550 #10 boost::iterators::detail::iterator_facade_base<boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> >, boost::signals2::detail::void_type, boost::iterators::single_pass_traversal_tag, boost::signals2::detail::void_type const&, long, false, false>::operator*() const (this=0x7f2f653ee5b0) at /usr/include/boost/iterator/iterator_facade.hpp:656 #11 boost::signals2::optional_last_value<void>::operator()<boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >(boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> >, boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> >) const (last=..., first=..., this=<optimized out>) at /usr/include/boost/signals2/optional_last_value.hpp:57 #12 boost::signals2::detail::combiner_invoker<void>::operator()<boost::signals2::optional_last_value<void>, boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >(boost::signals2::optional_last_value<void>&, boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> >, boost::signals2::detail::slot_call_iterator_t<boost::signals2::detail::variadic_slot_invoker<boost::signals2::detail::void_type, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason>, std::_List_iterator<boost::shared_ptr<boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> > >, boost::signals2::detail::connection_body<std::pair<boost::signals2::detail::slot_meta_group, boost::optional<int> >, boost::signals2::slot<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)> >, boost::signals2::mutex> >) const (this=<optimized out>, last=..., first=..., combiner=...) at /usr/include/boost/signals2/detail/result_type_wrapper.hpp:64 #13 boost::signals2::detail::signal_impl<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::signals2::optional_last_value<void>, int, std::less<int>, boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>, boost::function<void (boost::signals2::connection const&, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>, boost::signals2::mutex>::operator()(boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason) (this=0x7f2f48020480, args#0=..., args#1=<optimized out>, args#1@entry=ros::Connection::TransportDisconnect) at /usr/include/boost/signals2/detail/signal_template.hpp:243 #14 0x00007f2f68b42c53 in boost::signals2::signal<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason), boost::signals2::optional_last_value<void>, int, std::less<int>, boost::function<void (boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>, boost::function<void (boost::signals2::connection const&, boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason)>, boost::signals2::mutex>::operator()(boost::shared_ptr<ros::Connection> const&, ros::Connection::DropReason) (args#1=ros::Connection::TransportDisconnect, args#0=..., this=0x7f2f4802d230) at /usr/include/boost/smart_ptr/shared_ptr.hpp:726 #15 ros::Connection::drop (this=0x7f2f4802d0c0, reason=ros::Connection::TransportDisconnect) at ../../../src/roscpp/src/libros/connection.cpp:343 #16 0x00007f2f68b43251 in ros::Connection::onDisconnect (this=0x7f2f4802d0c0, transport=...) at ../../../src/roscpp/src/libros/connection.cpp:322 #17 0x00007f2f68bbcde4 in boost::function1<void, boost::shared_ptr<ros::Transport> const&>::operator() (a0=..., this=0x7f2f653eea30) at /usr/include/boost/function/function_template.hpp:677 #18 ros::TransportTCP::close (this=0x7f2f48022110) at ../../../src/roscpp/src/libros/transport/transport_tcp.cpp:481 #19 0x00007f2f68bbd709 in ros::TransportTCP::socketUpdate (this=0x7f2f48022110, events=8197) at ../../../src/roscpp/src/libros/transport/transport_tcp.cpp:758 #20 0x00007f2f68bf722c in boost::function1<void, int>::operator() (a0=<optimized out>, this=0x7f2f653eec50) at /usr/include/boost/function/function_template.hpp:677 #21 ros::PollSet::update (this=this@entry=0x55827833cce0, poll_timeout=poll_timeout@entry=100) at ../../../src/roscpp/src/libros/poll_set.cpp:255 #22 0x00007f2f68b7e2c3 in ros::PollManager::threadFunc (this=0x55827833cce0) at ../../../src/roscpp/src/libros/poll_manager.cpp:88 #23 0x00007f2f6837c43b in ?? () from /lib/x86_64-linux-gnu/libboost_thread.so.1.71.0 #24 0x00007f2f68a68609 in start_thread (arg=<optimized out>) at pthread_create.c:477 #25 0x00007f2f6872e293 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

Code Analysis

The following commit 5f8d76d introduced a lock prior to calling the signal when a rosservice socket drops: https://github.com/ros/ros_comm/blob/noetic-devel/clients/roscpp/src/libros/connection.cpp#L342
When a socket drops, the signaling thread eventually makes its way to cancelling the call that is in progress here: https://github.com/ros/ros_comm/blob/noetic-devel/clients/roscpp/src/libros/service_server_link.cpp#L81
If at the exact same time, another thread attempts to drop the socket itself via the processNextCall here: https://github.com/ros/ros_comm/blob/noetic-devel/clients/roscpp/src/libros/service_server_link.cpp#L309
The second thread will block forever.