-
Notifications
You must be signed in to change notification settings - Fork 5.3k
Closed
Description
Seeing this crash with newly deployed init manager code:
Backtrace:
#0 _M_bucket_begin (this=0x18, __bkt=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/hashtable.h:926
#0 _M_bucket_begin (this=0x18, __bkt=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/hashtable.h:926
#1 std::_Hashtable<std::string, std::pair<std::string const, std::unique_ptr<Envoy::Upstream::ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry, std::default_delete<Envoy::Upstream::ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry> > >, std::allocator<std::pair<std::string const, std::unique_ptr<Envoy::Upstream::ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry, std::default_delete<Envoy::Upstream::ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry> > > >, std::__detail::_Select1st, std::equal_to<std::string>, std::hash<std::string>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<true, false, true> >::count (this=0x18, __k=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/hashtable.h:1457
#2 0x00000000006a08b4 in count (this=0x18, __x=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/unordered_map.h:938
#3 operator() (this=0x606a720) at external/envoy/source/common/upstream/cluster_manager_impl.cc:514
#4 std::_Function_handler<void (), Envoy::Upstream::ClusterManagerImpl::createOrUpdateThreadLocalCluster(Envoy::Upstream::ClusterManagerImpl::ClusterData&)::$_10>::_M_invoke(std::_Any_data const&) (__functor=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/std_function.h:316
#5 0x0000000000600ae7 in operator() (this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/std_function.h:706
#6 Envoy::ThreadLocal::InstanceImpl::runOnAllThreads(std::function<void ()>) (this=<optimized out>, cb=...) at external/envoy/source/common/thread_local/thread_local_impl.cc:91
#7 0x0000000000601a62 in Envoy::ThreadLocal::InstanceImpl::SlotImpl::runOnAllThreads(std::function<void ()>) (this=<optimized out>, cb=...) at bazel-out/k8-opt/bin/external/envoy/source/common/thread_local/_virtual_includes/thread_local_lib/common/thread_local/thread_local_impl.h:37
#8 0x00000000006946ff in Envoy::Upstream::ClusterManagerImpl::createOrUpdateThreadLocalCluster (this=<optimized out>, cluster=...) at external/envoy/source/common/upstream/cluster_manager_impl.cc:509
#9 0x00000000006a01be in operator() (this=0x603a9c0) at external/envoy/source/common/upstream/cluster_manager_impl.cc:497
#10 std::_Function_handler<void (), Envoy::Upstream::ClusterManagerImpl::addOrUpdateCluster(envoy::api::v2::Cluster const&, std::string const&, std::function<void (std::string const&, Envoy::Upstream::ClusterManager::ClusterWarmingState)>)::$_9>::_M_invoke(std::_Any_data const&) (__functor=...) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/std_function.h:316
#11 0x00000000006ea62a in operator() (this=0x603a9c0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/std_function.h:706
#12 Envoy::Upstream::ClusterImplBase::finishInitialization (this=0x5fb76c0) at external/envoy/source/common/upstream/upstream_impl.cc:758
#13 0x00000000006ea562 in Envoy::Upstream::ClusterImplBase::onInitDone (this=0x5fb76c0) at external/envoy/source/common/upstream/upstream_impl.cc:740
#14 0x00000000007349a3 in operator() (this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/std_function.h:706
#15 Envoy::Init::WatcherHandleImpl::ready (this=<optimized out>) at external/envoy/source/common/init/watcher_impl.cc:15
#16 0x0000000000734200 in ready (this=<optimized out>) at external/envoy/source/common/init/manager_impl.cc:75
#17 Envoy::Init::ManagerImpl::initialize (this=0x5fb76c8, watcher=...) at external/envoy/source/common/init/manager_impl.cc:46
#18 0x00000000006ea3a2 in Envoy::Upstream::ClusterImplBase::onPreInitComplete (this=<optimized out>) at external/envoy/source/common/upstream/upstream_impl.cc:722
#19 0x0000000000638dc6 in Envoy::Config::GrpcMuxSubscriptionImpl::onConfigUpdateFailed (this=0x5fb7e68, e=0x0) at bazel-out/k8-opt/bin/external/envoy/source/common/config/_virtual_includes/grpc_mux_subscription_lib/common/config/grpc_mux_subscription_impl.h:80
#20 0x0000000000707270 in non-virtual thunk to Envoy::Config::GrpcMuxImpl::handleEstablishmentFailure() () at external/envoy/source/common/config/grpc_mux_impl.cc:204
#21 0x0000000000707413 in Envoy::Config::GrpcStream<envoy::api::v2::DiscoveryRequest, envoy::api::v2::DiscoveryResponse, std::string>::onRemoteClose (this=0x5fb7d50, status=Envoy::Grpc::Status::Unavailable, message=...) at bazel-out/k8-opt/bin/external/envoy/source/common/config/_virtual_includes/grpc_stream_lib/common/config/grpc_stream.h:104
#22 0x0000000000735f97 in Envoy::Grpc::AsyncStreamImpl::onTrailers(std::unique_ptr<Envoy::Http::HeaderMap, std::default_delete<Envoy::Http::HeaderMap> >&&) (this=0x5fa7680, trailers=<unknown type in /usr/sbin/envoy, CU 0x7e81c63, DIE 0x7f3faaf>) at external/envoy/source/common/grpc/async_client_impl.cc:160
#23 0x0000000000735c85 in Envoy::Grpc::AsyncStreamImpl::onHeaders(std::unique_ptr<Envoy::Http::HeaderMap, std::default_delete<Envoy::Http::HeaderMap> >&&, bool) (this=0x5fa7680, headers=<unknown type in /usr/sbin/envoy, CU 0x7e81c63, DIE 0x7f3efeb>, end_stream=true) at external/envoy/source/common/grpc/async_client_impl.cc:106
#24 0x000000000073aa85 in Envoy::Http::AsyncStreamImpl::encodeHeaders(std::unique_ptr<Envoy::Http::HeaderMap, std::default_delete<Envoy::Http::HeaderMap> >&&, bool) (this=0x5fd6700, headers=<unknown type in /usr/sbin/envoy, CU 0x7f615e3, DIE 0x8050ab6>, end_stream=true) at external/envoy/source/common/http/async_client_impl.cc:96
#25 0x00000000009472f6 in operator() (this=0x7ffcf927b538, __args=true, __args=true) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/std_function.h:706
#26 Envoy::Http::Utility::sendLocalReply(bool, std::function<void (std::unique_ptr<Envoy::Http::HeaderMap, std::default_delete<Envoy::Http::HeaderMap> >&&, bool)>, std::function<void (Envoy::Buffer::Instance&, bool)>, bool const&, Envoy::Http::Code, absl::string_view, absl::optional<Envoy::Grpc::Status::GrpcStatus>, bool) (is_grpc=<optimized out>, encode_headers=..., encode_data=..., is_reset=<optimized out>, response_code=<optimized out>, body_text=..., grpc_status=..., is_head_request=<optimized out>) at external/envoy/source/common/http/utility.cc:327
#27 0x000000000073baed in Envoy::Http::AsyncStreamImpl::sendLocalReply(Envoy::Http::Code, absl::string_view, std::function<void (Envoy::Http::HeaderMap&)>, absl::optional<Envoy::Grpc::Status::GrpcStatus>) (this=<optimized out>, code=Envoy::Http::ServiceUnavailable, body=..., modify_headers=..., grpc_status=...) at bazel-out/k8-opt/bin/external/envoy/source/common/http/_virtual_includes/async_client_lib/common/http/async_client_impl.h:324
#28 0x00000000007420ce in Envoy::Router::Filter::onUpstreamAbort (this=0x5fd6740, code=<optimized out>, response_flags=<optimized out>, body=..., dropped=<optimized out>) at external/envoy/source/common/router/router.cc:597
#29 0x0000000000742676 in Envoy::Router::Filter::onUpstreamReset (this=0x5fd6740, reset_reason=Envoy::Http::ConnectionTermination, transport_failure_reason=...) at external/envoy/source/common/router/router.cc:659
#30 0x00000000007b4ec5 in runResetCallbacks (reason=Envoy::Http::ConnectionTermination, this=<optimized out>) at bazel-out/k8-opt/bin/external/envoy/source/common/http/_virtual_includes/codec_helper_lib/common/http/codec_helper.h:49
#31 Envoy::Http::Http2::ConnectionImpl::StreamImpl::resetStream (this=0x5fd2600, reason=Envoy::Http::ConnectionTermination) at external/envoy/source/common/http/http2/codec_impl.cc:302
#32 0x000000000072ad24 in Envoy::Http::CodecClient::onEvent (this=0x4001900, event=<optimized out>) at external/envoy/source/common/http/codec_client.cc:87
#33 0x000000000065dafd in Envoy::Network::ConnectionImpl::raiseEvent (this=0x50f8000, event=Envoy::Network::LocalClose) at external/envoy/source/common/network/connection_impl.cc:329
#34 0x000000000065d4a4 in Envoy::Network::ConnectionImpl::closeSocket (this=0x50f8000, close_type=Envoy::Network::LocalClose) at external/envoy/source/common/network/connection_impl.cc:194
#35 0x000000000065d1c7 in Envoy::Network::ConnectionImpl::close (this=0x50f8000, type=Envoy::Network::NoFlush) at external/envoy/source/common/network/connection_impl.cc:110
#36 0x00000000006d56c4 in Envoy::Http::Http2::ConnPoolImpl::~ConnPoolImpl (this=0x3e01780) at external/envoy/source/common/http/http2/conn_pool.cc:26
#37 0x00000000006d590e in Envoy::Http::Http2::ProdConnPoolImpl::~ProdConnPoolImpl (this=0x3e01780) at bazel-out/k8-opt/bin/external/envoy/source/common/http/http2/_virtual_includes/conn_pool_lib/common/http/http2/conn_pool.h:103
#38 0x000000000069e62d in operator() (__ptr=0x3e9e4758d97bcb08, this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/unique_ptr.h:78
#39 ~unique_ptr (this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/unique_ptr.h:268
#40 ~pair (this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/stl_pair.h:193
#41 destroy<std::pair<std::vector<unsigned char, std::allocator<unsigned char> >, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > > > (this=<optimized out>, __p=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/ext/new_allocator.h:140
#42 destroy_impl<std::allocator<std::pair<std::vector<unsigned char, std::allocator<unsigned char> > const, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > > >, std::pair<std::vector<unsigned char, std::allocator<unsigned char> >, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > > > (a=..., p=<optimized out>) at external/com_google_absl/absl/memory/memory.h:590
#43 destroy<std::pair<std::vector<unsigned char, std::allocator<unsigned char> >, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > > > (a=..., p=<optimized out>) at external/com_google_absl/absl/memory/memory.h:548
#44 destroy<std::allocator<std::pair<std::vector<unsigned char, std::allocator<unsigned char> > const, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > > > > (alloc=<optimized out>, slot=<optimized out>) at external/com_google_absl/absl/container/internal/container_memory.h:347
#45 destroy<std::allocator<std::pair<std::vector<unsigned char, std::allocator<unsigned char> > const, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > > > > (alloc=<optimized out>, slot=<optimized out>) at external/com_google_absl/absl/container/flat_hash_map.h:501
#46 destroy<std::allocator<std::pair<std::vector<unsigned char, std::allocator<unsigned char> > const, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > > > > (alloc=<optimized out>, slot=<optimized out>) at external/com_google_absl/absl/container/internal/hash_policy_traits.h:83
#47 absl::container_internal::raw_hash_set<absl::container_internal::FlatHashMapPolicy<std::vector<unsigned char, std::allocator<unsigned char> >, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > >, absl::hash_internal::Hash<std::vector<unsigned char, std::allocator<unsigned char> > >, std::equal_to<std::vector<unsigned char, std::allocator<unsigned char> > >, std::allocator<std::pair<std::vector<unsigned char, std::allocator<unsigned char> > const, std::unique_ptr<Envoy::Http::ConnectionPool::Instance, std::default_delete<Envoy::Http::ConnectionPool::Instance> > > > >::clear (this=0x402b500) at external/com_google_absl/absl/container/internal/raw_hash_set.h:1040
#48 0x000000000069e2d8 in clearActivePools (this=0x402b500) at bazel-out/k8-opt/bin/external/envoy/source/common/upstream/_virtual_includes/conn_pool_map_impl_lib/common/upstream/conn_pool_map_impl.h:119
#49 Envoy::Upstream::ConnPoolMap<std::vector<unsigned char, std::allocator<unsigned char> >, Envoy::Http::ConnectionPool::Instance>::~ConnPoolMap (this=0x402b500) at bazel-out/k8-opt/bin/external/envoy/source/common/upstream/_virtual_includes/conn_pool_map_impl_lib/common/upstream/conn_pool_map_impl.h:19
#50 0x000000000069e09c in operator() (__ptr=0x402b500, this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/unique_ptr.h:78
#51 ~unique_ptr (this=0x330eff8) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/unique_ptr.h:268
#52 ~array (this=0x330eff0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/array:94
#53 ~PriorityConnPoolMap (this=0x330eff0) at bazel-out/k8-opt/bin/external/envoy/source/common/upstream/_virtual_includes/priority_conn_pool_map/common/upstream/priority_conn_pool_map.h:22
#54 destroy<Envoy::Upstream::PriorityConnPoolMap<std::vector<unsigned char, std::allocator<unsigned char> >, Envoy::Http::ConnectionPool::Instance> > (this=0x330eff0, __p=0x330eff0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/ext/new_allocator.h:140
#55 destroy<Envoy::Upstream::PriorityConnPoolMap<std::vector<unsigned char, std::allocator<unsigned char> >, Envoy::Http::ConnectionPool::Instance> > (__a=..., __p=0x330eff0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/alloc_traits.h:487
#56 std::_Sp_counted_ptr_inplace<Envoy::Upstream::PriorityConnPoolMap<std::vector<unsigned char, std::allocator<unsigned char> >, Envoy::Http::ConnectionPool::Instance>, std::allocator<Envoy::Upstream::PriorityConnPoolMap<std::vector<unsigned char, std::allocator<unsigned char> >, Envoy::Http::ConnectionPool::Instance> >, (__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x330efe0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/shared_ptr_base.h:535
#57 0x000000000069df95 in _M_release (this=0x330efe0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/shared_ptr_base.h:154
#58 ~__shared_count (this=0x3dd7be0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/shared_ptr_base.h:684
#59 ~__shared_ptr (this=0x3dd7bd8) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/shared_ptr_base.h:1123
#60 ~ConnPoolsContainer (this=0x3dd7bd8) at bazel-out/k8-opt/bin/external/envoy/source/common/upstream/_virtual_includes/cluster_manager_lib/common/upstream/cluster_manager_impl.h:238
#61 ~pair (this=0x3dd7bc8) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/stl_pair.h:193
#62 destroy<std::pair<std::shared_ptr<Envoy::Upstream::Host const> const, Envoy::Upstream::ClusterManagerImpl::ThreadLocalClusterManagerImpl::ConnPoolsContainer> > (this=<optimized out>, __p=0x3dd7bc8) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/ext/new_allocator.h:140
#63 destroy<std::pair<std::shared_ptr<Envoy::Upstream::Host const> const, Envoy::Upstream::ClusterManagerImpl::ThreadLocalClusterManagerImpl::ConnPoolsContainer> > (__a=..., __p=0x3dd7bc8) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/alloc_traits.h:487
#64 std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<std::shared_ptr<Envoy::Upstream::Host const> const, Envoy::Upstream::ClusterManagerImpl::ThreadLocalClusterManagerImpl::ConnPoolsContainer>, false> > >::_M_deallocate_node (this=<optimized out>, __n=0x3dd7bc0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/hashtable_policy.h:2084
#65 0x000000000069a263 in _M_deallocate_nodes (this=0x3334f60, __n=0x0) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/hashtable_policy.h:2097
#66 clear (this=0x3334f60) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/hashtable.h:2032
#67 clear (this=0x3334f60) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/unordered_map.h:842
#68 Envoy::Upstream::ClusterManagerImpl::ThreadLocalClusterManagerImpl::~ThreadLocalClusterManagerImpl (this=0x3334f10) at external/envoy/source/common/upstream/cluster_manager_impl.cc:831
#69 0x0000000000601777 in _M_release (this=0x3334f00) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/shared_ptr_base.h:154
#70 ~__shared_count (this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/shared_ptr_base.h:684
#71 ~__shared_ptr (this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/shared_ptr_base.h:1123
#72 reset (this=<optimized out>) at /usr/bin/../lib/gcc/x86_64-linux-gnu/7.4.0/../../../../include/c++/7.4.0/bits/shared_ptr_base.h:1235
#73 Envoy::ThreadLocal::InstanceImpl::shutdownThread (this=<optimized out>) at external/envoy/source/common/thread_local/thread_local_impl.cc:167
#74 0x0000000000607f8e in Envoy::Server::InstanceImpl::terminate (this=0x2a46500) at external/envoy/source/server/server.cc:520
#75 0x000000000060bfaf in Envoy::Server::InstanceImpl::run (this=0x2a46500) at external/envoy/source/server/server.cc:487
#76 0x0000000000459d39 in Envoy::MainCommonBase::run (this=0x2a8e530) at external/envoy/source/exe/main_common.cc:117
#77 0x0000000000458b84 in run (this=0x2a8e420) at bazel-out/k8-opt/bin/external/envoy/source/exe/_virtual_includes/envoy_main_common_lib/exe/main_common.h:88
#78 main (argc=17, argv=<optimized out>) at external/envoy/source/exe/main.cc:39
Reactions are currently unavailable