Ceph » crimson

https://jenkins.ceph.com/job/ceph-pull-requests/69856/consoleFull#1915030918d63714d2-c8d8-41fc-a9d4-8dee30be4c32

unittest-seastar-messenger: /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/include/seastar/core/sharded.hh:518: seastar::sharded<T>::~sharded() [with Service = crimson::common::ConfigProxy]: Assertion `_instances.empty()' failed.
=================================================================
==26366==ERROR: AddressSanitizer: heap-use-after-free on address 0x619000004b90 at pc 0x55617cbbda18 bp 0x7ffcdeb080a0 sp 0x7ffcdeb08090
READ of size 8 at 0x619000004b90 thread T0
    #0 0x55617cbbda17 in operator() /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/util/backtrace.cc:76
    #1 0x55617cbc5ed7 in operator()<__gnu_cxx::__normal_iterator<const seastar::shared_object*, std::vector<seastar::shared_object> > > /usr/include/c++/9/bits/predefined_ops.h:283
    #2 0x55617cbc5bb1 in __find_if<__gnu_cxx::__normal_iterator<const seastar::shared_object*, std::vector<seastar::shared_object> >, __gnu_cxx::__ops::_Iter_pred<seastar::decorate(uintptr_t)::<lambda(const seastar::shared_object&)> > > /usr/include/c++/9/bits/stl_algo.h:120
    #3 0x55617cbc55b5 in __find_if<__gnu_cxx::__normal_iterator<const seastar::shared_object*, std::vector<seastar::shared_object> >, __gnu_cxx::__ops::_Iter_pred<seastar::decorate(uintptr_t)::<lambda(const seastar::shared_object&)> > > /usr/include/c++/9/bits/stl_algo.h:161
    #4 0x55617cbc4957 in find_if<__gnu_cxx::__normal_iterator<const seastar::shared_object*, std::vector<seastar::shared_object> >, seastar::decorate(uintptr_t)::<lambda(const seastar::shared_object&)> > /usr/include/c++/9/bits/stl_algo.h:3963
    #5 0x55617cbbdc65 in seastar::decorate(unsigned long) /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/util/backtrace.cc:77
    #6 0x55617c58d553 in void seastar::backtrace<seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}>(seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}&&) /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/include/seastar/util/backtrace.hh:62
    #7 0x55617c565735 in seastar::backtrace_buffer::append_backtrace() /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:732
    #8 0x55617c416070 in print_with_backtrace /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:753
    #9 0x55617c4161e3 in print_with_backtrace /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:760
    #10 0x55617c479f83 in sigabrt_action /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:3513
    #11 0x55617c4b2b4e in operator() /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:3495
    #12 0x55617c4b2be9 in _FUN /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:3491
    #13 0x7f5cce71389f  (/lib/x86_64-linux-gnu/libpthread.so.0+0x1289f)
    #14 0x7f5ccce5af46 in gsignal (/lib/x86_64-linux-gnu/libc.so.6+0x3ef46)
    #15 0x7f5ccce5c8b0 in abort (/lib/x86_64-linux-gnu/libc.so.6+0x408b0)
    #16 0x7f5ccce4c429  (/lib/x86_64-linux-gnu/libc.so.6+0x30429)
    #17 0x7f5ccce4c4a1 in __assert_fail (/lib/x86_64-linux-gnu/libc.so.6+0x304a1)
    #18 0x55617bd2b9cb in seastar::sharded<crimson::common::ConfigProxy>::~sharded() /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/include/seastar/core/sharded.hh:518
    #19 0x7f5ccce5f0f0  (/lib/x86_64-linux-gnu/libc.so.6+0x430f0)
    #20 0x7f5ccce5f1e9 in exit (/lib/x86_64-linux-gnu/libc.so.6+0x431e9)
    #21 0x7f5ccce3db9d in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b9d)
    #22 0x556179b680a9 in _start (/home/jenkins-build/build/workspace/ceph-pull-requests/build/bin/unittest-seastar-messenger+0xb1a40a9)

0x619000004b90 is located 16 bytes inside of 1024-byte region [0x619000004b80,0x619000004f80)
freed by thread T0 here:
    #0 0x7f5cd0193cef in operator delete(void*) (/usr/lib/x86_64-linux-gnu/libasan.so.5+0x10ecef)
    #1 0x55617cbd31e3 in __gnu_cxx::new_allocator<seastar::shared_object>::deallocate(seastar::shared_object*, unsigned long) /usr/include/c++/9/ext/new_allocator.h:128
    #2 0x55617cbcfb33 in std::allocator_traits<std::allocator<seastar::shared_object> >::deallocate(std::allocator<seastar::shared_object>&, seastar::shared_object*, unsigned long) /usr/include/c++/9/bits/alloc_traits.h:470
    #3 0x55617cbcb019 in std::_Vector_base<seastar::shared_object, std::allocator<seastar::shared_object> >::_M_deallocate(seastar::shared_object*, unsigned long) /usr/include/c++/9/bits/stl_vector.h:351
    #4 0x55617cbc893c in std::_Vector_base<seastar::shared_object, std::allocator<seastar::shared_object> >::~_Vector_base() /usr/include/c++/9/bits/stl_vector.h:332
    #5 0x55617cbc8b59 in std::vector<seastar::shared_object, std::allocator<seastar::shared_object> >::~vector() /usr/include/c++/9/bits/stl_vector.h:680
    #6 0x7f5ccce5f0f0  (/lib/x86_64-linux-gnu/libc.so.6+0x430f0)

previously allocated by thread T0 here:
    #0 0x7f5cd0192d6f in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.5+0x10dd6f)
    #1 0x55617cbd5f6e in __gnu_cxx::new_allocator<seastar::shared_object>::allocate(unsigned long, void const*) /usr/include/c++/9/ext/new_allocator.h:114
    #2 0x55617cbd49b9 in std::allocator_traits<std::allocator<seastar::shared_object> >::allocate(std::allocator<seastar::shared_object>&, unsigned long) /usr/include/c++/9/bits/alloc_traits.h:444
    #3 0x55617cbd2cff in std::_Vector_base<seastar::shared_object, std::allocator<seastar::shared_object> >::_M_allocate(unsigned long) /usr/include/c++/9/bits/stl_vector.h:343
    #4 0x55617cbcf220 in void std::vector<seastar::shared_object, std::allocator<seastar::shared_object> >::_M_realloc_insert<seastar::shared_object>(__gnu_cxx::__normal_iterator<seastar::shared_object*, std::vector<seastar::shared_object, std::allocator<seastar::shared_object> > >, seastar::shared_object&&) /usr/include/c++/9/bits/vector.tcc:440
    #5 0x55617cbcadbb in seastar::shared_object& std::vector<seastar::shared_object, std::allocator<seastar::shared_object> >::emplace_back<seastar::shared_object>(seastar::shared_object&&) /usr/include/c++/9/bits/vector.tcc:121
    #6 0x55617cbc8710 in std::vector<seastar::shared_object, std::allocator<seastar::shared_object> >::push_back(seastar::shared_object&&) /usr/include/c++/9/bits/stl_vector.h:1201
    #7 0x55617cbbd619 in dl_iterate_phdr_callback /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/util/backtrace.cc:49
    #8 0x7f5cccf82160 in dl_iterate_phdr (/lib/x86_64-linux-gnu/libc.so.6+0x166160)

SUMMARY: AddressSanitizer: heap-use-after-free /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/util/backtrace.cc:76 in operator()
Shadow bytes around the buggy address:
  0x0c327fff8920: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8930: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8940: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8950: fd fd fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c327fff8960: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
=>0x0c327fff8970: fd fd[fd]fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8980: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8990: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff89a0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff89b0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff89c0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07 
  Heap left redzone:       fa
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
  Shadow gap:              cc
==26366==ABORTING

https://jenkins.ceph.com/job/ceph-pull-requests/69836/consoleFull#1962414319e840cee4-f4a4-4183-81dd-42855615f2c1

INFO  2021-02-23 17:43:59,258 [shard 0] ms - -- 0 --
INFO  2021-02-23 17:43:59,258 [shard 0] ms - [Test] setup connection...
INFO  2021-02-23 17:43:59,258 [shard 0] ms - [Test] peer_connect_me(v2:127.0.0.1:9012/2)
INFO  2021-02-23 17:43:59,271 [shard 0] ms - [TestPeer] connect_peer(v2:127.0.0.1:9012/2)
INFO  2021-02-23 17:43:59,272 [shard 0] ms - [osd.4(TestPeer) v2:127.0.0.1:9014/4 >> osd.? v2:127.0.0.1:9012/2] ProtocolV2::start_connect(): peer_addr=v2:127.0.0.1:9012/2, peer_name=osd.?, cc=1286576077112908087 policy(lossy=true, server=false, standby=false, resetcheck=false)
INFO  2021-02-23 17:43:59,273 [shard 0] ms - [TestPeer] this is a new session
INFO  2021-02-23 17:43:59,281 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> unknown.? -] new connection registered
INFO  2021-02-23 17:43:59,281 [shard 0] ms - [osd.2(Test) v2:127.0.0.1:9012/2 >> unknown.? -@57769] ProtocolV2::start_accept(): target_addr=127.0.0.1:57769/0
INFO  2021-02-23 17:43:59,282 [shard 0] ms - [Test] accept_attempts=1
INFO  2021-02-23 17:43:59,283 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> unknown.? -@57769] intercepted SOCKET_ACCEPTED(1)
INFO  2021-02-23 17:43:59,284 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> unknown.? -@57769] intercepted BANNER_WRITE(1)
INFO  2021-02-23 17:43:59,291 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> unknown.? -@57769] intercepted BANNER_READ(1)
INFO  2021-02-23 17:43:59,312 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> unknown.? -@57769] intercepted BANNER_PAYLOAD_READ(1)
INFO  2021-02-23 17:43:59,325 [shard 0] ms - [Test] send_peer() (pending 1)
INFO  2021-02-23 17:43:59,327 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> unknown.? -@57769] intercepted HELLO_WRITE(1)
INFO  2021-02-23 17:43:59,328 [shard 0] ms - [Test] peer_send_me()
INFO  2021-02-23 17:43:59,341 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> unknown.? -@57769] intercepted HELLO_READ(1)
INFO  2021-02-23 17:43:59,354 [shard 0] ms - [TestPeer] send_peer()
INFO  2021-02-23 17:43:59,354 [shard 0] ms - [osd.4(TestPeer) v2:127.0.0.1:9014/4@57769 >> osd.? v2:127.0.0.1:9012/2] write_event: delay ...
INFO  2021-02-23 17:43:59,359 [shard 0] ms - [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.? -@57769] UPDATE: peer_type=osd, policy(lossy=true server=true standby=false resetcheck=false)
INFO  2021-02-23 17:43:59,370 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.? -@57769] intercepted AUTH_REQUEST_READ(1)
INFO  2021-02-23 17:43:59,378 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.? -@57769] intercepted AUTH_DONE_WRITE(1)
INFO  2021-02-23 17:43:59,382 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.? -@57769] intercepted AUTH_SIGNATURE_WRITE(1)
INFO  2021-02-23 17:43:59,387 [shard 0] ms - [Test] wait_result(1) ...
INFO  2021-02-23 17:43:59,388 [shard 0] ms - [Test] wait_ready(): wait for connections, currently 1 out of 1, pending 1 ready ...
INFO  2021-02-23 17:43:59,388 [shard 0] ms - [Test] wait_ready(): wait for pending_send=1, pending_peer_receive=0, pending_receive=1, pending 1/0 ready/establish connections ...
INFO  2021-02-23 17:43:59,404 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.? -@57769] intercepted AUTH_SIGNATURE_READ(1)
INFO  2021-02-23 17:43:59,414 [shard 0] ms - [Test] server_connect_attemps=1
INFO  2021-02-23 17:43:59,414 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.? -@57769] intercepted CLIENT_IDENT_READ(1)
INFO  2021-02-23 17:43:59,419 [shard 0] ms - [Test] got accept (cnt_accept_dispatched=1), track [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.4 v2:127.0.0.1:9014/4@57769]
INFO  2021-02-23 17:43:59,420 [shard 0] ms - [Test] flush sending 1 ops
INFO  2021-02-23 17:43:59,420 [shard 0] ms - [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.4 v2:127.0.0.1:9014/4@57769] write_event: delay ...
INFO  2021-02-23 17:43:59,423 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.4 v2:127.0.0.1:9014/4@57769] intercepted SERVER_IDENT_WRITE(1)
INFO  2021-02-23 17:43:59,426 [shard 0] ms - [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.4 v2:127.0.0.1:9014/4@57769] established: gs=0, pgs=1, cs=0, client_cookie=1286576077112908087, server_cookie=0, in_seq=0, out_seq=0, out_q=1
INFO  2021-02-23 17:43:59,427 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.4 v2:127.0.0.1:9014/4@57769] ready
INFO  2021-02-23 17:43:59,433 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.4 v2:127.0.0.1:9014/4@57769] intercepted MESSAGE_WRITE(1)
INFO  2021-02-23 17:43:59,435 [shard 0] ms - [Test] wait_ready(): wait for pending_send=0, pending_peer_receive=1, pending_receive=1, pending 0/1 ready/establish connections ...
INFO  2021-02-23 17:43:59,442 [shard 0] ms - [osd.4(TestPeer) v2:127.0.0.1:9014/4@57769 >> osd.2 v2:127.0.0.1:9012/2] connected: gs=1, pgs=1, cs=0, client_cookie=1286576077112908087, server_cookie=0, in_seq=0, out_seq=0, out_q=1
INFO  2021-02-23 17:43:59,450 [shard 0] ms - [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.4 v2:127.0.0.1:9014/4@57769] intercepted MESSAGE_READ(1)
INFO  2021-02-23 17:43:59,459 [shard 0] ms - [Test] got op, left 0 ops -- [0] [osd.2(Test) v2:127.0.0.1:9012/2 >> osd.4 v2:127.0.0.1:9014/4@57769]
INFO  2021-02-23 17:43:59,462 [shard 0] ms - [Test] wait_ready(): wait for pending_send=0, pending_peer_receive=1, pending_receive=0, pending 0/1 ready/establish connections ...
INFO  2021-02-23 17:43:59,465 [shard 0] ms - [TestPeer] got op from Test
INFO  2021-02-23 17:43:59,476 [shard 0] ms - [Test] TestPeer said got op, left 0 ops
INFO  2021-02-23 17:43:59,478 [shard 0] ms - [Test] wait_ready(): wait done!
INFO  2021-02-23 17:43:59,480 [shard 0] ms - -- 1 --
INFO  2021-02-23 17:43:59,481 [shard 0] ms - [Test] client markdown...
INFO  2021-02-23 17:43:59,481 [shard 0] ms - [Test] markdown_peer()
ERROR 2021-02-23 17:43:59,491 [shard 0] ms - test_messenger timeout after 120s, abort! Consider to extend the period if the test is still running.
INFO  2021-02-23 17:43:59,496 [shard 0] ms - [TestPeer] markdown()
INFO  2021-02-23 17:43:59,497 [shard 0] ms - [osd.4(TestPeer) v2:127.0.0.1:9014/4@57769 >> osd.2 v2:127.0.0.1:9012/2] closing: reset no, replace no
unittest-seastar-messenger: /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/include/seastar/core/sharded.hh:518: seastar::sharded<T>::~sharded() [with Service = crimson::common::ConfigProxy]: Assertion `_instances.empty()' failed.
=================================================================
==25808==ERROR: AddressSanitizer: heap-use-after-free on address 0x619000004b90 at pc 0x55a1409d6a48 bp 0x7ffc5c40f460 sp 0x7ffc5c40f450
READ of size 8 at 0x619000004b90 thread T0
    #0 0x55a1409d6a47 in operator() /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/util/backtrace.cc:76
    #1 0x55a1409def07 in operator()<__gnu_cxx::__normal_iterator<const seastar::shared_object*, std::vector<seastar::shared_object> > > /usr/include/c++/9/bits/predefined_ops.h:283
    #2 0x55a1409debe1 in __find_if<__gnu_cxx::__normal_iterator<const seastar::shared_object*, std::vector<seastar::shared_object> >, __gnu_cxx::__ops::_Iter_pred<seastar::decorate(uintptr_t)::<lambda(const seastar::shared_object&)> > > /usr/include/c++/9/bits/stl_algo.h:120
    #3 0x55a1409de5e5 in __find_if<__gnu_cxx::__normal_iterator<const seastar::shared_object*, std::vector<seastar::shared_object> >, __gnu_cxx::__ops::_Iter_pred<seastar::decorate(uintptr_t)::<lambda(const seastar::shared_object&)> > > /usr/include/c++/9/bits/stl_algo.h:161
    #4 0x55a1409dd987 in find_if<__gnu_cxx::__normal_iterator<const seastar::shared_object*, std::vector<seastar::shared_object> >, seastar::decorate(uintptr_t)::<lambda(const seastar::shared_object&)> > /usr/include/c++/9/bits/stl_algo.h:3963
    #5 0x55a1409d6c95 in seastar::decorate(unsigned long) /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/util/backtrace.cc:77
    #6 0x55a1403a6583 in void seastar::backtrace<seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}>(seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}&&) /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/include/seastar/util/backtrace.hh:62
    #7 0x55a14037e765 in seastar::backtrace_buffer::append_backtrace() /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:732
    #8 0x55a14022f0a0 in print_with_backtrace /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:753
    #9 0x55a14022f213 in print_with_backtrace /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:760
    #10 0x55a140292fb3 in sigabrt_action /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:3513
    #11 0x55a1402cbb7e in operator() /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:3495
    #12 0x55a1402cbc19 in _FUN /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/core/reactor.cc:3491
    #13 0x7f555212f89f  (/lib/x86_64-linux-gnu/libpthread.so.0+0x1289f)
    #14 0x7f5550876f46 in gsignal (/lib/x86_64-linux-gnu/libc.so.6+0x3ef46)
    #15 0x7f55508788b0 in abort (/lib/x86_64-linux-gnu/libc.so.6+0x408b0)
    #16 0x7f5550868429  (/lib/x86_64-linux-gnu/libc.so.6+0x30429)
    #17 0x7f55508684a1 in __assert_fail (/lib/x86_64-linux-gnu/libc.so.6+0x304a1)
    #18 0x55a13fb446f1 in seastar::sharded<crimson::common::ConfigProxy>::~sharded() /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/include/seastar/core/sharded.hh:518
    #19 0x7f555087b0f0  (/lib/x86_64-linux-gnu/libc.so.6+0x430f0)
    #20 0x7f555087b1e9 in exit (/lib/x86_64-linux-gnu/libc.so.6+0x431e9)
    #21 0x7f5550859b9d in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x21b9d)
    #22 0x55a13d98fb59 in _start (/home/jenkins-build/build/workspace/ceph-pull-requests/build/bin/unittest-seastar-messenger+0xb135b59)

0x619000004b90 is located 16 bytes inside of 1024-byte region [0x619000004b80,0x619000004f80)
freed by thread T0 here:
    #0 0x7f5553bafcef in operator delete(void*) (/usr/lib/x86_64-linux-gnu/libasan.so.5+0x10ecef)
    #1 0x55a1409ec213 in __gnu_cxx::new_allocator<seastar::shared_object>::deallocate(seastar::shared_object*, unsigned long) /usr/include/c++/9/ext/new_allocator.h:128
    #2 0x55a1409e8b63 in std::allocator_traits<std::allocator<seastar::shared_object> >::deallocate(std::allocator<seastar::shared_object>&, seastar::shared_object*, unsigned long) /usr/include/c++/9/bits/alloc_traits.h:470
    #3 0x55a1409e4049 in std::_Vector_base<seastar::shared_object, std::allocator<seastar::shared_object> >::_M_deallocate(seastar::shared_object*, unsigned long) /usr/include/c++/9/bits/stl_vector.h:351
    #4 0x55a1409e196c in std::_Vector_base<seastar::shared_object, std::allocator<seastar::shared_object> >::~_Vector_base() /usr/include/c++/9/bits/stl_vector.h:332
    #5 0x55a1409e1b89 in std::vector<seastar::shared_object, std::allocator<seastar::shared_object> >::~vector() /usr/include/c++/9/bits/stl_vector.h:680
    #6 0x7f555087b0f0  (/lib/x86_64-linux-gnu/libc.so.6+0x430f0)

previously allocated by thread T0 here:
    #0 0x7f5553baed6f in operator new(unsigned long) (/usr/lib/x86_64-linux-gnu/libasan.so.5+0x10dd6f)
    #1 0x55a1409eef9e in __gnu_cxx::new_allocator<seastar::shared_object>::allocate(unsigned long, void const*) /usr/include/c++/9/ext/new_allocator.h:114
    #2 0x55a1409ed9e9 in std::allocator_traits<std::allocator<seastar::shared_object> >::allocate(std::allocator<seastar::shared_object>&, unsigned long) /usr/include/c++/9/bits/alloc_traits.h:444
    #3 0x55a1409ebd2f in std::_Vector_base<seastar::shared_object, std::allocator<seastar::shared_object> >::_M_allocate(unsigned long) /usr/include/c++/9/bits/stl_vector.h:343
    #4 0x55a1409e8250 in void std::vector<seastar::shared_object, std::allocator<seastar::shared_object> >::_M_realloc_insert<seastar::shared_object>(__gnu_cxx::__normal_iterator<seastar::shared_object*, std::vector<seastar::shared_object, std::allocator<seastar::shared_object> > >, seastar::shared_object&&) /usr/include/c++/9/bits/vector.tcc:440
    #5 0x55a1409e3deb in seastar::shared_object& std::vector<seastar::shared_object, std::allocator<seastar::shared_object> >::emplace_back<seastar::shared_object>(seastar::shared_object&&) /usr/include/c++/9/bits/vector.tcc:121
    #6 0x55a1409e1740 in std::vector<seastar::shared_object, std::allocator<seastar::shared_object> >::push_back(seastar::shared_object&&) /usr/include/c++/9/bits/stl_vector.h:1201
    #7 0x55a1409d6649 in dl_iterate_phdr_callback /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/util/backtrace.cc:49
    #8 0x7f555099e160 in dl_iterate_phdr (/lib/x86_64-linux-gnu/libc.so.6+0x166160)

SUMMARY: AddressSanitizer: heap-use-after-free /home/jenkins-build/build/workspace/ceph-pull-requests/src/seastar/src/util/backtrace.cc:76 in operator()
Shadow bytes around the buggy address:
  0x0c327fff8920: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8930: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8940: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8950: fd fd fa fa fa fa fa fa fa fa fa fa fa fa fa fa
  0x0c327fff8960: fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa fa
=>0x0c327fff8970: fd fd[fd]fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8980: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff8990: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff89a0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff89b0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
  0x0c327fff89c0: fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd fd
Shadow byte legend (one shadow byte represents 8 application bytes):
  Addressable:           00
  Partially addressable: 01 02 03 04 05 06 07 
  Heap left redzone:       fa
  Freed heap region:       fd
  Stack left redzone:      f1
  Stack mid redzone:       f2
  Stack right redzone:     f3
  Stack after return:      f5
  Stack use after scope:   f8
  Global redzone:          f9
  Global init order:       f6
  Poisoned by user:        f7
  Container overflow:      fc
  Array cookie:            ac
  Intra object redzone:    bb
  ASan internal:           fe
  Left alloca redzone:     ca
  Right alloca redzone:    cb
  Shadow gap:              cc
==25808==ABORTING

all of these failed tests were running on 172.21.3.232+irvingi03. i think they are caused by one or more seastar tests, which were competing with unittest-seastar-messenger for CPU cycles.

i just logged onto the test node and killed all seastar tests.