TSAN error (__tsan_atomic32_fetch_add) with current main branch #760
Description
Hi I just updated https://github.com/envoyproxy/envoy with the following abseil diff: 302b250...ce4bc92
I'm now seeing the following TSAN error:
WARNING: ThreadSanitizer: data race (pid=12)
Atomic write of size 4 at 0x7f5e1f8fc150 by thread T1:
#0 __tsan_atomic32_fetch_add /home/brian/src/final/llvm-project/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cpp:615:3 (xfcc_integration_test+0x2b57ac9)
#1 int std::__1::__cxx_atomic_fetch_add<int>(std::__1::__cxx_atomic_base_impl<int>*, int, std::__1::memory_order) /opt/llvm/bin/../include/c++/v1/atomic:1014:12 (xfcc_integration_test+0x52796cb)
#2 std::__1::__atomic_base<int, true>::fetch_add(int, std::__1::memory_order) /opt/llvm/bin/../include/c++/v1/atomic:1575:17 (xfcc_integration_test+0x5279600)
#3 absl::synchronization_internal::Waiter::Post() /proc/self/cwd/external/com_google_absl/absl/synchronization/internal/waiter.cc:174:14 (xfcc_integration_test+0x7dddc0c)
#4 AbslInternalPerThreadSemPost /proc/self/cwd/external/com_google_absl/absl/synchronization/internal/per_thread_sem.cc:73:64 (xfcc_integration_test+0x7ddcf70)
#5 absl::synchronization_internal::PerThreadSem::Post(absl::base_internal::ThreadIdentity*) /proc/self/cwd/external/com_google_absl/absl/synchronization/internal/per_thread_sem.h:107:3 (xfcc_integration_test+0x7dd8d48)
#6 absl::Mutex::IncrementSynchSem(absl::Mutex*, absl::base_internal::PerThreadSynch*) /proc/self/cwd/external/com_google_absl/absl/synchronization/mutex.cc:554:3 (xfcc_integration_test+0x7dd7fec)
#7 absl::Mutex::Wakeup(absl::base_internal::PerThreadSynch*) /proc/self/cwd/external/com_google_absl/absl/synchronization/mutex.cc:1146:3 (xfcc_integration_test+0x7dcba9d)
#8 absl::Mutex::UnlockSlow(absl::SynchWaitParams*) /proc/self/cwd/external/com_google_absl/absl/synchronization/mutex.cc:2309:19 (xfcc_integration_test+0x7dcf8bc)
#9 absl::Mutex::Unlock() /proc/self/cwd/external/com_google_absl/absl/synchronization/mutex.cc:1700:11 (xfcc_integration_test+0x7dd1a4d)
#10 Envoy::Thread::MutexBasicLockable::unlock() /proc/self/cwd/bazel-out/k8-dbg/bin/source/common/common/_virtual_includes/thread_lib/common/common/thread.h:25:58 (xfcc_integration_test+0x2bbecff)
#11 Envoy::Thread::LockGuard::~LockGuard() /proc/self/cwd/bazel-out/k8-dbg/bin/source/common/common/_virtual_includes/lock_guard_lib/common/common/lock_guard.h:109:47 (xfcc_integration_test+0x2bc45b2)
#12 Envoy::ConditionalInitializer::setReady() /proc/self/cwd/test/test_common/utility.cc:374:1 (xfcc_integration_test+0x57f50e6)
#13 Envoy::FakeUpstream::threadRoutine() /proc/self/cwd/test/integration/fake_upstream.cc:536:23 (xfcc_integration_test+0x2dfdeec)
#14 Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12::operator()() const /proc/self/cwd/test/integration/fake_upstream.cc:500:67 (xfcc_integration_test+0x2e2381b)
#15 decltype(std::__1::forward<Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12&>(fp)()) std::__1::__invoke<Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12&>(Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12&) /opt/llvm/bin/../include/c++/v1/type_traits:3539:1 (xfcc_integration_test+0x2e23770)
#16 void std::__1::__invoke_void_return_wrapper<void>::__call<Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12&>(Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12&) /opt/llvm/bin/../include/c++/v1/__functional_base:348:9 (xfcc_integration_test+0x2e236d0)
#17 std::__1::__function::__alloc_func<Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12, std::__1::allocator<Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12>, void ()>::operator()() /opt/llvm/bin/../include/c++/v1/functional:1540:16 (xfcc_integration_test+0x2e23670)
#18 std::__1::__function::__func<Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12, std::__1::allocator<Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool)::$_12>, void ()>::operator()() /opt/llvm/bin/../include/c++/v1/functional:1714:12 (xfcc_integration_test+0x2e2193f)
#19 std::__1::__function::__value_func<void ()>::operator()() const /opt/llvm/bin/../include/c++/v1/functional:1867:16 (xfcc_integration_test+0x2ddf296)
#20 std::__1::function<void ()>::operator()() const /opt/llvm/bin/../include/c++/v1/functional:2473:12 (xfcc_integration_test+0x2ddef28)
#21 Envoy::Thread::ThreadImplPosix::ThreadImplPosix(std::__1::function<void ()>, std::__1::optional<Envoy::Thread::Options> const&)::'lambda'(void*)::operator()(void*) const /proc/self/cwd/source/common/common/posix/thread_impl.cc:49:11 (xfcc_integration_test+0x7cdff6b)
#22 Envoy::Thread::ThreadImplPosix::ThreadImplPosix(std::__1::function<void ()>, std::__1::optional<Envoy::Thread::Options> const&)::'lambda'(void*)::__invoke(void*) /proc/self/cwd/source/common/common/posix/thread_impl.cc:48:9 (xfcc_integration_test+0x7cdfef8)
Previous write of size 8 at 0x7f5e1f8fc150 by main thread:
#0 mmap /home/brian/src/final/llvm-project/compiler-rt/lib/tsan/../sanitizer_common/sanitizer_common_interceptors.inc:7140:3 (xfcc_integration_test+0x2b3bc7a)
#1 absl::base_internal::DoAllocWithArena(unsigned long, absl::base_internal::LowLevelAlloc::Arena*) /proc/self/cwd/external/com_google_absl/absl/base/internal/low_level_alloc.cc:562:21 (xfcc_integration_test+0x7ed2296)
#2 absl::base_internal::LowLevelAlloc::Alloc(unsigned long) /proc/self/cwd/external/com_google_absl/absl/base/internal/low_level_alloc.cc:606:18 (xfcc_integration_test+0x7eff146)
#3 absl::synchronization_internal::NewThreadIdentity() /proc/self/cwd/external/com_google_absl/absl/synchronization/internal/create_thread_identity.cc:113:24 (xfcc_integration_test+0x7ddd489)
#4 absl::synchronization_internal::CreateThreadIdentity() /proc/self/cwd/external/com_google_absl/absl/synchronization/internal/create_thread_identity.cc:129:45 (xfcc_integration_test+0x7ddd365)
#5 absl::synchronization_internal::GetOrCreateCurrentThreadIdentity() /proc/self/cwd/external/com_google_absl/absl/synchronization/internal/create_thread_identity.h:51:12 (xfcc_integration_test+0x7dd8c76)
#6 absl::Synch_GetPerThread() /proc/self/cwd/external/com_google_absl/absl/synchronization/mutex.cc:526:30 (xfcc_integration_test+0x7dd5905)
#7 absl::Synch_GetPerThreadAnnotated(absl::Mutex*) /proc/self/cwd/external/com_google_absl/absl/synchronization/mutex.cc:534:23 (xfcc_integration_test+0x7dcdb34)
#8 absl::CondVar::WaitCommon(absl::Mutex*, absl::synchronization_internal::KernelTimeout) /proc/self/cwd/external/com_google_absl/absl/synchronization/mutex.cc:2537:25 (xfcc_integration_test+0x7dd410d)
#9 absl::CondVar::Wait(absl::Mutex*) /proc/self/cwd/external/com_google_absl/absl/synchronization/mutex.cc:2581:3 (xfcc_integration_test+0x7dd4672)
#10 Envoy::Thread::CondVar::wait(Envoy::Thread::MutexBasicLockable&) /proc/self/cwd/bazel-out/k8-dbg/bin/source/common/common/_virtual_includes/thread_lib/common/common/thread.h:60:14 (xfcc_integration_test+0x2f154df)
#11 Envoy::ConditionalInitializer::waitReady() /proc/self/cwd/test/test_common/utility.cc:383:7 (xfcc_integration_test+0x57f51fc)
#12 Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool) /proc/self/cwd/test/integration/fake_upstream.cc:501:23 (xfcc_integration_test+0x2dfc343)
#13 Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, unsigned int, Envoy::FakeHttpConnection::Type, Envoy::Network::Address::IpVersion, Envoy::Event::TestTimeSystem&) /proc/self/cwd/test/integration/fake_upstream.cc:483:7 (xfcc_integration_test+0x2dfd54c)
#14 Envoy::Xfcc::XfccIntegrationTest::createUpstreams() /proc/self/cwd/test/integration/xfcc_integration_test.cc:121:36 (xfcc_integration_test+0x2b9add4)
#15 non-virtual thunk to Envoy::Xfcc::XfccIntegrationTest::createUpstreams() /proc/self/cwd/test/integration/xfcc_integration_test.cc (xfcc_integration_test+0x2b9aecf)
#16 Envoy::BaseIntegrationTest::initialize() /proc/self/cwd/test/integration/integration.cc:325:3 (xfcc_integration_test+0x2e7fc71)
#17 Envoy::Xfcc::XfccIntegrationTest::initialize() /proc/self/cwd/test/integration/xfcc_integration_test.cc:153:24 (xfcc_integration_test+0x2b9b15f)
#18 Envoy::Xfcc::XfccIntegrationTest_MtlsForwardOnly_Test::TestBody() /proc/self/cwd/test/integration/xfcc_integration_test.cc:196:3 (xfcc_integration_test+0x2b9d392)
#19 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2433:10 (xfcc_integration_test+0x7ea525c)
#20 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2469:14 (xfcc_integration_test+0x7e8998e)
#21 testing::Test::Run() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2508:5 (xfcc_integration_test+0x7e6edb1)
#22 testing::TestInfo::Run() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2684:11 (xfcc_integration_test+0x7e6fc53)
#23 testing::TestSuite::Run() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2816:28 (xfcc_integration_test+0x7e7068a)
#24 testing::internal::UnitTestImpl::RunAllTests() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:5338:44 (xfcc_integration_test+0x7e7fe2d)
#25 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2433:10 (xfcc_integration_test+0x7eac7dc)
#26 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2469:14 (xfcc_integration_test+0x7e8d69e)
#27 testing::UnitTest::Run() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:4925:10 (xfcc_integration_test+0x7e7f6eb)
#28 RUN_ALL_TESTS() /proc/self/cwd/external/com_google_googletest/googletest/include/gtest/gtest.h:2473:46 (xfcc_integration_test+0x5727ba7)
#29 Envoy::TestRunner::RunTests(int, char**) /proc/self/cwd/test/test_runner.cc:154:10 (xfcc_integration_test+0x5726e5f)
#30 main /proc/self/cwd/test/main.cc:34:10 (xfcc_integration_test+0x5725154)
Thread T1 (tid=19, running) created by main thread at:
#0 pthread_create /home/brian/src/final/llvm-project/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:962:3 (xfcc_integration_test+0x2b0cbab)
#1 Envoy::Thread::ThreadImplPosix::ThreadImplPosix(std::__1::function<void ()>, std::__1::optional<Envoy::Thread::Options> const&) /proc/self/cwd/source/common/common/posix/thread_impl.cc:46:20 (xfcc_integration_test+0x7cde977)
#2 std::__1::__unique_if<Envoy::Thread::ThreadImplPosix>::__unique_single std::__1::make_unique<Envoy::Thread::ThreadImplPosix, std::__1::function<void ()>&, std::__1::optional<Envoy::Thread::Options> const&>(std::__1::function<void ()>&, std::__1::optional<Envoy::Thread::Options> const&) /opt/llvm/bin/../include/c++/v1/memory:3028:32 (xfcc_integration_test+0x7cde2cf)
#3 Envoy::Thread::ThreadFactoryImplPosix::createThread(std::__1::function<void ()>, std::__1::optional<Envoy::Thread::Options> const&) /proc/self/cwd/source/common/common/posix/thread_impl.cc:121:10 (xfcc_integration_test+0x7cde013)
#4 Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, std::__1::unique_ptr<Envoy::Network::Socket, std::__1::default_delete<Envoy::Network::Socket> >&&, Envoy::FakeHttpConnection::Type, Envoy::Event::TestTimeSystem&, bool) /proc/self/cwd/test/integration/fake_upstream.cc:500:35 (xfcc_integration_test+0x2dfc2db)
#5 Envoy::FakeUpstream::FakeUpstream(std::__1::unique_ptr<Envoy::Network::TransportSocketFactory, std::__1::default_delete<Envoy::Network::TransportSocketFactory> >&&, unsigned int, Envoy::FakeHttpConnection::Type, Envoy::Network::Address::IpVersion, Envoy::Event::TestTimeSystem&) /proc/self/cwd/test/integration/fake_upstream.cc:483:7 (xfcc_integration_test+0x2dfd54c)
#6 Envoy::Xfcc::XfccIntegrationTest::createUpstreams() /proc/self/cwd/test/integration/xfcc_integration_test.cc:121:36 (xfcc_integration_test+0x2b9add4)
#7 non-virtual thunk to Envoy::Xfcc::XfccIntegrationTest::createUpstreams() /proc/self/cwd/test/integration/xfcc_integration_test.cc (xfcc_integration_test+0x2b9aecf)
#8 Envoy::BaseIntegrationTest::initialize() /proc/self/cwd/test/integration/integration.cc:325:3 (xfcc_integration_test+0x2e7fc71)
#9 Envoy::Xfcc::XfccIntegrationTest::initialize() /proc/self/cwd/test/integration/xfcc_integration_test.cc:153:24 (xfcc_integration_test+0x2b9b15f)
#10 Envoy::Xfcc::XfccIntegrationTest_MtlsForwardOnly_Test::TestBody() /proc/self/cwd/test/integration/xfcc_integration_test.cc:196:3 (xfcc_integration_test+0x2b9d392)
#11 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2433:10 (xfcc_integration_test+0x7ea525c)
#12 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2469:14 (xfcc_integration_test+0x7e8998e)
#13 testing::Test::Run() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2508:5 (xfcc_integration_test+0x7e6edb1)
#14 testing::TestInfo::Run() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2684:11 (xfcc_integration_test+0x7e6fc53)
#15 testing::TestSuite::Run() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2816:28 (xfcc_integration_test+0x7e7068a)
#16 testing::internal::UnitTestImpl::RunAllTests() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:5338:44 (xfcc_integration_test+0x7e7fe2d)
#17 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2433:10 (xfcc_integration_test+0x7eac7dc)
#18 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:2469:14 (xfcc_integration_test+0x7e8d69e)
#19 testing::UnitTest::Run() /proc/self/cwd/external/com_google_googletest/googletest/src/gtest.cc:4925:10 (xfcc_integration_test+0x7e7f6eb)
#20 RUN_ALL_TESTS() /proc/self/cwd/external/com_google_googletest/googletest/include/gtest/gtest.h:2473:46 (xfcc_integration_test+0x5727ba7)
#21 Envoy::TestRunner::RunTests(int, char**) /proc/self/cwd/test/test_runner.cc:154:10 (xfcc_integration_test+0x5726e5f)
#22 main /proc/self/cwd/test/main.cc:34:10 (xfcc_integration_test+0x5725154)
SUMMARY: ThreadSanitizer: data race /opt/llvm/bin/../include/c++/v1/atomic:1014:12 in int std::__1::__cxx_atomic_fetch_add<int>(std::__1::__cxx_atomic_base_impl<int>*, int, std::__1::memory_order)
Ironically I pulled this to fix a different TSAN issue which I think was fixed with:
9fc78436565eb3b204d4aa425ee3773354392f45 by Derek Mauro <dmauro@google.com>:
Use auto-detected sanitizer attributes for ASAN, MSAN, and TSAN builds
Or related. Not sure if this is part of the problem here. (I'm no longer seeing another TSAN issue I was seeing within mutex but am now seeing this one.)
Any ideas on what is going on here? Thank you.