search

open-telemetry / opentelemetry-cpp

The OpenTelemetry C++ Client
https://opentelemetry.io/
Apache License 2.0
843 stars 404 forks source link

race condition in attributes_hashmap_benchmark #1542

Closed esigo closed 2 years ago

esigo commented 2 years ago

We have a race condition in attributes_hashmap_benchmark:

WARNING: ThreadSanitizer: data race (pid=12)
  Write of size 8 at 0x7ffd627125c0 by main thread:
    #0 (anonymous namespace)::BM_AttributseHashMap(benchmark::State&) <null> (attributes_hashmap_benchmark_smoketest+0x9b0d9)
    #1 benchmark::internal::FunctionBenchmark::Run(benchmark::State&) <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xa7c52)
    #2 benchmark::internal::BenchmarkInstance::Run(unsigned long, int, benchmark::internal::ThreadTimer*, benchmark::internal::ThreadManager*, benchmark::internal::PerfCountersMeasurement*) const <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xa3a3a)
    #3 benchmark::internal::(anonymous namespace)::RunInThread(benchmark::internal::BenchmarkInstance const*, unsigned long, int, benchmark::internal::ThreadManager*, benchmark::internal::PerfCountersMeasurement*) <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xba4bb)
    #4 benchmark::internal::BenchmarkRunner::DoNIterations() <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xbb225)
    #5 benchmark::internal::BenchmarkRunner::DoOneRepetition() <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xbbd04)
    #6 benchmark::internal::(anonymous namespace)::RunBenchmarks(std::vector<benchmark::internal::BenchmarkInstance, std::allocator<benchmark::internal::BenchmarkInstance> > const&, benchmark::BenchmarkReporter*, benchmark::BenchmarkReporter*) <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0x87539)
    #7 benchmark::RunSpecifiedBenchmarks(benchmark::BenchmarkReporter*, benchmark::BenchmarkReporter*) <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0x8818b)
    #8 benchmark::RunSpecifiedBenchmarks() <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0x87c1d)
    #9 main <null> (attributes_hashmap_benchmark_smoketest+0x9b300)

  Previous read of size 8 at 0x7ffd627125c0 by thread T1:
    #0 (anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}::operator()() const <null> (attributes_hashmap_benchmark_smoketest+0x9aaa7)
    #1 void std::__invoke_impl<void, (anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}>(std::__invoke_other, (anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}&&) <null> (attributes_hashmap_benchmark_smoketest+0x9bf4e)
    #2 std::__invoke_result<(anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}>::type std::__invoke<(anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}>(std::__invoke_result&&, ((anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}&&)...) <null> (attributes_hashmap_benchmark_smoketest+0x9beb5)
    #3 void std::thread::_Invoker<std::tuple<(anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) <null> (attributes_hashmap_benchmark_smoketest+0x9be14)
    #4 std::thread::_Invoker<std::tuple<(anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}> >::operator()() <null> (attributes_hashmap_benchmark_smoketest+0x9bdba)
    #5 std::thread::_State_impl<std::thread::_Invoker<std::tuple<(anonymous namespace)::BM_AttributseHashMap(benchmark::State&)::{lambda()#2}> > >::_M_run() <null> (attributes_hashmap_benchmark_smoketest+0x9bd70)
    #6 <null> <null> (libstdc++.so.6+0xd6de3)

  Location is stack of main thread.

  Location is global '<null>' at 0x000000000000 ([stack]+0x00000001b5c0)

  Thread T1 (tid=19, finished) created by main thread at:
    #0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:962 (libtsan.so.0+0x5ea79)
    #1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xd70a8)
    #2 (anonymous namespace)::BM_AttributseHashMap(benchmark::State&) <null> (attributes_hashmap_benchmark_smoketest+0x9b088)
    #3 benchmark::internal::FunctionBenchmark::Run(benchmark::State&) <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xa7c52)
    #4 benchmark::internal::BenchmarkInstance::Run(unsigned long, int, benchmark::internal::ThreadTimer*, benchmark::internal::ThreadManager*, benchmark::internal::PerfCountersMeasurement*) const <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xa3a3a)
    #5 benchmark::internal::(anonymous namespace)::RunInThread(benchmark::internal::BenchmarkInstance const*, unsigned long, int, benchmark::internal::ThreadManager*, benchmark::internal::PerfCountersMeasurement*) <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xba4bb)
    #6 benchmark::internal::BenchmarkRunner::DoNIterations() <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xbb225)
    #7 benchmark::internal::BenchmarkRunner::DoOneRepetition() <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0xbbd04)
    #8 benchmark::internal::(anonymous namespace)::RunBenchmarks(std::vector<benchmark::internal::BenchmarkInstance, std::allocator<benchmark::internal::BenchmarkInstance> > const&, benchmark::BenchmarkReporter*, benchmark::BenchmarkReporter*) <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0x87539)
    #9 benchmark::RunSpecifiedBenchmarks(benchmark::BenchmarkReporter*, benchmark::BenchmarkReporter*) <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0x8818b)
    #10 benchmark::RunSpecifiedBenchmarks() <null> (libexternal_Scom_Ugithub_Ugoogle_Ubenchmark_Slibbenchmark.so+0x87c1d)
    #11 main <null> (attributes_hashmap_benchmark_smoketest+0x9b300)

SUMMARY: ThreadSanitizer: data race (/home/runner/.cache/bazel/e6b8d6759295e14b76bc8cb98b604748/execroot/io_opentelemetry_cpp/bazel-out/k8-tsan-fastbuild/bin/sdk/test/metrics/attributes_hashmap_benchmark_smoketest+0x9b0d9) in (anonymous namespace)::BM_AttributseHashMap(benchmark::State&)
==================

hash_map.GetOrSetDefault returns a pointer to an Aggregation (DropAggregation), for this type it should be ok as we have no read/write. But for some other type, the reads are not protected see for example: std::unique_ptr<Aggregation> LongSumAggregation::Merge std::unique_ptr<Aggregation> LongSumAggregation::ToPoint We need to review reads/writes for all the aggregations.

lalitb commented 2 years ago

Thanks for debugging it :)

esigo commented 2 years ago

one more here:

WARNING: ThreadSanitizer: data race (pid=12)
  Write of size 8 at 0x7b4800000150 by main thread:
    #0 pthread_cond_destroy ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1208 (libtsan.so.0+0x467aa)
    #1 opentelemetry::v1::exporter::otlp::OtlpHttpClient::~OtlpHttpClient() <null> (libexporters_Sotlp_Slibotlp_Uhttp_Uclient.so+0x795ba)
    #2 std::default_delete<opentelemetry::v1::exporter::otlp::OtlpHttpClient>::operator()(opentelemetry::v1::exporter::otlp::OtlpHttpClient*) const <null> (otlp_http_exporter_test+0xf2977)
    #3 std::unique_ptr<opentelemetry::v1::exporter::otlp::OtlpHttpClient, std::default_delete<opentelemetry::v1::exporter::otlp::OtlpHttpClient> >::~unique_ptr() <null> (otlp_http_exporter_test+0xe623a)
    #4 opentelemetry::v1::exporter::otlp::OtlpHttpExporter::~OtlpHttpExporter() <null> (otlp_http_exporter_test+0xfb289)
    #5 opentelemetry::v1::exporter::otlp::OtlpHttpExporter::~OtlpHttpExporter() <null> (otlp_http_exporter_test+0xfb2db)
    #6 std::default_delete<opentelemetry::v1::sdk::trace::SpanExporter>::operator()(opentelemetry::v1::sdk::trace::SpanExporter*) const <null> (otlp_http_exporter_test+0xf2b5e)
    #7 std::unique_ptr<opentelemetry::v1::sdk::trace::SpanExporter, std::default_delete<opentelemetry::v1::sdk::trace::SpanExporter> >::~unique_ptr() <null> (otlp_http_exporter_test+0xe6318)
    #8 opentelemetry::v1::sdk::trace::BatchSpanProcessor::~BatchSpanProcessor() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xaeb63)
    #9 opentelemetry::v1::sdk::trace::BatchSpanProcessor::~BatchSpanProcessor() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xaeba5)
    #10 std::default_delete<opentelemetry::v1::sdk::trace::SpanProcessor>::operator()(opentelemetry::v1::sdk::trace::SpanProcessor*) const <null> (otlp_http_exporter_test+0xf7d16)
    #11 std::unique_ptr<opentelemetry::v1::sdk::trace::SpanProcessor, std::default_delete<opentelemetry::v1::sdk::trace::SpanProcessor> >::~unique_ptr() <null> (otlp_http_exporter_test+0xe7ab6)
    #12 opentelemetry::v1::sdk::trace::MultiSpanProcessor::ProcessorNode::~ProcessorNode() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xcdac1)
    #13 opentelemetry::v1::sdk::trace::MultiSpanProcessor::Cleanup() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xcdbe3)
    #14 opentelemetry::v1::sdk::trace::MultiSpanProcessor::~MultiSpanProcessor() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xcd9ba)
    #15 opentelemetry::v1::sdk::trace::MultiSpanProcessor::~MultiSpanProcessor() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xcd9fb)
    #16 std::default_delete<opentelemetry::v1::sdk::trace::SpanProcessor>::operator()(opentelemetry::v1::sdk::trace::SpanProcessor*) const <null> (otlp_http_exporter_test+0xf7d16)
    #17 std::unique_ptr<opentelemetry::v1::sdk::trace::SpanProcessor, std::default_delete<opentelemetry::v1::sdk::trace::SpanProcessor> >::~unique_ptr() <null> (otlp_http_exporter_test+0xe7ab6)
    #18 opentelemetry::v1::sdk::trace::TracerContext::~TracerContext() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe7141)
    #19 void __gnu_cxx::new_allocator<opentelemetry::v1::sdk::trace::TracerContext>::destroy<opentelemetry::v1::sdk::trace::TracerContext>(opentelemetry::v1::sdk::trace::TracerContext*) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe71a1)
    #20 void std::allocator_traits<std::allocator<opentelemetry::v1::sdk::trace::TracerContext> >::destroy<opentelemetry::v1::sdk::trace::TracerContext>(std::allocator<opentelemetry::v1::sdk::trace::TracerContext>&, opentelemetry::v1::sdk::trace::TracerContext*) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe706a)
    #21 std::_Sp_counted_ptr_inplace<opentelemetry::v1::sdk::trace::TracerContext, std::allocator<opentelemetry::v1::sdk::trace::TracerContext>, (__gnu_cxx::_Lock_policy)2>::_M_dispose() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe6dde)
    #22 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() <null> (otlp_http_exporter_test+0xefda0)
    #23 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() <null> (otlp_http_exporter_test+0xe42d4)
    #24 std::__shared_ptr<opentelemetry::v1::sdk::trace::TracerContext, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xc3d65)
    #25 std::shared_ptr<opentelemetry::v1::sdk::trace::TracerContext>::~shared_ptr() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xc3dd1)
    #26 opentelemetry::v1::sdk::trace::Tracer::~Tracer() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xca5c9)
    #27 opentelemetry::v1::sdk::trace::Tracer::~Tracer() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xca62b)
    #28 std::_Sp_counted_ptr<opentelemetry::v1::sdk::trace::Tracer*, (__gnu_cxx::_Lock_policy)2>::_M_dispose() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe6f2e)
    #29 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() <null> (otlp_http_exporter_test+0xefda0)
    #30 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() <null> (otlp_http_exporter_test+0xe42d4)
    #31 std::__shared_ptr<opentelemetry::v1::sdk::trace::Tracer, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xc012f)
    #32 std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>::~shared_ptr() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xc015f)
    #33 void std::_Destroy<std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer> >(std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe4c70)
    #34 void std::_Destroy_aux<false>::__destroy<std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*>(std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*, std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe40ed)
    #35 void std::_Destroy<std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*>(std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*, std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe3405)
    #36 void std::_Destroy<std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*, std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer> >(std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*, std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>*, std::allocator<std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer> >&) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe299f)
    #37 std::vector<std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer>, std::allocator<std::shared_ptr<opentelemetry::v1::sdk::trace::Tracer> > >::~vector() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe1ce6)
    #38 opentelemetry::v1::sdk::trace::TracerProvider::~TracerProvider() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe0927)
    #39 opentelemetry::v1::sdk::trace::TracerProvider::~TracerProvider() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xe0969)
    #40 std::_Sp_counted_ptr<opentelemetry::v1::trace::TracerProvider*, (__gnu_cxx::_Lock_policy)2>::_M_dispose() <null> (otlp_http_exporter_test+0x1626ea)
    #41 std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release() <null> (otlp_http_exporter_test+0xefda0)
    #42 std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count() <null> (otlp_http_exporter_test+0xe42d4)
    #43 std::__shared_ptr<opentelemetry::v1::trace::TracerProvider, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr() <null> (otlp_http_exporter_test+0xe5d09)
    #44 std::shared_ptr<opentelemetry::v1::trace::TracerProvider>::~shared_ptr() <null> (otlp_http_exporter_test+0xe5d39)
    #45 opentelemetry::v1::nostd::shared_ptr<opentelemetry::v1::trace::TracerProvider>::shared_ptr_wrapper::~shared_ptr_wrapper() <null> (otlp_http_exporter_test+0xf1ae9)
    #46 opentelemetry::v1::nostd::shared_ptr<opentelemetry::v1::trace::TracerProvider>::~shared_ptr() <null> (otlp_http_exporter_test+0xe5cce)
    #47 opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync() <null> (otlp_http_exporter_test+0xdfcfb)
    #48 opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer_ExportBinaryIntegrationTestAsync_Test::TestBody() <null> (otlp_http_exporter_test+0xca758)
    #49 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x130d23)
    #50 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x1287f5)
    #51 testing::Test::Run() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x1080d3)
    #52 testing::TestInfo::Run() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x108d68)
    #53 testing::TestSuite::Run() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x109a13)
    #54 testing::internal::UnitTestImpl::RunAllTests() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x118a74)
    #55 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x132a6f)
    #56 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x12a2b5)
    #57 testing::UnitTest::Run() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x116b79)
    #58 RUN_ALL_TESTS() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest_Umain.so+0xe1b)
    #59 main <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest_Umain.so+0xd16)

  Previous read of size 8 at 0x7b4800000150 by thread T6:
    #0 pthread_cond_broadcast ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:1201 (libtsan.so.0+0x3e306)
    #1 std::condition_variable::notify_all() <null> (libstdc++.so.6+0xd0e6c)
    #2 opentelemetry::v1::exporter::otlp::(anonymous namespace)::ResponseHandler::Unbind(opentelemetry::v1::sdk::common::ExportResult) <null> (libexporters_Sotlp_Slibotlp_Uhttp_Uclient.so+0x77cc4)
    #3 opentelemetry::v1::exporter::otlp::(anonymous namespace)::ResponseHandler::OnResponse(opentelemetry::v1::ext::http::client::Response&) <null> (libexporters_Sotlp_Slibotlp_Uhttp_Uclient.so+0x76305)
    #4 opentelemetry::v1::ext::http::client::nosend::Response::Finish(opentelemetry::v1::ext::http::client::EventHandler&) <null> (otlp_http_exporter_test+0xd41bc)
    #5 opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}::operator()() const <null> (otlp_http_exporter_test+0xde7d9)
    #6 void std::__invoke_impl<void, opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}>(std::__invoke_other, opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}&&) <null> (otlp_http_exporter_test+0x164a08)
    #7 std::__invoke_result<opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}>::type std::__invoke<opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}>(opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}&&, (std::__invoke_result&&)...) <null> (otlp_http_exporter_test+0x1645f3)
    #8 void std::thread::_Invoker<std::tuple<opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) <null> (otlp_http_exporter_test+0x163f08)
    #9 std::thread::_Invoker<std::tuple<opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}> >::operator()() <null> (otlp_http_exporter_test+0x163920)
    #10 std::thread::_State_impl<std::thread::_Invoker<std::tuple<opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const::{lambda()#1}> > >::_M_run() <null> (otlp_http_exporter_test+0x1628ee)
    #11 <null> <null> (libstdc++.so.6+0xd6de3)

  Location is heap block of size 384 at 0x7b4800000000 allocated by main thread:
    #0 operator new(unsigned long) ../../../../src/libsanitizer/tsan/tsan_new_delete.cpp:64 (libtsan.so.0+0x8c032)
    #1 opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::GetMockOtlpHttpClient(opentelemetry::v1::exporter::otlp::HttpRequestContentType, bool) <null> (otlp_http_exporter_test+0xd69f7)
    #2 opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync() <null> (otlp_http_exporter_test+0xde8c4)
    #3 opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer_ExportBinaryIntegrationTestAsync_Test::TestBody() <null> (otlp_http_exporter_test+0xca758)
    #4 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x130d23)
    #5 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x1287f5)
    #6 testing::Test::Run() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x1080d3)
    #7 testing::TestInfo::Run() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x108d68)
    #8 testing::TestSuite::Run() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x109a13)
    #9 testing::internal::UnitTestImpl::RunAllTests() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x118a74)
    #10 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x132a6f)
    #11 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x12a2b5)
    #12 testing::UnitTest::Run() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0x116b79)
    #13 RUN_ALL_TESTS() <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest_Umain.so+0xe1b)
    #14 main <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest_Umain.so+0xd16)

  Thread T6 (tid=24, running) created by thread T5 at:
    #0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:962 (libtsan.so.0+0x5ea79)
    #1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xd70a8)
    #2 opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const <null> (otlp_http_exporter_test+0xde508)
    #3 std::_Function_handler<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>), opentelemetry::v1::exporter::otlp::OtlpHttpExporterTestPeer::ExportBinaryIntegrationTestAsync()::{lambda(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)#1}>::_M_invoke(std::_Any_data const&, std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>&&) <null> (otlp_http_exporter_test+0x13a6fa)
    #4 std::function<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)>::operator()(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) const <null> (libext_Ssrc_Shttp_Sclient_Snosend_Slibhttp_Uclient_Unosend.so+0x38650)
    #5 decltype (((forward<std::function<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)> const&>)({parm#1}))((get<0ul>)((forward<std::tuple<std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler> > >)({parm#2})))) testing::internal::ApplyImpl<std::function<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)> const&, std::tuple<std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler> >, 0ul>(std::function<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)> const&, std::tuple<std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler> >&&, testing::internal::IndexSequence<0ul>) <null> (libext_Ssrc_Shttp_Sclient_Snosend_Slibhttp_Uclient_Unosend.so+0x37ed4)
    #6 decltype (ApplyImpl((forward<std::function<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)> const&>)({parm#1}), (forward<std::tuple<std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler> > >)({parm#2}), (testing::internal::MakeIndexSequenceImpl<std::tuple_size<std::remove_reference<std::tuple<std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler> > >::type>::value>::type)())) testing::internal::Apply<std::function<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)> const&, std::tuple<std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler> > >(std::function<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)> const&, std::tuple<std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler> >&&) <null> (libext_Ssrc_Shttp_Sclient_Snosend_Slibhttp_Uclient_Unosend.so+0x372e2)
    #7 testing::Action<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)>::Perform(std::tuple<std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler> >) const <null> (libext_Ssrc_Shttp_Sclient_Snosend_Slibhttp_Uclient_Unosend.so+0x3680d)
    #8 testing::internal::ActionResultHolder<void>* testing::internal::ActionResultHolder<void>::PerformAction<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)>(testing::Action<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)> const&, testing::internal::Function<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)>::ArgumentTuple&&) <null> (libext_Ssrc_Shttp_Sclient_Snosend_Slibhttp_Uclient_Unosend.so+0x3606c)
    #9 testing::internal::FunctionMocker<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)>::UntypedPerformAction(void const*, void*) const <null> (libext_Ssrc_Shttp_Sclient_Snosend_Slibhttp_Uclient_Unosend.so+0x35b60)
    #10 testing::internal::UntypedFunctionMockerBase::UntypedInvokeWith(void*) <null> (libexternal_Scom_Ugoogle_Ugoogletest_Slibgtest.so+0xcdb2e)
    #11 testing::internal::FunctionMocker<void (std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>)>::Invoke(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) <null> (libext_Ssrc_Shttp_Sclient_Snosend_Slibhttp_Uclient_Unosend.so+0x2f97d)
    #12 opentelemetry::v1::ext::http::client::nosend::Session::SendRequest(std::shared_ptr<opentelemetry::v1::ext::http::client::EventHandler>) <null> (libext_Ssrc_Shttp_Sclient_Snosend_Slibhttp_Uclient_Unosend.so+0x2e539)
    #13 opentelemetry::v1::exporter::otlp::OtlpHttpClient::addSession(opentelemetry::v1::exporter::otlp::OtlpHttpClient::HttpSessionData&&) <null> (libexporters_Sotlp_Slibotlp_Uhttp_Uclient.so+0x7b2b8)
    #14 opentelemetry::v1::exporter::otlp::OtlpHttpClient::Export(google::protobuf::Message const&, std::function<bool (opentelemetry::v1::sdk::common::ExportResult)>&&, unsigned long) <null> (libexporters_Sotlp_Slibotlp_Uhttp_Uclient.so+0x79ab7)
    #15 opentelemetry::v1::exporter::otlp::OtlpHttpClient::Export(google::protobuf::Message const&, std::function<bool (opentelemetry::v1::sdk::common::ExportResult)>&&) <null> (libexporters_Sotlp_Slibotlp_Uhttp_Uclient.so+0x798dc)
    #16 opentelemetry::v1::exporter::otlp::OtlpHttpExporter::Export(opentelemetry::v1::nostd::span<std::unique_ptr<opentelemetry::v1::sdk::trace::Recordable, std::default_delete<opentelemetry::v1::sdk::trace::Recordable> >, 18446744073709551615ul> const&) <null> (libexporters_Sotlp_Slibotlp_Uhttp_Uexporter.so+0x330ab)
    #17 opentelemetry::v1::sdk::trace::BatchSpanProcessor::Export() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xae614)
    #18 opentelemetry::v1::sdk::trace::BatchSpanProcessor::DoBackgroundWork() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xae2ce)
    #19 void std::__invoke_impl<void, void (opentelemetry::v1::sdk::trace::BatchSpanProcessor::*)(), opentelemetry::v1::sdk::trace::BatchSpanProcessor*>(std::__invoke_memfun_deref, void (opentelemetry::v1::sdk::trace::BatchSpanProcessor::*&&)(), opentelemetry::v1::sdk::trace::BatchSpanProcessor*&&) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xb844c)
    #20 std::__invoke_result<void (opentelemetry::v1::sdk::trace::BatchSpanProcessor::*)(), opentelemetry::v1::sdk::trace::BatchSpanProcessor*>::type std::__invoke<void (opentelemetry::v1::sdk::trace::BatchSpanProcessor::*)(), opentelemetry::v1::sdk::trace::BatchSpanProcessor*>(void (opentelemetry::v1::sdk::trace::BatchSpanProcessor::*&&)(), opentelemetry::v1::sdk::trace::BatchSpanProcessor*&&) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xb82b7)
    #21 void std::thread::_Invoker<std::tuple<void (opentelemetry::v1::sdk::trace::BatchSpanProcessor::*)(), opentelemetry::v1::sdk::trace::BatchSpanProcessor*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xb8008)
    #22 std::thread::_Invoker<std::tuple<void (opentelemetry::v1::sdk::trace::BatchSpanProcessor::*)(), opentelemetry::v1::sdk::trace::BatchSpanProcessor*> >::operator()() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xb7d44)
    #23 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (opentelemetry::v1::sdk::trace::BatchSpanProcessor::*)(), opentelemetry::v1::sdk::trace::BatchSpanProcessor*> > >::_M_run() <null> (libsdk_Ssrc_Strace_Slibtrace.so+0xb7b32)
    #24 <null> <null> (libstdc++.so.6+0xd6de3)