simsong / bulk_extractor

This is the development tree. Production downloads are at:
https://github.com/simsong/bulk_extractor/releases
Other
1.11k stars 187 forks source link

Fix threadSanitizer race condition #303

Closed simsong closed 2 years ago

simsong commented 2 years ago

c.f. https://github.com/simsong/bulk_extractor/runs/4438876180?check_suite_focus=true

==================
WARNING: ThreadSanitizer: data race (pid=29742)
  Write of size 8 at 0x7b1000008bc0 by thread T5:
    #0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:823 (libtsan.so.0+0x42313)
    #1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:815 (libtsan.so.0+0x42313)
    #2 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) <null> (libstdc++.so.6+0x143284)
    #3 scanner_set::process_sbuf(sbuf_t const*, void (*)(scanner_params&)) be13_api/scanner_set.cpp:844 (test_be+0x9877e)
    #4 worker::run() be13_api/threadpool.cpp:195 (test_be+0xbbf87)
    #5 worker::start_worker(void*) be13_api/threadpool.cpp:142 (test_be+0xbbabe)
    #6 void* std::__invoke_impl<void*, void* (*)(void*), void*>(std::__invoke_other, void* (*&&)(void*), void*&&) /usr/include/c++/9/bits/invoke.h:60 (test_be+0xc4144)
    #7 std::__invoke_result<void* (*)(void*), void*>::type std::__invoke<void* (*)(void*), void*>(void* (*&&)(void*), void*&&) /usr/include/c++/9/bits/invoke.h:95 (test_be+0xc4031)
    #8 void* std::thread::_Invoker<std::tuple<void* (*)(void*), void*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/include/c++/9/thread:244 (test_be+0xc3f24)
    #9 std::thread::_Invoker<std::tuple<void* (*)(void*), void*> >::operator()() /usr/include/c++/9/thread:251 (test_be+0xc3ea6)
    #10 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (*)(void*), void*> > >::_M_run() /usr/include/c++/9/thread:195 (test_be+0xc3e58)
    #11 <null> <null> (libstdc++.so.6+0xd6de3)

  Previous read of size 1 at 0x7b1000008bc3 by thread T7:
    #0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:823 (libtsan.so.0+0x42313)
    #1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:815 (libtsan.so.0+0x42313)
    #2 void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char*>(char*, char*, std::forward_iterator_tag) /usr/include/c++/9/bits/basic_string.tcc:225 (test_be+0x36418)
    #3 scanner_set::get_realtime_stats[abi:cxx11]() const be13_api/scanner_set.cpp:257 (test_be+0x93e72)
    #4 notify_thread::run() /home/runner/work/bulk_extractor/bulk_extractor/src/notify_thread.cpp:77 (test_be+0x12b078)
    #5 void* std::__invoke_impl<void*, void* (notify_thread::*)(), notify_thread*>(std::__invoke_memfun_deref, void* (notify_thread::*&&)(), notify_thread*&&) /usr/include/c++/9/bits/invoke.h:73 (test_be+0x12e4bd)
    #6 std::__invoke_result<void* (notify_thread::*)(), notify_thread*>::type std::__invoke<void* (notify_thread::*)(), notify_thread*>(void* (notify_thread::*&&)(), notify_thread*&&) /usr/include/c++/9/bits/invoke.h:95 (test_be+0x12e35d)
    #7 void* std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/include/c++/9/thread:244 (test_be+0x12e250)
    #8 std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> >::operator()() /usr/include/c++/9/thread:251 (test_be+0x12e1d2)
    #9 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> > >::_M_run() /usr/include/c++/9/thread:195 (test_be+0x12e184)
    #10 <null> <null> (libstdc++.so.6+0xd6de3)

  Location is heap block of size 61 at 0x7b1000008bc0 allocated by thread T5:
    #0 operator new(unsigned long) ../../../../src/libsanitizer/tsan/tsan_new_delete.cpp:64 (libtsan.so.0+0x8c032)
    #1 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) <null> (libstdc++.so.6+0x14329e)
    #2 scanner_set::process_sbuf(sbuf_t const*) be13_api/scanner_set.cpp:1015 (test_be+0x9a792)
    #3 worker::run() be13_api/threadpool.cpp:198 (test_be+0xbbfcd)
    #4 worker::start_worker(void*) be13_api/threadpool.cpp:142 (test_be+0xbbabe)
    #5 void* std::__invoke_impl<void*, void* (*)(void*), void*>(std::__invoke_other, void* (*&&)(void*), void*&&) /usr/include/c++/9/bits/invoke.h:60 (test_be+0xc4144)
    #6 std::__invoke_result<void* (*)(void*), void*>::type std::__invoke<void* (*)(void*), void*>(void* (*&&)(void*), void*&&) /usr/include/c++/9/bits/invoke.h:95 (test_be+0xc4031)
    #7 void* std::thread::_Invoker<std::tuple<void* (*)(void*), void*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/include/c++/9/thread:244 (test_be+0xc3f24)
    #8 std::thread::_Invoker<std::tuple<void* (*)(void*), void*> >::operator()() /usr/include/c++/9/thread:251 (test_be+0xc3ea6)
    #9 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (*)(void*), void*> > >::_M_run() /usr/include/c++/9/thread:195 (test_be+0xc3e58)
    #10 <null> <null> (libstdc++.so.6+0xd6de3)

  Thread T5 (tid=29751, running) created by main thread at:
    #0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:962 (libtsan.so.0+0x5ea79)
    #1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xd70a8)
    #2 thread_pool::launch_workers(unsigned long) be13_api/threadpool.cpp:15 (test_be+0xbae4d)
    #3 scanner_set::launch_workers(int) be13_api/scanner_set.cpp:144 (test_be+0x93206)
    #4 bulk_extractor_main(std::ostream&, std::ostream&, int, char* const*) /home/runner/work/bulk_extractor/bulk_extractor/src/bulk_extractor.cpp:575 (test_be+0xd627b)
    #5 ____C_A_T_C_H____T_E_S_T____10 /home/runner/work/bulk_extractor/bulk_extractor/src/test_be2.cpp:206 (test_be+0x2df71e)
    #6 Catch::TestInvokerAsFunction::invoke() const be13_api/catch.hpp:14317 (test_be+0x1fb904)
    #7 Catch::TestCase::invoke() const be13_api/catch.hpp:14156 (test_be+0x1fa744)
    #8 Catch::RunContext::invokeActiveTestCase() be13_api/catch.hpp:13016 (test_be+0x1f3207)
    #9 Catch::RunContext::runCurrentTest(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) be13_api/catch.hpp:12989 (test_be+0x1f2e48)
    #10 Catch::RunContext::runTest(Catch::TestCase const&) be13_api/catch.hpp:12750 (test_be+0x1f0f10)
    #11 execute be13_api/catch.hpp:13343 (test_be+0x1f5349)
    #12 Catch::Session::runInternal() be13_api/catch.hpp:13549 (test_be+0x1f6b8b)
    #13 Catch::Session::run() be13_api/catch.hpp:13505 (test_be+0x1f676f)
    #14 int Catch::Session::run<char>(int, char const* const*) be13_api/catch.hpp:13227 (test_be+0x268ceb)
    #15 main be13_api/catch.hpp:17504 (test_be+0x21335e)

  Thread T7 (tid=29753, running) created by main thread at:
    #0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:962 (libtsan.so.0+0x5ea79)
    #1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xd70a8)
    #2 notify_thread::start_notify_thread() /home/runner/work/bulk_extractor/bulk_extractor/src/notify_thread.cpp:130 (test_be+0x12bcb5)
    #3 bulk_extractor_main(std::ostream&, std::ostream&, int, char* const*) /home/runner/work/bulk_extractor/bulk_extractor/src/bulk_extractor.cpp:585 (test_be+0xd63c7)
    #4 ____C_A_T_C_H____T_E_S_T____10 /home/runner/work/bulk_extractor/bulk_extractor/src/test_be2.cpp:206 (test_be+0x2df71e)
    #5 Catch::TestInvokerAsFunction::invoke() const be13_api/catch.hpp:14317 (test_be+0x1fb904)
    #6 Catch::TestCase::invoke() const be13_api/catch.hpp:14156 (test_be+0x1fa744)
    #7 Catch::RunContext::invokeActiveTestCase() be13_api/catch.hpp:13016 (test_be+0x1f3207)
    #8 Catch::RunContext::runCurrentTest(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) be13_api/catch.hpp:12989 (test_be+0x1f2e48)
    #9 Catch::RunContext::runTest(Catch::TestCase const&) be13_api/catch.hpp:12750 (test_be+0x1f0f10)
    #10 execute be13_api/catch.hpp:13343 (test_be+0x1f5349)
    #11 Catch::Session::runInternal() be13_api/catch.hpp:13549 (test_be+0x1f6b8b)
    #12 Catch::Session::run() be13_api/catch.hpp:13505 (test_be+0x1f676f)
    #13 int Catch::Session::run<char>(int, char const* const*) be13_api/catch.hpp:13227 (test_be+0x268ceb)
    #14 main be13_api/catch.hpp:17504 (test_be+0x21335e)

SUMMARY: ThreadSanitizer: data race (/lib/x86_64-linux-gnu/libstdc++.so.6+0x143284) in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)
==================
==================
WARNING: ThreadSanitizer: data race (pid=29742)
  Read of size 8 at 0x7b0800003800 by thread T7:
    #0 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:823 (libtsan.so.0+0x42313)
    #1 memcpy ../../../../src/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:815 (libtsan.so.0+0x42313)
    #2 void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char*>(char*, char*, std::forward_iterator_tag) /usr/include/c++/9/bits/basic_string.tcc:225 (test_be+0x36418)
    #3 scanner_set::get_realtime_stats[abi:cxx11]() const be13_api/scanner_set.cpp:257 (test_be+0x93e72)
    #4 notify_thread::run() /home/runner/work/bulk_extractor/bulk_extractor/src/notify_thread.cpp:77 (test_be+0x12b078)
    #5 void* std::__invoke_impl<void*, void* (notify_thread::*)(), notify_thread*>(std::__invoke_memfun_deref, void* (notify_thread::*&&)(), notify_thread*&&) /usr/include/c++/9/bits/invoke.h:73 (test_be+0x12e4bd)
    #6 std::__invoke_result<void* (notify_thread::*)(), notify_thread*>::type std::__invoke<void* (notify_thread::*)(), notify_thread*>(void* (notify_thread::*&&)(), notify_thread*&&) /usr/include/c++/9/bits/invoke.h:95 (test_be+0x12e35d)
    #7 void* std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/include/c++/9/thread:244 (test_be+0x12e250)
    #8 std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> >::operator()() /usr/include/c++/9/thread:251 (test_be+0x12e1d2)
    #9 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> > >::_M_run() /usr/include/c++/9/thread:195 (test_be+0x12e184)
    #10 <null> <null> (libstdc++.so.6+0xd6de3)

  Previous write of size 8 at 0x7b0800003800 by thread T6:
    [failed to restore the stack]

  As if synchronized via sleep:
    #0 nanosleep ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:362 (libtsan.so.0+0x63360)
    #1 void std::this_thread::sleep_for<long, std::ratio<1l, 1l> >(std::chrono::duration<long, std::ratio<1l, 1l> > const&) /usr/include/c++/9/thread:378 (test_be+0xa0eee)
    #2 notify_thread::run() /home/runner/work/bulk_extractor/bulk_extractor/src/notify_thread.cpp:123 (test_be+0x12ba8d)
    #3 void* std::__invoke_impl<void*, void* (notify_thread::*)(), notify_thread*>(std::__invoke_memfun_deref, void* (notify_thread::*&&)(), notify_thread*&&) /usr/include/c++/9/bits/invoke.h:73 (test_be+0x12e4bd)
    #4 std::__invoke_result<void* (notify_thread::*)(), notify_thread*>::type std::__invoke<void* (notify_thread::*)(), notify_thread*>(void* (notify_thread::*&&)(), notify_thread*&&) /usr/include/c++/9/bits/invoke.h:95 (test_be+0x12e35d)
    #5 void* std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/include/c++/9/thread:244 (test_be+0x12e250)
    #6 std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> >::operator()() /usr/include/c++/9/thread:251 (test_be+0x12e1d2)
    #7 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (notify_thread::*)(), notify_thread*> > >::_M_run() /usr/include/c++/9/thread:195 (test_be+0x12e184)
    #8 <null> <null> (libstdc++.so.6+0xd6de3)

  Location is heap block of size 31 at 0x7b0800003800 allocated by thread T6:
    #0 operator new(unsigned long) ../../../../src/libsanitizer/tsan/tsan_new_delete.cpp:64 (libtsan.so.0+0x8c032)
    #1 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_assign(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) <null> (libstdc++.so.6+0x14329e)
    #2 scanner_set::process_sbuf(sbuf_t const*, void (*)(scanner_params&)) be13_api/scanner_set.cpp:844 (test_be+0x9877e)
    #3 worker::run() be13_api/threadpool.cpp:195 (test_be+0xbbf87)
    #4 worker::start_worker(void*) be13_api/threadpool.cpp:142 (test_be+0xbbabe)
    #5 void* std::__invoke_impl<void*, void* (*)(void*), void*>(std::__invoke_other, void* (*&&)(void*), void*&&) /usr/include/c++/9/bits/invoke.h:60 (test_be+0xc4144)
    #6 std::__invoke_result<void* (*)(void*), void*>::type std::__invoke<void* (*)(void*), void*>(void* (*&&)(void*), void*&&) /usr/include/c++/9/bits/invoke.h:95 (test_be+0xc4031)
    #7 void* std::thread::_Invoker<std::tuple<void* (*)(void*), void*> >::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/include/c++/9/thread:244 (test_be+0xc3f24)
    #8 std::thread::_Invoker<std::tuple<void* (*)(void*), void*> >::operator()() /usr/include/c++/9/thread:251 (test_be+0xc3ea6)
    #9 std::thread::_State_impl<std::thread::_Invoker<std::tuple<void* (*)(void*), void*> > >::_M_run() /usr/include/c++/9/thread:195 (test_be+0xc3e58)
    #10 <null> <null> (libstdc++.so.6+0xd6de3)

  Thread T7 (tid=29753, running) created by main thread at:
    #0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:962 (libtsan.so.0+0x5ea79)
    #1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xd70a8)
    #2 notify_thread::start_notify_thread() /home/runner/work/bulk_extractor/bulk_extractor/src/notify_thread.cpp:130 (test_be+0x12bcb5)
    #3 bulk_extractor_main(std::ostream&, std::ostream&, int, char* const*) /home/runner/work/bulk_extractor/bulk_extractor/src/bulk_extractor.cpp:585 (test_be+0xd63c7)
    #4 ____C_A_T_C_H____T_E_S_T____10 /home/runner/work/bulk_extractor/bulk_extractor/src/test_be2.cpp:206 (test_be+0x2df71e)
    #5 Catch::TestInvokerAsFunction::invoke() const be13_api/catch.hpp:14317 (test_be+0x1fb904)
    #6 Catch::TestCase::invoke() const be13_api/catch.hpp:14156 (test_be+0x1fa744)
    #7 Catch::RunContext::invokeActiveTestCase() be13_api/catch.hpp:13016 (test_be+0x1f3207)
    #8 Catch::RunContext::runCurrentTest(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) be13_api/catch.hpp:12989 (test_be+0x1f2e48)
    #9 Catch::RunContext::runTest(Catch::TestCase const&) be13_api/catch.hpp:12750 (test_be+0x1f0f10)
    #10 execute be13_api/catch.hpp:13343 (test_be+0x1f5349)
    #11 Catch::Session::runInternal() be13_api/catch.hpp:13549 (test_be+0x1f6b8b)
    #12 Catch::Session::run() be13_api/catch.hpp:13505 (test_be+0x1f676f)
    #13 int Catch::Session::run<char>(int, char const* const*) be13_api/catch.hpp:13227 (test_be+0x268ceb)
    #14 main be13_api/catch.hpp:17504 (test_be+0x21335e)

  Thread T6 (tid=29752, running) created by main thread at:
    #0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:962 (libtsan.so.0+0x5ea79)
    #1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xd70a8)
    #2 thread_pool::launch_workers(unsigned long) be13_api/threadpool.cpp:15 (test_be+0xbae4d)
    #3 scanner_set::launch_workers(int) be13_api/scanner_set.cpp:144 (test_be+0x93206)
    #4 bulk_extractor_main(std::ostream&, std::ostream&, int, char* const*) /home/runner/work/bulk_extractor/bulk_extractor/src/bulk_extractor.cpp:575 (test_be+0xd627b)
    #5 ____C_A_T_C_H____T_E_S_T____10 /home/runner/work/bulk_extractor/bulk_extractor/src/test_be2.cpp:206 (test_be+0x2df71e)
    #6 Catch::TestInvokerAsFunction::invoke() const be13_api/catch.hpp:14317 (test_be+0x1fb904)
    #7 Catch::TestCase::invoke() const be13_api/catch.hpp:14156 (test_be+0x1fa744)
    #8 Catch::RunContext::invokeActiveTestCase() be13_api/catch.hpp:13016 (test_be+0x1f3207)
    #9 Catch::RunContext::runCurrentTest(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&) be13_api/catch.hpp:12989 (test_be+0x1f2e48)
    #10 Catch::RunContext::runTest(Catch::TestCase const&) be13_api/catch.hpp:12750 (test_be+0x1f0f10)
    #11 execute be13_api/catch.hpp:13343 (test_be+0x1f5349)
    #12 Catch::Session::runInternal() be13_api/catch.hpp:13549 (test_be+0x1f6b8b)
    #13 Catch::Session::run() be13_api/catch.hpp:13505 (test_be+0x1f676f)
    #14 int Catch::Session::run<char>(int, char const* const*) be13_api/catch.hpp:13227 (test_be+0x268ceb)
    #15 main be13_api/catch.hpp:17504 (test_be+0x21335e)

SUMMARY: ThreadSanitizer: data race /usr/include/c++/9/bits/basic_string.tcc:225 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char*>(char*, char*, std::forward_iterator_tag)
==================
All data read; waiting for threads to finish...
Computing final histograms and shutting down...
*** More time spent waiting for workers. You need faster CPU or more cores for improved performance.
testing with command line:
bulk_extractor --notify_async -1o /tmp/be_tmp5e608f9d064126f5 ./tests/CFReDS001.E01 
opening ./tests/CFReDS001.E01