Closed ostash closed 7 years ago
Boost 1.65, GCC 7.2.0
Clang 5.0 reports same
This is under Linux x86_64
Managed to minimize it a bit more:
#include <boost/interprocess/segment_manager.hpp>
#include <boost/interprocess/mem_algo/rbtree_best_fit.hpp>
#include <boost/interprocess/indexes/null_index.hpp>
#include <boost/interprocess/sync/mutex_family.hpp>
#include <boost/container/vector.hpp>
#include <iostream>
#include <thread>
namespace bi = boost::interprocess;
using MemoryAlgorithm = bi::rbtree_best_fit<bi::mutex_family, void*, 0ul>;
using SegmentManager = bi::segment_manager<char, MemoryAlgorithm, bi::null_index>;
template <typename T>
using Allocator = typename SegmentManager::allocator<T>::type;
using VectorInt = boost::container::vector<unsigned, Allocator<unsigned>>;
void fillVector(SegmentManager* sm)
{
try {
void* ptr = sm->allocate(sizeof(VectorInt));
VectorInt* vecPtr = new (ptr) VectorInt(sm);
std::cerr << pthread_self() << " vector at " << vecPtr << '\n';
auto oldData = vecPtr->data();
auto oldCap = vecPtr->capacity();
while (true)
{
vecPtr->push_back(0xffffffff);
auto data = vecPtr->data();
auto cap = vecPtr->capacity();
if (data != oldData || cap != oldCap)
{
std::cerr << pthread_self() << " vect data " << data << ", till " << data + vecPtr->capacity() -1 << '\n';
oldData = data;
oldCap = cap;
}
}
}
catch (const std::exception& ex)
{
std::cerr << pthread_self() << ": " << ex.what() << '\n';
}
}
int main()
{
constexpr size_t blockSize = 2*1024*1024ul;
void* mem = malloc(blockSize);
SegmentManager* sm = new (mem) SegmentManager(blockSize);
std::thread th1([sm]{ fillVector(sm);});
std::thread th2([sm]{ fillVector(sm);});
th1.join();
th2.join();
return 0;
}
which gives following output from ThreadSanitizer:
140042383124224 vector at 0x7f5e288000a0
140042383124224 vect data 0x7f5e288000d0, till 0x7f5e288000f4
140042383124224 vect data 0x7f5e288000d0, till 0x7f5e28800124
140042383124224 vect data 0x7f5e288000d0, till 0x7f5e28800184
140042383124224 vect data 0x7f5e288000d0, till 0x7f5e28800244
140042383124224 vect data 0x7f5e288000d0, till 0x7f5e288003c4
140042383124224 vect data 0x7f5e288000d0, till 0x7f5e288006c4
140042374731520 vector at 0x7f5e288006d0
140042374731520 vect data 0x7f5e28800700, till 0x7f5e28800724
140042374731520 vect data 0x7f5e28800700, till 0x7f5e28800754
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288007b4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e28800874
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288009f4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e28800cf4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288012f4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e28801ef4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288036f4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288066f4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e2880c6f4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288186f4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288306f4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288606f4
140042374731520 vect data 0x7f5e28800700, till 0x7f5e288c06f4
==================
WARNING: ThreadSanitizer: data race (pid=19471)
Write of size 4 at 0x7f5e288006c0 by thread T1:
#0 void boost::interprocess::allocator<unsigned int, boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index> >::construct<unsigned int>(unsigned int* const&, unsigned int&&) /usr/include/boost/interprocess/allocators/allocator.hpp:264:7 (ip_tsan+0x4c5a41)
#1 void boost::container::allocator_traits<boost::interprocess::allocator<unsigned int, boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index> > >::priv_construct<unsigned int, unsigned int>(boost::move_detail::integral_constant<bool, true>, boost::interprocess::allocator<unsigned int, boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index> >&, unsigned int*, unsigned int&&) /usr/include/boost/container/allocator_traits.hpp:411:12 (ip_tsan+0x4c596b)
#2 void boost::container::allocator_traits<boost::interprocess::allocator<unsigned int, boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index> > >::construct<unsigned int, unsigned int>(boost::interprocess::allocator<unsigned int, boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index> >&, unsigned int*, unsigned int&&) /usr/include/boost/container/allocator_traits.hpp:360:10 (ip_tsan+0x4c52d3)
#3 void boost::container::vector<unsigned int, boost::interprocess::allocator<unsigned int, boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index> > >::priv_push_back<unsigned int>(unsigned int&&) /usr/include/boost/container/vector.hpp:2572:10 (ip_tsan+0x4c50e0)
#4 boost::container::vector<unsigned int, boost::interprocess::allocator<unsigned int, boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index> > >::push_back(unsigned int&&) /usr/include/boost/container/vector.hpp:1857:4 (ip_tsan+0x4b7f0b)
#5 fillVector(boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index>*) /home/ostash/tmp/ip/ip_tsan.cpp:31:15 (ip_tsan+0x4b62db)
#6 main::$_0::operator()() const /home/ostash/tmp/ip/ip_tsan.cpp:54:25 (ip_tsan+0x4b7098)
#7 void std::__invoke_impl<void, main::$_0>(std::__invoke_other, main::$_0&&) /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/bits/invoke.h:60:14 (ip_tsan+0x4b7030)
#8 std::__invoke_result<main::$_0>::type std::__invoke<main::$_0>(main::$_0&&) /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/bits/invoke.h:95:14 (ip_tsan+0x4b6f40)
#9 _ZNSt6thread8_InvokerISt5tupleIJZ4mainE3$_0EEE9_M_invokeIJLm0EEEEDTclsr3stdE8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/thread:234:13 (ip_tsan+0x4b6ee8)
#10 std::thread::_Invoker<std::tuple<main::$_0> >::operator()() /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/thread:243:11 (ip_tsan+0x4b6e88)
#11 std::thread::_State_impl<std::thread::_Invoker<std::tuple<main::$_0> > >::_M_run() /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/thread:186:13 (ip_tsan+0x4b6c6c)
#12 <null> <null> (libstdc++.so.6+0xbad2e)
Previous write of size 8 at 0x7f5e288006c0 by thread T2 (mutexes: write M9):
#0 boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>::priv_check_and_allocate(unsigned long, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>::block_ctrl*, unsigned long&) /usr/include/boost/interprocess/mem_algo/rbtree_best_fit.hpp:1257:21 (ip_tsan+0x4bad57)
#1 boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>::priv_allocate(int, unsigned long, unsigned long&, void*&, unsigned long) /usr/include/boost/interprocess/mem_algo/rbtree_best_fit.hpp:976:38 (ip_tsan+0x4b8927)
#2 boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>::allocate(unsigned long) /usr/include/boost/interprocess/mem_algo/rbtree_best_fit.hpp:673:11 (ip_tsan+0x4b82b5)
#3 boost::interprocess::segment_manager_base<boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul> >::allocate(unsigned long) /usr/include/boost/interprocess/segment_manager.hpp:177:37 (ip_tsan+0x4b7ca8)
#4 fillVector(boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index>*) /home/ostash/tmp/ip/ip_tsan.cpp:24:21 (ip_tsan+0x4b61d2)
#5 main::$_1::operator()() const /home/ostash/tmp/ip/ip_tsan.cpp:55:25 (ip_tsan+0x4b7a48)
#6 void std::__invoke_impl<void, main::$_1>(std::__invoke_other, main::$_1&&) /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/bits/invoke.h:60:14 (ip_tsan+0x4b79e0)
#7 std::__invoke_result<main::$_1>::type std::__invoke<main::$_1>(main::$_1&&) /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/bits/invoke.h:95:14 (ip_tsan+0x4b78f0)
#8 _ZNSt6thread8_InvokerISt5tupleIJZ4mainE3$_1EEE9_M_invokeIJLm0EEEEDTclsr3stdE8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/thread:234:13 (ip_tsan+0x4b7898)
#9 std::thread::_Invoker<std::tuple<main::$_1> >::operator()() /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/thread:243:11 (ip_tsan+0x4b7838)
#10 std::thread::_State_impl<std::thread::_Invoker<std::tuple<main::$_1> > >::_M_run() /usr/lib/gcc/x86_64-pc-linux-gnu/7.2.0/include/g++-v7/thread:186:13 (ip_tsan+0x4b761c)
#11 <null> <null> (libstdc++.so.6+0xbad2e)
Location is heap block of size 2097152 at 0x7f5e28800000 allocated by main thread:
#0 malloc /var/tmp/portage/sys-libs/compiler-rt-sanitizers-5.0.0/work/compiler-rt-5.0.0.src/lib/tsan/rtl/tsan_interceptors.cc:578 (ip_tsan+0x44b52b)
#1 main /home/ostash/tmp/ip/ip_tsan.cpp:51:15 (ip_tsan+0x4b65a2)
Mutex M9 (0x7f5e28800000) created at:
#0 pthread_mutex_init /var/tmp/portage/sys-libs/compiler-rt-sanitizers-5.0.0/work/compiler-rt-5.0.0.src/lib/tsan/rtl/tsan_interceptors.cc:1105 (ip_tsan+0x44c70a)
#1 boost::interprocess::ipcdetail::mutex_initializer::mutex_initializer(pthread_mutex_t&, pthread_mutexattr_t&) /usr/include/boost/interprocess/sync/posix/pthread_helpers.hpp:85:13 (ip_tsan+0x4c96b8)
#2 boost::interprocess::ipcdetail::posix_mutex::posix_mutex() /usr/include/boost/interprocess/sync/posix/mutex.hpp:84:22 (ip_tsan+0x4c9445)
#3 boost::interprocess::interprocess_mutex::interprocess_mutex() /usr/include/boost/interprocess/sync/interprocess_mutex.hpp:153:28 (ip_tsan+0x4c9328)
#4 boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>::header_t::header_t() /usr/include/boost/interprocess/mem_algo/rbtree_best_fit.hpp:150:11 (ip_tsan+0x4c8c5c)
#5 boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>::rbtree_best_fit(unsigned long, unsigned long) /usr/include/boost/interprocess/mem_algo/rbtree_best_fit.hpp:172:4 (ip_tsan+0x4c8abe)
#6 boost::interprocess::segment_manager_base<boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul> >::segment_manager_base(unsigned long, unsigned long) /usr/include/boost/interprocess/segment_manager.hpp:105:10 (ip_tsan+0x4c88f8)
#7 boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index>::segment_manager(unsigned long) /usr/include/boost/interprocess/segment_manager.hpp:418:10 (ip_tsan+0x4b7fc3)
#8 main /home/ostash/tmp/ip/ip_tsan.cpp:52:34 (ip_tsan+0x4b65ba)
Thread T1 (tid=19473, running) created by main thread at:
#0 pthread_create /var/tmp/portage/sys-libs/compiler-rt-sanitizers-5.0.0/work/compiler-rt-5.0.0.src/lib/tsan/rtl/tsan_interceptors.cc:889 (ip_tsan+0x44c326)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xbaff4)
#2 main /home/ostash/tmp/ip/ip_tsan.cpp:54:15 (ip_tsan+0x4b65dc)
Thread T2 (tid=19474, running) created by main thread at:
#0 pthread_create /var/tmp/portage/sys-libs/compiler-rt-sanitizers-5.0.0/work/compiler-rt-5.0.0.src/lib/tsan/rtl/tsan_interceptors.cc:889 (ip_tsan+0x44c326)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xbaff4)
#2 main /home/ostash/tmp/ip/ip_tsan.cpp:55:15 (ip_tsan+0x4b65f6)
SUMMARY: ThreadSanitizer: data race /usr/include/boost/interprocess/allocators/allocator.hpp:264:7 in void boost::interprocess::allocator<unsigned int, boost::interprocess::segment_manager<char, boost::interprocess::rbtree_best_fit<boost::interprocess::mutex_family, void*, 0ul>, boost::interprocess::null_index> >::construct<unsigned int>(unsigned int* const&, unsigned int&&)
==================
140042383124224 vect data 0x7f5e288c0700, till 0x7f5e288c12f4
...
It looks like rbtree_best_fit gives a bit of memory from another block during priv_expand OR boost::container::vector thinks that it can use that memory.
Ok, I think I found it, this is a race on bit-field (https://github.com/google/sanitizers/wiki/ThreadSanitizerPopularDataRaces#race-on-bit-field).
If we take a look on boost::interprocess::rbtree_best_fit::SizeHolder
:
struct SizeHolder
{
//!This block's memory size (including block_ctrl
//!header) in Alignment units
size_type m_prev_size : sizeof(size_type)*CHAR_BIT;
size_type m_size : sizeof(size_type)*CHAR_BIT - 2;
size_type m_prev_allocated : 1;
size_type m_allocated : 1;
};
non-synchronized access to m_prev_size
and m_size
is a race.
From what I understand from reading source, it is allowed to write UsableByPreviousChunk
bytes past allocated block, or to m_prev_size
of next block. But during allocation reads and writes to m_size
are performed.
Following minimal test case:
is considered as racy by ThreadSanitizer: