Race condition on concurrent awaitable operators on Windows

[jothalha]

We encountered crashes with awaitable operators. This is a reduced (but probably not minimal) example that still fails for me after an indefinite number of loop runs on windows. So far I couldn't reproduce a segfault on linux or with single-threaded execution.

If anyone has some advice on how to better debug it I'm all ears as I'm missing some understanding of what should happen when.

Sample

#include <boost/asio/co_spawn.hpp>
#include <boost/asio/experimental/awaitable_operators.hpp>
#include <boost/asio/steady_timer.hpp>
#include <boost/asio/thread_pool.hpp>
#include <boost/asio/use_future.hpp>

using boost::asio::awaitable;
using boost::asio::co_spawn;
using boost::asio::use_awaitable;

using namespace boost::asio::experimental::awaitable_operators;

awaitable<int> foo() {
  auto timer = boost::asio::use_awaitable_t<>::as_default_on(
      boost::asio::steady_timer(co_await boost::asio::this_coro::executor));
  timer.expires_after(std::chrono::milliseconds(500));
  co_await timer.async_wait();
  co_return 42;
}

awaitable<void> composed() {
  auto timer = boost::asio::use_awaitable_t<>::as_default_on(
      boost::asio::steady_timer(co_await boost::asio::this_coro::executor));
  timer.expires_after(std::chrono::milliseconds(500));
  co_await (foo() || timer.async_wait());
}

int main() {
  boost::asio::thread_pool pool(2);

  while (true) {
    co_spawn(pool, composed(), boost::asio::use_future).get();
  }

  pool.join();
  return 0;
}

Compiled with

As an environment we use MSYS2 clang64 with Boost 1.83

clang++ -std=c++20 -stdlib=libc++ -g -O2 -lWs2_32 main.cpp

Stacktrace

* thread #5, stop reason = Exception 0xc0000005 encountered at address 0x7ff7b57431b5: Access violation reading location 0xffffffffffffffff
    frame #0: 0x00007ff7b57431b5 main.exe`void boost::asio::detail::executor_function::complete<boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator()(boost::asio::cancellation_type):
:'lambda'()>, std::__1::allocator<void>>(boost::asio::detail::executor_function::impl_base*, bool) [inlined] boost::asio::cancellation_signal::emit(this=<unavailable>, type=terminal) at cancellation_signal.hpp:119:17
   116    void emit(cancellation_type_t type)
   117    {
   118      if (handler_)
-> 119        handler_->call(type);
   120    }
   121
   122    /// Returns the single slot associated with the signal.
(lldb) bt
* thread #5, stop reason = Exception 0xc0000005 encountered at address 0x7ff7b57431b5: Access violation reading location 0xffffffffffffffff
  * frame #0: 0x00007ff7b57431b5 main.exe`void boost::asio::detail::executor_function::complete<boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator()(boost::asio::cancellation_type):
:'lambda'()>, std::__1::allocator<void>>(boost::asio::detail::executor_function::impl_base*, bool) [inlined] boost::asio::cancellation_signal::emit(this=<unavailable>, type=terminal) at cancellation_signal.hpp:119:17
    frame #1: 0x00007ff7b57431a8 main.exe`void boost::asio::detail::executor_function::complete<boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator()(boost::asio::cancellation_type):
:'lambda'()>, std::__1::allocator<void>>(boost::asio::detail::executor_function::impl_base*, bool) [inlined] boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator(this=<unavailable>)(boost::asio::cancellation_type
)::'lambda'()::operator()() const at co_spawn.hpp:250:50
    frame #2: 0x00007ff7b57431a3 main.exe`void boost::asio::detail::executor_function::complete<boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator()(boost::asio::cancellation_type):
:'lambda'()>, std::__1::allocator<void>>(boost::asio::detail::executor_function::impl_base*, bool) [inlined] boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator()(boost::asio::cancel
lation_type)::'lambda'()>::operator()(this=<unavailable>) at bind_handler.hpp:60:5
    frame #3: 0x00007ff7b57431a3 main.exe`void boost::asio::detail::executor_function::complete<boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator()(boost::asio::cancellation_type):
:'lambda'()>, std::__1::allocator<void>>(boost::asio::detail::executor_function::impl_base*, bool) [inlined] void boost::asio::asio_handler_invoke<boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor,
 void>::operator()(boost::asio::cancellation_type)::'lambda'()>>(function=<unavailable>) at handler_invoke_hook.hpp:88:3
    frame #4: 0x00007ff7b57431a3 main.exe`void boost::asio::detail::executor_function::complete<boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator()(boost::asio::cancellation_type):
:'lambda'()>, std::__1::allocator<void>>(boost::asio::detail::executor_function::impl_base*, bool) [inlined] void boost_asio_handler_invoke_helpers::invoke<boost::asio::detail::binder0<boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_
executor, void>::operator()(boost::asio::cancellation_type)::'lambda'()>, boost::asio::detail::co_spawn_cancellation_handler<boost::asio::detail::awaitable_handler<boost::asio::any_io_executor, std::exception_ptr>, boost::asio::any_io_executor, void>::operator()(boost::asio::cancellation_type)::'lambda'()>(function=<unavailable>, context=<unava
ilable>) at handler_invoke_helpers.hpp:54:3
...

Full stacktrace:

stacktrace.txt

[jothalha]

Might be related to #1348

[klemens-morgenstern]

Sounds like you need to use a strand.

[jothalha]

Thanks for the reply! So this seems to fix the problem and the tsan warning:

co_spawn(make_strand(pool), composed(), boost::asio::use_future).get();

Could there be other pitfalls depending on what happens in composed or other coros? The only problem that I see at the moment would be to spawn a new coro with a different executor somewhere, ie

co_await (foo() || co_spawn(some_executor, [&] () { return timer.async_wait(); }, boost::asio::use_awaitable))

which would circumvent the strand again - or is there a way around that?