ballard26
commented
7 months ago Here are a few task traces all from oversized allocs with this same backtrace;
WARN 2024-03-07 07:01:58,435 [shard 0:main] seastar_memory - oversized allocation: 524288 bytes. This is non-fatal, but could lead to latency and/or fragmentation issues. Please report: at 0x94ad7a3 0x91152cc 0x91225a0 0x3c16c65 0x2e634aa 0x91e27cf 0x91e5f41 0x91e30d6 0x90e2250 0x90e0648 0x2d13b26 0x9503d59 /opt/redpanda/lib/libc.so.6+0x2d58f /opt/redpanda/lib/libc.so.6+0 x2d648 0x2d0bae4
--------
seastar::internal::coroutine_traits_base<void>::promise_type
--------
seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::future<void> seastar::future<void>::handle_exception<raft::replicate_batcher::cache_and_wait_for_result(s eastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consist ency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)>(raft::replicate_batcher::c ache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_b atch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&& )::'lambda'(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_c onstant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const: :'lambda'(std::exception_ptr const&)&&), seastar::futurize<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model ::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1 ::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)>::type seastar::future<void>::then_wrapped_nrvo<seastar::future<void>, seastar::future<void> seastar::fu ture<void>::handle_exception<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std ::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::op erator()() const::'lambda'(std::exception_ptr const&)>(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::mo del_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ra tio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)::'lambda'(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<d etail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::ch rono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)>(seastar::future<void> seastar::future<void>::handle_exception<raft ::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, tr ue>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::e xception_ptr const&)>(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1:: integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator( )() const::'lambda'(std::exception_ptr const&)&&)::'lambda'(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, mode l::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__ 1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)&&)::'lambda'(seastar::internal::promise_base_with_type<void>&&, seastar::future<void> seastar::futur e<void>::handle_exception<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::_ _1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::opera tor()() const::'lambda'(std::exception_ptr const&)>(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model _raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio <1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)::'lambda'(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<deta il::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chron o::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)&, seastar::future_state<seastar::internal::monostate>&&), void>
--------
seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<raft::replicate_batcher::cache_a nd_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_r eader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>(seastar::gate::holder&&, raft::replicate_batcher::cache_and_wai t_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0&&)::'lambda'(), false>, seastar::futurize<raft::replicate_batcher::cac he_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_bat ch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>::type seastar::future<void>::then_wrapped_nrvo<seastar::fut ure<void>, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<d etail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::ch rono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>(seastar::gate::holder&&, raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail: :base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono:: duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0&&)::'lambda'(), false>>(seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<raft::replicate_batcher::ca che_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_ba tch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>(seastar::gate::holder&&, raft::replicate_batcher::cache_an d_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_re ader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0&&)::'lambda'(), false>&&)::'lambda'(seastar::internal::promise_ba se_with_type<void>&&, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1: :optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<s td::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>(seastar::gate::holder&&, raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optio nal<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__ 1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0&&)::'lambda'(), false>&, seastar::future_state<seastar::internal::monostate>&&), void>
-------- seastar::internal::coroutine_traits_base<void>::promise_type
WARN 2024-03-07 07:01:58,436 [shard 0:main] seastar_memory - oversized allocation: 524288 bytes. This is non-fatal, but could lead to latency and/or fragmentation issues. Please report: at 0x94ad7a3 0x91152cc 0x91225a0 0x8926afb 0x91e27cf 0x91e5f41 0x91e30d6 0x90e2250 0x90e0648 0x2d13b26 0x9503d59 /opt/redpanda/lib/libc.so.6+0x2d58f /opt/redpanda/lib/libc.so.6+0x2d648 0x2 d0bae4
--------
seastar::internal::repeater<auto model::record_batch_reader::impl::do_action<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clo ck, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer, auto model::record_batch_reader::impl::do_consume<model::consume_reader_to_memory(mode l::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>(model::consume _reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_co nsumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::'lambda'(model::consume_reader_to_memory(model::reco rd_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&)>(model::consume_read er_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consume r&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>, auto model::record_batch_reader::impl::do_consume<model::c onsume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_ba tch_consumer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000 000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::'lambda'(model::consume_ reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_con sumer&)&&)::'lambda'()>
--------
seastar::continuation<seastar::internal::promise_base_with_type<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>, auto model::record_batch_reader ::impl::do_action<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer, auto model::record_batch_reader::impl::do_consume<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar:: lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono ::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, s td::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::'lambda'(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_ clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&)>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::tim e_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::_ _1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>, auto model::record_batch_reader::impl::do_consume<model::consume_reader_to_memory(model::record_batch_reader, std::__1:: chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>(model::consume_reader_to_memory(model::record_ba tch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_po int<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::'lambda'(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono: :time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&)&&)::'lambda0'(), auto model::record_batch_reader::impl ::do_consume<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 10000 00000l>>>)::memory_batch_consumer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std ::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::'la mbda'(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> >>)::memory_batch_consumer&) seastar::future<void>::then_impl_nrvo<auto model::record_batch_reader::impl::do_action<model::consume_reader_to_memory(model::record_batch_reader, std::__1::ch rono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer, auto model::record_batch_reader::impl::do_consume< model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::m emory_batch_consumer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio< 1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::'lambda'(model:: consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_b atch_consumer&)>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1 000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>, auto model::record_bat ch_reader::impl::do_consume<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1:: ratio<1l, 1000000000l>>>)::memory_batch_consumer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration <long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 10000 00000l>>>)::'lambda'(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1 l, 1000000000l>>>)::memory_batch_consumer&)&&)::'lambda0'(), seastar::future<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>>(model::consume_reader _to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer& &)::'lambda'(seastar::internal::promise_base_with_type<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>&&, auto model::record_batch_reader::impl::do _action<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000 l>>>)::memory_batch_consumer, auto model::record_batch_reader::impl::do_consume<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clo ck, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_poi nt<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::c hrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::'lambda'(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std ::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&)>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<se astar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono ::duration<long long, std::__1::ratio<1l, 1000000000l>>>, auto model::record_batch_reader::impl::do_consume<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::ti me_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>(model::consume_reader_to_memory(model::record_batch_reader , std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&, std::__1::chrono::time_point<seasta r::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::'lambda'(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_poin t<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer&)&&)::'lambda0'()&, seastar::future_state<seastar::internal::monos tate>&&), void>
--------
seastar::internal::do_with_state<std::__1::tuple<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::durati on<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>, seastar::future<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>>
--------
--------
seastar::continuation<seastar::internal::promise_base_with_type<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>, seastar::future<seastar::circul ar_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>::finally_body<auto model::record_batch_reader::consume<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>(model::consume_reader_to_memory(model ::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer, std::__1::chron o::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>) &&::'lambda'(), true>, seastar::futurize<model::consume_reader_to_memory(mode l::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>::type seastar: :future<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>::then_wrapped_nrvo<seastar::future<seastar::circular_buffer<model::record_batch, std::__1:: allocator<model::record_batch>>>, seastar::future<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>::finally_body<auto model::record_batch_reader::co nsume<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l> >>)::memory_batch_consumer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1:: ratio<1l, 1000000000l>>>)::memory_batch_consumer, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>) &&::'lambda' (), true>>(seastar::future<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>::finally_body<auto model::record_batch_reader::consume<model::consume_re ader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consu mer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>> >)::memory_batch_consumer, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>) &&::'lambda'(), true>&&)::'lambda'( seastar::internal::promise_base_with_type<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>&&, seastar::future<seastar::circular_buffer<model::record _batch, std::__1::allocator<model::record_batch>>>::finally_body<auto model::record_batch_reader::consume<model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time _point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer>(model::consume_reader_to_memory(model::record_batch_reader, std::__1::chrono::time_point<seastar::lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>)::memory_batch_consumer, std::__1::chrono::time_point<seastar:: lowres_clock, std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000000000l>>>) &&::'lambda'(), true>&, seastar::future_state<seastar::circular_buffer<model::record_batch, std::__1 ::allocator<model::record_batch>>>&&), seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>
--------
seastar::continuation<seastar::internal::promise_base_with_type<std::__1::vector<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>, std::__1::alloc ator<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>>>, raft::details::share_reader(model::record_batch_reader, unsigned long, bool)::$_0, seastar: :future<std::__1::vector<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>, std::__1::allocator<seastar::circular_buffer<model::record_batch, std::__1 ::allocator<model::record_batch>>>>> seastar::future<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>::then_impl_nrvo<raft::details::share_reader(mo del::record_batch_reader, unsigned long, bool)::$_0, seastar::future<std::__1::vector<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>, std::__1::all ocator<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>>>>(raft::details::share_reader(model::record_batch_reader, unsigned long, bool)::$_0&&)::'la mbda'(seastar::internal::promise_base_with_type<std::__1::vector<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>, std::__1::allocator<seastar::circu lar_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>>>&&, raft::details::share_reader(model::record_batch_reader, unsigned long, bool)::$_0&, seastar::future_state<se astar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>&&), seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>
--------
seastar::continuation<seastar::internal::promise_base_with_type<std::__1::vector<model::record_batch_reader, std::__1::allocator<model::record_batch_reader>>>, raft::details::share_read er(model::record_batch_reader, unsigned long, bool)::$_1, seastar::future<std::__1::vector<model::record_batch_reader, std::__1::allocator<model::record_batch_reader>>> seastar::future<std ::__1::vector<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>, std::__1::allocator<seastar::circular_buffer<model::record_batch, std::__1::allocator <model::record_batch>>>>>::then_impl_nrvo<raft::details::share_reader(model::record_batch_reader, unsigned long, bool)::$_1, seastar::future<std::__1::vector<model::record_batch_reader, st d::__1::allocator<model::record_batch_reader>>>>(raft::details::share_reader(model::record_batch_reader, unsigned long, bool)::$_1&&)::'lambda'(seastar::internal::promise_base_with_type<st d::__1::vector<model::record_batch_reader, std::__1::allocator<model::record_batch_reader>>>&&, raft::details::share_reader(model::record_batch_reader, unsigned long, bool)::$_1&, seastar: :future_state<std::__1::vector<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>, std::__1::allocator<seastar::circular_buffer<model::record_batch, st d::__1::allocator<model::record_batch>>>>>&&), std::__1::vector<seastar::circular_buffer<model::record_batch, std::__1::allocator<model::record_batch>>, std::__1::allocator<seastar::circul ar_buffer<model::record_batch, std::__1::allocator<model::record_batch>>>>>
--------
seastar::internal::coroutine_traits_base<model::record_batch_reader>::promise_type
--------
seastar::continuation<seastar::internal::promise_base_with_type<storage::append_result>, raft::replicate_entries_stm::append_to_self()::$_0, seastar::future<storage::append_result> seas tar::future<model::record_batch_reader>::then_impl_nrvo<raft::replicate_entries_stm::append_to_self()::$_0, seastar::future<storage::append_result>>(raft::replicate_entries_stm::append_to_ self()::$_0&&)::'lambda'(seastar::internal::promise_base_with_type<storage::append_result>&&, raft::replicate_entries_stm::append_to_self()::$_0&, seastar::future_state<model::record_batch _reader>&&), model::record_batch_reader>
--------
seastar::continuation<seastar::internal::promise_base_with_type<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw _as_system_error<storage::append_result, std::__1::error_code, void>>>, raft::replicate_entries_stm::append_to_self()::$_1, seastar::future<boost::outcome_v2::basic_result<storage::append_ result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>> seastar::future<storage::append_result>::the n_impl_nrvo<raft::replicate_entries_stm::append_to_self()::$_1, seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::err or_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>>>(raft::replicate_entries_stm::append_to_self()::$_1&&)::'lambda'(seastar::internal::promise_base_with_ty pe<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<storage::append_result, std::__1::error_code, v oid>>>&&, raft::replicate_entries_stm::append_to_self()::$_1&, seastar::future_state<storage::append_result>&&), storage::append_result>
--------
seastar::continuation<seastar::internal::promise_base_with_type<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw _as_system_error<storage::append_result, std::__1::error_code, void>>>, seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::pol icy::error_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>> seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, bo ost::outcome_v2::policy::error_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>>::handle_exception<raft::replicate_entries_stm::append_to_self()::$_2>(raft:: replicate_entries_stm::append_to_self()::$_2&&)::'lambda'(raft::replicate_entries_stm::append_to_self()::$_2&&), seastar::futurize<raft::replicate_entries_stm::append_to_self()::$_2>::type seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<storage::append_result, std::__1: :error_code, void>>>::then_wrapped_nrvo<seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_ error<storage::append_result, std::__1::error_code, void>>>, seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_ code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>> seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcom e_v2::policy::error_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>>::handle_exception<raft::replicate_entries_stm::append_to_self()::$_2>(raft::replicate_e ntries_stm::append_to_self()::$_2&&)::'lambda'(raft::replicate_entries_stm::append_to_self()::$_2&&)>(seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::erro r_code, boost::outcome_v2::policy::error_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>> seastar::future<boost::outcome_v2::basic_result<storage::append_re sult, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>>::handle_exception<raft::replicate_entries_stm: :append_to_self()::$_2>(raft::replicate_entries_stm::append_to_self()::$_2&&)::'lambda'(raft::replicate_entries_stm::append_to_self()::$_2&&)&&)::'lambda'(seastar::internal::promise_base_w ith_type<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<storage::append_result, std::__1::error_c ode, void>>>&&, seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<storage::append_re sult, std::__1::error_code, void>>> seastar::future<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_erro r<storage::append_result, std::__1::error_code, void>>>::handle_exception<raft::replicate_entries_stm::append_to_self()::$_2>(raft::replicate_entries_stm::append_to_self()::$_2&&)::'lambda '(raft::replicate_entries_stm::append_to_self()::$_2&&)&, seastar::future_state<boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::policy::err or_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>>&&), boost::outcome_v2::basic_result<storage::append_result, std::__1::error_code, boost::outcome_v2::pol icy::error_code_throw_as_system_error<storage::append_result, std::__1::error_code, void>>>
--------
seastar::internal::coroutine_traits_base<boost::outcome_v2::basic_result<raft::replicate_result, std::__1::error_code, boost::outcome_v2::policy::error_code_throw_as_system_error<raft:: replicate_result, std::__1::error_code, void>>>::promise_type
--------
seastar::internal::coroutine_traits_base<void>::promise_type
--------
seastar::internal::coroutine_traits_base<void>::promise_type
--------
seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::future<void> seastar::future<void>::handle_exception<raft::replicate_batcher::cache_and_wait_for_result(s eastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consist ency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)>(raft::replicate_batcher::c ache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_b atch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&& )::'lambda'(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_c onstant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const: :'lambda'(std::exception_ptr const&)&&), seastar::futurize<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model ::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1 ::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)>::type seastar::future<void>::then_wrapped_nrvo<seastar::future<void>, seastar::future<void> seastar::fu ture<void>::handle_exception<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std ::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::op erator()() const::'lambda'(std::exception_ptr const&)>(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::mo del_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ra tio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)::'lambda'(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<d etail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::ch rono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)>(seastar::future<void> seastar::future<void>::handle_exception<raft ::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, tr ue>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::e xception_ptr const&)>(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1:: integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator( )() const::'lambda'(std::exception_ptr const&)&&)::'lambda'(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, mode l::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__ 1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)&&)::'lambda'(seastar::internal::promise_base_with_type<void>&&, seastar::future<void> seastar::futur e<void>::handle_exception<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::_ _1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::opera tor()() const::'lambda'(std::exception_ptr const&)>(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model _raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio <1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)::'lambda'(raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<deta il::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chron o::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0::operator()() const::'lambda'(std::exception_ptr const&)&&)&, seastar::future_state<seastar::internal::monostate>&&), void>
--------
seastar::continuation<seastar::internal::promise_base_with_type<void>, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<raft::replicate_batcher::cache_a nd_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_r eader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>(seastar::gate::holder&&, raft::replicate_batcher::cache_and_wai t_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0&&)::'lambda'(), false>, seastar::futurize<raft::replicate_batcher::cac he_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_bat ch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>::type seastar::future<void>::then_wrapped_nrvo<seastar::fut ure<void>, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<d etail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::ch rono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>(seastar::gate::holder&&, raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail: :base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono:: duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0&&)::'lambda'(), false>>(seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<raft::replicate_batcher::ca che_and_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_ba tch_reader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>(seastar::gate::holder&&, raft::replicate_batcher::cache_an d_wait_for_result(seastar::promise<void>, std::__1::optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_re ader, raft::consistency_level, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0&&)::'lambda'(), false>&&)::'lambda'(seastar::internal::promise_ba se_with_type<void>&&, seastar::future<void>::finally_body<auto seastar::internal::invoke_func_with_gate<raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1: :optional<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<s td::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0>(seastar::gate::holder&&, raft::replicate_batcher::cache_and_wait_for_result(seastar::promise<void>, std::__1::optio nal<detail::base_named_type<long, model::model_raft_term_id_type, std::__1::integral_constant<bool, true>>>, model::record_batch_reader, raft::consistency_level, std::__1::optional<std::__ 1::chrono::duration<long long, std::__1::ratio<1l, 1000l>>>)::$_0&&)::'lambda'(), false>&, seastar::future_state<seastar::internal::monostate>&&), void>
--------
From topic scale testing.
The issue is from a one off run of a test in https://github.com/redpanda-data/redpanda/pull/16843
Redpanda version;
Backtrace;