Open mcches opened 1 month ago
Hm, this stack trace is for one of the later panics, just about poison. Do you have one for the dangling store key?
I can't imagine how you even got a store key with a stream ID of 0. I've looked through and any frame type that relates to streams has a check for the ID.... Unless we missed it somewhere.
This is all I got for this one, as we flush tracing and exit the process from the panic hook.
Here is another one, which I think is from the initial panic:
panicked at [..]/h2-0.3.26/src/proto/streams/store.rs:203:17:
dangling store key for stream_id=StreamId(0)","panic.file":"[..]/h2-0.3.26/src/proto/streams/store.rs","panic.line":203,"panic.column":17,"backtrace":" 0: {closure#0}
at [..]/src/panic.rs:45:25
1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2036:9
std::panicking::rust_panic_with_hook
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:799:13
2: std::panicking::begin_panic_handler::{{closure}}
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:664:13
3: std::sys_common::backtrace::__rust_end_short_backtrace
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:171:18
4: rust_begin_unwind
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:652:5
5: core::panicking::panic_fmt
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panicking.rs:72:14
6: drop_stream_ref
drop
at [..]/h2-0.3.26/src/proto/streams/streams.rs:1420:9
7: drop_in_place<h2::proto::streams::streams::OpaqueStreamRef>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
drop_in_place<h2::proto::streams::streams::StreamRef<hyper::proto::h2::SendBuf<bytes::bytes::Bytes>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
8: drop_in_place<h2::share::SendStream<hyper::proto::h2::SendBuf<bytes::bytes::Bytes>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
drop_in_place<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
drop_in_place<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
drop_in_place<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
9: drop<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>
at [..]/pin-project-lite-0.2.14/src/lib.rs:1700:17
drop_in_place<pin_project_lite::__private::UnsafeDropInPlaceGuard<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
drop_in_place<(pin_project_lite::__private::UnsafeDropInPlaceGuard<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>, ())>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
project_replace<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
at [..]/pin-project-lite-0.2.14/src/lib.rs:1173:29
poll<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, ()>
at [..]/futures-util-0.3.30/src/future/future/map.rs:56:23
poll<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
at [..]/futures-util-0.3.30/src/lib.rs:91:13
poll_pipe<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
at [..]/hyper-0.14.30/src/proto/h2/client.rs:259:43
10: poll<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
11: poll<tonic::transport::service::io::BoxedIo, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
at [..]/hyper-0.14.30/src/client/conn.rs:1110:47
poll<tonic::transport::service::io::BoxedIo, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
at [..]/hyper-0.14.30/src/client/conn.rs:559:22
12: {async_block#0}<tonic::transport::service::connector::Connector<hyper::client::connect::http::HttpConnector<hyper::client::connect::dns::GaiResolver>>, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>, http::uri::Uri>
at [..]/hyper-0.14.30/src/client/service.rs:78:50
13: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
{closure#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>
at [..]/tokio-1.38.1/src/runtime/task/core.rs:328:17
with_mut<tokio::runtime::task::core::Stage<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>
at [..]/tokio-1.38.1/src/loom/std/unsafe_cell.rs:16:9
poll<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>
at [..]/tokio-1.38.1/src/runtime/task/core.rs:317:30
14: {closure#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
at [..]/tokio-1.38.1/src/runtime/task/harness.rs:485:19
call_once<core::task::poll::Poll<()>, tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
poll_future<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
at [..]/tokio-1.38.1/src/runtime/task/harness.rs:473:18
poll_inner<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
at [..]/tokio-1.38.1/src/runtime/task/harness.rs:208:27
poll<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
at [..]/tokio-1.38.1/src/runtime/task/harness.rs:153:15
15: poll
at [..]/tokio-1.38.1/src/runtime/task/raw.rs:201:18
run<alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
at [..]/tokio-1.38.1/src/runtime/task/mod.rs:427:9
{closure#0}
at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:585:18
with_budget<core::result::Result<alloc::boxed::Box<tokio::runtime::scheduler::multi_thread::worker::Core, alloc::alloc::Global>, ()>, tokio::runtime::scheduler::multi_thread::worker::{impl#1}::run_task::{closure_env#0}>
at [..]/tokio-1.38.1/src/runtime/coop.rs:107:5
budget<core::result::Result<alloc::boxed::Box<tokio::runtime::scheduler::multi_thread::worker::Core, alloc::alloc::Global>, ()>, tokio::runtime::scheduler::multi_thread::worker::{impl#1}::run_task::{closure_env#0}>
at [..]/tokio-1.38.1/src/runtime/coop.rs:73:5
run_task
at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:584:9
16: run
17: {closure#0}
at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:500:21
set<tokio::runtime::scheduler::Context, tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}, ()>
at [..]/tokio-1.38.1/src/runtime/context/scoped.rs:40:9
{closure#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>
at [..]/tokio-1.38.1/src/runtime/context.rs:180:26
try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>, ()>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:286:12
with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>, ()>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:262:9
set_scheduler<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>
at [..]/tokio-1.38.1/src/runtime/context.rs:180:17
18: {closure#0}
at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:495:9
enter_runtime<tokio::runtime::scheduler::multi_thread::worker::run::{closure_env#0}, ()>
at [..]/tokio-1.38.1/src/runtime/context/runtime.rs:65:16
19: run
at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:487:5
20: {closure#0}
at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:455:45
poll<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}, ()>
at [..]/tokio-1.38.1/src/runtime/blocking/task.rs:42:21
{closure#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
at [..]/tokio-1.38.1/src/runtime/task/core.rs:328:17
with_mut<tokio::runtime::task::core::Stage<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>
at [..]/tokio-1.38.1/src/loom/std/unsafe_cell.rs:16:9
poll<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
at [..]/tokio-1.38.1/src/runtime/task/core.rs:317:30
21: {closure#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
at [..]/tokio-1.38.1/src/runtime/task/harness.rs:485:19
call_once<core::task::poll::Poll<()>, tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>, core::task::poll::Poll<()>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>, core::task::poll::Poll<()>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
poll_future<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
at [..]/tokio-1.38.1/src/runtime/task/harness.rs:473:18
poll_inner<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
at [..]/tokio-1.38.1/src/runtime/task/harness.rs:208:27
poll<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
at [..]/tokio-1.38.1/src/runtime/task/harness.rs:153:15
22: poll
at [..]/tokio-1.38.1/src/runtime/task/raw.rs:201:18
run<tokio::runtime::blocking::schedule::BlockingSchedule>
at [..]/tokio-1.38.1/src/runtime/task/mod.rs:464:9
run
at [..]/tokio-1.38.1/src/runtime/blocking/pool.rs:159:9
run
at [..]/tokio-1.38.1/src/runtime/blocking/pool.rs:513:17
23: {closure#0}
at [..]/tokio-1.38.1/src/runtime/blocking/pool.rs:471:13
__rust_begin_short_backtrace<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:155:18
24: {closure#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:542:17
call_once<(), std::thread::{impl#0}::spawn_unchecked_::{closure#2}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
do_call<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#2}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>, ()>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
try<(), core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#2}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#2}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>, ()>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
{closure#2}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:541:30
call_once<std::thread::{impl#0}::spawn_unchecked_::{closure_env#2}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>, ()>
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:250:5
25: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
<alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
std::sys::pal::unix::thread::Thread::new::thread_start
at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys/pal/unix/thread.rs:108:17
26: start_thread
27: thread_start
Version
hyper = "0.14"
viatonic = "0.11"
h2 = "0.3.26"
Platform
Linux 6.1.97-104.177.amzn2023.aarch64
Description
We encountered a rare request failure in the wild. We do not have a local repro for this situation. Our setup is a tonic gRPC service using unix socket transport.
error:
panic messages:
backtrace: