hyper::Error(User(DispatchGone) on client request using tonic

[mcches]

Version

hyper = "0.14" via tonic = "0.11" h2 = "0.3.26"

Platform

Linux 6.1.97-104.177.amzn2023.aarch64

Description

We encountered a rare request failure in the wild. We do not have a local repro for this situation. Our setup is a tonic gRPC service using unix socket transport.

error:

Status { code: Unknown, message: "transport error", source: Some(tonic::transport::Error(Transport, hyper::Error(User(DispatchGone), "user code panicked"))) }

panic messages:

panicked at [..]/h2-0.3.26/src/proto/streams/store.rs:203:17:
dangling store key for stream_id=StreamId(0)

thread 'tokio-runtime-worker' panicked at [...]/h2-0.3.26/src/proto/streams/streams.rs:1001:36:
called `Result::unwrap()` on an `Err` value: PoisonError { .. }

thread 'tokio-runtime-worker' panicked at [...]/h2-0.3.26/src/proto/streams/streams.rs:1433:17:
StreamRef::drop; mutex poisoned

backtrace:

0: {closure#0}
            at [...]/src/panic.rs:45:25
1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2034:9
    std::panicking::rust_panic_with_hook
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:783:13
2: std::panicking::begin_panic_handler::{{closure}}
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:657:13
3: std::sys_common::backtrace::__rust_end_short_backtrace
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:171:18
4: rust_begin_unwind
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:645:5
5: core::panicking::panic_fmt
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panicking.rs:72:14
6: drop_stream_ref
    drop
            at [...]/h2-0.3.26/src/proto/streams/streams.rs:1420:9
7: drop_in_place<h2::proto::streams::streams::OpaqueStreamRef>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ptr/mod.rs:515:1
    drop_in_place<h2::proto::streams::streams::StreamRef<hyper::proto::h2::SendBuf<bytes::bytes::Bytes>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ptr/mod.rs:515:1
8: drop_in_place<h2::share::SendStream<hyper::proto::h2::SendBuf<bytes::bytes::Bytes>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ptr/mod.rs:515:1
    drop_in_place<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ptr/mod.rs:515:1
    drop_in_place<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ptr/mod.rs:515:1
    drop_in_place<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ptr/mod.rs:515:1
9: drop<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>
            at [...]/pin-project-lite-0.2.14/src/lib.rs:1700:17
    drop_in_place<pin_project_lite::__private::UnsafeDropInPlaceGuard<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ptr/mod.rs:515:1
    drop_in_place<(pin_project_lite::__private::UnsafeDropInPlaceGuard<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>, ())>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ptr/mod.rs:515:1
    project_replace<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
            at [...]/pin-project-lite-0.2.14/src/lib.rs:1173:29
    poll<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, ()>
            at [...]/futures-util-0.3.30/src/future/future/map.rs:56:23
10: poll<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
            at [...]/futures-util-0.3.30/src/future/future/mod.rs:48:1
    poll_pipe<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
            at [...]/hyper-0.14.29/src/proto/h2/client.rs:259:43
11: poll<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
12: poll<tonic::transport::service::io::BoxedIo, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
            at [...]/hyper-0.14.29/src/client/conn.rs:1110:47
    poll<tonic::transport::service::io::BoxedIo, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
            at [...]/hyper-0.14.29/src/client/conn.rs:559:22
13: {async_block#0}<tonic::transport::service::connector::Connector<hyper::client::connect::http::HttpConnector<hyper::client::connect::dns::GaiResolver>>, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>, http::uri::Uri>
            at [...]/hyper-0.14.29/src/client/service.rs:78:50
14: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/future/future.rs:123:9
    {closure#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>
            at [...]/tokio-1.38.0/src/runtime/task/core.rs:328:17
    with_mut<tokio::runtime::task::core::Stage<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>
            at [...]/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
    poll<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>
            at [...]/tokio-1.38.0/src/runtime/task/core.rs:317:30
15: {closure#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
            at [...]/tokio-1.38.0/src/runtime/task/harness.rs:485:19
    call_once<core::task::poll::Poll<()>, tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
    do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
    try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
    catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
    poll_future<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
            at [...]/tokio-1.38.0/src/runtime/task/harness.rs:473:18
    poll_inner<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
            at [...]/tokio-1.38.0/src/runtime/task/harness.rs:208:27
    poll<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
            at [...]/tokio-1.38.0/src/runtime/task/harness.rs:153:15
16: poll
            at [...]/tokio-1.38.0/src/runtime/task/raw.rs:201:18
    run<alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
            at [...]/tokio-1.38.0/src/runtime/task/mod.rs:427:9
    {closure#0}
            at [...]/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:585:18
    with_budget<core::result::Result<alloc::boxed::Box<tokio::runtime::scheduler::multi_thread::worker::Core, alloc::alloc::Global>, ()>, tokio::runtime::scheduler::multi_thread::worker::{impl#1}::run_task::{closure_env#0}>
            at [...]/tokio-1.38.0/src/runtime/coop.rs:107:5
    budget<core::result::Result<alloc::boxed::Box<tokio::runtime::scheduler::multi_thread::worker::Core, alloc::alloc::Global>, ()>, tokio::runtime::scheduler::multi_thread::worker::{impl#1}::run_task::{closure_env#0}>
            at [...]/tokio-1.38.0/src/runtime/coop.rs:73:5
    run_task
            at [...]/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:584:9
17: run
18: {closure#0}
            at [...]/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:500:21
    set<tokio::runtime::scheduler::Context, tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}, ()>
            at [...]/tokio-1.38.0/src/runtime/context/scoped.rs:40:9
    {closure#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>
            at [...]/tokio-1.38.0/src/runtime/context.rs:180:26
    try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>, ()>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/local.rs:284:16
    with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>, ()>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/local.rs:260:9
    set_scheduler<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>
            at [...]/tokio-1.38.0/src/runtime/context.rs:180:17
19: {closure#0}
            at [...]/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:495:9
    enter_runtime<tokio::runtime::scheduler::multi_thread::worker::run::{closure_env#0}, ()>
            at [...]/tokio-1.38.0/src/runtime/context/runtime.rs:65:16
20: run
            at [...]/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:487:5
21: {closure#0}
            at [...]/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:455:45
    poll<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}, ()>
            at [...]/tokio-1.38.0/src/runtime/blocking/task.rs:42:21
    {closure#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
            at [...]/tokio-1.38.0/src/runtime/task/core.rs:328:17
    with_mut<tokio::runtime::task::core::Stage<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>
            at [...]/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
    poll<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
            at [...]/tokio-1.38.0/src/runtime/task/core.rs:317:30
22: {closure#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
            at [...]/tokio-1.38.0/src/runtime/task/harness.rs:485:19
    call_once<core::task::poll::Poll<()>, tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
    do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>, core::task::poll::Poll<()>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
    try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
    catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>, core::task::poll::Poll<()>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
    poll_future<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
            at [...]/tokio-1.38.0/src/runtime/task/harness.rs:473:18
    poll_inner<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
            at [...]/tokio-1.38.0/src/runtime/task/harness.rs:208:27
    poll<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
            at [...]/tokio-1.38.0/src/runtime/task/harness.rs:153:15
23: poll
            at [...]/tokio-1.38.0/src/runtime/task/raw.rs:201:18
    run<tokio::runtime::blocking::schedule::BlockingSchedule>
            at [...]/tokio-1.38.0/src/runtime/task/mod.rs:464:9
    run
            at [...]/tokio-1.38.0/src/runtime/blocking/pool.rs:159:9
    run
            at [...]/tokio-1.38.0/src/runtime/blocking/pool.rs:513:17
    {closure#0}
            at [...]/tokio-1.38.0/src/runtime/blocking/pool.rs:471:13
    __rust_begin_short_backtrace<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys_common/backtrace.rs:155:18
24: {closure#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/mod.rs:528:17
    call_once<(), std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/panic/unwind_safe.rs:272:9
    do_call<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>, ()>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:552:40
    try<(), core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panicking.rs:516:19
    catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#1}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>, ()>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/panic.rs:146:14
    {closure#1}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/thread/mod.rs:527:30
    call_once<std::thread::{impl#0}::spawn_unchecked_::{closure_env#1}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>, ()>
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/core/src/ops/function.rs:250:5
25: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2020:9
    <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/alloc/src/boxed.rs:2020:9
    std::sys::pal::unix::thread::Thread::new::thread_start
            at /rustc/9b00956e56009bab2aa15d7bff10916599e3d6d6/library/std/src/sys/pal/unix/thread.rs:108:17
26: start_thread
27: thread_start

[seanmonstar]

Hm, this stack trace is for one of the later panics, just about poison. Do you have one for the dangling store key?

I can't imagine how you even got a store key with a stream ID of 0. I've looked through and any frame type that relates to streams has a check for the ID.... Unless we missed it somewhere.

[mcches]

This is all I got for this one, as we flush tracing and exit the process from the panic hook.

[mcches]

Here is another one, which I think is from the initial panic:

panicked at [..]/h2-0.3.26/src/proto/streams/store.rs:203:17:
dangling store key for stream_id=StreamId(0)","panic.file":"[..]/h2-0.3.26/src/proto/streams/store.rs","panic.line":203,"panic.column":17,"backtrace":"   0: {closure#0}
             at [..]/src/panic.rs:45:25
   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2036:9
      std::panicking::rust_panic_with_hook
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:799:13
   2: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:664:13
   3: std::sys_common::backtrace::__rust_end_short_backtrace
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:171:18
   4: rust_begin_unwind
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:652:5
   5: core::panicking::panic_fmt
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panicking.rs:72:14
   6: drop_stream_ref
      drop
             at [..]/h2-0.3.26/src/proto/streams/streams.rs:1420:9
   7: drop_in_place<h2::proto::streams::streams::OpaqueStreamRef>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
      drop_in_place<h2::proto::streams::streams::StreamRef<hyper::proto::h2::SendBuf<bytes::bytes::Bytes>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
   8: drop_in_place<h2::share::SendStream<hyper::proto::h2::SendBuf<bytes::bytes::Bytes>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
      drop_in_place<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
      drop_in_place<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
      drop_in_place<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
   9: drop<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>
             at [..]/pin-project-lite-0.2.14/src/lib.rs:1700:17
      drop_in_place<pin_project_lite::__private::UnsafeDropInPlaceGuard<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
      drop_in_place<(pin_project_lite::__private::UnsafeDropInPlaceGuard<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>>, ())>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ptr/mod.rs:514:1
      project_replace<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
             at [..]/pin-project-lite-0.2.14/src/lib.rs:1173:29
      poll<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, ()>
             at [..]/futures-util-0.3.30/src/future/future/map.rs:56:23
      poll<core::pin::Pin<alloc::boxed::Box<hyper::proto::h2::PipeToSendStream<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>, alloc::alloc::Global>>, hyper::proto::h2::client::{impl#3}::poll_pipe::{closure_env#0}<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>>
             at [..]/futures-util-0.3.30/src/lib.rs:91:13
      poll_pipe<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
             at [..]/hyper-0.14.30/src/proto/h2/client.rs:259:43
  10: poll<http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
  11: poll<tonic::transport::service::io::BoxedIo, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
             at [..]/hyper-0.14.30/src/client/conn.rs:1110:47
      poll<tonic::transport::service::io::BoxedIo, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>>
             at [..]/hyper-0.14.30/src/client/conn.rs:559:22
  12: {async_block#0}<tonic::transport::service::connector::Connector<hyper::client::connect::http::HttpConnector<hyper::client::connect::dns::GaiResolver>>, http_body::combinators::box_body::UnsyncBoxBody<bytes::bytes::Bytes, tonic::status::Status>, http::uri::Uri>
             at [..]/hyper-0.14.30/src/client/service.rs:78:50
  13: poll<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
      {closure#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>
             at [..]/tokio-1.38.1/src/runtime/task/core.rs:328:17
      with_mut<tokio::runtime::task::core::Stage<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>>
             at [..]/tokio-1.38.1/src/loom/std/unsafe_cell.rs:16:9
      poll<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::current_thread::Handle, alloc::alloc::Global>>
             at [..]/tokio-1.38.1/src/runtime/task/core.rs:317:30
  14: {closure#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
             at [..]/tokio-1.38.1/src/runtime/task/harness.rs:485:19
      call_once<core::task::poll::Poll<()>, tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
      do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
      try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
      catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>>, core::task::poll::Poll<()>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
      poll_future<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
             at [..]/tokio-1.38.1/src/runtime/task/harness.rs:473:18
      poll_inner<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
             at [..]/tokio-1.38.1/src/runtime/task/harness.rs:208:27
      poll<core::pin::Pin<alloc::boxed::Box<(dyn core::future::future::Future<Output=()> + core::marker::Send), alloc::alloc::Global>>, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
             at [..]/tokio-1.38.1/src/runtime/task/harness.rs:153:15
  15: poll
             at [..]/tokio-1.38.1/src/runtime/task/raw.rs:201:18
      run<alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle, alloc::alloc::Global>>
             at [..]/tokio-1.38.1/src/runtime/task/mod.rs:427:9
      {closure#0}
             at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:585:18
      with_budget<core::result::Result<alloc::boxed::Box<tokio::runtime::scheduler::multi_thread::worker::Core, alloc::alloc::Global>, ()>, tokio::runtime::scheduler::multi_thread::worker::{impl#1}::run_task::{closure_env#0}>
             at [..]/tokio-1.38.1/src/runtime/coop.rs:107:5
      budget<core::result::Result<alloc::boxed::Box<tokio::runtime::scheduler::multi_thread::worker::Core, alloc::alloc::Global>, ()>, tokio::runtime::scheduler::multi_thread::worker::{impl#1}::run_task::{closure_env#0}>
             at [..]/tokio-1.38.1/src/runtime/coop.rs:73:5
      run_task
             at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:584:9
  16: run
  17: {closure#0}
             at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:500:21
      set<tokio::runtime::scheduler::Context, tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}, ()>
             at [..]/tokio-1.38.1/src/runtime/context/scoped.rs:40:9
      {closure#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>
             at [..]/tokio-1.38.1/src/runtime/context.rs:180:26
      try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>, ()>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:286:12
      with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>, ()>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:262:9
      set_scheduler<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure_env#0}>
             at [..]/tokio-1.38.1/src/runtime/context.rs:180:17
  18: {closure#0}
             at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:495:9
      enter_runtime<tokio::runtime::scheduler::multi_thread::worker::run::{closure_env#0}, ()>
             at [..]/tokio-1.38.1/src/runtime/context/runtime.rs:65:16
  19: run
             at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:487:5
  20: {closure#0}
             at [..]/tokio-1.38.1/src/runtime/scheduler/multi_thread/worker.rs:455:45
      poll<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}, ()>
             at [..]/tokio-1.38.1/src/runtime/blocking/task.rs:42:21
      {closure#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
             at [..]/tokio-1.38.1/src/runtime/task/core.rs:328:17
      with_mut<tokio::runtime::task::core::Stage<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>>, core::task::poll::Poll<()>, tokio::runtime::task::core::{impl#6}::poll::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>
             at [..]/tokio-1.38.1/src/loom/std/unsafe_cell.rs:16:9
      poll<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
             at [..]/tokio-1.38.1/src/runtime/task/core.rs:317:30
  21: {closure#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
             at [..]/tokio-1.38.1/src/runtime/task/harness.rs:485:19
      call_once<core::task::poll::Poll<()>, tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
      do_call<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>, core::task::poll::Poll<()>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
      try<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
      catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future::{closure_env#0}<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>, core::task::poll::Poll<()>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
      poll_future<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
             at [..]/tokio-1.38.1/src/runtime/task/harness.rs:473:18
      poll_inner<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
             at [..]/tokio-1.38.1/src/runtime/task/harness.rs:208:27
      poll<tokio::runtime::blocking::task::BlockingTask<tokio::runtime::scheduler::multi_thread::worker::{impl#0}::launch::{closure_env#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
             at [..]/tokio-1.38.1/src/runtime/task/harness.rs:153:15
  22: poll
             at [..]/tokio-1.38.1/src/runtime/task/raw.rs:201:18
      run<tokio::runtime::blocking::schedule::BlockingSchedule>
             at [..]/tokio-1.38.1/src/runtime/task/mod.rs:464:9
      run
             at [..]/tokio-1.38.1/src/runtime/blocking/pool.rs:159:9
      run
             at [..]/tokio-1.38.1/src/runtime/blocking/pool.rs:513:17
  23: {closure#0}
             at [..]/tokio-1.38.1/src/runtime/blocking/pool.rs:471:13
      __rust_begin_short_backtrace<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:155:18
  24: {closure#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:542:17
      call_once<(), std::thread::{impl#0}::spawn_unchecked_::{closure#2}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
      do_call<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#2}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>, ()>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
      try<(), core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#2}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
      catch_unwind<core::panic::unwind_safe::AssertUnwindSafe<std::thread::{impl#0}::spawn_unchecked_::{closure#2}::{closure_env#0}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>>, ()>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
      {closure#2}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:541:30
      call_once<std::thread::{impl#0}::spawn_unchecked_::{closure_env#2}<tokio::runtime::blocking::pool::{impl#6}::spawn_thread::{closure_env#0}, ()>, ()>
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:250:5
  25: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys/pal/unix/thread.rs:108:17
  26: start_thread
  27: thread_start