search

hyperium / h2

HTTP 2.0 client & server implementation for Rust.
MIT License
1.34k stars 266 forks source link

assertion failed: self.num_remote_reset_streams > 0 #684

Closed jeromegn closed 11 months ago

jeromegn commented 1 year ago

We are running h2 v0.3.18.

Here's the backtrace from Sentry:

assertion failed: self.num_remote_reset_streams > 0

Thread 0 Crashed:
0   std                             0x55c96d2f6c5c      std::sys_common::backtrace::__rust_end_short_backtrace (backtrace.rs:137)
1   <unknown>                       0x55c96d2f8142      rust_begin_unwind (panicking.rs:575)
2   core                            0x55c96d31dc13      core::panicking::panic_fmt (panicking.rs:64)
3   core                            0x55c96d31dced      core::panicking::panic (panicking.rs:111)
4   <unknown>                       0x55c96cf4afff      h2[c00baf226614d5c]::proto::streams::counts::Counts::dec_num_remote_reset_streams (counts.rs:137)
5   <unknown>                       0x55c96c61f83e      [inlined] h2[c00baf226614d5c]::proto::streams::streams::Streams<T>::next_incoming::{closure#0} (streams.rs:146)
6   <unknown>                       0x55c96c61f83e      [inlined] core[cf0a937d1125fa47]::option::Option<T>::map:: (option.rs:969)
7   <unknown>                       0x55c96c61f83e      h2[c00baf226614d5c]::proto::streams::streams::Streams<T>::next_incoming (streams.rs:133)
8   <unknown>                       0x55c96c245f64      [inlined] <h2[c00baf226614d5c]::proto::connection::Connection<hyper[39e87571ecf802b8]::common::io::rewind::Rewind<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]... (connection.rs:552)
9   <unknown>                       0x55c96c245f64      <h2[c00baf226614d5c]::server::Connection<hyper[39e87571ecf802b8]::common::io::rewind::Rewind<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Bo... (server.rs:426)
10  <unknown>                       0x55c96c43f29d      [inlined] <hyper[39e87571ecf802b8]::proto::h2::server::Serving<hyper[39e87571ecf802b8]::common::io::rewind::Rewind<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa... (server.rs:284)
11  <unknown>                       0x55c96c43f29d      <hyper[39e87571ecf802b8]::proto::h2::server::Server<hyper[39e87571ecf802b8]::common::io::rewind::Rewind<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]... (server.rs:218)
12  <unknown>                       0x55c96c252c35      [inlined] <hyper[39e87571ecf802b8]::server::conn::ProtoServer<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<fp_tcp_idle_timeout[691300eef0d93111]::... (conn.rs:988)
13  <unknown>                       0x55c96c252c35      <hyper[39e87571ecf802b8]::server::conn::upgrades::UpgradeableConnection<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<fp_tcp_idle_timeout... (conn.rs:1046)
14  <unknown>                       0x55c96c2e9186      [inlined] <&mut hyper[39e87571ecf802b8]::server::conn::upgrades::UpgradeableConnection<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<fp_tcp_idle_ti... (future.rs:112)
15  <unknown>                       0x55c96c2e9186      [inlined] <tracing[f7f30f32cc34498a]::instrument::Instrumented<&mut hyper[39e87571ecf802b8]::server::conn::upgrades::UpgradeableConnection<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pi... (instrument.rs:272)
16  <unknown>                       0x55c96c2e9186      <opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumented<&mut hyper[39e87571ecf802b8]::server::conn::upgrades::UpgradeableConnection<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_met... (context.rs:373)
17  <unknown>                       0x55c96c2871d6      [inlined] <tripwire[24f4131df546c329]::preempt::PreemptibleFuture<&mut tripwire[24f4131df546c329]::tripwire::Tripwire, opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumented<&mut hyper[39e87571ecf802b8... (preempt.rs:56)
18  <unknown>                       0x55c96c2871d6      [inlined] <fp_http[4f776ff130c93902]::service::HttpService as tower_service[51fcaf158879a154]::Service<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Bo... (service.rs:129)
19  <unknown>                       0x55c96c2871d6      <tracing[f7f30f32cc34498a]::instrument::Instrumented<<fp_http[4f776ff130c93902]::service::HttpService as tower_service[51fcaf158879a154]::Service<tokio_rustls[2b4471a888ff256]::server::TlsStream<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1... (instrument.rs:272)
20  <unknown>                       0x55c96c24461f      [inlined] <core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = core[cf0a937d1125fa47]::result::Result<(), fp_errors[4e01595fafbed632]::proxy::ProxyError>> + core[cf0a937d1125fa47]::marke... (future.rs:124)
21  <unknown>                       0x55c96c24461f      <fp_tls[2bc31ed72f4ace2]::layers::tls::TlsLayeredService<fp_http[4f776ff130c93902]::service::HttpService> as tower_service[51fcaf158879a154]::Service<fp_tcp_metered[5283a7fef24b1328]::MeteredIo<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::b... (tls.rs:109)
22  <unknown>                       0x55c96c44645c      [inlined] <core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = core[cf0a937d1125fa47]::result::Result<(), fp_errors[4e01595fafbed632]::proxy::ProxyError>> + core[cf0a937d1125fa47]::marke... (future.rs:124)
23  <unknown>                       0x55c96c44645c      [inlined] <tower[92830710c1249629]::limit::concurrency::future::ResponseFuture<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = core[cf0a937d1125fa47]::result::Result<(), fp_errors[4e... (future.rs:39)
24  <unknown>                       0x55c96c44645c      <fp_layers[fd136ed4dda1018c]::load_shed::ResponseFuture<tower[92830710c1249629]::limit::concurrency::future::ResponseFuture<core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = ... (load_shed.rs:152)
25  <unknown>                       0x55c96c30636b      [inlined] <core[cf0a937d1125fa47]::pin::Pin<alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::future::future::Future<Output = core[cf0a937d1125fa47]::result::Result<(), alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::error::Error + c... (future.rs:124)
26  <unknown>                       0x55c96c30636b      <tower[92830710c1249629]::util::oneshot::Oneshot<tower[92830710c1249629]::util::boxed::sync::BoxService<fp_tcp[27e50990d6be481a]::permitted::PermittedTcpStream, (), alloc[e89cf99e2daa46aa]::boxed::Box<dyn core[cf0a937d1125fa47]::error::Error + core[cf0a... (oneshot.rs:97)
27  <unknown>                       0x55c96c27c958      [inlined] fp_edge[71fb797d61ca5443]::handle_edge_incoming::{closure#0}::{closure#0} (lib.rs:104)
28  <unknown>                       0x55c96c27c958      tracing[f7f30f32cc34498a]::instrument::Instrumented<T>::poll (instrument.rs:272)
29  <unknown>                       0x55c96c276c3c      [inlined] opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<T>::poll (context.rs:373)
30  <unknown>                       0x55c96c276c3c      [inlined] <spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumented<fp_edge[71fb797d61ca5443]::handle_edge_incoming::{closure#0}::{closure#0}>>> as core[cf0a937d1125fa... (lib.rs:103)
31  <unknown>                       0x55c96c276c3c      <tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumented<fp_edge[71fb797d61ca5443]::handle_edge_incoming... (instrument.rs:272)
32  <unknown>                       0x55c96c333c0c      [inlined] <tokio[1aa1d45b4aef9d67]::runtime::task::core::Core<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumen... (core.rs:223)
33  <unknown>                       0x55c96c333c0c      [inlined] <tokio[1aa1d45b4aef9d67]::loom::std::unsafe_cell::UnsafeCell<tokio[1aa1d45b4aef9d67]::runtime::task::core::Stage<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context:... (unsafe_cell.rs:14)
34  <unknown>                       0x55c96c333c0c      <tokio[1aa1d45b4aef9d67]::runtime::task::core::Core<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Instrumen... (core.rs:212)
35  <unknown>                       0x55c96c4240a9      [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future:: (harness.rs:476)
36  <unknown>                       0x55c96c4240a9      [inlined] <core[cf0a937d1125fa47]::panic::unwind_safe::AssertUnwindSafe<tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace... (unwind_safe.rs:271)
37  <unknown>                       0x55c96c4240a9      [inlined] std[645413f22396c11]::panicking::try::do_call:: (panicking.rs:483)
38  <unknown>                       0x55c96c4240a9      [inlined] std[645413f22396c11]::panicking::try:: (panicking.rs:447)
39  <unknown>                       0x55c96c4240a9      [inlined] std[645413f22396c11]::panic::catch_unwind:: (panic.rs:137)
40  <unknown>                       0x55c96c4240a9      [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future:: (harness.rs:464)
41  <unknown>                       0x55c96c4240a9      [inlined] <tokio[1aa1d45b4aef9d67]::runtime::task::harness::Harness<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Ins... (harness.rs:198)
42  <unknown>                       0x55c96c4240a9      <tokio[1aa1d45b4aef9d67]::runtime::task::harness::Harness<tracing[f7f30f32cc34498a]::instrument::Instrumented<spawn[84a4156df4e192c9]::CountedFut<opentelemetry_api[71a91f7da52bd67b]::trace::context::WithContext<tracing[f7f30f32cc34498a]::instrument::Ins... (harness.rs:152)
43  <unknown>                       0x55c96cfd4c0c      [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::raw::RawTask::poll (raw.rs:200)
44  <unknown>                       0x55c96cfd4c0c      [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::LocalNotified<T>::run (mod.rs:394)
45  <unknown>                       0x55c96cfd4c0c      [inlined] tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::Context::run_task::{closure#0} (worker.rs:464)
46  <unknown>                       0x55c96cfd4c0c      [inlined] tokio[1aa1d45b4aef9d67]::runtime::coop::with_budget:: (coop.rs:107)
47  <unknown>                       0x55c96cfd4c0c      [inlined] tokio[1aa1d45b4aef9d67]::runtime::coop::budget:: (coop.rs:73)
48  <unknown>                       0x55c96cfd4c0c      tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::Context::run_task (worker.rs:463)
49  <unknown>                       0x55c96cfd4691      tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::Context::run (worker.rs:433)
50  <unknown>                       0x55c96cfe0f19      [inlined] tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::run::{closure#0} (worker.rs:406)
51  <unknown>                       0x55c96cfe0f19      tokio[1aa1d45b4aef9d67]::macros::scoped_tls::ScopedKey<T>::set::<T> (scoped_tls.rs:61)
52  <unknown>                       0x55c96cfd3a64      tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::run (worker.rs:403)
53  <unknown>                       0x55c96c2a611b      [inlined] tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place::<T>::{closure#0}::{closure#0} (worker.rs:343)
54  <unknown>                       0x55c96c2a611b      [inlined] <tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::read::{closure#0}::{closure#0}, core[cf0a937d1125fa47]::resu... (task.rs:42)
55  <unknown>                       0x55c96c2a611b      [inlined] <tokio[1aa1d45b4aef9d67]::runtime::task::core::Core<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::read::{cl... (core.rs:223)
56  <unknown>                       0x55c96c2a611b      <tokio[1aa1d45b4aef9d67]::loom::std::unsafe_cell::UnsafeCell<tokio[1aa1d45b4aef9d67]::runtime::task::core::Stage<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_p... (unsafe_cell.rs:14)
57  <unknown>                       0x55c96c334e5c      <tokio[1aa1d45b4aef9d67]::runtime::task::core::Core<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::read::{cl... (core.rs:212)
58  <unknown>                       0x55c96c426c99      [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future:: (harness.rs:476)
59  <unknown>                       0x55c96c426c99      [inlined] <core[cf0a937d1125fa47]::panic::unwind_safe::AssertUnwindSafe<tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::... (unwind_safe.rs:271)
60  <unknown>                       0x55c96c426c99      [inlined] std[645413f22396c11]::panicking::try::do_call:: (panicking.rs:483)
61  <unknown>                       0x55c96c426c99      [inlined] std[645413f22396c11]::panicking::try:: (panicking.rs:447)
62  <unknown>                       0x55c96c426c99      [inlined] std[645413f22396c11]::panic::catch_unwind:: (panic.rs:137)
63  <unknown>                       0x55c96c426c99      [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::harness::poll_future:: (harness.rs:464)
64  <unknown>                       0x55c96c426c99      [inlined] <tokio[1aa1d45b4aef9d67]::runtime::task::harness::Harness<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::rea... (harness.rs:198)
65  <unknown>                       0x55c96c426c99      <tokio[1aa1d45b4aef9d67]::runtime::task::harness::Harness<tokio[1aa1d45b4aef9d67]::runtime::blocking::task::BlockingTask<tokio[1aa1d45b4aef9d67]::runtime::scheduler::multi_thread::worker::block_in_place<estate_corrosion[4f14d3214435355a]::instances::rea... (harness.rs:152)
66  <unknown>                       0x55c96cfc5abc      [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::raw::RawTask::poll (raw.rs:200)
67  <unknown>                       0x55c96cfc5abc      [inlined] tokio[1aa1d45b4aef9d67]::runtime::task::UnownedTask<T>::run (mod.rs:431)
68  <unknown>                       0x55c96cfc5abc      [inlined] tokio[1aa1d45b4aef9d67]::runtime::blocking::pool::Task::run (pool.rs:159)
69  <unknown>                       0x55c96cfc5abc      tokio[1aa1d45b4aef9d67]::runtime::blocking::pool::Inner::run (pool.rs:511)
70  <unknown>                       0x55c96cfc6dfd      [inlined] tokio[1aa1d45b4aef9d67]::runtime::blocking::pool::Spawner::spawn_thread::{closure#0} (pool.rs:469)
71  <unknown>                       0x55c96cfc6dfd      std[645413f22396c11]::sys_common::backtrace::__rust_begin_short_backtrace::<T> (backtrace.rs:121)
72  <unknown>                       0x55c96cfd1347      [inlined] std[645413f22396c11]::thread::Builder::spawn_unchecked_::<T>::{closure#1}::{closure#0} (mod.rs:550)
73  <unknown>                       0x55c96cfd1347      [inlined] <core[cf0a937d1125fa47]::panic::unwind_safe::AssertUnwindSafe<<std[645413f22396c11]::thread::Builder>::spawn_unchecked_<<tokio[1aa1d45b4aef9d67]::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>::{closure#1}::{closure#0}> as core[cf0a93... (unwind_safe.rs:271)
74  <unknown>                       0x55c96cfd1347      [inlined] std[645413f22396c11]::panicking::try::do_call:: (panicking.rs:483)
75  <unknown>                       0x55c96cfd1347      [inlined] std[645413f22396c11]::panicking::try:: (panicking.rs:447)
76  <unknown>                       0x55c96cfd1347      [inlined] std[645413f22396c11]::panic::catch_unwind:: (panic.rs:137)
77  <unknown>                       0x55c96cfd1347      [inlined] std[645413f22396c11]::thread::Builder::spawn_unchecked_::<T>::{closure#1} (mod.rs:549)
78  <unknown>                       0x55c96cfd1347      std[645413f22396c11]::thread::Builder::spawn_unchecked_<T>::{closure#1}::call_once::{shim:vtable#0} (function.rs:507)
79  alloc                           0x55c96d2fdb63      [inlined] alloc::boxed::Box<T>::call_once (boxed.rs:2000)
80  alloc                           0x55c96d2fdb63      [inlined] alloc::boxed::Box<T>::call_once (boxed.rs:2000)
81  std                             0x55c96d2fdb63      std::sys::unix::thread::Thread::new::thread_start (thread.rs:108)
82  <unknown>                       0x7fcc4412f6db      start_thread (pthread_create.c:463)
83  <unknown>                       0x7fcc438b671f      __GI___clone (clone.S:95)
84  <unknown>                       0x0                 <unknown>

There are 3 subsequent PoisonError unwrap panics, but I assume they were caused by this assertion. If not I can also provide their backtrace.

seanmonstar commented 11 months ago

I believe this was fixed in https://github.com/hyperium/h2/pull/682, released in v0.3.19. (If not, we can re-open.)