Failing to send

[futurepaul]

WARN  client::net::api              > Unusual PeerError error=Rpc error: The background task closed i/o error: Operation timed out (os error 60); restart required peer_id=0
 WARN  db                            > DatabaseTransaction has writes and has not called commit. location=   0: backtrace::backtrace::libunwind::trace
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/backtrace-0.3.71/src/backtrace/libunwind.rs:105:5
      backtrace::backtrace::trace_unsynchronized
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/backtrace-0.3.71/src/backtrace/mod.rs:66:5
   1: backtrace::backtrace::trace
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/backtrace-0.3.71/src/backtrace/mod.rs:53:14
   2: backtrace::capture::Backtrace::create
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/backtrace-0.3.71/src/capture.rs:193:9
   3: backtrace::capture::Backtrace::new
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/backtrace-0.3.71/src/capture.rs:158:22
   4: <fedimint_core::db::CommitTracker as core::ops::drop::Drop>::drop
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fedimint-core-0.3.1/src/db/mod.rs:1255:33
   5: core::ptr::drop_in_place<fedimint_core::db::CommitTracker>
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/core/src/ptr/mod.rs:514:1
   6: core::ptr::drop_in_place<fedimint_core::db::MaybeRef<fedimint_core::db::CommitTracker>>
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/core/src/ptr/mod.rs:514:1
   7: core::ptr::drop_in_place<fedimint_core::db::DatabaseTransaction<fedimint_core::db::Committable>>
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/core/src/ptr/mod.rs:514:1
   8: fedimint_ln_client::LightningClientModule::pay_bolt11_invoice::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/fedimint-ln-client-0.3.1/src/lib.rs:1118:5
   9: harbor::core::HarborCore::send::{{closure}}
             at /Users/futurepaul/dev/rust/harbor/src/core.rs:89:14
  10: harbor::core::run_core::{{closure}}::{{closure}}
             at /Users/futurepaul/dev/rust/harbor/src/core.rs:225:72
  11: <futures_util::stream::once::Once<Fut> as futures_core::stream::Stream>::poll_next
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/once.rs:46:33
  12: <futures_util::stream::stream::map::Map<St,F> as futures_core::stream::Stream>::poll_next
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/map.rs:58:26
  13: futures_util::stream::select_with_strategy::poll_side
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/select_with_strategy.rs:220:28
  14: futures_util::stream::select_with_strategy::poll_inner
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/select_with_strategy.rs:243:11
  15: <futures_util::stream::select_with_strategy::SelectWithStrategy<St1,St2,Clos,State> as futures_core::stream::Stream>::poll_next
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/select_with_strategy.rs:270:17
  16: <futures_util::stream::select::Select<St1,St2> as futures_core::stream::Stream>::poll_next
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/select.rs:115:9
  17: <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  18: futures_util::stream::stream::StreamExt::poll_next_unpin
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  19: <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  20: futures_util::future::future::FutureExt::poll_unpin
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/future/mod.rs:558:9
  21: <futures_util::future::select::Select<A,B> as core::future::future::Future>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/select.rs:118:35
  22: iced_futures::subscription::tracker::Tracker::update::{{closure}}
             at /Users/futurepaul/.cargo/git/checkouts/iced-f01cba4d5e61fd0a/b30d34f/futures/src/subscription/tracker.rs:102:34
  23: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/core/src/future/future.rs:123:9
  24: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
  25: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/core.rs:328:17
  26: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/loom/std/unsafe_cell.rs:16:9
      tokio::runtime::task::core::Core<T,S>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/core.rs:317:13
  27: tokio::runtime::task::harness::poll_future::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:485:19
  28: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/core/src/panic/unwind_safe.rs:272:9
  29: std::panicking::try::do_call
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panicking.rs:559:40
  30: ___rust_try
  31: std::panicking::try
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panicking.rs:523:19
  32: std::panic::catch_unwind
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panic.rs:149:14
  33: tokio::runtime::task::harness::poll_future
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:473:18
  34: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:208:27
  35: tokio::runtime::task::harness::Harness<T,S>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:153:15
  36: tokio::runtime::task::raw::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/raw.rs:271:5
  37: tokio::runtime::task::raw::RawTask::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/raw.rs:201:18
  38: tokio::runtime::task::LocalNotified<S>::run
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/mod.rs:427:9
  39: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs:639:17
  40: tokio::runtime::coop::with_budget
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/coop.rs:107:5
      tokio::runtime::coop::budget
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/coop.rs:73:5
      tokio::runtime::scheduler::multi_thread::worker::Context::run_task
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs:575:9
  41: tokio::runtime::scheduler::multi_thread::worker::Context::run
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs:526:24
  42: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs:491:21
  43: tokio::runtime::context::scoped::Scoped<T>::set
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/context/scoped.rs:40:9
  44: tokio::runtime::context::set_scheduler::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/context.rs:176:26
  45: std::thread::local::LocalKey<T>::try_with
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/thread/local.rs:286:12
  46: std::thread::local::LocalKey<T>::with
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/thread/local.rs:262:9
  47: tokio::runtime::context::set_scheduler
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/context.rs:176:9
  48: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs:486:9
  49: tokio::runtime::context::runtime::enter_runtime
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/context/runtime.rs:65:16
  50: tokio::runtime::scheduler::multi_thread::worker::run
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs:478:5
  51: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/scheduler/multi_thread/worker.rs:447:45
  52: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/blocking/task.rs:42:21
  53: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
  54: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/core.rs:328:17
  55: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/loom/std/unsafe_cell.rs:16:9
      tokio::runtime::task::core::Core<T,S>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/core.rs:317:13
  56: tokio::runtime::task::harness::poll_future::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:485:19
  57: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/core/src/panic/unwind_safe.rs:272:9
  58: std::panicking::try::do_call
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panicking.rs:559:40
  59: ___rust_try
  60: std::panicking::try
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panicking.rs:523:19
  61: std::panic::catch_unwind
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panic.rs:149:14
  62: tokio::runtime::task::harness::poll_future
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:473:18
  63: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:208:27
  64: tokio::runtime::task::harness::Harness<T,S>::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/harness.rs:153:15
  65: tokio::runtime::task::raw::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/raw.rs:271:5
  66: tokio::runtime::task::raw::RawTask::poll
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/raw.rs:201:18
  67: tokio::runtime::task::UnownedTask<S>::run
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/task/mod.rs:464:9
  68: tokio::runtime::blocking::pool::Task::run
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/blocking/pool.rs:159:9
  69: tokio::runtime::blocking::pool::Inner::run
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/blocking/pool.rs:513:17
  70: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
             at /Users/futurepaul/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.37.0/src/runtime/blocking/pool.rs:471:13
  71: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/sys_common/backtrace.rs:155:18
  72: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/thread/mod.rs:542:17
  73: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/core/src/panic/unwind_safe.rs:272:9
  74: std::panicking::try::do_call
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panicking.rs:559:40
  75: ___rust_try
  76: std::panicking::try
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panicking.rs:523:19
  77: std::panic::catch_unwind
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/panic.rs:149:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/thread/mod.rs:541:30
  78: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/core/src/ops/function.rs:250:5
  79: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/alloc/src/boxed.rs:2022:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/alloc/src/boxed.rs:2022:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/ef0027897d2e9014766fb47dce9ddbb925d2f540/library/std/src/sys/pal/unix/thread.rs:108:17
  80: __pthread_joiner_wake

 ERROR harbor::core                  > Error sending: Federation rpc error {general => Received errors from 1 peers: peer-0: Rpc error: The background task closed i/o error: Operation timed out (os error 60); restart required), 0 => Rpc error: The background task closed i/o error: Operation timed out (os error 60); restart required), }