Liveness 100% and sync is good, but

[jovar95]

I have 16 CPU; 62 GB RAM DVS on Hetzner AX51. Set up up with docker (offiicial guide) OWNER ADDRESS: 0x21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42

consensus_proposals_count increasing very slovly and no rewards for last 3 EPOCH and reward performance it still 49% LOGS

testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/peer/mod.rs:384"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.415732Z","message":"[validator,Validator,21b4ef9b] Closed connection to peer: 28dde00b","data":{"connection_id":294,"connection_origin":"inbound","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"28dde00b49a2c5ec8b46a1fbd855afd0fac64ea347a3dffc803ad672e7810186"}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/rpc/mod.rs:558"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.616514Z","message":"[validator,Validator,21b4ef9b] Error making outbound rpc request with request_id 6 to 9f9ae72a: Error: Rpc timed out\n\nStack backtrace:\n   0: network::protocols::rpc::OutboundRpcs::handle_outbound_request::{{closure}}::{{closure}}\n             at ./aptos/network/src/protocols/rpc/mod.rs:477:57\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      network::protocols::rpc::OutboundRpcs::handle_outbound_request::{{closure}}::{{closure}}\n             at ./aptos/network/src/protocols/rpc/mod.rs:490:37\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: <core::pin::Pin<P> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/future.rs:124:9\n      <futures_util::stream::futures_unordered::FuturesUnordered<Fut> as futures_core::stream::Stream>::poll_next\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/futures_unordered/mod.rs:514:17\n      futures_util::stream::stream::StreamExt::poll_next_unpin\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/stream/mod.rs:1626:9\n   2: <futures_util::stream::stream::select_next_some::SelectNextSome<St> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/stream/select_next_some.rs:34:36\n      <core::pin::Pin<P> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/future.rs:124:9\n      futures_util::future::future::FutureExt::poll_unpin\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/future/future/mod.rs:562:9\n      network::peer::Peer<TSocket>::start::{{closure}}::{{closure}}::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/async_await/select_mod.rs:321:13\n   3: core::ops::function::impls::<impl core::ops::function::FnMut<A> for &mut F>::call_mut\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:290:13\n      network::peer::Peer<TSocket>::start::{{closure}}::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/async_await/select_mod.rs:321:13\n      <futures_util::future::poll_fn::PollFn<F> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/future/poll_fn.rs:56:9\n   4: network::peer::Peer<TSocket>::start::{{closure}}\n             at ./aptos/network/src/peer/mod.rs:254:13\n   5: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   6: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   7: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   8: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   9: tokio::runtime::thread_pool::worker::Context::run\n  10: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n  11: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n  12: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  13: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  14: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  15: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  16: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  17: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  18: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  19: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","data":{"network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1"}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/health_checker/mod.rs:381"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.619311Z","message":"[validator,Validator,21b4ef9b] Ping failed for peer: 9f9ae72a round: 31 with error: TimedOut","data":{"error":"TimedOut","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1","round":31}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/protocols/health_checker/mod.rs:421"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.619563Z","message":"[validator,Validator,21b4ef9b] Disconnecting from peer: 9f9ae72a","data":{"network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/peer/mod.rs:350"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.619830Z","message":"[validator,Validator,21b4ef9b] Closing connection to peer: 9f9ae72a","data":{"connection_id":279,"connection_origin":"inbound","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/peer_manager/mod.rs:370"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.619842Z","message":"[validator,Validator,21b4ef9b] Connection [9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1,/ip4/167.235.56.58/tcp/39520/noise-ik/0x4df1d67ecd21fff7647ca8abf9ccb3264b068e111fe03bc7def455a882dda30a/handshake/0,inbound,V1,ProtocolIdSet(BitVec { inner: [231, 152] }),validator] closed due to Requested","data":{"connection_id":279,"connection_origin":"inbound","disconnection_reason":"Requested","network_address":"/ip4/167.235.56.58/tcp/39520/noise-ik/0x4df1d67ecd21fff7647ca8abf9ccb3264b068e111fe03bc7def455a882dda30a/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1"}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/rpc/mod.rs:558"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.619994Z","message":"[validator,Validator,21b4ef9b] Error making outbound rpc request with request_id 5 to 5e841137: Error: Rpc timed out\n\nStack backtrace:\n   0: network::protocols::rpc::OutboundRpcs::handle_outbound_request::{{closure}}::{{closure}}\n             at ./aptos/network/src/protocols/rpc/mod.rs:477:57\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      network::protocols::rpc::OutboundRpcs::handle_outbound_request::{{closure}}::{{closure}}\n             at ./aptos/network/src/protocols/rpc/mod.rs:490:37\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: <core::pin::Pin<P> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/future.rs:124:9\n      <futures_util::stream::futures_unordered::FuturesUnordered<Fut> as futures_core::stream::Stream>::poll_next\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/futures_unordered/mod.rs:514:17\n      futures_util::stream::stream::StreamExt::poll_next_unpin\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/stream/mod.rs:1626:9\n   2: <futures_util::stream::stream::select_next_some::SelectNextSome<St> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/stream/select_next_some.rs:34:36\n      <core::pin::Pin<P> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/future.rs:124:9\n      futures_util::future::future::FutureExt::poll_unpin\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/future/future/mod.rs:562:9\n      network::peer::Peer<TSocket>::start::{{closure}}::{{closure}}::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/async_await/select_mod.rs:321:13\n   3: core::ops::function::impls::<impl core::ops::function::FnMut<A> for &mut F>::call_mut\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:290:13\n      network::peer::Peer<TSocket>::start::{{closure}}::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/async_await/select_mod.rs:321:13\n      <futures_util::future::poll_fn::PollFn<F> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/future/poll_fn.rs:56:9\n   4: network::peer::Peer<TSocket>::start::{{closure}}\n             at ./aptos/network/src/peer/mod.rs:254:13\n   5: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   6: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   7: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   8: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   9: tokio::runtime::thread_pool::worker::Context::run\n  10: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n  11: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n  12: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  13: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  14: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  15: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  16: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  17: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  18: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  19: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","data":{"network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094"}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/health_checker/mod.rs:381"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620073Z","message":"[validator,Validator,21b4ef9b] Ping failed for peer: 5e841137 round: 31 with error: TimedOut","data":{"error":"TimedOut","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094","round":31}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/protocols/health_checker/mod.rs:421"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620111Z","message":"[validator,Validator,21b4ef9b] Disconnecting from peer: 5e841137","data":{"network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/connectivity_manager/mod.rs:802"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620125Z","message":"[validator,Validator,21b4ef9b] Removing peer '9f9ae72a' metadata: [9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1,/ip4/167.235.56.58/tcp/39520/noise-ik/0x4df1d67ecd21fff7647ca8abf9ccb3264b068e111fe03bc7def455a882dda30a/handshake/0,inbound,V1,ProtocolIdSet(BitVec { inner: [231, 152] }),validator], vs event metadata: [9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1,/ip4/167.235.56.58/tcp/39520/noise-ik/0x4df1d67ecd21fff7647ca8abf9ccb3264b068e111fe03bc7def455a882dda30a/handshake/0,inbound,V1,ProtocolIdSet(BitVec { inner: [231, 152] }),validator]","data":{"connection_id":279,"connection_origin":"inbound","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1","stored_metadata":{"remote_peer_id":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1","connection_id":279,"addr":"/ip4/167.235.56.58/tcp/39520/noise-ik/0x4df1d67ecd21fff7647ca8abf9ccb3264b068e111fe03bc7def455a882dda30a/handshake/0","origin":"Inbound","messaging_protocol":"V1","application_protocols":{"inner":[231,152]},"role":"Validator"}}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/peer/mod.rs:350"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620274Z","message":"[validator,Validator,21b4ef9b] Closing connection to peer: 5e841137","data":{"connection_id":280,"connection_origin":"inbound","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/peer_manager/mod.rs:370"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620285Z","message":"[validator,Validator,21b4ef9b] Connection [5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094,/ip4/211.91.254.188/tcp/35442/noise-ik/0x848f4e17d02ad1c906955924a8482ea7fb51c56de42002a253af02780a51bf69/handshake/0,inbound,V1,ProtocolIdSet(BitVec { inner: [231, 152] }),validator] closed due to Requested","data":{"connection_id":280,"connection_origin":"inbound","disconnection_reason":"Requested","network_address":"/ip4/211.91.254.188/tcp/35442/noise-ik/0x848f4e17d02ad1c906955924a8482ea7fb51c56de42002a253af02780a51bf69/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094"}}
testnet-validator-1  | {"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620308Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer d801e1b2 at /ip4/95.188.77.87/tcp/6181/noise-ik/0xcc98cf612f641136ff744bcfd0a3228219fa815e9269556b6d992c23236b462f/handshake/0: Transport error: timeout elapsed","data":{"error":"Transport error: timeout elapsed","network_address":"/ip4/95.188.77.87/tcp/6181/noise-ik/0xcc98cf612f641136ff744bcfd0a3228219fa815e9269556b6d992c23236b462f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/peer/mod.rs:384"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620389Z","message":"[validator,Validator,21b4ef9b] Closed connection to peer: 5e841137","data":{"connection_id":280,"connection_origin":"inbound","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/connectivity_manager/mod.rs:802"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620456Z","message":"[validator,Validator,21b4ef9b] Removing peer '5e841137' metadata: [5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094,/ip4/211.91.254.188/tcp/35442/noise-ik/0x848f4e17d02ad1c906955924a8482ea7fb51c56de42002a253af02780a51bf69/handshake/0,inbound,V1,ProtocolIdSet(BitVec { inner: [231, 152] }),validator], vs event metadata: [5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094,/ip4/211.91.254.188/tcp/35442/noise-ik/0x848f4e17d02ad1c906955924a8482ea7fb51c56de42002a253af02780a51bf69/handshake/0,inbound,V1,ProtocolIdSet(BitVec { inner: [231, 152] }),validator]","data":{"connection_id":280,"connection_origin":"inbound","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094","stored_metadata":{"remote_peer_id":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094","connection_id":280,"addr":"/ip4/211.91.254.188/tcp/35442/noise-ik/0x848f4e17d02ad1c906955924a8482ea7fb51c56de42002a253af02780a51bf69/handshake/0","origin":"Inbound","messaging_protocol":"V1","application_protocols":{"inner":[231,152]},"role":"Validator"}}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/peer/mod.rs:384"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620473Z","message":"[validator,Validator,21b4ef9b] Closed connection to peer: 9f9ae72a","data":{"connection_id":279,"connection_origin":"inbound","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1"}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/health_checker/mod.rs:381"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620503Z","message":"[validator,Validator,21b4ef9b] Ping failed for peer: 9f9ae72a round: 33 with error: UnexpectedResponseChannelCancel","data":{"error":"UnexpectedResponseChannelCancel","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1","round":33}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/health_checker/mod.rs:381"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620524Z","message":"[validator,Validator,21b4ef9b] Ping failed for peer: 9f9ae72a round: 32 with error: UnexpectedResponseChannelCancel","data":{"error":"UnexpectedResponseChannelCancel","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1","round":32}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/connectivity_manager/mod.rs:869"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620530Z","message":"[validator,Validator,21b4ef9b] Failed to connect to peer: d801e1b2 at address: /ip4/95.188.77.87/tcp/6181/noise-ik/0xcc98cf612f641136ff744bcfd0a3228219fa815e9269556b6d992c23236b462f/handshake/0; error: Transport error: timeout elapsed","data":{"error":"Transport error: timeout elapsed","network_address":"/ip4/95.188.77.87/tcp/6181/noise-ik/0xcc98cf612f641136ff744bcfd0a3228219fa815e9269556b6d992c23236b462f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3"}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/health_checker/mod.rs:381"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620537Z","message":"[validator,Validator,21b4ef9b] Ping failed for peer: 5e841137 round: 33 with error: UnexpectedResponseChannelCancel","data":{"error":"UnexpectedResponseChannelCancel","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094","round":33}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/health_checker/mod.rs:381"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.620549Z","message":"[validator,Validator,21b4ef9b] Ping failed for peer: 5e841137 round: 32 with error: UnexpectedResponseChannelCancel","data":{"error":"UnexpectedResponseChannelCancel","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5e8411375edd94cec20e313abaa9a4694f0af87df41942f74e8d7d432ffd3094","round":32}}
testnet-validator-1  | {"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.692236Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer b4046267 at /ip4/92.255.176.91/tcp/6180/noise-ik/0xabac71d7faba0e2d26daffb8365b12706f6a09d24805b98259e197b1b30c9429/handshake/0: Transport error: timeout elapsed","data":{"error":"Transport error: timeout elapsed","network_address":"/ip4/92.255.176.91/tcp/6180/noise-ik/0xabac71d7faba0e2d26daffb8365b12706f6a09d24805b98259e197b1b30c9429/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"b404626788b5ed7e6940200acfe6884f91cf636c62751bd12508d51bdb80e13a"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/connectivity_manager/mod.rs:869"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:04.692282Z","message":"[validator,Validator,21b4ef9b] Failed to connect to peer: b4046267 at address: /ip4/92.255.176.91/tcp/6180/noise-ik/0xabac71d7faba0e2d26daffb8365b12706f6a09d24805b98259e197b1b30c9429/handshake/0; error: Transport error: timeout elapsed","data":{"error":"Transport error: timeout elapsed","network_address":"/ip4/92.255.176.91/tcp/6180/noise-ik/0xabac71d7faba0e2d26daffb8365b12706f6a09d24805b98259e197b1b30c9429/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"b404626788b5ed7e6940200acfe6884f91cf636c62751bd12508d51bdb80e13a"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"aptos_data_client","file":"state-sync/aptos-data-client/src/aptosnet/mod.rs:728"},"thread_name":"aptos-data-client","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:05.005710Z","data":{"event":"priority_and_regular_peers","message":"Number of priority peers: 215. Number of regular peers: 1","name":"peer_states"}}
testnet-validator-1  | {"level":"INFO","source":{"package":"aptos_data_client","file":"state-sync/aptos-data-client/src/aptosnet/mod.rs:805"},"thread_name":"aptos-data-client","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:05.170794Z","data":{"event":"aggregate_summary","message":"Global data summary: AdvertisedData { Highest epoch ending ledger info, epoch: Some(34). Highest synced ledger info, epoch: Some(35), version: Some(79518532). Lowest transaction version: Some(0), Lowest transaction output version: Some(0), Lowest states version: Some(534412) }, OptimalChunkSizes { epoch_chunk_size: 100, state_chunk_size: 2000, transaction_chunk_size: 2000, transaction_output_chunk_size: 2000 }","name":"peer_states"}}
testnet-validator-1  | {"level":"ERROR","source":{"package":"aptos_data_client","file":"state-sync/aptos-data-client/src/aptosnet/mod.rs:434"},"thread_name":"data-streaming-service","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:07.308771Z","data":{"error":"TimeoutWaitingForResponse(\"Rpc timed out\")","event":"response_error","name":"storage_service_response","peer":"Validator:afea3644","request_id":2837,"request_type":"get_transaction_outputs_with_proof_compressed"}}
testnet-validator-1  | {"level":"WARN","source":{"package":"network","file":"network/src/protocols/rpc/mod.rs:558"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:07.309091Z","message":"[validator,Validator,21b4ef9b] Error making outbound rpc request with request_id 44 to afea3644: Error: Rpc timed out\n\nStack backtrace:\n   0: network::protocols::rpc::OutboundRpcs::handle_outbound_request::{{closure}}::{{closure}}\n             at ./aptos/network/src/protocols/rpc/mod.rs:477:57\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      network::protocols::rpc::OutboundRpcs::handle_outbound_request::{{closure}}::{{closure}}\n             at ./aptos/network/src/protocols/rpc/mod.rs:490:37\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: <core::pin::Pin<P> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/future.rs:124:9\n      <futures_util::stream::futures_unordered::FuturesUnordered<Fut> as futures_core::stream::Stream>::poll_next\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/futures_unordered/mod.rs:514:17\n      futures_util::stream::stream::StreamExt::poll_next_unpin\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/stream/mod.rs:1626:9\n   2: <futures_util::stream::stream::select_next_some::SelectNextSome<St> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/stream/stream/select_next_some.rs:34:36\n      <core::pin::Pin<P> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/future.rs:124:9\n      futures_util::future::future::FutureExt::poll_unpin\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/future/future/mod.rs:562:9\n      network::peer::Peer<TSocket>::start::{{closure}}::{{closure}}::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/async_await/select_mod.rs:321:13\n   3: core::ops::function::impls::<impl core::ops::function::FnMut<A> for &mut F>::call_mut\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:290:13\n      network::peer::Peer<TSocket>::start::{{closure}}::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/async_await/select_mod.rs:321:13\n      <futures_util::future::poll_fn::PollFn<F> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/futures-util-0.3.21/src/future/poll_fn.rs:56:9\n   4: network::peer::Peer<TSocket>::start::{{closure}}\n             at ./aptos/network/src/peer/mod.rs:254:13\n   5: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   6: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   7: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   8: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   9: tokio::runtime::thread_pool::worker::Context::run\n  10: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n  11: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n  12: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  13: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  14: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  15: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  16: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  17: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  18: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  19: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","data":{"network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"afea364425825255938bbfdb1704f06ec56d5c914833208b96fc47b94aff1671"}}
testnet-validator-1  | {"level":"ERROR","source":{"package":"data_streaming_service","file":"state-sync/state-sync-v2/data-streaming-service/src/data_stream.rs:413"},"thread_name":"data-streaming-service","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:07.406818Z","data":{"error":{"AptosDataClientError":"Timed out waiting for a response: Rpc timed out"},"event":"error","message":"Encountered a data client error!","name":"received_data_response","stream_id":2}}
testnet-validator-1  | {"level":"INFO","source":{"package":"network","file":"network/src/peer_manager/mod.rs:359"},"thread_name":"network-Validator","hostname":"8de8354f765c","timestamp":"2022-09-01T11:25:07.485882Z","message":"[validator,Validator,21b4ef9b] New connection established: [28dde00b49a2c5ec8b46a1fbd855afd0fac64ea347a3dffc803ad672e7810186,/ip4/5.161.137.197/tcp/36574/noise-ik/0xe26f5cdc2b89545b3481c055fac34781b585b4fa5b1d93bbf2865f7cb20b130c/handshake/0,inbound,V1,ProtocolIdSet(BitVec { inner: [231, 152] }),validator]","data":{"connection_id":298,"connection_origin":"inbound","network_address":"/ip4/5.161.137.197/tcp/36574/noise-ik/0xe26f5cdc2b89545b3481c055fac34781b585b4fa5b1d93bbf2865f7cb20b130c/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"28dde00b49a2c5ec8b46a1fbd855afd0fac64ea347a3dffc803ad672e7810186"}}

[JoshLind]

Can you upload more logs please @jovar95? To determine what the exact issue is, we'll need to look further in the past 😄

[jovar95]

@JoshLind here logs after docker logs testnet-validator-1 | grep ERROR

{"level":"INFO","source":{"package":"aptos_node","file":"aptos-node/src/lib.rs:201"},"thread_name":"main","hostname":"8378f8a71795","timestamp":"2022-09-01T14:27:45.545382Z","message":"Loaded AptosNode config","data":{"config":{"base":{"data_dir":"/opt/aptos/data","role":"validator","waypoint":{"from_file":"/opt/aptos/genesis/waypoint.txt"}},"consensus":{"max_block_txns":6000,"max_block_bytes":10485760,"max_pruned_blocks_in_mem":100,"mempool_executed_txn_timeout_ms":1000,"mempool_txn_pull_timeout_ms":1000,"round_initial_timeout_ms":1500,"safety_rules":{"backend":{"type":"on_disk_storage","path":"/opt/aptos/data/secure-data.json","namespace":null},"logger":{"chan_size":10000,"enable_backtrace":false,"is_async":true,"level":"INFO","console_port":6669,"enable_telemetry_remote_log":true,"enable_telemetry_flush":true,"telemetry_level":"ERROR"},"service":{"type":"local"},"test":null,"network_timeout_ms":30000,"enable_cached_safety_data":true,"initial_safety_rules_config":{"from_file":{"identity_blob_path":"/opt/aptos/genesis/validator-identity.yaml","waypoint":{"from_file":"/opt/aptos/genesis/waypoint.txt"}}}},"sync_only":false,"channel_size":30,"use_quorum_store":false,"quorum_store_pull_timeout_ms":1000,"quorum_store_poll_count":1,"intra_consensus_channel_buffer_size":10},"execution":{"genesis_file_location":"/opt/aptos/genesis/genesis.blob","network_timeout_ms":30000,"concurrency_level":4,"num_proof_reading_threads":32},"full_node_networks":[{"max_connection_delay_ms":60000,"connection_backoff_base":2,"connectivity_check_interval_ms":5000,"network_channel_size":1024,"max_concurrent_network_reqs":100,"discovery_method":"none","discovery_methods":[],"identity":{"type":"from_config","key":"0xb0f405a3e75516763c43a2ae1d70423699f34cd68fa9f8c6bb2d67aa87d0af69","peer_id":"00000000000000000000000000000000d58bc7bb154b38039bc9096ce04e1237"},"listen_address":"/ip4/0.0.0.0/tcp/6181","mutual_authentication":false,"network_id":{"private":"vfn"},"runtime_threads":null,"seed_addrs":{},"seeds":{},"max_frame_size":4194304,"enable_proxy_protocol":false,"ping_interval_ms":10000,"ping_timeout_ms":20000,"ping_failures_tolerated":5,"max_outbound_connections":4,"max_inbound_connections":100,"inbound_rate_limit_config":null,"outbound_rate_limit_config":null,"max_message_size":67108864}],"inspection_service":{"address":"0.0.0.0","port":9101,"expose_configuration":false,"expose_system_information":true},"logger":{"chan_size":10000,"enable_backtrace":false,"is_async":true,"level":"INFO","console_port":6669,"enable_telemetry_remote_log":true,"enable_telemetry_flush":true,"telemetry_level":"ERROR"},"mempool":{"capacity":2000000,"capacity_per_user":100,"default_failovers":3,"max_broadcasts_per_peer":1,"mempool_snapshot_interval_secs":180,"shared_mempool_ack_timeout_ms":2000,"shared_mempool_backoff_interval_ms":30000,"shared_mempool_batch_size":100,"shared_mempool_max_batch_bytes":64880640,"shared_mempool_max_concurrent_inbound_syncs":4,"shared_mempool_tick_interval_ms":50,"system_transaction_timeout_secs":600,"system_transaction_gc_interval_ms":60000,"shared_mempool_validator_broadcast":true},"metrics":{},"peer_monitoring_service":{"max_concurrent_requests":1000,"max_network_channel_size":1000},"api":{"enabled":true,"address":"0.0.0.0:8080","failpoints_enabled":false,"json_output_enabled":true,"bcs_output_enabled":true,"encode_submission_enabled":true,"transaction_submission_enabled":true,"transaction_simulation_enabled":true,"max_submit_transaction_batch_size":100},"state_sync":{"data_streaming_service":{"global_summary_refresh_interval_ms":50,"max_concurrent_requests":3,"max_concurrent_state_requests":6,"max_data_stream_channel_sizes":300,"max_request_retry":3,"max_notification_id_mappings":300,"progress_check_interval_ms":100},"aptos_data_client":{"max_num_in_flight_priority_polls":10,"max_num_in_flight_regular_polls":10,"response_timeout_ms":5000,"summary_poll_interval_ms":200,"use_compression":true},"state_sync_driver":{"bootstrapping_mode":"ApplyTransactionOutputsFromGenesis","commit_notification_timeout_ms":5000,"continuous_syncing_mode":"ApplyTransactionOutputs","progress_check_interval_ms":100,"max_connection_deadline_secs":10,"max_consecutive_stream_notifications":10,"max_pending_data_chunks":100,"max_stream_wait_time_ms":5000,"num_versions_to_skip_snapshot_sync":10000000},"storage_service":{"max_concurrent_requests":4000,"max_epoch_chunk_size":100,"max_lru_cache_size":100,"max_network_channel_size":4000,"max_network_chunk_bytes":64880640,"max_state_chunk_size":2000,"max_subscription_period_ms":5000,"max_transaction_chunk_size":2000,"max_transaction_output_chunk_size":2000,"storage_summary_refresh_interval_ms":50}},"firehose_stream":{"enabled":false},"storage":{"address":"127.0.0.1:6666","backup_service_address":"127.0.0.1:6186","dir":"db","storage_pruner_config":{"ledger_pruner_config":{"enable":true,"prune_window":150000000,"batch_size":500,"user_pruning_window_offset":200000},"state_merkle_pruner_config":{"enable":true,"prune_window":100000,"batch_size":1000},"epoch_snapshot_pruner_config":{"enable":true,"prune_window":80000000,"batch_size":1000}},"target_snapshot_size":100000,"max_num_nodes_per_lru_cache_shard":8192,"rocksdb_configs":{"ledger_db_config":{"max_open_files":5000,"max_total_wal_size":1073741824,"max_background_jobs":16,"block_cache_size":8388608,"block_size":4096,"cache_index_and_filter_blocks":false},"state_merkle_db_config":{"max_open_files":5000,"max_total_wal_size":1073741824,"max_background_jobs":16,"block_cache_size":8388608,"block_size":4096,"cache_index_and_filter_blocks":false},"index_db_config":{"max_open_files":1000,"max_total_wal_size":1073741824,"max_background_jobs":16,"block_cache_size":8388608,"block_size":4096,"cache_index_and_filter_blocks":false}},"enable_indexer":false},"test":null,"validator_network":{"max_connection_delay_ms":60000,"connection_backoff_base":2,"connectivity_check_interval_ms":5000,"network_channel_size":1024,"max_concurrent_network_reqs":100,"discovery_method":"onchain","discovery_methods":[],"identity":{"type":"from_file","path":"/opt/aptos/genesis/validator-identity.yaml"},"listen_address":"/ip4/0.0.0.0/tcp/6180","mutual_authentication":true,"network_id":"validator","runtime_threads":null,"seed_addrs":{},"seeds":{},"max_frame_size":4194304,"enable_proxy_protocol":false,"ping_interval_ms":10000,"ping_timeout_ms":20000,"ping_failures_tolerated":5,"max_outbound_connections":4,"max_inbound_connections":100,"inbound_rate_limit_config":null,"outbound_rate_limit_config":null,"max_message_size":67108864},"failpoints":null}}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network.rs:163"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:27:59.495521Z","message":"Failed to send a msg to peer","data":{"error":"NetworkError(Peer not connected\n\nStack backtrace:\n   0: consensus::network_interface::ConsensusNetworkSender::supported_protocols\n      consensus::network_interface::ConsensusNetworkSender::preferred_protocol_for_peer\n             at ./aptos/consensus/src/network_interface.rs:151:32\n   1: <consensus::network_interface::ConsensusNetworkSender as network::protocols::network::ApplicationNetworkSender<consensus::network_interface::ConsensusMsg>>::send_to\n             at ./aptos/consensus/src/network_interface.rs:165:24\n   2: consensus::network::NetworkSender::send::{{closure}}\n             at ./aptos/consensus/src/network.rs:162:29\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   3: consensus::network::NetworkSender::send_vote::{{closure}}\n             at ./aptos/consensus/src/network.rs:212:35\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::process_proposal::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:554:58\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::process_proposal_msg::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:309:64\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:772:65\n   4: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   5: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   6: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   7: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   8: tokio::runtime::thread_pool::worker::Context::run\n   9: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n  10: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n  11: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  12: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  13: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  14: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  15: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  16: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  17: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  18: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95)","remote_peer":"f399f294c725bbcc8840845c4286a85324cd65e45271d88ed6d72074f25c1598"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:188"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:27:59.499222Z","message":"Unavailable peers: [17e6eeec824c4ebe71f276aeb2115b63defcd8b5898663074d02c6db6494991b, 5e7ed6a3d40cb5c8831edbfefb4bbfd4874e5566556eea08dde72c14514af7b5, 8d7766a61a01a33199db8412362805e9550d7dcdce6ab74fecb2ddf9899f74d0, ae91b5f2c5a13f5394c46bcfc7e67e4db1b3659dc6c6faa08e4b9a76a8dde53d, bfeac9b2da698dee2d9b882322d20dc7508bb6cf2df4165c8938ffd748e11337, ff79fc4bf0b39c4b8a9810b57d311527700639b661198f4d8db77d7c6d64f0c9, e0aff63df5c4b9ce3d7e415e2c0b8aa2f036f0a0d730ab31546bf45a2569d925, 4b904c781e92c118f45b3bd4897fdeb769cd1b638c81841c8bcdba98985698ee, 7246f46a4b1a0dfaff0687358794736f9e6c24faefcf72cebc7791c2c00f7810, 69268e38beda87f1ff5bf22881f22b9deb564c89028e7af89ee8b911fc616f95, 89d079d0935ed64d8a755bfe97938150bd56131336d656b31db0eacdbf03b545, f399f294c725bbcc8840845c4286a85324cd65e45271d88ed6d72074f25c1598, bd2d5cb253a3ae97ee84039b34ec753c3bca038a41ab953b9f6df4ac38bf7d35, a4e0e382397c09a69408b088f021a475d1e9d55c3f708660102c837440e2ef1b, 0a0df6771891d5d1171a3c808aeb5914487944e82863017e500f290a377368bc, 309da7c8f94a4454364810dfc0f6f317079cc382d872e73a9e32fbf4fbf70ef1, 23f422eb4212100c47cdc28c242cab052b65ace49d1429c1197f4a1b153b178e, 84699c4425469b5610f8c30d66026a57b9f7d7442cdcac7b2bf11875d80b05c2, 62562701d5f396a2f8e7bc44e8700214d0cbc38c46d78dbf9dcab949f1dec6c5, 8e4d6379237ec2576f9b71513ee566e524793310027ae6271977d481d6d07bf4, 365e00da83dd53c167be1c08437daf01146a71bd8dc859df037c6bcd27ca37fb, 442a4f67e45a025612221d7710d3ab7bb8918d3d4bc888303baf7e6709588a1b, 4428fefbc147d0170f9bbdfd7310adb642f9a50fcf7df338e4a95f2ea2033ed2, 87a572ffb4a0c22b32d41b91d1df342da0d4477b234faa394f0328bed498a1be, 8a147943e71c0c9d2d167492905d5e27e6af4ee4788badde9299120abc231b7b, 5ee6ee045f34d87bcfac120c45aeb8e8973aee4196643341dac5e6b34df496fa, 87b30b548452c5a3200a1aec4251a1a01b05b15cb74d41e49701665ef4d36804, 7ed9a1107545401206e1ae1758fa2ad86e2772c49a0e0ca7383286e3e81a8e34, 802f9bbd78574afe79289edf5adcce044ab23535dcd8e79289abf2cb492a3c57, 0e3557dd416faf22f7efa010cbaf9f47d40cdfdf98d2fe26145ec01b3d45c2c0, 050c8d865560ff370970f4652d1c08c376ed85d68647a3b34863004d84b6388e, b178f79f9bb08355fad69e6ddd7cd5daff672e89c6f8f1921fa03d53adc52e46, 8485b0e4779190c2e0cefc899ac853cd3d878930e66587df8d867c0c941267a6, b77026ce272a63b7261d20e5d0d9ca8cddd81b42b3432891668c43c03dbd1b73, 6f929de7b48e002849eaa3aff478dc8ac427283dc185891c457180b7395c5f16, 337925eaba30d718f87ddc251c8c3314322f8e0bb87cf2e6c77cc4211b97e493, 03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, 1b0867a303b9c2c975b3c874866c06fd53e5ee0e538c301b66ffcecf08e63d47, ea1272b8a1bc5a1fb9a1c77c9dc96e1a02a691d1d6073a1dc06d51d770b3f2e2, 234c957655cc9f4a5f8f0d1ca8f99baf8af3ea57740cd71fa2ab3d14fa3a4ede, c3bb0a1e3406f82dae0ac03161b6816bd27bf5a861ce122e8c64e42373f1a124, 07d7df40b1b25dcb7e8fb6caa76ee2272f081a8992629e145aead916b8713f3d, 479107df9ac8bb68a12900e6df3b928a1a6fcf99fbf64ac6f0c1b6b37a2c7b3a, d62ae400c7622e8469c2ee52b1e0af5e5c11f6a72560bc4434694eeeb3c1ffd4, fcfa1efb940e23936915c2a6814ca557dbf3c6972d4ff0154d8e6aacda905c64, b41568279f81fccc440bdbb807f68eb0cbdd494c74de5cccf545b0a47d890e93, bec5d849323f0e3b633bdd47283ada03ca085f455e6bbca056fe91de941db4ef, e586f8a8c30b8cc949b9642c5c8d41548dc4558385656e46cd5b5be45a2c9d2a, 96b148b5ba586d3b0406450e793ddd416fe6f7bdd21ee8d4f0b1ff245303aa4a, c11fffec54e78cb992ff9d34e46ea601d5a84c35985ae516f125bd4c0c5449e3, e36800ba3761489f69de5ed4f0d3023d352209fe62c6fd67b3498e0914e45bb7, 26a5f68e39a2543aef7ceaec7dcd047510d856b495f95979879bd278e4725e07, 873cc8a29a8ad6afe1d13ffbaa512a91be22bb1b513ac4bd0b1077200d6be96f, e3370ffd180416064f3c8e529da8cbe14e2567506834d8d90be3ab7fbb50ef6d, 8d1f8a22eb447833aa978f611c13d0fa327bdda2bed8ca169b4417edbbdf4aaf, f988e9f477963de81075449affe13bb81beddb91d7bb5ab29071ce1c5f0bc400, f2e802753bbaa2b3e0861180dfd2dd3fe8dc9bb0d49e0094376f6d565bab9884, f4cb45b3d657a4957d723cfaaeca5803c91ea38981bd306abf7a8c44790c894f, 62c89435fbbe74628844c05c3429e4116a3c4fc0a276a25901f7c40432f31925, b1bc346da7756a51dae71ddbd8d74228604598fa6cc35a52fb78e86ca3915e26, 57be1bdaf544178f3e754dd23f84c74a95785c694066f6c1d48762b8c888cbee, 614d8a2a081e4e51bd216648ec4fafd6c3d0ff03f6f7e83a044b989c738438cf, a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, c310249c3f3cc990e95b6dfc4da612373e66fd300d707390b4719665a1e7e52c, 19105a85b8d455e51a1b0f917dbb2464d47c47a325c20e5aaa64916299e06aa4, 44194bedcb5bfac13f003eddbc6567e27ffbc5c2cb98090294b90419c522c0e7, a4079618c9f4cbbf11d69d7a12b7f5a6deadef4aed991c00e4fd412e2765b713, b12cf85a0a28c36b5703ff9aa5fcc995a3c64f8869f68f61ffbc17b1b6814d25, 8aa4d82016b1a0356421c22cda47990d8ca5ffc0f5a012754817db871d34ae90, c9b19c9b6afb63f3e9fd8c0491fa6ca9fa6a008cd5f2b2dc3b5d7a984b9c5a7f, 916926a34055b6402d115dbd5f6b8eeebad4ba70d775e0e6da864f3f919b9100, 7b689638b3a5473309f49325f87284ffca126e126f80495ad6c185e43eecb7d3, de102b1247b8b89979b5f58dc29da624d01b0381fd725908464e02a2d3bcec94, ef5bbb8237eeac0d4d1764f1802e123ae3d3deacc42d39a2e0884a0245263c32, c725187215cf9ba7a54f3dfca3472d986eec830065eab4d667dc271483ac5728, b1a242d87bdc7a948db36f9c25dc8e92fcc2c66c7f090ad369d6362d70024988, 0851fbcc396e403f8f910870f05d4307b275f73f96cfaaae2c06ca0b93aa77b5, 848750a4781af9d181cbbf9ecd0ee8eb04c62c048f207a4ef23864334e39b01e, 0e41c1afa2edc48280f24f0b456311086fa3f777f25b2db7d9d12a91526214a2, 8ba19a694947afb53a3b79923f0c0bd25e8c0aa7cab71b29be9b16173d67e6df, a7f779852a2739af1d9d7abdf661827dd5edb4b9e3db716c30b76105676f68f1, 168625c7aca6358fe94af73c52d65615c009683f372b5b0e7068d5f306aea764, 5bd7c59fc0f31a87ab9633a228fd7a9d6b8081a792abb688047621cdce99b012, e5207ae13fce7f9cfdf3e388e415d97cbf55de727d0e7dcb97e7ca2f209fff76, b10a392979c9de9e2fb696dbb01c33d65036349e46f5ac51e1e4266c3b8847e0, dcef38c93319415362a06c921481e70887d87d0650bf31ad40fc858d72d80cb5, 3406fda6b62a60cae162193b9dcb59c871b9cf8e150a792ca2d433bf8a040e49, 4449959032d51d85aae24c37ad6a9c5a27499b7d7b67dfd5327e61eb2bc3051c, be2ac53a4dba0325cf3d65eca93c68e5239c6fc5b4ca013e6ec7089235c43775, c54fc020dbf12a2f04c184e62ffd53f64a1c82f9d2bd6f4e074bed4f512148c4, f9e184c68cfa53ea4ba0d7354c5ecc512c0d2fa32a13c479c5560b293b7f6a35, 35a254e3c5a42534b8e7f08e5ce3cd9fa28ba20e6b1b9f228dc38b4275ade609, b14350297a3d2e2caf1a887213b4aebf6b120fa2ae87e9ddd1d34d9ad7beb44c, dc10d8a940d909cd3c26b8d2c0378d6aab4165b5d98a1f5bdc592aeaa6cdca5f, 5be2700c076c27678184fad6af04a0541fd0f2f1ed66c49cca5fe16a466807f9, 8089a7bd4bbdcd6f0ba7812313b49e99f569880644e8f4ad9fd3ce584e0c22c4, 454ae62855c24140f39398de478b49ac53dd072edea2b54f07d272c9008ddc12, 33e6210e459009bd91061a50355ad1837c2467640a5c50eec23fbc223d9ae788, 66bdb52ecd77bace3b5ae4c014b69d0a29957f27b3dcb7406c806b4052e05f4c, 750a7e5997bcb7cc87917880524d91c7a59847801b6fe0a8d2c632c0a10c9441, 06b15cee0fddb86956e883d0156eb7b5da441e7139ea00ef39bcf0cbf9dc52b5, a2445e86428742f04b3ac622343c44fd3321f352ce8bc42399605b32cc190a7e, 08c3f260d82e0aa572b0f58eb85a6edeb9ca5022fab829a5fb3a21952710f63f, b2243c6234285ca4ddba4927603750a0b2571d9453b1021c4cd4a0c34e045ecd, 202c77e0e8b6f7c17428c989d3138a75c989e7fa61c1bb3d38fdbbef0b5bca79, 71eb495f8fb7d8f9912dc7ba1406559ad6a7fa54d8abb2bdf5eac78184719798, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42, afea364425825255938bbfdb1704f06ec56d5c914833208b96fc47b94aff1671, 87a92d3c58a7b644f0efc520f210f590573d10fad04ba9aa0ccb343dfdc36439, 6ebbe079ff81335cb9b1ab2c667a15c9994b60001768b6a2236b19b4c273b71d, a6665a2c9c2ed477121ecea1047b803b1e0daa96748ee58aa66f3762dbeb2fac, 1dd17527c01616b252c9801366c158fbc0ff67b5ee76436e3cec04399d8de5d0, 81bd0427d12a5240dede6df96b99f1b70fabdd8acc05a79dad4c71910564bcdd, ae289fd9643d98f1c486bbf983a0400043206c9e84c920957487910988817a78, 000c5b2258a85d4ea8a313f09ff68165cb87e6e1a917f245a94949f0895fc290, 943bdde7612aecb92e745facfdf732b2392b1d27f9fd852d83d0f8619fd49382, d947abce2dab0abd4e0b705db96db38e0444c7347232ffed1b4d29cca74f86ef, d15c01e90855d4e89a8497c74f5915656138817ba29c56cfca85f8b5f5b7b07b, b92890f0c60eaa2b79814615402ebbaf6371052e296049f1788870a9887a2dc7, c654a30871b504309c94fac9a164b2273b8be5d71149ed79b304ba8d89fd5c88, da26d0b97726a315035486da1562a1e7d42c1f8bd50fee123402a1e5d4e3af92, 39ac041fa2f9f85bb615d12dcd628e4b06d2b7f03fe7c52af5e2b250d27c252a, b9eab269389f080a221e1811493284ff7141b542dd2bdcaf4fe3e4f4518e8f43, 86b5e89746c45648735ef61c722debec0c35e051db941c783a82756d77a54f8c, 093cc8991c3092c9a9a6262d233cfad2550abdc6f5a58422e30675ad08ccfb08, 5c8af906ab13573a0a1a3c34f3308c0afb9e72f1b6e5d7845c7f57ba2f02c7ca, a3fdaacd44c39f1d548aaac3fb6a25336c37fc01bec7233505d50a77fddc84ae, 94d9705a8a3333eb52c96889c50e6e02294d7c55ac24b3d327dd2bb79aff1974, b10dc0857cfb72773f71f11cb2a466eba3e1947d1b8cd1bf2b3124e936fb5468, 3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0, b6eb384b229e17db7b1f4f6fa5a692b91cf8d371e1e97e442ae7b60a0f4308d6, b404626788b5ed7e6940200acfe6884f91cf636c62751bd12508d51bdb80e13a, 0a5d60ca5c94b53c75dfc19d8d1ecc624b10565143ca90199c071089eb86173c, dc0253a45c8e56c91425f1415b3227fc016e207eedb1536cebf50c9e7baec92d, f758b0bf7a9bfb700b84b83779bb92ef8d7a4c22587bf7a05481abd77713df65, 6e86e31f81c541b4f7aaafb1170db28ba1d64fbde5ed453637c51377f7de518e, c4a5af5fa6c1ae769157675c8fbe55c49e2cbd7ce89a8dec5f09a6ced957568d, d78ecbd74f9e224a80374ca807e480c4d24f88d0599a263e7949cd7129dff377, 1ce1a4ea4755a027843adc6ea9f13e158cc45edcb26be55b194b0fd278cfc041, 49bbd1c4b5313d97a9218d97d48020392d3991e978886802d51d36bb747f7bf1, 631bc1f429ca72def1ebb108c442211b6a2e9337188342ef7b1eb132aa91efc4, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, e0c9b84c58ba4234aeebbb7203746c6f275025ceee94bb86d4573289565bbdde, 9eaa6c899597ca4d691cf453a1b0b17b534ed8fa91eac78a9e9bc0f457ee8cba, 1a86526cab3a14ce5d3ea14d55fa313c9b284ab1543df434b9ea67f90f4ff682, 0f33a11bc2cc943b451e17e4d74dfc7fcc3af2bf5a6ba6dfefffc4f7c7e95d04, ae050e3c60e8f4e7817e0aaed4007e64a3d10b713f85f7ff28a7a7fc0d3f80e8, a89b6feeccb1e459737a3e0bd69628b15aa795f68deb2a113cd9177c86984c39, 170c36cfb0294b4e50ea54c66a501ecd776ee9611f0dcce314e9416626619b70, 528a460b6482d633fbfc9d9e9a6b5a2e47f2749c35a50d2416124674cb46bc1f, 6165c3ef6ac4c0d8ef7496a0cacd3de25384bc7909130482da3e7369bee2da24, c3f2c9dc502dbe9de4d3d3ee472fa679f578d978e53c1242a6906f894d435bd1, 00fa666ddf3c506305d2f6226d19f2df4c25c09f8537cecd4fd8a5b381366d3b, 840880fc27f01706d134397dcbeb14cd54899fc8dc93f3f9f1b1a454840829fe, d2f842ded238f6ba4629650863dc3ebbd506f857c6e1050a4ac923fc60e2e4a1, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da, 9314ef791e942598a729e85037a59f851c9c25ce3a22babda48a17988c87f7e5, b33d3f37718f3a22f5d682206ed861f7c94fa4359890070b953f5a22ecb43a44, d16b8e15be631f0d22199f8b91b7338862fef9ee0a3af419477bd148a26cd2ff, b5aeeb91fefdc7785dfea6f14ca5d492d837514cb39043dd4159df9e1b1c4e06, f1d4c93aa7e20e194e339a7d23c63b09a5501595870910c666213afea6ea1239, 414f7f91f6d6618b126d2c93097142817efa75666a32fdde62c34ed01134b164, 46774d7bda97925b0f7dc7eecc927fe4f2f17739f087243043694d2bea153cf5]"}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network.rs:163"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:00.137237Z","message":"Failed to send a msg to peer","data":{"error":"NetworkError(Peer not connected\n\nStack backtrace:\n   0: consensus::network_interface::ConsensusNetworkSender::supported_protocols\n      consensus::network_interface::ConsensusNetworkSender::preferred_protocol_for_peer\n             at ./aptos/consensus/src/network_interface.rs:151:32\n   1: <consensus::network_interface::ConsensusNetworkSender as network::protocols::network::ApplicationNetworkSender<consensus::network_interface::ConsensusMsg>>::send_to\n             at ./aptos/consensus/src/network_interface.rs:165:24\n   2: consensus::network::NetworkSender::send::{{closure}}\n             at ./aptos/consensus/src/network.rs:162:29\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   3: consensus::network::NetworkSender::send_vote::{{closure}}\n             at ./aptos/consensus/src/network.rs:212:35\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::process_proposal::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:554:58\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::process_proposal_msg::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:309:64\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:772:65\n   4: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   5: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   6: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   7: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   8: tokio::runtime::thread_pool::worker::Context::run\n   9: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n  10: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n  11: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  12: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  13: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  14: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  15: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  16: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  17: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  18: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95)","remote_peer":"4428fefbc147d0170f9bbdfd7310adb642f9a50fcf7df338e4a95f2ea2033ed2"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network.rs:163"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.092022Z","message":"Failed to send a msg to peer","data":{"error":"NetworkError(Peer not connected\n\nStack backtrace:\n   0: consensus::network_interface::ConsensusNetworkSender::supported_protocols\n      consensus::network_interface::ConsensusNetworkSender::preferred_protocol_for_peer\n             at ./aptos/consensus/src/network_interface.rs:151:32\n   1: <consensus::network_interface::ConsensusNetworkSender as network::protocols::network::ApplicationNetworkSender<consensus::network_interface::ConsensusMsg>>::send_to\n             at ./aptos/consensus/src/network_interface.rs:165:24\n   2: consensus::network::NetworkSender::send::{{closure}}\n             at ./aptos/consensus/src/network.rs:162:29\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   3: consensus::network::NetworkSender::send_vote::{{closure}}\n             at ./aptos/consensus/src/network.rs:212:35\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::process_proposal::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:554:58\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::process_proposal_msg::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:309:64\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:772:65\n   4: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   5: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   6: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   7: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   8: tokio::runtime::thread_pool::worker::Context::run\n   9: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n  10: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n  11: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  12: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  13: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  14: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  15: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  16: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  17: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  18: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95)","remote_peer":"4b904c781e92c118f45b3bd4897fdeb769cd1b638c81841c8bcdba98985698ee"}}

[jovar95]

`{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.189321Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 03c02943 at /ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.217163Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 1dd17527 at /ip4/167.235.26.108/tcp/6180/noise-ik/0xa6bf1ee023a3a51c554752e9db07d554a69c36e7938c5404d4e33917f9004a5c/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/167.235.26.108/tcp/6180/noise-ik/0xa6bf1ee023a3a51c554752e9db07d554a69c36e7938c5404d4e33917f9004a5c/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"1dd17527c01616b252c9801366c158fbc0ff67b5ee76436e3cec04399d8de5d0"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.252490Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer e10db9a9 at /ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.260792Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 802f9bbd at /dns/validator.it3.aptos.scw.testnet.kiln.fi/tcp/6180/noise-ik/0xfa01ddaad94935025a1fc57b494504fc8b38fbd836e1240b530bb4827e83a868/handshake/0: Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","data":{"error":"Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","network_address":"/dns/validator.it3.aptos.scw.testnet.kiln.fi/tcp/6180/noise-ik/0xfa01ddaad94935025a1fc57b494504fc8b38fbd836e1240b530bb4827e83a868/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"802f9bbd78574afe79289edf5adcce044ab23535dcd8e79289abf2cb492a3c57"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.404770Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 5ee6ee04 at /ip4/44.242.152.37/tcp/6180/noise-ik/0x614912d50b34e9781c9cec6ff8bec115710273df8ded723ea8f22a37be3a627b/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/44.242.152.37/tcp/6180/noise-ik/0x614912d50b34e9781c9cec6ff8bec115710273df8ded723ea8f22a37be3a627b/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"5ee6ee045f34d87bcfac120c45aeb8e8973aee4196643341dac5e6b34df496fa"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.408807Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer ed06553d at /ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.501337Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 3a9bcf67 at /ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0: Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","data":{"error":"Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","network_address":"/ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:803"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:01.725382Z","data":{"committed_round":1870,"error":"from peer 064ca224990b2bec38cc2db62fd079dadf36b9f03aa98582d5d7d75f98850abc\n\nCaused by:\n    0: [RoundManager] Process proposal\n    1: [RoundManager] sync_only flag is set, stop voting\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::execute_and_vote::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:577:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::process_proposal::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:539:40\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::process_proposal_msg::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:309:64\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:772:65\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: []","round":1872}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:474"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:07.832390Z","data":{"event":"Timeout","remote_peer":"f326fd3007939a18e96d9cbb68da732e06a160759ae12b1fe3d4bc0ef7066a81","round":1878,"voted_nil":false}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:803"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:07.834580Z","data":{"committed_round":1876,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 1878 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:480:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:786:54\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: [LI ff6ac0aa has 3 votes [5be2700c076c27678184fad6af04a0541fd0f2f1ed66c49cca5fe16a466807f9, 8089a7bd4bbdcd6f0ba7812313b49e99f569880644e8f4ad9fd3ce584e0c22c4, 89d079d0935ed64d8a755bfe97938150bd56131336d656b31db0eacdbf03b545] 3 timeout [5be2700c076c27678184fad6af04a0541fd0f2f1ed66c49cca5fe16a466807f9, 8089a7bd4bbdcd6f0ba7812313b49e99f569880644e8f4ad9fd3ce584e0c22c4, 89d079d0935ed64d8a755bfe97938150bd56131336d656b31db0eacdbf03b545]]","round":1878,"self_vote":"Vote: [vote data: VoteData: [block id: 475e47ae, epoch: 37, round: 1878, timestamp: 1662042485615006,parent_block_id: 3729ccf4, parent_block_round: 1877, parent_timestamp: 1662042484745614], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 37, round: 1877, id: 3729ccf4, executed_state_id: 41434355, version: 0, timestamp (us): 1662042484745614, next_epoch_state: None]]]"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:08.168234Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 1dd17527 at /ip4/167.235.26.108/tcp/6180/noise-ik/0xa6bf1ee023a3a51c554752e9db07d554a69c36e7938c5404d4e33917f9004a5c/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/167.235.26.108/tcp/6180/noise-ik/0xa6bf1ee023a3a51c554752e9db07d554a69c36e7938c5404d4e33917f9004a5c/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"1dd17527c01616b252c9801366c158fbc0ff67b5ee76436e3cec04399d8de5d0"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:08.260081Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer e10db9a9 at /ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:08.272340Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 03c02943 at /ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:08.374724Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer ed06553d at /ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:08.395145Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 3a9bcf67 at /ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0: Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","data":{"error":"Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","network_address":"/ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:188"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:10.128460Z","message":"Unavailable peers: [03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42, 1dd17527c01616b252c9801366c158fbc0ff67b5ee76436e3cec04399d8de5d0, 3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da]"}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:474"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:10.128668Z","data":{"event":"Timeout","remote_peer":"168625c7aca6358fe94af73c52d65615c009683f372b5b0e7068d5f306aea764","round":1879,"voted_nil":true}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:803"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:10.129039Z","data":{"committed_round":1877,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 1879 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:480:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:786:54\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: []","round":1879,"self_vote":"Vote: [vote data: VoteData: [block id: 9c099b9c, epoch: 37, round: 1879, timestamp: 1662042485615006,parent_block_id: 475e47ae, parent_block_round: 1878, parent_timestamp: 1662042485615006], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 37, round: 1878, id: 475e47ae, executed_state_id: 41434355, version: 0, timestamp (us): 1662042485615006, next_epoch_state: None]]]"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:17.230742Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer e10db9a9 at /ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:17.232083Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 1dd17527 at /ip4/167.235.26.108/tcp/6180/noise-ik/0xa6bf1ee023a3a51c554752e9db07d554a69c36e7938c5404d4e33917f9004a5c/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/167.235.26.108/tcp/6180/noise-ik/0xa6bf1ee023a3a51c554752e9db07d554a69c36e7938c5404d4e33917f9004a5c/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"1dd17527c01616b252c9801366c158fbc0ff67b5ee76436e3cec04399d8de5d0"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:17.251809Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 03c02943 at /ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:17.387193Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 3a9bcf67 at /ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0: Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","data":{"error":"Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","network_address":"/ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:17.405930Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer ed06553d at /ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:188"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:24.226664Z","message":"Unavailable peers: [03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42, 1dd17527c01616b252c9801366c158fbc0ff67b5ee76436e3cec04399d8de5d0, 3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da]"}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:31.148577Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer d801e1b2 at /ip4/95.188.77.87/tcp/6181/noise-ik/0xcc98cf612f641136ff744bcfd0a3228219fa815e9269556b6d992c23236b462f/handshake/0: Transport error: timeout elapsed","data":{"error":"Transport error: timeout elapsed","network_address":"/ip4/95.188.77.87/tcp/6181/noise-ik/0xcc98cf612f641136ff744bcfd0a3228219fa815e9269556b6d992c23236b462f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:31.164779Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 414f7f91 at /ip4/35.232.235.205/tcp/6180/noise-ik/0x9f28e814486908229ad6e87854f0f736f2d90707cda9d0912078db8213ac162c/handshake/0: Transport error: timeout elapsed","data":{"error":"Transport error: timeout elapsed","network_address":"/ip4/35.232.235.205/tcp/6180/noise-ik/0x9f28e814486908229ad6e87854f0f736f2d90707cda9d0912078db8213ac162c/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"414f7f91f6d6618b126d2c93097142817efa75666a32fdde62c34ed01134b164"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:31.194503Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 0851fbcc at /ip4/35.232.235.205/tcp/6180/noise-ik/0xb1251f6b9f8c7d25ffae475297da56ede376d192e82983e668134b1525abd151/handshake/0: Transport error: timeout elapsed","data":{"error":"Transport error: timeout elapsed","network_address":"/ip4/35.232.235.205/tcp/6180/noise-ik/0xb1251f6b9f8c7d25ffae475297da56ede376d192e82983e668134b1525abd151/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"0851fbcc396e403f8f910870f05d4307b275f73f96cfaaae2c06ca0b93aa77b5"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:188"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:34.375569Z","message":"Unavailable peers: [03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42, 3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da]"}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:35.199893Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer e10db9a9 at /ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:35.213800Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 03c02943 at /ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:35.373317Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer ed06553d at /ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:35.445861Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 3a9bcf67 at /ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0: Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","data":{"error":"Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","network_address":"/ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:188"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:28:54.495485Z","message":"Unavailable peers: [03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42, 3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da]"}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:188"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:04.519994Z","message":"Unavailable peers: [03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42, 3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da]"}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:08.152541Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer d801e1b2 at /ip4/95.188.77.87/tcp/6181/noise-ik/0xcc98cf612f641136ff744bcfd0a3228219fa815e9269556b6d992c23236b462f/handshake/0: Transport error: timeout elapsed","data":{"error":"Transport error: timeout elapsed","network_address":"/ip4/95.188.77.87/tcp/6181/noise-ik/0xcc98cf612f641136ff744bcfd0a3228219fa815e9269556b6d992c23236b462f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:11.226534Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer e10db9a9 at /ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/149.102.134.38/tcp/6180/noise-ik/0xb2b4c8e5011535b30fc1444a8a19a471e33a502a78a448a5a53a9866bb885a2e/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:11.261753Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 03c02943 at /ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/146.19.24.64/tcp/6180/noise-ik/0xbe3be653dc956aeba4f0efbaf7b5854f88fd5c99b790ca386398e8248424f31f/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:11.429456Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer 3a9bcf67 at /ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0: Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","data":{"error":"Transport error: noise client: error reading server handshake response message, server probably rejected our handshake message: unexpected end of file","network_address":"/ip4/34.134.37.36/tcp/6180/noise-ik/0xa5c30c777336df4716887460a63ff7fa441129f5ec6071c57d84ce61e15df625/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0"}}
{"level":"ERROR","source":{"package":"network","file":"network/src/peer_manager/transport.rs:258"},"thread_name":"network-Validator","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:11.455562Z","message":"[validator,Validator,21b4ef9b] Outbound connection failed for peer ed06553d at /ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0: Transport error: Connection refused (os error 111)","data":{"error":"Transport error: Connection refused (os error 111)","network_address":"/ip4/200.58.100.107/tcp/6180/noise-ik/0x0586ccc0c4b30f8f83c52f3faa041dd93442ebe35bbd62aecaec913c7e4fd620/handshake/0","network_context":{"role":"validator","network_id":"Validator","peer_id":"21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42"},"remote_peer":"ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/epoch_manager.rs:789"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:14.173542Z","data":{"epoch":37,"error":"[EpochManager] Failed to send epoch proof to 5bd7c59fc0f31a87ab9633a228fd7a9d6b8081a792abb688047621cdce99b012\n\nCaused by:\n    Peer not connected\n\nStack backtrace:\n   0: consensus::epoch_manager::EpochManager::process_epoch_retrieval::{{closure}}\n             at ./aptos/consensus/src/epoch_manager.rs:336:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::epoch_manager::EpochManager::process_different_epoch::{{closure}}\n             at ./aptos/consensus/src/epoch_manager.rs:362:18\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::epoch_manager::EpochManager::check_epoch::{{closure}}\n             at ./aptos/consensus/src/epoch_manager.rs:683:77\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::epoch_manager::EpochManager::process_message::{{closure}}\n             at ./aptos/consensus/src/epoch_manager.rs:642:78\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      consensus::epoch_manager::EpochManager::start::{{closure}}\n             at ./aptos/consensus/src/epoch_manager.rs:788:68\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   2: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:458:21\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:188"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:15.279906Z","message":"Unavailable peers: [03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, 168625c7aca6358fe94af73c52d65615c009683f372b5b0e7068d5f306aea764, 5bd7c59fc0f31a87ab9633a228fd7a9d6b8081a792abb688047621cdce99b012, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42, 3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da]"}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:188"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:25.129339Z","message":"Unavailable peers: [03c02943185bd29f525828fc418a80cf2efb1da7a8e44d7d06e045422176b824, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42, 3a9bcf67be83271b05a2b3587e92b9a86834f7b281b35b636f9a2637398a9bf0, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da]"}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:474"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:28.829376Z","data":{"event":"Timeout","remote_peer":"064ca224990b2bec38cc2db62fd079dadf36b9f03aa98582d5d7d75f98850abc","round":1991,"voted_nil":false}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:803"},"thread_name":"consensus","hostname":"8378f8a71795","timestamp":"2022-09-01T14:29:28.829895Z","data":{"committed_round":1989,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 1991 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:480:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:786:54\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:435:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:434:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:386:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:383:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/thread_pool/worker.rs:362:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: [LI c9441e1a has 21 votes [0a0df6771891d5d1171a3c808aeb5914487944e82863017e500f290a377368bc, 170c36cfb0294b4e50ea54c66a501ecd776ee9611f0dcce314e9416626619b70, 1b0867a303b9c2c975b3c874866c06fd53e5ee0e538c301b66ffcecf08e63d47, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 309da7c8f94a4454364810dfc0f6f317079cc382d872e73a9e32fbf4fbf70ef1, 5e7ed6a3d40cb5c8831edbfefb4bbfd4874e5566556eea08dde72c14514af7b5, 5ee6ee045f34d87bcfac120c45aeb8e8973aee4196643341dac5e6b34df496fa, 5eea1639b99765ff96502180949fac3662ffd89d358e4adff5dcd31b60ed72c0, 840880fc27f01706d134397dcbeb14cd54899fc8dc93f3f9f1b1a454840829fe, 8aa4d82016b1a0356421c22cda47990d8ca5ffc0f5a012754817db871d34ae90, a4e0e382397c09a69408b088f021a475d1e9d55c3f708660102c837440e2ef1b, b6eb384b229e17db7b1f4f6fa5a692b91cf8d371e1e97e442ae7b60a0f4308d6, b7aa84d8be549026574559735403a79b001ca2c3be54d405b8ec7c4df79c1b15, b92890f0c60eaa2b79814615402ebbaf6371052e296049f1788870a9887a2dc7, b9eab269389f080a221e1811493284ff7141b542dd2bdcaf4fe3e4f4518e8f43, bec1cf784c7b93744687fe7899f7aeafe6fc9a7fc9c758f3f933ec9cf1668f70, c11fffec54e78cb992ff9d34e46ea601d5a84c35985ae516f125bd4c0c5449e3, e38cec9883de5eaa58452d4860e13efabeb986876a7e51cc0e055f540f1a9bf4, eb15bc1d36806e5079d8699c404e3aba86cb136751be943969d6373570c88b7e, f399f294c725bbcc8840845c4286a85324cd65e45271d88ed6d72074f25c1598, f9e184c68cfa53ea4ba0d7354c5ecc512c0d2fa32a13c479c5560b293b7f6a35] 21 timeout [0a0df6771891d5d1171a3c808aeb5914487944e82863017e500f290a377368bc, 170c36cfb0294b4e50ea54c66a501ecd776ee9611f0dcce314e9416626619b70, 1b0867a303b9c2c975b3c874866c06fd53e5ee0e538c301b66ffcecf08e63d47, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 309da7c8f94a4454364810dfc0f6f317079cc382d872e73a9e32fbf4fbf70ef1, 5e7ed6a3d40cb5c8831edbfefb4bbfd4874e5566556eea08dde72c14514af7b5, 5ee6ee045f34d87bcfac120c45aeb8e8973aee4196643341dac5e6b34df496fa, 5eea1639b99765ff96502180949fac3662ffd89d358e4adff5dcd31b60ed72c0, 840880fc27f01706d134397dcbeb14cd54899fc8dc93f3f9f1b1a454840829fe, 8aa4d82016b1a0356421c22cda47990d8ca5ffc0f5a012754817db871d34ae90, a4e0e382397c09a69408b088f021a475d1e9d55c3f708660102c837440e2ef1b, b6eb384b229e17db7b1f4f6fa5a692b91cf8d371e1e97e442ae7b60a0f4308d6, b7aa84d8be549026574559735403a79b001ca2c3be54d405b8ec7c4df79c1b15, b92890f0c60eaa2b79814615402ebbaf6371052e296049f1788870a9887a2dc7, b9eab269389f080a221e1811493284ff7141b542dd2bdcaf4fe3e4f4518e8f43, bec1cf784c7b93744687fe7899f7aeafe6fc9a7fc9c758f3f933ec9cf1668f70, c11fffec54e78cb992ff9d34e46ea601d5a84c35985ae516f125bd4c0c5449e3, e38cec9883de5eaa58452d4860e13efabeb986876a7e51cc0e055f540f1a9bf4, eb15bc1d36806e5079d8699c404e3aba86cb136751be943969d6373570c88b7e, f399f294c725bbcc8840845c4286a85324cd65e45271d88ed6d72074f25c1598, f9e184c68cfa53ea4ba0d7354c5ecc512c0d2fa32a13c479c5560b293b7f6a35]]","round":1991,"self_vote":"Vote: [vote data: VoteData: [block id: ec074e0f, epoch: 37, round: 1991, timestamp: 1662042566361091,parent_block_id: 22c76dc4, parent_block_round: 1990, parent_timestamp: 1662042565242604], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 37, round: 1990, id: 22c76dc4, executed_state_id: 41434355, version: 0, timestamp (us): 1662042565242604, next_epoch_state: None]]]"}}
`

[jovar95]

@JoshLind enough or do you need more? :)

[jovar95]

{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:473"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:30.764549Z","data":{"event":"Timeout","remote_peer":"4247098897780ab7b0c02864dd8ebbeb4dbc0df85c03f82edbb0c00a78fce685","round":767,"voted_nil":false}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:800"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:30.765777Z","data":{"committed_round":765,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 767 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:479:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:789:62\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:431:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:430:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:382:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:379:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:358:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: [LI 439595db has 66 votes [01a95d1f40c04eecf33d1ca9ddf2e077cb3cdc8f39132251dd26dd1560d719d8, 06b15cee0fddb86956e883d0156eb7b5da441e7139ea00ef39bcf0cbf9dc52b5, 0851fbcc396e403f8f910870f05d4307b275f73f96cfaaae2c06ca0b93aa77b5, 0a5d60ca5c94b53c75dfc19d8d1ecc624b10565143ca90199c071089eb86173c, 0e3557dd416faf22f7efa010cbaf9f47d40cdfdf98d2fe26145ec01b3d45c2c0, 0e41c1afa2edc48280f24f0b456311086fa3f777f25b2db7d9d12a91526214a2, 1614c0510f246f089e63cb0dbf1c30529d6389d4b6591d209afd821065bf3541, 168625c7aca6358fe94af73c52d65615c009683f372b5b0e7068d5f306aea764, 19105a85b8d455e51a1b0f917dbb2464d47c47a325c20e5aaa64916299e06aa4, 202c77e0e8b6f7c17428c989d3138a75c989e7fa61c1bb3d38fdbbef0b5bca79, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 22cbd4218363ab29c03b98b129f19c7c5188668514bffa99186d8689123af3bc, 234c957655cc9f4a5f8f0d1ca8f99baf8af3ea57740cd71fa2ab3d14fa3a4ede, 2ec5fe60d0e96c4791826635be764ad9e24db9918642aff078244dcb03b24203, 35eaae0c2966bd7b6d9e8fb1eb4f81740693b932b2c26136c463dfc85c67c1a2, 394f58ff583315bf52b055d34c8635329f4426626b57da5c709a81aa6a68d83a, 442a4f67e45a025612221d7710d3ab7bb8918d3d4bc888303baf7e6709588a1b, 4449959032d51d85aae24c37ad6a9c5a27499b7d7b67dfd5327e61eb2bc3051c, 454ae62855c24140f39398de478b49ac53dd072edea2b54f07d272c9008ddc12, 528a460b6482d633fbfc9d9e9a6b5a2e47f2749c35a50d2416124674cb46bc1f, 5badd632abd92d673892e42578151881d644472326b6c021603f1110f10f74b1, 5c8af906ab13573a0a1a3c34f3308c0afb9e72f1b6e5d7845c7f57ba2f02c7ca, 62562701d5f396a2f8e7bc44e8700214d0cbc38c46d78dbf9dcab949f1dec6c5, 62c89435fbbe74628844c05c3429e4116a3c4fc0a276a25901f7c40432f31925, 631bc1f429ca72def1ebb108c442211b6a2e9337188342ef7b1eb132aa91efc4, 6503030aaf4e9be90bf06a172e4980a520064f839288c89b39f21d180d973c6c, 66bdb52ecd77bace3b5ae4c014b69d0a29957f27b3dcb7406c806b4052e05f4c, 71eb495f8fb7d8f9912dc7ba1406559ad6a7fa54d8abb2bdf5eac78184719798, 750a7e5997bcb7cc87917880524d91c7a59847801b6fe0a8d2c632c0a10c9441, 802f9bbd78574afe79289edf5adcce044ab23535dcd8e79289abf2cb492a3c57, 85c351605b52d46c016e04876239647644bf1a83619f8ff6c0b36f3f344d79ff, 87b30b548452c5a3200a1aec4251a1a01b05b15cb74d41e49701665ef4d36804, 8aa4d82016b1a0356421c22cda47990d8ca5ffc0f5a012754817db871d34ae90, 8c91f8c74f48784f9a5c873d372e52bfbce6fee6ae420e04ef346e9bffae9d63, 8d7766a61a01a33199db8412362805e9550d7dcdce6ab74fecb2ddf9899f74d0, 9314ef791e942598a729e85037a59f851c9c25ce3a22babda48a17988c87f7e5, 943bdde7612aecb92e745facfdf732b2392b1d27f9fd852d83d0f8619fd49382, 975757242ebe7b3b679b25795fa93b1ed07521003c007faf0e53708a4a0a2bbb, 97a0efea1c4e3a9aeb9ac4a9cfa9025bb602b01d8fdaa14c9b1ab03c7be05cb8, 9ab325358eafa5b89b016647e08bc6ecc7b4f070a13d0c167809c279a6a5b07d, a2445e86428742f04b3ac622343c44fd3321f352ce8bc42399605b32cc190a7e, a4e0e382397c09a69408b088f021a475d1e9d55c3f708660102c837440e2ef1b, ae289fd9643d98f1c486bbf983a0400043206c9e84c920957487910988817a78, ae576d308450419f8ff71e72c550da533720271c7c3693b3a3bb4596a7475942, ae91b5f2c5a13f5394c46bcfc7e67e4db1b3659dc6c6faa08e4b9a76a8dde53d, afd409797b862bb1e10cfb3060986461f206c74dbb95534c963a90d04216f5d6, b10dc0857cfb72773f71f11cb2a466eba3e1947d1b8cd1bf2b3124e936fb5468, b12cf85a0a28c36b5703ff9aa5fcc995a3c64f8869f68f61ffbc17b1b6814d25, b1a242d87bdc7a948db36f9c25dc8e92fcc2c66c7f090ad369d6362d70024988, b7aa84d8be549026574559735403a79b001ca2c3be54d405b8ec7c4df79c1b15, bb356acd63be6e072284d210b7e8e2eeae911ad512f1bc24631dcbc2c22a4a98, c54fc020dbf12a2f04c184e62ffd53f64a1c82f9d2bd6f4e074bed4f512148c4, c654a30871b504309c94fac9a164b2273b8be5d71149ed79b304ba8d89fd5c88, c725187215cf9ba7a54f3dfca3472d986eec830065eab4d667dc271483ac5728, c9b19c9b6afb63f3e9fd8c0491fa6ca9fa6a008cd5f2b2dc3b5d7a984b9c5a7f, cce4914c224ee34157924f910bb08ca30d5e66b56ca094d50dba2bd2adb8b80b, d52684b0f57370239beeac25d0c9ad1db6e224a96c725eb5ffbc070231f31cc8, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, da26d0b97726a315035486da1562a1e7d42c1f8bd50fee123402a1e5d4e3af92, dc10d8a940d909cd3c26b8d2c0378d6aab4165b5d98a1f5bdc592aeaa6cdca5f, e0aff63df5c4b9ce3d7e415e2c0b8aa2f036f0a0d730ab31546bf45a2569d925, e3370ffd180416064f3c8e529da8cbe14e2567506834d8d90be3ab7fbb50ef6d, ea1272b8a1bc5a1fb9a1c77c9dc96e1a02a691d1d6073a1dc06d51d770b3f2e2, ed8f98dd4c607ea196fdd0edb9aafa68e7fe31f9eb736f93e9dc744954d960e5, ff79fc4bf0b39c4b8a9810b57d311527700639b661198f4d8db77d7c6d64f0c9] 66 timeout [01a95d1f40c04eecf33d1ca9ddf2e077cb3cdc8f39132251dd26dd1560d719d8, 06b15cee0fddb86956e883d0156eb7b5da441e7139ea00ef39bcf0cbf9dc52b5, 0851fbcc396e403f8f910870f05d4307b275f73f96cfaaae2c06ca0b93aa77b5, 0a5d60ca5c94b53c75dfc19d8d1ecc624b10565143ca90199c071089eb86173c, 0e3557dd416faf22f7efa010cbaf9f47d40cdfdf98d2fe26145ec01b3d45c2c0, 0e41c1afa2edc48280f24f0b456311086fa3f777f25b2db7d9d12a91526214a2, 1614c0510f246f089e63cb0dbf1c30529d6389d4b6591d209afd821065bf3541, 168625c7aca6358fe94af73c52d65615c009683f372b5b0e7068d5f306aea764, 19105a85b8d455e51a1b0f917dbb2464d47c47a325c20e5aaa64916299e06aa4, 202c77e0e8b6f7c17428c989d3138a75c989e7fa61c1bb3d38fdbbef0b5bca79, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 22cbd4218363ab29c03b98b129f19c7c5188668514bffa99186d8689123af3bc, 234c957655cc9f4a5f8f0d1ca8f99baf8af3ea57740cd71fa2ab3d14fa3a4ede, 2ec5fe60d0e96c4791826635be764ad9e24db9918642aff078244dcb03b24203, 35eaae0c2966bd7b6d9e8fb1eb4f81740693b932b2c26136c463dfc85c67c1a2, 394f58ff583315bf52b055d34c8635329f4426626b57da5c709a81aa6a68d83a, 442a4f67e45a025612221d7710d3ab7bb8918d3d4bc888303baf7e6709588a1b, 4449959032d51d85aae24c37ad6a9c5a27499b7d7b67dfd5327e61eb2bc3051c, 454ae62855c24140f39398de478b49ac53dd072edea2b54f07d272c9008ddc12, 528a460b6482d633fbfc9d9e9a6b5a2e47f2749c35a50d2416124674cb46bc1f, 5badd632abd92d673892e42578151881d644472326b6c021603f1110f10f74b1, 5c8af906ab13573a0a1a3c34f3308c0afb9e72f1b6e5d7845c7f57ba2f02c7ca, 62562701d5f396a2f8e7bc44e8700214d0cbc38c46d78dbf9dcab949f1dec6c5, 62c89435fbbe74628844c05c3429e4116a3c4fc0a276a25901f7c40432f31925, 631bc1f429ca72def1ebb108c442211b6a2e9337188342ef7b1eb132aa91efc4, 6503030aaf4e9be90bf06a172e4980a520064f839288c89b39f21d180d973c6c, 66bdb52ecd77bace3b5ae4c014b69d0a29957f27b3dcb7406c806b4052e05f4c, 71eb495f8fb7d8f9912dc7ba1406559ad6a7fa54d8abb2bdf5eac78184719798, 750a7e5997bcb7cc87917880524d91c7a59847801b6fe0a8d2c632c0a10c9441, 802f9bbd78574afe79289edf5adcce044ab23535dcd8e79289abf2cb492a3c57, 85c351605b52d46c016e04876239647644bf1a83619f8ff6c0b36f3f344d79ff, 87b30b548452c5a3200a1aec4251a1a01b05b15cb74d41e49701665ef4d36804, 8aa4d82016b1a0356421c22cda47990d8ca5ffc0f5a012754817db871d34ae90, 8c91f8c74f48784f9a5c873d372e52bfbce6fee6ae420e04ef346e9bffae9d63, 8d7766a61a01a33199db8412362805e9550d7dcdce6ab74fecb2ddf9899f74d0, 9314ef791e942598a729e85037a59f851c9c25ce3a22babda48a17988c87f7e5, 943bdde7612aecb92e745facfdf732b2392b1d27f9fd852d83d0f8619fd49382, 975757242ebe7b3b679b25795fa93b1ed07521003c007faf0e53708a4a0a2bbb, 97a0efea1c4e3a9aeb9ac4a9cfa9025bb602b01d8fdaa14c9b1ab03c7be05cb8, 9ab325358eafa5b89b016647e08bc6ecc7b4f070a13d0c167809c279a6a5b07d, a2445e86428742f04b3ac622343c44fd3321f352ce8bc42399605b32cc190a7e, a4e0e382397c09a69408b088f021a475d1e9d55c3f708660102c837440e2ef1b, ae289fd9643d98f1c486bbf983a0400043206c9e84c920957487910988817a78, ae576d308450419f8ff71e72c550da533720271c7c3693b3a3bb4596a7475942, ae91b5f2c5a13f5394c46bcfc7e67e4db1b3659dc6c6faa08e4b9a76a8dde53d, afd409797b862bb1e10cfb3060986461f206c74dbb95534c963a90d04216f5d6, b10dc0857cfb72773f71f11cb2a466eba3e1947d1b8cd1bf2b3124e936fb5468, b12cf85a0a28c36b5703ff9aa5fcc995a3c64f8869f68f61ffbc17b1b6814d25, b1a242d87bdc7a948db36f9c25dc8e92fcc2c66c7f090ad369d6362d70024988, b7aa84d8be549026574559735403a79b001ca2c3be54d405b8ec7c4df79c1b15, bb356acd63be6e072284d210b7e8e2eeae911ad512f1bc24631dcbc2c22a4a98, c54fc020dbf12a2f04c184e62ffd53f64a1c82f9d2bd6f4e074bed4f512148c4, c654a30871b504309c94fac9a164b2273b8be5d71149ed79b304ba8d89fd5c88, c725187215cf9ba7a54f3dfca3472d986eec830065eab4d667dc271483ac5728, c9b19c9b6afb63f3e9fd8c0491fa6ca9fa6a008cd5f2b2dc3b5d7a984b9c5a7f, cce4914c224ee34157924f910bb08ca30d5e66b56ca094d50dba2bd2adb8b80b, d52684b0f57370239beeac25d0c9ad1db6e224a96c725eb5ffbc070231f31cc8, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, da26d0b97726a315035486da1562a1e7d42c1f8bd50fee123402a1e5d4e3af92, dc10d8a940d909cd3c26b8d2c0378d6aab4165b5d98a1f5bdc592aeaa6cdca5f, e0aff63df5c4b9ce3d7e415e2c0b8aa2f036f0a0d730ab31546bf45a2569d925, e3370ffd180416064f3c8e529da8cbe14e2567506834d8d90be3ab7fbb50ef6d, ea1272b8a1bc5a1fb9a1c77c9dc96e1a02a691d1d6073a1dc06d51d770b3f2e2, ed8f98dd4c607ea196fdd0edb9aafa68e7fe31f9eb736f93e9dc744954d960e5, ff79fc4bf0b39c4b8a9810b57d311527700639b661198f4d8db77d7c6d64f0c9]]","round":767,"self_vote":"Vote: [vote data: VoteData: [block id: b475372f, epoch: 91, round: 767, timestamp: 1662297326846533,parent_block_id: d02fac7b, parent_block_round: 766, parent_timestamp: 1662297324727802], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 91, round: 766, id: d02fac7b, executed_state_id: 41434355, version: 0, timestamp (us): 1662297324727802, next_epoch_state: None]]]"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:473"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:32.281603Z","data":{"event":"Timeout","remote_peer":"4247098897780ab7b0c02864dd8ebbeb4dbc0df85c03f82edbb0c00a78fce685","round":767,"voted_nil":false}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:800"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:32.282804Z","data":{"committed_round":765,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 767 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:479:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:789:62\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:431:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:430:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:382:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:379:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:358:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: [LI 439595db has 125 votes [01a95d1f40c04eecf33d1ca9ddf2e077cb3cdc8f39132251dd26dd1560d719d8, 0289311ada457c33abfaaabff61177c935a3705ac7ac624a43e767615cad2548, 050c8d865560ff370970f4652d1c08c376ed85d68647a3b34863004d84b6388e, 06b15cee0fddb86956e883d0156eb7b5da441e7139ea00ef39bcf0cbf9dc52b5, 07d7df40b1b25dcb7e8fb6caa76ee2272f081a8992629e145aead916b8713f3d, 0851fbcc396e403f8f910870f05d4307b275f73f96cfaaae2c06ca0b93aa77b5, 0a0df6771891d5d1171a3c808aeb5914487944e82863017e500f290a377368bc, 0a5d60ca5c94b53c75dfc19d8d1ecc624b10565143ca90199c071089eb86173c, 0e3557dd416faf22f7efa010cbaf9f47d40cdfdf98d2fe26145ec01b3d45c2c0, 0e41c1afa2edc48280f24f0b456311086fa3f777f25b2db7d9d12a91526214a2, 1614c0510f246f089e63cb0dbf1c30529d6389d4b6591d209afd821065bf3541, 168625c7aca6358fe94af73c52d65615c009683f372b5b0e7068d5f306aea764, 17e6eeec824c4ebe71f276aeb2115b63defcd8b5898663074d02c6db6494991b, 19105a85b8d455e51a1b0f917dbb2464d47c47a325c20e5aaa64916299e06aa4, 1a86526cab3a14ce5d3ea14d55fa313c9b284ab1543df434b9ea67f90f4ff682, 1e04892640120d294c35cc85033634af8e93c04574771c1335d4ea6d89dbfb6e, 202c77e0e8b6f7c17428c989d3138a75c989e7fa61c1bb3d38fdbbef0b5bca79, 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 22cbd4218363ab29c03b98b129f19c7c5188668514bffa99186d8689123af3bc, 234c957655cc9f4a5f8f0d1ca8f99baf8af3ea57740cd71fa2ab3d14fa3a4ede, 2ec5fe60d0e96c4791826635be764ad9e24db9918642aff078244dcb03b24203, 309da7c8f94a4454364810dfc0f6f317079cc382d872e73a9e32fbf4fbf70ef1, 35eaae0c2966bd7b6d9e8fb1eb4f81740693b932b2c26136c463dfc85c67c1a2, 394f58ff583315bf52b055d34c8635329f4426626b57da5c709a81aa6a68d83a, 4247098897780ab7b0c02864dd8ebbeb4dbc0df85c03f82edbb0c00a78fce685, 42fff934f0321f6235c81a2ef8baf77245fb7578cc572acc3928c6f25b78b39e, 44194bedcb5bfac13f003eddbc6567e27ffbc5c2cb98090294b90419c522c0e7, 4428fefbc147d0170f9bbdfd7310adb642f9a50fcf7df338e4a95f2ea2033ed2, 442a4f67e45a025612221d7710d3ab7bb8918d3d4bc888303baf7e6709588a1b, 4449959032d51d85aae24c37ad6a9c5a27499b7d7b67dfd5327e61eb2bc3051c, 44c6dc9efba8be28f10f8c44cd318c5656721c1d88ed0e8e5a3609ac8ed4d63f, 454ae62855c24140f39398de478b49ac53dd072edea2b54f07d272c9008ddc12, 46774d7bda97925b0f7dc7eecc927fe4f2f17739f087243043694d2bea153cf5, 4a38e50e00b576cf037248dd6151d0f927c63205bff0ec4d8752d5a203afecc8, 4b904c781e92c118f45b3bd4897fdeb769cd1b638c81841c8bcdba98985698ee, 528a460b6482d633fbfc9d9e9a6b5a2e47f2749c35a50d2416124674cb46bc1f, 568e99f8422a6a49d7e648db5e58747103e8495989e2a4680faa5cbeb296a4ca, 57be1bdaf544178f3e754dd23f84c74a95785c694066f6c1d48762b8c888cbee, 5badd632abd92d673892e42578151881d644472326b6c021603f1110f10f74b1, 5c8af906ab13573a0a1a3c34f3308c0afb9e72f1b6e5d7845c7f57ba2f02c7ca, 5e7ed6a3d40cb5c8831edbfefb4bbfd4874e5566556eea08dde72c14514af7b5, 5eea1639b99765ff96502180949fac3662ffd89d358e4adff5dcd31b60ed72c0, 62562701d5f396a2f8e7bc44e8700214d0cbc38c46d78dbf9dcab949f1dec6c5, 62c89435fbbe74628844c05c3429e4116a3c4fc0a276a25901f7c40432f31925, 631bc1f429ca72def1ebb108c442211b6a2e9337188342ef7b1eb132aa91efc4, 6503030aaf4e9be90bf06a172e4980a520064f839288c89b39f21d180d973c6c, 66bdb52ecd77bace3b5ae4c014b69d0a29957f27b3dcb7406c806b4052e05f4c, 69268e38beda87f1ff5bf22881f22b9deb564c89028e7af89ee8b911fc616f95, 699b1fe83796492513a17f276607a4d3dbc250ac26e9d5f455ea790776b6337e, 71eb495f8fb7d8f9912dc7ba1406559ad6a7fa54d8abb2bdf5eac78184719798, 750a7e5997bcb7cc87917880524d91c7a59847801b6fe0a8d2c632c0a10c9441, 802f9bbd78574afe79289edf5adcce044ab23535dcd8e79289abf2cb492a3c57, 84699c4425469b5610f8c30d66026a57b9f7d7442cdcac7b2bf11875d80b05c2, 8485b0e4779190c2e0cefc899ac853cd3d878930e66587df8d867c0c941267a6, 85c351605b52d46c016e04876239647644bf1a83619f8ff6c0b36f3f344d79ff, 87a572ffb4a0c22b32d41b91d1df342da0d4477b234faa394f0328bed498a1be, 87a92d3c58a7b644f0efc520f210f590573d10fad04ba9aa0ccb343dfdc36439, 87b30b548452c5a3200a1aec4251a1a01b05b15cb74d41e49701665ef4d36804, 8a147943e71c0c9d2d167492905d5e27e6af4ee4788badde9299120abc231b7b, 8aa4d82016b1a0356421c22cda47990d8ca5ffc0f5a012754817db871d34ae90, 8ba19a694947afb53a3b79923f0c0bd25e8c0aa7cab71b29be9b16173d67e6df, 8c91f8c74f48784f9a5c873d372e52bfbce6fee6ae420e04ef346e9bffae9d63, 8d1f8a22eb447833aa978f611c13d0fa327bdda2bed8ca169b4417edbbdf4aaf, 8d7766a61a01a33199db8412362805e9550d7dcdce6ab74fecb2ddf9899f74d0, 9314ef791e942598a729e85037a59f851c9c25ce3a22babda48a17988c87f7e5, 935719a60ad09f4a6b74baf20133760c720b895a3c4a59d2cb3258d8b5b055a1, 943bdde7612aecb92e745facfdf732b2392b1d27f9fd852d83d0f8619fd49382, 975757242ebe7b3b679b25795fa93b1ed07521003c007faf0e53708a4a0a2bbb, 97a0efea1c4e3a9aeb9ac4a9cfa9025bb602b01d8fdaa14c9b1ab03c7be05cb8, 9991b734f39f2f22d8bc071db5ec2d42cd074fa4fe9e9f19a12873fd7cca34df, 9ab325358eafa5b89b016647e08bc6ecc7b4f070a13d0c167809c279a6a5b07d, a2445e86428742f04b3ac622343c44fd3321f352ce8bc42399605b32cc190a7e, a4079618c9f4cbbf11d69d7a12b7f5a6deadef4aed991c00e4fd412e2765b713, a4e0e382397c09a69408b088f021a475d1e9d55c3f708660102c837440e2ef1b, a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, ae289fd9643d98f1c486bbf983a0400043206c9e84c920957487910988817a78, ae576d308450419f8ff71e72c550da533720271c7c3693b3a3bb4596a7475942, ae91b5f2c5a13f5394c46bcfc7e67e4db1b3659dc6c6faa08e4b9a76a8dde53d, afd409797b862bb1e10cfb3060986461f206c74dbb95534c963a90d04216f5d6, b0cc635492330102e2e0592033045f0b14381a9cbd8f43f21223fc1fa3b58ac6, b10dc0857cfb72773f71f11cb2a466eba3e1947d1b8cd1bf2b3124e936fb5468, b12cf85a0a28c36b5703ff9aa5fcc995a3c64f8869f68f61ffbc17b1b6814d25, b14350297a3d2e2caf1a887213b4aebf6b120fa2ae87e9ddd1d34d9ad7beb44c, b1a242d87bdc7a948db36f9c25dc8e92fcc2c66c7f090ad369d6362d70024988, b1bc346da7756a51dae71ddbd8d74228604598fa6cc35a52fb78e86ca3915e26, b2243c6234285ca4ddba4927603750a0b2571d9453b1021c4cd4a0c34e045ecd, b6eb384b229e17db7b1f4f6fa5a692b91cf8d371e1e97e442ae7b60a0f4308d6, b7aa84d8be549026574559735403a79b001ca2c3be54d405b8ec7c4df79c1b15, b92890f0c60eaa2b79814615402ebbaf6371052e296049f1788870a9887a2dc7, bb356acd63be6e072284d210b7e8e2eeae911ad512f1bc24631dcbc2c22a4a98, be2ac53a4dba0325cf3d65eca93c68e5239c6fc5b4ca013e6ec7089235c43775, bec1cf784c7b93744687fe7899f7aeafe6fc9a7fc9c758f3f933ec9cf1668f70, bfeac9b2da698dee2d9b882322d20dc7508bb6cf2df4165c8938ffd748e11337, c2371a6ccfdc035723f58b67df18fdcc28b1579522670680732c741063eb55ff, c388fe9c9186b87c894dde602023d4650f8fe677067ab5227a15caa2baf528fd, c4a5af5fa6c1ae769157675c8fbe55c49e2cbd7ce89a8dec5f09a6ced957568d, c54fc020dbf12a2f04c184e62ffd53f64a1c82f9d2bd6f4e074bed4f512148c4, c654a30871b504309c94fac9a164b2273b8be5d71149ed79b304ba8d89fd5c88, c725187215cf9ba7a54f3dfca3472d986eec830065eab4d667dc271483ac5728, c9b19c9b6afb63f3e9fd8c0491fa6ca9fa6a008cd5f2b2dc3b5d7a984b9c5a7f, cce4914c224ee34157924f910bb08ca30d5e66b56ca094d50dba2bd2adb8b80b, d52684b0f57370239beeac25d0c9ad1db6e224a96c725eb5ffbc070231f31cc8, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, da26d0b97726a315035486da1562a1e7d42c1f8bd50fee123402a1e5d4e3af92, dc10d8a940d909cd3c26b8d2c0378d6aab4165b5d98a1f5bdc592aeaa6cdca5f, de102b1247b8b89979b5f58dc29da624d01b0381fd725908464e02a2d3bcec94, e0aff63df5c4b9ce3d7e415e2c0b8aa2f036f0a0d730ab31546bf45a2569d925, e0c9b84c58ba4234aeebbb7203746c6f275025ceee94bb86d4573289565bbdde, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da, e3370ffd180416064f3c8e529da8cbe14e2567506834d8d90be3ab7fbb50ef6d, e38cec9883de5eaa58452d4860e13efabeb986876a7e51cc0e055f540f1a9bf4, e8278487a145eadfa3b07d7a711efe1606a7184045337b27259d6ab73e89aa5a, ea1272b8a1bc5a1fb9a1c77c9dc96e1a02a691d1d6073a1dc06d51d770b3f2e2, eb15bc1d36806e5079d8699c404e3aba86cb136751be943969d6373570c88b7e, ed8f98dd4c607ea196fdd0edb9aafa68e7fe31f9eb736f93e9dc744954d960e5, f2e802753bbaa2b3e0861180dfd2dd3fe8dc9bb0d49e0094376f6d565bab9884, f326fd3007939a18e96d9cbb68da732e06a160759ae12b1fe3d4bc0ef7066a81, f399f294c725bbcc8840845c4286a85324cd65e45271d88ed6d72074f25c1598, f4cb45b3d657a4957d723cfaaeca5803c91ea38981bd306abf7a8c44790c894f, f7c045f174d8527bd300eb5e9e84fb6db43a9e80190f7b9597db095fa0995658, f9e184c68cfa53ea4ba0d7354c5ecc512c0d2fa32a13c479c5560b293b7f6a35, fcfa1efb940e23936915c2a6814ca557dbf3c6972d4ff0154d8e6aacda905c64, ff79fc4bf0b39c4b8a9810b57d311527700639b661198f4d8db77d7c6d64f0c9] 125 timeout [01a95d1f40c04eecf33d1ca9ddf2e077cb3cdc8f39132251dd26dd1560d719d8, 0289311ada457c33abfaaabff61177c935a3705ac7ac624a43e767615cad2548, 050c8d865560ff370970f4652d1c08c376ed85d68647a3b34863004d84b6388e, 06b15cee0fddb86956e883d0156eb7b5da441e7139ea00ef39bcf0cbf9dc52b5, 07d7df40b1b25dcb7e8fb6caa76ee2272f081a8992629e145aead916b8713f3d, 0851fbcc396e403f8f910870f05d4307b275f73f96cfaaae2c06ca0b93aa77b5, 0a0df6771891d5d1171a3c808aeb5914487944e82863017e500f290a377368bc, 0a5d60ca5c94b53c75dfc19d8d1ecc624b10565143ca90199c071089eb86173c, 0e3557dd416faf22f7efa010cbaf9f47d40cdfdf98d2fe26145ec01b3d45c2c0, 0e41c1afa2edc48280f24f0b456311086fa3f777f25b2db7d9d12a91526214a2, 1614c0510f246f089e63cb0dbf1c30529d6389d4b6591d209afd821065bf3541, 168625c7aca6358fe94af73c52d65615c009683f372b5b0e7068d5f306aea764, 17e6eeec824c4ebe71f276aeb2115b63defcd8b5898663074d02c6db6494991b, 19105a85b8d455e51a1b0f917dbb2464d47c47a325c20e5aaa64916299e06aa4, 1a86526cab3a14ce5d3ea14d55fa313c9b284ab1543df434b9ea67f90f4ff682, 1e04892640120d294c35cc85033634af8e93c04574771c1335d4ea6d89dbfb6e, 202c77e0e8b6f7c17428c989d3138a75c989e7fa61c1bb3d38fdbbef0b5bca79, 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 22cbd4218363ab29c03b98b129f19c7c5188668514bffa99186d8689123af3bc, 234c957655cc9f4a5f8f0d1ca8f99baf8af3ea57740cd71fa2ab3d14fa3a4ede, 2ec5fe60d0e96c4791826635be764ad9e24db9918642aff078244dcb03b24203, 309da7c8f94a4454364810dfc0f6f317079cc382d872e73a9e32fbf4fbf70ef1, 35eaae0c2966bd7b6d9e8fb1eb4f81740693b932b2c26136c463dfc85c67c1a2, 394f58ff583315bf52b055d34c8635329f4426626b57da5c709a81aa6a68d83a, 4247098897780ab7b0c02864dd8ebbeb4dbc0df85c03f82edbb0c00a78fce685, 42fff934f0321f6235c81a2ef8baf77245fb7578cc572acc3928c6f25b78b39e, 44194bedcb5bfac13f003eddbc6567e27ffbc5c2cb98090294b90419c522c0e7, 4428fefbc147d0170f9bbdfd7310adb642f9a50fcf7df338e4a95f2ea2033ed2, 442a4f67e45a025612221d7710d3ab7bb8918d3d4bc888303baf7e6709588a1b, 4449959032d51d85aae24c37ad6a9c5a27499b7d7b67dfd5327e61eb2bc3051c, 44c6dc9efba8be28f10f8c44cd318c5656721c1d88ed0e8e5a3609ac8ed4d63f, 454ae62855c24140f39398de478b49ac53dd072edea2b54f07d272c9008ddc12, 46774d7bda97925b0f7dc7eecc927fe4f2f17739f087243043694d2bea153cf5, 4a38e50e00b576cf037248dd6151d0f927c63205bff0ec4d8752d5a203afecc8, 4b904c781e92c118f45b3bd4897fdeb769cd1b638c81841c8bcdba98985698ee, 528a460b6482d633fbfc9d9e9a6b5a2e47f2749c35a50d2416124674cb46bc1f, 568e99f8422a6a49d7e648db5e58747103e8495989e2a4680faa5cbeb296a4ca, 57be1bdaf544178f3e754dd23f84c74a95785c694066f6c1d48762b8c888cbee, 5badd632abd92d673892e42578151881d644472326b6c021603f1110f10f74b1, 5c8af906ab13573a0a1a3c34f3308c0afb9e72f1b6e5d7845c7f57ba2f02c7ca, 5e7ed6a3d40cb5c8831edbfefb4bbfd4874e5566556eea08dde72c14514af7b5, 5eea1639b99765ff96502180949fac3662ffd89d358e4adff5dcd31b60ed72c0, 62562701d5f396a2f8e7bc44e8700214d0cbc38c46d78dbf9dcab949f1dec6c5, 62c89435fbbe74628844c05c3429e4116a3c4fc0a276a25901f7c40432f31925, 631bc1f429ca72def1ebb108c442211b6a2e9337188342ef7b1eb132aa91efc4, 6503030aaf4e9be90bf06a172e4980a520064f839288c89b39f21d180d973c6c, 66bdb52ecd77bace3b5ae4c014b69d0a29957f27b3dcb7406c806b4052e05f4c, 69268e38beda87f1ff5bf22881f22b9deb564c89028e7af89ee8b911fc616f95, 699b1fe83796492513a17f276607a4d3dbc250ac26e9d5f455ea790776b6337e, 71eb495f8fb7d8f9912dc7ba1406559ad6a7fa54d8abb2bdf5eac78184719798, 750a7e5997bcb7cc87917880524d91c7a59847801b6fe0a8d2c632c0a10c9441, 802f9bbd78574afe79289edf5adcce044ab23535dcd8e79289abf2cb492a3c57, 84699c4425469b5610f8c30d66026a57b9f7d7442cdcac7b2bf11875d80b05c2, 8485b0e4779190c2e0cefc899ac853cd3d878930e66587df8d867c0c941267a6, 85c351605b52d46c016e04876239647644bf1a83619f8ff6c0b36f3f344d79ff, 87a572ffb4a0c22b32d41b91d1df342da0d4477b234faa394f0328bed498a1be, 87a92d3c58a7b644f0efc520f210f590573d10fad04ba9aa0ccb343dfdc36439, 87b30b548452c5a3200a1aec4251a1a01b05b15cb74d41e49701665ef4d36804, 8a147943e71c0c9d2d167492905d5e27e6af4ee4788badde9299120abc231b7b, 8aa4d82016b1a0356421c22cda47990d8ca5ffc0f5a012754817db871d34ae90, 8ba19a694947afb53a3b79923f0c0bd25e8c0aa7cab71b29be9b16173d67e6df, 8c91f8c74f48784f9a5c873d372e52bfbce6fee6ae420e04ef346e9bffae9d63, 8d1f8a22eb447833aa978f611c13d0fa327bdda2bed8ca169b4417edbbdf4aaf, 8d7766a61a01a33199db8412362805e9550d7dcdce6ab74fecb2ddf9899f74d0, 9314ef791e942598a729e85037a59f851c9c25ce3a22babda48a17988c87f7e5, 935719a60ad09f4a6b74baf20133760c720b895a3c4a59d2cb3258d8b5b055a1, 943bdde7612aecb92e745facfdf732b2392b1d27f9fd852d83d0f8619fd49382, 975757242ebe7b3b679b25795fa93b1ed07521003c007faf0e53708a4a0a2bbb, 97a0efea1c4e3a9aeb9ac4a9cfa9025bb602b01d8fdaa14c9b1ab03c7be05cb8, 9991b734f39f2f22d8bc071db5ec2d42cd074fa4fe9e9f19a12873fd7cca34df, 9ab325358eafa5b89b016647e08bc6ecc7b4f070a13d0c167809c279a6a5b07d, a2445e86428742f04b3ac622343c44fd3321f352ce8bc42399605b32cc190a7e, a4079618c9f4cbbf11d69d7a12b7f5a6deadef4aed991c00e4fd412e2765b713, a4e0e382397c09a69408b088f021a475d1e9d55c3f708660102c837440e2ef1b, a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, ae289fd9643d98f1c486bbf983a0400043206c9e84c920957487910988817a78, ae576d308450419f8ff71e72c550da533720271c7c3693b3a3bb4596a7475942, ae91b5f2c5a13f5394c46bcfc7e67e4db1b3659dc6c6faa08e4b9a76a8dde53d, afd409797b862bb1e10cfb3060986461f206c74dbb95534c963a90d04216f5d6, b0cc635492330102e2e0592033045f0b14381a9cbd8f43f21223fc1fa3b58ac6, b10dc0857cfb72773f71f11cb2a466eba3e1947d1b8cd1bf2b3124e936fb5468, b12cf85a0a28c36b5703ff9aa5fcc995a3c64f8869f68f61ffbc17b1b6814d25, b14350297a3d2e2caf1a887213b4aebf6b120fa2ae87e9ddd1d34d9ad7beb44c, b1a242d87bdc7a948db36f9c25dc8e92fcc2c66c7f090ad369d6362d70024988, b1bc346da7756a51dae71ddbd8d74228604598fa6cc35a52fb78e86ca3915e26, b2243c6234285ca4ddba4927603750a0b2571d9453b1021c4cd4a0c34e045ecd, b6eb384b229e17db7b1f4f6fa5a692b91cf8d371e1e97e442ae7b60a0f4308d6, b7aa84d8be549026574559735403a79b001ca2c3be54d405b8ec7c4df79c1b15, b92890f0c60eaa2b79814615402ebbaf6371052e296049f1788870a9887a2dc7, bb356acd63be6e072284d210b7e8e2eeae911ad512f1bc24631dcbc2c22a4a98, be2ac53a4dba0325cf3d65eca93c68e5239c6fc5b4ca013e6ec7089235c43775, bec1cf784c7b93744687fe7899f7aeafe6fc9a7fc9c758f3f933ec9cf1668f70, bfeac9b2da698dee2d9b882322d20dc7508bb6cf2df4165c8938ffd748e11337, c2371a6ccfdc035723f58b67df18fdcc28b1579522670680732c741063eb55ff, c388fe9c9186b87c894dde602023d4650f8fe677067ab5227a15caa2baf528fd, c4a5af5fa6c1ae769157675c8fbe55c49e2cbd7ce89a8dec5f09a6ced957568d, c54fc020dbf12a2f04c184e62ffd53f64a1c82f9d2bd6f4e074bed4f512148c4, c654a30871b504309c94fac9a164b2273b8be5d71149ed79b304ba8d89fd5c88, c725187215cf9ba7a54f3dfca3472d986eec830065eab4d667dc271483ac5728, c9b19c9b6afb63f3e9fd8c0491fa6ca9fa6a008cd5f2b2dc3b5d7a984b9c5a7f, cce4914c224ee34157924f910bb08ca30d5e66b56ca094d50dba2bd2adb8b80b, d52684b0f57370239beeac25d0c9ad1db6e224a96c725eb5ffbc070231f31cc8, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, da26d0b97726a315035486da1562a1e7d42c1f8bd50fee123402a1e5d4e3af92, dc10d8a940d909cd3c26b8d2c0378d6aab4165b5d98a1f5bdc592aeaa6cdca5f, de102b1247b8b89979b5f58dc29da624d01b0381fd725908464e02a2d3bcec94, e0aff63df5c4b9ce3d7e415e2c0b8aa2f036f0a0d730ab31546bf45a2569d925, e0c9b84c58ba4234aeebbb7203746c6f275025ceee94bb86d4573289565bbdde, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da, e3370ffd180416064f3c8e529da8cbe14e2567506834d8d90be3ab7fbb50ef6d, e38cec9883de5eaa58452d4860e13efabeb986876a7e51cc0e055f540f1a9bf4, e8278487a145eadfa3b07d7a711efe1606a7184045337b27259d6ab73e89aa5a, ea1272b8a1bc5a1fb9a1c77c9dc96e1a02a691d1d6073a1dc06d51d770b3f2e2, eb15bc1d36806e5079d8699c404e3aba86cb136751be943969d6373570c88b7e, ed8f98dd4c607ea196fdd0edb9aafa68e7fe31f9eb736f93e9dc744954d960e5, f2e802753bbaa2b3e0861180dfd2dd3fe8dc9bb0d49e0094376f6d565bab9884, f326fd3007939a18e96d9cbb68da732e06a160759ae12b1fe3d4bc0ef7066a81, f399f294c725bbcc8840845c4286a85324cd65e45271d88ed6d72074f25c1598, f4cb45b3d657a4957d723cfaaeca5803c91ea38981bd306abf7a8c44790c894f, f7c045f174d8527bd300eb5e9e84fb6db43a9e80190f7b9597db095fa0995658, f9e184c68cfa53ea4ba0d7354c5ecc512c0d2fa32a13c479c5560b293b7f6a35, fcfa1efb940e23936915c2a6814ca557dbf3c6972d4ff0154d8e6aacda905c64, ff79fc4bf0b39c4b8a9810b57d311527700639b661198f4d8db77d7c6d64f0c9]]","round":767,"self_vote":"Vote: [vote data: VoteData: [block id: b475372f, epoch: 91, round: 767, timestamp: 1662297326846533,parent_block_id: d02fac7b, parent_block_round: 766, parent_timestamp: 1662297324727802], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 91, round: 766, id: d02fac7b, executed_state_id: 41434355, version: 0, timestamp (us): 1662297324727802, next_epoch_state: None]]]"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/network_interface.rs:189"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:33.947644Z","message":"Unavailable peers: [e342f41235dcfed6016769285e5f918530e9dbe6832e553df653e050d7347168, 96b148b5ba586d3b0406450e793ddd416fe6f7bdd21ee8d4f0b1ff245303aa4a, c11fffec54e78cb992ff9d34e46ea601d5a84c35985ae516f125bd4c0c5449e3, 9f9ae72a7539640b4ffc169e080a07b3276bca2527193f53232f27cbf12182d1, a3fdaacd44c39f1d548aaac3fb6a25336c37fc01bec7233505d50a77fddc84ae, b404626788b5ed7e6940200acfe6884f91cf636c62751bd12508d51bdb80e13a, 0d8fee8284ef09fe6fb272d357e31f142a8d2cef2c089aa618a6e6de76f53703, 1ce1a4ea4755a027843adc6ea9f13e158cc45edcb26be55b194b0fd278cfc041, c3f2c9dc502dbe9de4d3d3ee472fa679f578d978e53c1242a6906f894d435bd1, d16b8e15be631f0d22199f8b91b7338862fef9ee0a3af419477bd148a26cd2ff]"}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:473"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:33.948172Z","data":{"event":"Timeout","remote_peer":"7246f46a4b1a0dfaff0687358794736f9e6c24faefcf72cebc7791c2c00f7810","round":768,"voted_nil":false}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:800"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:33.949353Z","data":{"committed_round":766,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 768 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:479:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:789:62\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:431:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:430:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:382:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:379:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:358:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: [LI a9b1efb1 has 40 votes [01a95d1f40c04eecf33d1ca9ddf2e077cb3cdc8f39132251dd26dd1560d719d8, 06b15cee0fddb86956e883d0156eb7b5da441e7139ea00ef39bcf0cbf9dc52b5, 0851fbcc396e403f8f910870f05d4307b275f73f96cfaaae2c06ca0b93aa77b5, 19105a85b8d455e51a1b0f917dbb2464d47c47a325c20e5aaa64916299e06aa4, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 22cbd4218363ab29c03b98b129f19c7c5188668514bffa99186d8689123af3bc, 4449959032d51d85aae24c37ad6a9c5a27499b7d7b67dfd5327e61eb2bc3051c, 454ae62855c24140f39398de478b49ac53dd072edea2b54f07d272c9008ddc12, 528a460b6482d633fbfc9d9e9a6b5a2e47f2749c35a50d2416124674cb46bc1f, 5badd632abd92d673892e42578151881d644472326b6c021603f1110f10f74b1, 5c8af906ab13573a0a1a3c34f3308c0afb9e72f1b6e5d7845c7f57ba2f02c7ca, 5e7ed6a3d40cb5c8831edbfefb4bbfd4874e5566556eea08dde72c14514af7b5, 62562701d5f396a2f8e7bc44e8700214d0cbc38c46d78dbf9dcab949f1dec6c5, 62c89435fbbe74628844c05c3429e4116a3c4fc0a276a25901f7c40432f31925, 631bc1f429ca72def1ebb108c442211b6a2e9337188342ef7b1eb132aa91efc4, 71eb495f8fb7d8f9912dc7ba1406559ad6a7fa54d8abb2bdf5eac78184719798, 750a7e5997bcb7cc87917880524d91c7a59847801b6fe0a8d2c632c0a10c9441, 85c351605b52d46c016e04876239647644bf1a83619f8ff6c0b36f3f344d79ff, 87b30b548452c5a3200a1aec4251a1a01b05b15cb74d41e49701665ef4d36804, 8d7766a61a01a33199db8412362805e9550d7dcdce6ab74fecb2ddf9899f74d0, 943bdde7612aecb92e745facfdf732b2392b1d27f9fd852d83d0f8619fd49382, 95ce20e79294e79fca729d40db823144c677a27eec7a53c5cd021dd28602c6d7, 975757242ebe7b3b679b25795fa93b1ed07521003c007faf0e53708a4a0a2bbb, 97a0efea1c4e3a9aeb9ac4a9cfa9025bb602b01d8fdaa14c9b1ab03c7be05cb8, 9ab325358eafa5b89b016647e08bc6ecc7b4f070a13d0c167809c279a6a5b07d, ae289fd9643d98f1c486bbf983a0400043206c9e84c920957487910988817a78, ae576d308450419f8ff71e72c550da533720271c7c3693b3a3bb4596a7475942, ae91b5f2c5a13f5394c46bcfc7e67e4db1b3659dc6c6faa08e4b9a76a8dde53d, b10dc0857cfb72773f71f11cb2a466eba3e1947d1b8cd1bf2b3124e936fb5468, b1a242d87bdc7a948db36f9c25dc8e92fcc2c66c7f090ad369d6362d70024988, bec1cf784c7b93744687fe7899f7aeafe6fc9a7fc9c758f3f933ec9cf1668f70, c54fc020dbf12a2f04c184e62ffd53f64a1c82f9d2bd6f4e074bed4f512148c4, c725187215cf9ba7a54f3dfca3472d986eec830065eab4d667dc271483ac5728, c9b19c9b6afb63f3e9fd8c0491fa6ca9fa6a008cd5f2b2dc3b5d7a984b9c5a7f, d52684b0f57370239beeac25d0c9ad1db6e224a96c725eb5ffbc070231f31cc8, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da, ea1272b8a1bc5a1fb9a1c77c9dc96e1a02a691d1d6073a1dc06d51d770b3f2e2, ed8f98dd4c607ea196fdd0edb9aafa68e7fe31f9eb736f93e9dc744954d960e5, ff79fc4bf0b39c4b8a9810b57d311527700639b661198f4d8db77d7c6d64f0c9] 40 timeout [01a95d1f40c04eecf33d1ca9ddf2e077cb3cdc8f39132251dd26dd1560d719d8, 06b15cee0fddb86956e883d0156eb7b5da441e7139ea00ef39bcf0cbf9dc52b5, 0851fbcc396e403f8f910870f05d4307b275f73f96cfaaae2c06ca0b93aa77b5, 19105a85b8d455e51a1b0f917dbb2464d47c47a325c20e5aaa64916299e06aa4, 21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 22cbd4218363ab29c03b98b129f19c7c5188668514bffa99186d8689123af3bc, 4449959032d51d85aae24c37ad6a9c5a27499b7d7b67dfd5327e61eb2bc3051c, 454ae62855c24140f39398de478b49ac53dd072edea2b54f07d272c9008ddc12, 528a460b6482d633fbfc9d9e9a6b5a2e47f2749c35a50d2416124674cb46bc1f, 5badd632abd92d673892e42578151881d644472326b6c021603f1110f10f74b1, 5c8af906ab13573a0a1a3c34f3308c0afb9e72f1b6e5d7845c7f57ba2f02c7ca, 5e7ed6a3d40cb5c8831edbfefb4bbfd4874e5566556eea08dde72c14514af7b5, 62562701d5f396a2f8e7bc44e8700214d0cbc38c46d78dbf9dcab949f1dec6c5, 62c89435fbbe74628844c05c3429e4116a3c4fc0a276a25901f7c40432f31925, 631bc1f429ca72def1ebb108c442211b6a2e9337188342ef7b1eb132aa91efc4, 71eb495f8fb7d8f9912dc7ba1406559ad6a7fa54d8abb2bdf5eac78184719798, 750a7e5997bcb7cc87917880524d91c7a59847801b6fe0a8d2c632c0a10c9441, 85c351605b52d46c016e04876239647644bf1a83619f8ff6c0b36f3f344d79ff, 87b30b548452c5a3200a1aec4251a1a01b05b15cb74d41e49701665ef4d36804, 8d7766a61a01a33199db8412362805e9550d7dcdce6ab74fecb2ddf9899f74d0, 943bdde7612aecb92e745facfdf732b2392b1d27f9fd852d83d0f8619fd49382, 95ce20e79294e79fca729d40db823144c677a27eec7a53c5cd021dd28602c6d7, 975757242ebe7b3b679b25795fa93b1ed07521003c007faf0e53708a4a0a2bbb, 97a0efea1c4e3a9aeb9ac4a9cfa9025bb602b01d8fdaa14c9b1ab03c7be05cb8, 9ab325358eafa5b89b016647e08bc6ecc7b4f070a13d0c167809c279a6a5b07d, ae289fd9643d98f1c486bbf983a0400043206c9e84c920957487910988817a78, ae576d308450419f8ff71e72c550da533720271c7c3693b3a3bb4596a7475942, ae91b5f2c5a13f5394c46bcfc7e67e4db1b3659dc6c6faa08e4b9a76a8dde53d, b10dc0857cfb72773f71f11cb2a466eba3e1947d1b8cd1bf2b3124e936fb5468, b1a242d87bdc7a948db36f9c25dc8e92fcc2c66c7f090ad369d6362d70024988, bec1cf784c7b93744687fe7899f7aeafe6fc9a7fc9c758f3f933ec9cf1668f70, c54fc020dbf12a2f04c184e62ffd53f64a1c82f9d2bd6f4e074bed4f512148c4, c725187215cf9ba7a54f3dfca3472d986eec830065eab4d667dc271483ac5728, c9b19c9b6afb63f3e9fd8c0491fa6ca9fa6a008cd5f2b2dc3b5d7a984b9c5a7f, d52684b0f57370239beeac25d0c9ad1db6e224a96c725eb5ffbc070231f31cc8, d801e1b2c79395df00ff26f63f60e2f11d541d18d6755831ec1a6537fd3ad6d3, e10db9a98a2351837475fc89e3e7adaa69a6e4f83c78b2948ebcf5e0551446da, ea1272b8a1bc5a1fb9a1c77c9dc96e1a02a691d1d6073a1dc06d51d770b3f2e2, ed8f98dd4c607ea196fdd0edb9aafa68e7fe31f9eb736f93e9dc744954d960e5, ff79fc4bf0b39c4b8a9810b57d311527700639b661198f4d8db77d7c6d64f0c9]]","round":768,"self_vote":"Vote: [vote data: VoteData: [block id: 464a3b3f, epoch: 91, round: 768, timestamp: 1662297329730024,parent_block_id: b475372f, parent_block_round: 767, parent_timestamp: 1662297326846533], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 91, round: 767, id: b475372f, executed_state_id: 41434355, version: 0, timestamp (us): 1662297326846533, next_epoch_state: None]]]"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:473"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:36.525582Z","data":{"event":"Timeout","remote_peer":"234c957655cc9f4a5f8f0d1ca8f99baf8af3ea57740cd71fa2ab3d14fa3a4ede","round":769,"voted_nil":false}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:800"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:36.526927Z","data":{"committed_round":767,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 769 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:479:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:789:62\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:431:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:430:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:382:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:379:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:358:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: [LI 0abda983 has 7 votes [21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 394f58ff583315bf52b055d34c8635329f4426626b57da5c709a81aa6a68d83a, a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, b2243c6234285ca4ddba4927603750a0b2571d9453b1021c4cd4a0c34e045ecd, bb356acd63be6e072284d210b7e8e2eeae911ad512f1bc24631dcbc2c22a4a98, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2, e8278487a145eadfa3b07d7a711efe1606a7184045337b27259d6ab73e89aa5a] 7 timeout [21d0a3f424d63546ede086a3fb950a5d6bccd431b991374e1f037914e2e45bfe, 394f58ff583315bf52b055d34c8635329f4426626b57da5c709a81aa6a68d83a, a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, b2243c6234285ca4ddba4927603750a0b2571d9453b1021c4cd4a0c34e045ecd, bb356acd63be6e072284d210b7e8e2eeae911ad512f1bc24631dcbc2c22a4a98, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2, e8278487a145eadfa3b07d7a711efe1606a7184045337b27259d6ab73e89aa5a]]","round":769,"self_vote":"Vote: [vote data: VoteData: [block id: f6df3abe, epoch: 91, round: 769, timestamp: 1662297332449384,parent_block_id: 464a3b3f, parent_block_round: 768, parent_timestamp: 1662297329730024], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 91, round: 768, id: 464a3b3f, executed_state_id: 41434355, version: 0, timestamp (us): 1662297329730024, next_epoch_state: None]]]"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:473"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:39.587853Z","data":{"event":"Timeout","remote_peer":"b9eab269389f080a221e1811493284ff7141b542dd2bdcaf4fe3e4f4518e8f43","round":770,"voted_nil":true}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:800"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:39.588899Z","data":{"committed_round":768,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 770 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:479:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:789:62\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:431:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:430:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:382:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:379:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:358:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: [LI 3c65895c has 5 votes [394f58ff583315bf52b055d34c8635329f4426626b57da5c709a81aa6a68d83a, a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, b2243c6234285ca4ddba4927603750a0b2571d9453b1021c4cd4a0c34e045ecd, bb356acd63be6e072284d210b7e8e2eeae911ad512f1bc24631dcbc2c22a4a98, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2] 5 timeout [394f58ff583315bf52b055d34c8635329f4426626b57da5c709a81aa6a68d83a, a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, b2243c6234285ca4ddba4927603750a0b2571d9453b1021c4cd4a0c34e045ecd, bb356acd63be6e072284d210b7e8e2eeae911ad512f1bc24631dcbc2c22a4a98, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2]]","round":770,"self_vote":"Vote: [vote data: VoteData: [block id: 0864c3c2, epoch: 91, round: 770, timestamp: 1662297332449384,parent_block_id: f6df3abe, parent_block_round: 769, parent_timestamp: 1662297332449384], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 91, round: 769, id: f6df3abe, executed_state_id: 41434355, version: 0, timestamp (us): 1662297332449384, next_epoch_state: None]]]"}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:473"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:42.620774Z","data":{"event":"Timeout","remote_peer":"b10dc0857cfb72773f71f11cb2a466eba3e1947d1b8cd1bf2b3124e936fb5468","round":771,"voted_nil":false}}
{"level":"ERROR","source":{"package":"consensus","file":"consensus/src/round_manager.rs:800"},"thread_name":"consensus","hostname":"f9cf7f20b1ff","timestamp":"2022-09-04T13:15:42.621554Z","data":{"committed_round":768,"error":"from peer 21b4ef9b8512ec0b801e78942580f04e10d3fc44148d0d6acbba0dd21cf1ce42\n\nCaused by:\n    Round 771 timeout, broadcast to all peers\n\nStack backtrace:\n   0: consensus::round_manager::RoundManager::process_local_timeout::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:479:9\n      <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n   1: consensus::round_manager::RoundManager::start::{{closure}}\n             at ./aptos/consensus/src/round_manager.rs:789:62\n   2: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/future/mod.rs:91:19\n      tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n   3: tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n   4: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::LocalNotified<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:376:9\n      tokio::runtime::thread_pool::worker::Context::run_task::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:431:13\n      tokio::coop::with_budget::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:102:9\n      std::thread::local::LocalKey<T>::try_with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:445:16\n      std::thread::local::LocalKey<T>::with\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/local.rs:421:9\n   5: tokio::coop::with_budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:95:5\n      tokio::coop::budget\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/coop.rs:72:5\n      tokio::runtime::thread_pool::worker::Context::run_task\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:430:9\n   6: tokio::runtime::thread_pool::worker::Context::run\n   7: tokio::runtime::thread_pool::worker::run::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:382:17\n      tokio::macros::scoped_tls::ScopedKey<T>::set\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/macros/scoped_tls.rs:61:9\n   8: tokio::runtime::thread_pool::worker::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:379:5\n   9: tokio::runtime::thread_pool::worker::Launch::launch::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/thread_pool/worker.rs:358:45\n      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/task.rs:42:21\n  10: tokio::runtime::task::core::CoreStage<T>::poll::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:165:17\n      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/loom/std/unsafe_cell.rs:14:9\n      tokio::runtime::task::core::CoreStage<T>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/core.rs:155:13\n      tokio::runtime::task::harness::poll_future::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:470:19\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      tokio::runtime::task::harness::poll_future\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:458:18\n      tokio::runtime::task::harness::Harness<T,S>::poll_inner\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:104:27\n      tokio::runtime::task::harness::Harness<T,S>::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/harness.rs:57:15\n  11: tokio::runtime::task::raw::RawTask::poll\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/raw.rs:84:18\n      tokio::runtime::task::UnownedTask<S>::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/task/mod.rs:413:9\n      tokio::runtime::blocking::pool::Task::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:91:9\n      tokio::runtime::blocking::pool::Inner::run\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:325:17\n  12: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}\n             at ./usr/local/cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.18.2/src/runtime/blocking/pool.rs:300:13\n      std::sys_common::backtrace::__rust_begin_short_backtrace\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys_common/backtrace.rs:122:18\n  13: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:505:17\n      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/panic/unwind_safe.rs:271:9\n      std::panicking::try::do_call\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:492:40\n      std::panicking::try\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panicking.rs:456:19\n      std::panic::catch_unwind\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/panic.rs:137:14\n      std::thread::Builder::spawn_unchecked_::{{closure}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/thread/mod.rs:504:30\n      core::ops::function::FnOnce::call_once{{vtable.shim}}\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/core/src/ops/function.rs:248:5\n  14: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/alloc/src/boxed.rs:1951:9\n      std::sys::unix::thread::Thread::new::thread_start\n             at ./rustc/4b91a6ea7258a947e59c6522cd5898e7c0a6a88f/library/std/src/sys/unix/thread.rs:108:17\n  15: start_thread\n             at ./build/glibc-fWwxX8/glibc-2.28/nptl/pthread_create.c:486:8\n  16: clone\n             at ./build/glibc-fWwxX8/glibc-2.28/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:95","kind":"InternalError","pending_votes":"PendingVotes: [LI 246dccb1 has 1 votes [bb9451dce59fc99144c23c37ce0adc50fc93c2b50ab316fb4aeaa83647244bc9] LI 8f8fbe53 has 3 votes [a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42] 4 timeout [a9e136449be1ace0702284b34da3ce1742231cb844c1848c2ea95e2fc9b18ecd, bb9451dce59fc99144c23c37ce0adc50fc93c2b50ab316fb4aeaa83647244bc9, d6e173c7029f5af061f458062cfdb5a57f78b528675c988df9fa11b7c72130e2, ed06553d095aec80bbe77c3d86004b2bfa3f14e85969fe769fe5b7928af9db42]]","round":771,"self_vote":"Vote: [vote data: VoteData: [block id: a3092c9d, epoch: 91, round: 771, timestamp: 1662297339049634,parent_block_id: f6df3abe, parent_block_round: 769, parent_timestamp: 1662297332449384], author: 21b4ef9b, is_timeout: true, LedgerInfo: [commit_info: BlockInfo: [epoch: 0, round: 0, id: 00000000, executed_state_id: 00000000, version: 0, timestamp (us): 0, next_epoch_state: None]]]"}}

[jovar95]

@JoshLind @sherry-x added more error logs

[jovar95]

Issue is solved, thanks!