Ensure database can't be left in an inconsistent state

[JamesHinshelwood]

We insert a block, then its transaction receipts. Sometimes that can leave the DB in an inconsistent state. Either use a transaction for both or move the block insert after.

@theo-zil to add details.

[JamesHinshelwood]

When restarting one of the proto-testnet nodes we observed another DB consistency problem:

thread 'tokio-runtime-worker' panicked at zilliqa/src/consensus.rs:1998:52:
called `Result::unwrap()` on an `Err` value: database error: trie error: missing node

Caused by:
    trie error: missing node

Stack backtrace:
   0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14
   1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/result.rs:1964:27
   2: zilliqa::exec::<impl zilliqa::state::State>::apply_transaction_evm
             at ./zilliqa/zilliqa/src/exec.rs:457:17
   3: zilliqa::exec::<impl zilliqa::state::State>::call_contract
             at ./zilliqa/zilliqa/src/exec.rs:945:51
   4: zilliqa::exec::<impl zilliqa::state::State>::leader_raw
             at ./zilliqa/zilliqa/src/exec.rs:657:22
   5: zilliqa::consensus::Consensus::leader_at_block_raw
             at ./zilliqa/zilliqa/src/consensus.rs:1998:26
   6: zilliqa::consensus::Consensus::leader_at_block
             at ./zilliqa/zilliqa/src/consensus.rs:1982:9
   7: zilliqa::consensus::Consensus::check_block
             at ./zilliqa/zilliqa/src/consensus.rs:1614:24
   8: zilliqa::consensus::Consensus::proposal
             at ./zilliqa/zilliqa/src/consensus.rs:498:15
   9: zilliqa::consensus::Consensus::receive_block
             at ./zilliqa/zilliqa/src/consensus.rs:1748:22
  10: zilliqa::node::Node::handle_block_response
             at ./zilliqa/zilliqa/src/node.rs:862:28
  11: zilliqa::node::Node::handle_network_message
             at ./zilliqa/zilliqa/src/node.rs:217:17
  12: zilliqa::node_launcher::NodeLauncher::start_shard_node::{{closure}}
             at ./zilliqa/zilliqa/src/node_launcher.rs:147:21
  13: zilliqa::p2p_node::P2pNode::add_shard_node::{{closure}}::{{closure}}
             at ./zilliqa/zilliqa/src/p2p_node.rs:188:57

[JamesHinshelwood]

Saw something similar again many times in the devnet:

called `Result::unwrap()` on an `Err` value: database error: trie error: missing node

Caused by:
    trie error: missing node

Stack backtrace:
   0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.86/src/backtrace.rs:27:14
   1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/result.rs:1964:27
   2: zilliqa::exec::<impl zilliqa::state::State>::apply_transaction_evm
             at ./zilliqa/zilliqa/src/exec.rs:461:17
   3: zilliqa::exec::<impl zilliqa::state::State>::call_contract
             at ./zilliqa/zilliqa/src/exec.rs:949:51
   4: zilliqa::exec::<impl zilliqa::state::State>::leader_raw
             at ./zilliqa/zilliqa/src/exec.rs:661:22
   5: zilliqa::consensus::Consensus::leader_at_block_raw
             at ./zilliqa/zilliqa/src/consensus.rs:2119:26
   6: zilliqa::consensus::Consensus::leader_at_block
             at ./zilliqa/zilliqa/src/consensus.rs:2103:9
   7: zilliqa::consensus::Consensus::check_block
             at ./zilliqa/zilliqa/src/consensus.rs:1731:24
   8: zilliqa::consensus::Consensus::proposal
             at ./zilliqa/zilliqa/src/consensus.rs:506:15
   9: zilliqa::consensus::Consensus::receive_block
             at ./zilliqa/zilliqa/src/consensus.rs:1865:22
  10: zilliqa::node::Node::handle_block_response
             at ./zilliqa/zilliqa/src/node.rs:906:28
  11: zilliqa::node::Node::handle_response
             at ./zilliqa/zilliqa/src/node.rs:288:50
  12: zilliqa::node_launcher::NodeLauncher::start_shard_node::{{closure}}
             at ./zilliqa/zilliqa/src/node_launcher.rs:185:21
  13: zilliqa::p2p_node::P2pNode::add_shard_node::{{closure}}::{{closure}}
             at ./zilliqa/zilliqa/src/p2p_node.rs:198:57
  14: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:328:17
  15: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/loom/std/unsafe_cell.rs:16:9
  16: tokio::runtime::task::core::Core<T,S>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:317:30
  17: tokio::runtime::task::harness::poll_future::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:485:19
  18: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
  19: std::panicking::try::do_call
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  20: std::panicking::try
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  21: std::panic::catch_unwind
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  22: tokio::runtime::task::harness::poll_future
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:473:18
  23: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:208:27
  24: tokio::runtime::task::harness::Harness<T,S>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:153:15
  25: tokio::runtime::task::raw::RawTask::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/raw.rs:201:18
  26: tokio::runtime::task::LocalNotified<S>::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/mod.rs:427:9
  27: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:594:18
  28: tokio::runtime::coop::with_budget
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/coop.rs:107:5
  29: tokio::runtime::coop::budget
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/coop.rs:73:5
  30: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:593:9
  31: tokio::runtime::scheduler::multi_thread::worker::Context::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:544:24
  32: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:509:21
  33: tokio::runtime::context::scoped::Scoped<T>::set
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context/scoped.rs:40:9
  34: tokio::runtime::context::set_scheduler::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context.rs:180:26
  35: std::thread::local::LocalKey<T>::try_with
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:286:12
  36: std::thread::local::LocalKey<T>::with
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:262:9
  37: tokio::runtime::context::set_scheduler
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context.rs:180:17
  38: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:504:9
  39: tokio::runtime::context::runtime::enter_runtime
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context/runtime.rs:65:16
  40: tokio::runtime::scheduler::multi_thread::worker::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:496:5
  41: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:462:45
  42: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/task.rs:42:21
  43: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:328:17
  44: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/loom/std/unsafe_cell.rs:16:9
  45: tokio::runtime::task::core::Core<T,S>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:317:30
  46: tokio::runtime::task::harness::poll_future::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:485:19
  47: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
  48: std::panicking::try::do_call
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  49: std::panicking::try
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  50: std::panic::catch_unwind
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  51: tokio::runtime::task::harness::poll_future
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:473:18
  52: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:208:27
  53: tokio::runtime::task::harness::Harness<T,S>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:153:15
  54: tokio::runtime::task::raw::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/raw.rs:271:5
  55: tokio::runtime::task::raw::RawTask::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/raw.rs:201:18
  56: tokio::runtime::task::UnownedTask<S>::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/mod.rs:464:9
  57: tokio::runtime::blocking::pool::Task::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:160:9
  58: tokio::runtime::blocking::pool::Inner::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:518:17
  59: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:476:13
  60: std::sys_common::backtrace::__rust_begin_short_backtrace
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:155:18
  61: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:542:17
  62: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
  63: std::panicking::try::do_call
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  64: std::panicking::try
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  65: std::panic::catch_unwind
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  66: std::thread::Builder::spawn_unchecked_::{{closure}}
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:541:30
  67: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:250:5
  68: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
  69: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
  70: std::sys::pal::unix::thread::Thread::new::thread_start
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys/pal/unix/thread.rs:108:17
  71: <unknown>
  72: __clone",

  0: zilliqa::main::{{closure}}::{{closure}}
             at ./zilliqa/zilliqa/src/bin/zilliqa.rs:50:25
   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2036:9
   2: std::panicking::rust_panic_with_hook
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:799:13
   3: std::panicking::begin_panic_handler::{{closure}}
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:664:13
   4: std::sys_common::backtrace::__rust_end_short_backtrace
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:171:18
   5: rust_begin_unwind
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:652:5
   6: core::panicking::panic_fmt
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panicking.rs:72:14
   7: core::result::unwrap_failed
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/result.rs:1654:5
   8: core::result::Result<T,E>::unwrap
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/result.rs:1077:23
   9: zilliqa::consensus::Consensus::leader_at_block_raw
             at ./zilliqa/zilliqa/src/consensus.rs:2119:26
  10: zilliqa::consensus::Consensus::leader_at_block
             at ./zilliqa/zilliqa/src/consensus.rs:2103:9
  11: zilliqa::consensus::Consensus::check_block
             at ./zilliqa/zilliqa/src/consensus.rs:1731:24
  12: zilliqa::consensus::Consensus::proposal
             at ./zilliqa/zilliqa/src/consensus.rs:506:15
  13: zilliqa::consensus::Consensus::receive_block
             at ./zilliqa/zilliqa/src/consensus.rs:1865:22
  14: zilliqa::node::Node::handle_block_response
             at ./zilliqa/zilliqa/src/node.rs:906:28
  15: zilliqa::node::Node::handle_response
             at ./zilliqa/zilliqa/src/node.rs:288:50
  16: zilliqa::node_launcher::NodeLauncher::start_shard_node::{{closure}}
             at ./zilliqa/zilliqa/src/node_launcher.rs:185:21
  17: zilliqa::p2p_node::P2pNode::add_shard_node::{{closure}}::{{closure}}
             at ./zilliqa/zilliqa/src/p2p_node.rs:198:57
  18: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:328:17
  19: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/loom/std/unsafe_cell.rs:16:9
  20: tokio::runtime::task::core::Core<T,S>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:317:30
  21: tokio::runtime::task::harness::poll_future::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:485:19
  22: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
  23: std::panicking::try::do_call
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  24: std::panicking::try
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  25: std::panic::catch_unwind
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  26: tokio::runtime::task::harness::poll_future
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:473:18
  27: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:208:27
  28: tokio::runtime::task::harness::Harness<T,S>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:153:15
  29: tokio::runtime::task::raw::RawTask::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/raw.rs:201:18
  30: tokio::runtime::task::LocalNotified<S>::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/mod.rs:427:9
  31: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:594:18
  32: tokio::runtime::coop::with_budget
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/coop.rs:107:5
  33: tokio::runtime::coop::budget
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/coop.rs:73:5
  34: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:593:9
  35: tokio::runtime::scheduler::multi_thread::worker::Context::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:544:24
  36: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:509:21
  37: tokio::runtime::context::scoped::Scoped<T>::set
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context/scoped.rs:40:9
  38: tokio::runtime::context::set_scheduler::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context.rs:180:26
  39: std::thread::local::LocalKey<T>::try_with
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:286:12
  40: std::thread::local::LocalKey<T>::with
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:262:9
  41: tokio::runtime::context::set_scheduler
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context.rs:180:17
  42: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:504:9
  43: tokio::runtime::context::runtime::enter_runtime
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context/runtime.rs:65:16
  44: tokio::runtime::scheduler::multi_thread::worker::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:496:5
  45: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:462:45
  46: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/task.rs:42:21
  47: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:328:17
  48: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/loom/std/unsafe_cell.rs:16:9
  49: tokio::runtime::task::core::Core<T,S>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:317:30
  50: tokio::runtime::task::harness::poll_future::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:485:19
  51: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
  52: std::panicking::try::do_call
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  53: std::panicking::try
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  54: std::panic::catch_unwind
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  55: tokio::runtime::task::harness::poll_future
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:473:18
  56: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:208:27
  57: tokio::runtime::task::harness::Harness<T,S>::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:153:15
  58: tokio::runtime::task::raw::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/raw.rs:271:5
  59: tokio::runtime::task::raw::RawTask::poll
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/raw.rs:201:18
  60: tokio::runtime::task::UnownedTask<S>::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/mod.rs:464:9
  61: tokio::runtime::blocking::pool::Task::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:160:9
  62: tokio::runtime::blocking::pool::Inner::run
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:518:17
  63: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
             at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:476:13
  64: std::sys_common::backtrace::__rust_begin_short_backtrace
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:155:18
  65: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:542:17
  66: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
  67: std::panicking::try::do_call
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  68: std::panicking::try
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  69: std::panic::catch_unwind
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  70: std::thread::Builder::spawn_unchecked_::{{closure}}
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:541:30
  71: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:250:5
  72: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
  73: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
  74: std::sys::pal::unix::thread::Thread::new::thread_start
             at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys/pal/unix/thread.rs:108:17
  75: <unknown>
  76: __clone