search

linera-io / linera-protocol

Main repository for the Linera protocol
Apache License 2.0
112 stars 94 forks source link

Make the reconfiguration end-to-end test work with normal weight for the 6th validator. #2212

Open afck opened 2 days ago

afck commented 2 days ago

The test currently fails if the weight is the same as for the other validators.

afck commented 2 days ago

This is the failure I'm seeing locally:

2024-07-03T15:02:48.534210Z  INFO linera: Starting operations to change validator set
2024-07-03T15:02:48.553311Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:48.625737Z  INFO linera::client_context: Saved user chain states
2024-07-03T15:02:48.625761Z  INFO linera: Subscribed 0 chains to new committees
2024-07-03T15:02:48.864813Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:49.100686Z  INFO linera::client_context: Saved user chain states
2024-07-03T15:02:49.100722Z  INFO linera: Finalizing committee:
Certificate { value: HashedCertificateValue { value: ConfirmedBlock { executed_block: ExecutedBlock { block: Block { chain_id: e476187f6ddfeb9d, epoch: Epoch(3), incoming_messages: [], operations: [System(Admin(CreateCommittee { epoch: Epoch(4), committee: Committee { validators: {ValidatorName(02a9b7afb957f4d5): ValidatorState { network_address: "Tcp:127.0.0.1:9200", votes: 100 }, ValidatorName(67a478657f61d97a): ValidatorState { network_address: "tcp:127.0.0.1:9500", votes: 100 }, ValidatorName(95fe9f76cdcd8e36): ValidatorState { network_address: "Tcp:127.0.0.1:9300", votes: 100 }, ValidatorName(f83b11500e9874d2): ValidatorState { network_address: "Tcp:127.0.0.1:9100", votes: 100 }}, total_votes: 400, quorum_threshold: 267, validity_threshold: 134, policy: ResourceControlPolicy { block: Amount(1000000000000000), fuel_unit: Amount(10000000000), read_operation: Amount(10000000000000), write_operation: Amount(20000000000000), byte_read: Amount(10000000000), byte_written: Amount(100000000000), byte_stored: Amount(10000000000), operation: Amount(10000000000000), operation_byte: Amount(10000000000), message: Amount(10000000000000), message_byte: Amount(100000000000), maximum_bytes_read_per_block: 100000000, maximum_bytes_written_per_block: 10000000 } } }))], height: BlockHeight(8), timestamp: Timestamp(1720018968627262), authenticated_signer: Some(Owner(7136460f0c87ae46)), previous_block_hash: Some(05b5d742ddd55784) }, outcome: BlockExecutionOutcome { messages: [[OutgoingMessage { destination: Subscribers(ChannelName([0])), authenticated_signer: None, grant: Amount(0), refund_grant_to: None, kind: Protected, message: System(SetCommittees { epoch: Epoch(4), committees: {Epoch(3): Committee { validators: {ValidatorName(02a9b7afb957f4d5): ValidatorState { network_address: "Tcp:127.0.0.1:9200", votes: 100 }, ValidatorName(67a478657f61d97a): ValidatorState { network_address: "tcp:127.0.0.1:9500", votes: 100 }, ValidatorName(8f00a9a78f770417): ValidatorState { network_address: "Tcp:127.0.0.1:9000", votes: 100 }, ValidatorName(95fe9f76cdcd8e36): ValidatorState { network_address: "Tcp:127.0.0.1:9300", votes: 100 }, ValidatorName(f83b11500e9874d2): ValidatorState { network_address: "Tcp:127.0.0.1:9100", votes: 100 }}, total_votes: 500, quorum_threshold: 334, validity_threshold: 167, policy: ResourceControlPolicy { block: Amount(1000000000000000), fuel_unit: Amount(10000000000), read_operation: Amount(10000000000000), write_operation: Amount(20000000000000), byte_read: Amount(10000000000), byte_written: Amount(100000000000), byte_stored: Amount(10000000000), operation: Amount(10000000000000), operation_byte: Amount(10000000000), message: Amount(10000000000000), message_byte: Amount(100000000000), maximum_bytes_read_per_block: 100000000, maximum_bytes_written_per_block: 10000000 } }, Epoch(4): Committee { validators: {ValidatorName(02a9b7afb957f4d5): ValidatorState { network_address: "Tcp:127.0.0.1:9200", votes: 100 }, ValidatorName(67a478657f61d97a): ValidatorState { network_address: "tcp:127.0.0.1:9500", votes: 100 }, ValidatorName(95fe9f76cdcd8e36): ValidatorState { network_address: "Tcp:127.0.0.1:9300", votes: 100 }, ValidatorName(f83b11500e9874d2): ValidatorState { network_address: "Tcp:127.0.0.1:9100", votes: 100 }}, total_votes: 400, quorum_threshold: 267, validity_threshold: 134, policy: ResourceControlPolicy { block: Amount(1000000000000000), fuel_unit: Amount(10000000000), read_operation: Amount(10000000000000), write_operation: Amount(20000000000000), byte_read: Amount(10000000000), byte_written: Amount(100000000000), byte_stored: Amount(10000000000), operation: Amount(10000000000000), operation_byte: Amount(10000000000), message: Amount(10000000000000), message_byte: Amount(100000000000), maximum_bytes_read_per_block: 100000000, maximum_bytes_written_per_block: 10000000 } }} }) }]], state_hash: 03fcb929dd453418, oracle_records: [OracleRecord { responses: [] }] } } }, hash: 435be8aa2bd121cc }, round: Fast, signatures: [(ValidatorName(02a9b7afb957f4d5), 331cc130b68d7020), (ValidatorName(67a478657f61d97a), ab276b3f3324aed7), (ValidatorName(8f00a9a78f770417), d76b39d5173e1e56), (ValidatorName(95fe9f76cdcd8e36), 7b08cc7a68b385ee), (ValidatorName(f83b11500e9874d2), 9f89c68c05de33df)] }
2024-07-03T15:02:49.450640Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:49.459458Z  INFO linera::client_context: Saved user chain states
2024-07-03T15:02:49.460451Z  INFO linera::client_context: Saved user chain states
2024-07-03T15:02:49.460499Z  INFO linera: Operations confirmed after 926 ms
2024-07-03T15:02:49.529387Z  INFO linera: Processing the inbox of chain 1db1936dad0717597a7743a8353c9c0191c14c3a129b258e9743aec2b4f05d03
2024-07-03T15:02:49.538944Z  WARN linera_core::local_node: Ignoring error from validator: Network error while querying service: Connection refused (os error 111)
2024-07-03T15:02:49.895307Z ERROR linera_core::updater: Failed to query validator about missing blocks name=ValidatorName(8f00a9a78f770417) chain_id=1db1936dad071759 error=Network error while querying service: Connection refused (os error 111)
2024-07-03T15:02:50.026220Z ERROR linera_core::updater: Failed to query validator about missing blocks name=ValidatorName(8f00a9a78f770417) chain_id=1db1936dad071759 error=Network error while querying service: Connection refused (os error 111)
2024-07-03T15:02:50.141818Z  INFO linera::client_context: Saved user chain states
2024-07-03T15:02:50.141840Z  INFO linera: Processed incoming messages with 1 blocks in 612 ms
2024-07-03T15:02:50.200921Z  INFO linera: Starting operations to change validator set
2024-07-03T15:02:50.242457Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:50.276497Z  INFO linera::client_context: Saved user chain states
2024-07-03T15:02:50.276534Z  INFO linera: Subscribed 0 chains to new committees
2024-07-03T15:02:50.688467Z  INFO linera::client_context: Saved user chain states
2024-07-03T15:02:50.688508Z  INFO linera: Finalizing committee:
Certificate { value: HashedCertificateValue { value: ConfirmedBlock { executed_block: ExecutedBlock { block: Block { chain_id: e476187f6ddfeb9d, epoch: Epoch(4), incoming_messages: [], operations: [System(Admin(CreateCommittee { epoch: Epoch(5), committee: Committee { validators: {ValidatorName(02a9b7afb957f4d5): ValidatorState { network_address: "Tcp:127.0.0.1:9200", votes: 100 }, ValidatorName(67a478657f61d97a): ValidatorState { network_address: "tcp:127.0.0.1:9500", votes: 100 }, ValidatorName(95fe9f76cdcd8e36): ValidatorState { network_address: "Tcp:127.0.0.1:9300", votes: 100 }}, total_votes: 300, quorum_threshold: 201, validity_threshold: 100, policy: ResourceControlPolicy { block: Amount(1000000000000000), fuel_unit: Amount(10000000000), read_operation: Amount(10000000000000), write_operation: Amount(20000000000000), byte_read: Amount(10000000000), byte_written: Amount(100000000000), byte_stored: Amount(10000000000), operation: Amount(10000000000000), operation_byte: Amount(10000000000), message: Amount(10000000000000), message_byte: Amount(100000000000), maximum_bytes_read_per_block: 100000000, maximum_bytes_written_per_block: 10000000 } } }))], height: BlockHeight(10), timestamp: Timestamp(1720018970278025), authenticated_signer: Some(Owner(7136460f0c87ae46)), previous_block_hash: Some(40cc4bd4f69687f5) }, outcome: BlockExecutionOutcome { messages: [[OutgoingMessage { destination: Subscribers(ChannelName([0])), authenticated_signer: None, grant: Amount(0), refund_grant_to: None, kind: Protected, message: System(SetCommittees { epoch: Epoch(5), committees: {Epoch(4): Committee { validators: {ValidatorName(02a9b7afb957f4d5): ValidatorState { network_address: "Tcp:127.0.0.1:9200", votes: 100 }, ValidatorName(67a478657f61d97a): ValidatorState { network_address: "tcp:127.0.0.1:9500", votes: 100 }, ValidatorName(95fe9f76cdcd8e36): ValidatorState { network_address: "Tcp:127.0.0.1:9300", votes: 100 }, ValidatorName(f83b11500e9874d2): ValidatorState { network_address: "Tcp:127.0.0.1:9100", votes: 100 }}, total_votes: 400, quorum_threshold: 267, validity_threshold: 134, policy: ResourceControlPolicy { block: Amount(1000000000000000), fuel_unit: Amount(10000000000), read_operation: Amount(10000000000000), write_operation: Amount(20000000000000), byte_read: Amount(10000000000), byte_written: Amount(100000000000), byte_stored: Amount(10000000000), operation: Amount(10000000000000), operation_byte: Amount(10000000000), message: Amount(10000000000000), message_byte: Amount(100000000000), maximum_bytes_read_per_block: 100000000, maximum_bytes_written_per_block: 10000000 } }, Epoch(5): Committee { validators: {ValidatorName(02a9b7afb957f4d5): ValidatorState { network_address: "Tcp:127.0.0.1:9200", votes: 100 }, ValidatorName(67a478657f61d97a): ValidatorState { network_address: "tcp:127.0.0.1:9500", votes: 100 }, ValidatorName(95fe9f76cdcd8e36): ValidatorState { network_address: "Tcp:127.0.0.1:9300", votes: 100 }}, total_votes: 300, quorum_threshold: 201, validity_threshold: 100, policy: ResourceControlPolicy { block: Amount(1000000000000000), fuel_unit: Amount(10000000000), read_operation: Amount(10000000000000), write_operation: Amount(20000000000000), byte_read: Amount(10000000000), byte_written: Amount(100000000000), byte_stored: Amount(10000000000), operation: Amount(10000000000000), operation_byte: Amount(10000000000), message: Amount(10000000000000), message_byte: Amount(100000000000), maximum_bytes_read_per_block: 100000000, maximum_bytes_written_per_block: 10000000 } }} }) }]], state_hash: 478c64f60f80e305, oracle_records: [OracleRecord { responses: [] }] } } }, hash: 39a6dff730787cac }, round: Fast, signatures: [(ValidatorName(02a9b7afb957f4d5), f8bd83946872cee9), (ValidatorName(67a478657f61d97a), b4a1fb16f760df33), (ValidatorName(95fe9f76cdcd8e36), 24cbb2f7071f6a06), (ValidatorName(f83b11500e9874d2), 0bd0db5137467386)] }
2024-07-03T15:02:50.756744Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:51.720902Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:52.159195Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:52.866219Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:53.292309Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:54.473961Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:02:54.889820Z ERROR handle_message{chain_id=Some(e476187f6ddfeb9d)}: linera_proxy: Failed to proxy message error=deadline has elapsed
2024-07-03T15:03:00.893386Z  INFO linera::client_context: Saved user chain states
Error: Failed to finalize committee

Caused by:
    Failed to communicate with a quorum of validators: Network error while querying service: timed out

Stack backtrace:
   0: anyhow::error::<impl core::convert::From<E> for anyhow::Error>::from
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.80/src/backtrace.rs:27:14
   1: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/result.rs:1964:27
   2: linera::client_context::ClientContext<S>::apply_client_command::{{closure}}
             at /mnt/data/git/linera-protocol/linera-service/src/linera/client_context.rs:378:46
   3: <linera::Job as linera_service::storage::Runnable>::run::{{closure}}
             at /mnt/data/git/linera-protocol/linera-service/src/linera/main.rs:628:22
   4: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
   5: linera_service::storage::run_with_storage::{{closure}}
             at /mnt/data/git/linera-protocol/linera-service/src/storage.rs:534:30
   6: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
   7: linera::client_options::ClientOptions::run_command_with_storage::{{closure}}
             at /mnt/data/git/linera-protocol/linera-service/src/linera/client_options.rs:144:10
   8: linera::run::{{closure}}
             at /mnt/data/git/linera-protocol/linera-service/src/linera/main.rs:1490:49
   9: tokio::runtime::park::CachedParkThread::block_on::{{closure}}
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/park.rs:281:63
  10: tokio::runtime::coop::with_budget
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/coop.rs:107:5
  11: tokio::runtime::coop::budget
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/coop.rs:73:5
  12: tokio::runtime::park::CachedParkThread::block_on
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/park.rs:281:31
  13: tokio::runtime::context::blocking::BlockingRegionGuard::block_on
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/context/blocking.rs:66:9
  14: tokio::runtime::scheduler::multi_thread::MultiThread::block_on::{{closure}}
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/multi_thread/mod.rs:87:13
  15: tokio::runtime::context::runtime::enter_runtime
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/context/runtime.rs:65:16
  16: tokio::runtime::scheduler::multi_thread::MultiThread::block_on
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/multi_thread/mod.rs:86:9
  17: tokio::runtime::runtime::Runtime::block_on
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/runtime.rs:350:45
  18: linera::main
             at /mnt/data/git/linera-protocol/linera-service/src/linera/main.rs:1224:5
  19: core::ops::function::FnOnce::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:250:5
  20: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:155:18
  21: std::rt::lang_start::{{closure}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/rt.rs:159:18
  22: core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:284:13
  23: std::panicking::try::do_call
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  24: std::panicking::try
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  25: std::panic::catch_unwind
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  26: std::rt::lang_start_internal::{{closure}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/rt.rs:141:48
  27: std::panicking::try::do_call
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  28: std::panicking::try
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  29: std::panic::catch_unwind
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  30: std::rt::lang_start_internal
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/rt.rs:141:20
  31: std::rt::lang_start
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/rt.rs:158:17
  32: main
  33: __libc_start_call_main
             at ./csu/../sysdeps/nptl/libc_start_call_main.h:58:16
  34: __libc_start_main_impl
             at ./csu/../csu/libc-start.c:360:3
  35: _start
test test_end_to_end_reconfiguration::storage_service_tcp ... FAILED

failures:

---- test_end_to_end_reconfiguration::storage_service_tcp stdout ----
Error: While executing Command { std: cd "/tmp/.tmpxj379N" && "/mnt/data/git/linera-protocol/target/debug/linera" "--wallet" "wallet_0.json" "--storage" "rocksdb:/tmp/.tmpxj379N/client_0.db" "--max-pending-messages" "10000" "--send-timeout-ms" "10000" "--recv-timeout-ms" "10000" "--wait-for-outgoing-messages" "remove-validator" "--name" "f83b11500e9874d20d6cfe020def3f004949cb1653ae045755b7cc59f7785200", kill_on_drop: true }: got non-zero error code exit status: 1

Stack backtrace:
   0: anyhow::error::<impl anyhow::Error>::msg
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/anyhow-1.0.80/src/backtrace.rs:27:14
   1: <tokio::process::Command as linera_base::command::CommandExt>::spawn_and_wait_for_stdout::{{closure}}
             at /mnt/data/git/linera-protocol/linera-base/src/command.rs:164:9
   2: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
   3: linera_service::cli_wrappers::wallet::ClientWrapper::remove_validator::{{closure}}
             at ./src/cli_wrappers/wallet.rs:734:14
   4: end_to_end_tests::test_end_to_end_reconfiguration::{{closure}}
             at ./tests/end_to_end_tests.rs:2510:39
   5: end_to_end_tests::test_end_to_end_reconfiguration::storage_service_tcp::{{closure}}
             at ./tests/end_to_end_tests.rs:2441:41
   6: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
   7: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
   8: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}}
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:659:57
   9: tokio::runtime::coop::with_budget
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/coop.rs:107:5
  10: tokio::runtime::coop::budget
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/coop.rs:73:5
  11: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:659:25
  12: tokio::runtime::scheduler::current_thread::Context::enter
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:404:19
  13: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:658:36
  14: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:737:68
  15: tokio::runtime::context::scoped::Scoped<T>::set
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/context/scoped.rs:40:9
  16: tokio::runtime::context::set_scheduler::{{closure}}
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/context.rs:176:26
  17: std::thread::local::LocalKey<T>::try_with
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:286:12
  18: std::thread::local::LocalKey<T>::with
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/local.rs:262:9
  19: tokio::runtime::context::set_scheduler
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/context.rs:176:9
  20: tokio::runtime::scheduler::current_thread::CoreGuard::enter
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:737:27
  21: tokio::runtime::scheduler::current_thread::CoreGuard::block_on
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:646:19
  22: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}}
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:175:28
  23: tokio::runtime::context::runtime::enter_runtime
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/context/runtime.rs:65:16
  24: tokio::runtime::scheduler::current_thread::CurrentThread::block_on
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/scheduler/current_thread/mod.rs:167:9
  25: tokio::runtime::runtime::Runtime::block_on
             at /home/andreas/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.36.0/src/runtime/runtime.rs:348:47
  26: end_to_end_tests::test_end_to_end_reconfiguration::storage_service_tcp
             at ./tests/end_to_end_tests.rs:2441:41
  27: end_to_end_tests::test_end_to_end_reconfiguration::storage_service_tcp::{{closure}}
             at ./tests/end_to_end_tests.rs:2448:69
  28: core::ops::function::FnOnce::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:250:5
  29: core::ops::function::FnOnce::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:250:5
  30: test::__rust_begin_short_backtrace
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/test/src/lib.rs:623:18
  31: test::run_test_in_process::{{closure}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/test/src/lib.rs:646:60
  32: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
  33: std::panicking::try::do_call
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  34: std::panicking::try
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  35: std::panic::catch_unwind
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  36: test::run_test_in_process
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/test/src/lib.rs:646:27
  37: test::run_test::{{closure}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/test/src/lib.rs:569:43
  38: test::run_test::{{closure}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/test/src/lib.rs:597:41
  39: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys_common/backtrace.rs:155:18
  40: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:542:17
  41: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panic/unwind_safe.rs:272:9
  42: std::panicking::try::do_call
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:559:40
  43: std::panicking::try
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:523:19
  44: std::panic::catch_unwind
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panic.rs:149:14
  45: std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/thread/mod.rs:541:30
  46: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/ops/function.rs:250:5
  47: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
  48: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/alloc/src/boxed.rs:2022:9
  49: std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/sys/pal/unix/thread.rs:108:17
  50: start_thread
             at ./nptl/pthread_create.c:447:8
  51: __GI___clone3
             at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:78

failures:
    test_end_to_end_reconfiguration::storage_service_tcp