xline-kv / Xline

A geo-distributed KV store for metadata management
https://xline.cloud
Apache License 2.0
589 stars 74 forks source link

[Bug]: Log would panic sometimes #968

Open bsbds opened 2 weeks ago

bsbds commented 2 weeks ago

Description about the bug

The get_prev_entry_info in the log submodule would panic sometimes in tests. See the log output for details.

Version

0.1.0

Relevant log output

SIGABRT [   4.392s] xline::it cluster_test::xline_remove_node
--- STDOUT:              xline::it cluster_test::xline_remove_node ---
running 1 test
test panic! 
@info:
panicked at /__w/Xline/Xline/crates/curp/src/server/raw_curp/log.rs:497:17:
internal error: entered unreachable code: get log[2] when base_index is 0
@stackTrace:
   0: __covrec_3152F8625DB7F2EF
   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/alloc/src/boxed.rs:2021:9
   2: std::panicking::rust_panic_with_hook
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panicking.rs:735:13
   3: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panicking.rs:609:13
   4: std::sys_common::backtrace::__rust_end_short_backtrace
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/sys_common/backtrace.rs:170:18
   5: rust_begin_unwind
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panicking.rs:597:5
   6: core::panicking::panic_fmt
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/panicking.rs:72:14
   7: <curp::server::raw_curp::log::Log<xlineapi::command::Command>>::get_prev_entry_info::{closure#0}
   8: <core::option::Option<&alloc::sync::Arc<curp::log_entry::LogEntry<xlineapi::command::Command>>>>::unwrap_or_else::<<curp::server::raw_curp::log::Log<xlineapi::command::Command>>::get_prev_entry_info::{closure#0}>
   9: <curp::server::raw_curp::log::Log<xlineapi::command::Command>>::get_prev_entry_info
  10: __covrec_A72253ECA7F0102E
  11: <curp::server::curp_node::CurpNode<xlineapi::command::Command, xline::server::command::CommandExecutor, xline::state::State<alloc::sync::Arc<dyn curp::client::ClientApi<Cmd = xlineapi::command::Command, Error = tonic::status::Status> + core::marker::Sync + core::marker::Send>>>>::sync_follower_task::{closure#0}
  12: <unknown>
  13: __covrec_ADFE4DD0EADE82C8
  14: __covrec_8D4A3A4CCEE4F84u
  15: <core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<<curp::server::curp_node::CurpNode<xlineapi::command::Command, xline::server::command::CommandExecutor, xline::state::State<alloc::sync::Arc<dyn curp::client::ClientApi<Cmd = xlineapi::command::Command, Error = tonic::status::Status> + core::marker::Sync + core::marker::Send>>>>::sync_follower_task::{closure#0}, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>::{closure#0}> as core::ops::function::FnOnce<()>>::call_once
  16: std::panicking::try::do_call::<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<<curp::server::curp_node::CurpNode<xlineapi::command::Command, xline::server::command::CommandExecutor, xline::state::State<alloc::sync::Arc<dyn curp::client::ClientApi<Cmd = xlineapi::command::Command, Error = tonic::status::Status> + core::marker::Sync + core::marker::Send>>>>::sync_follower_task::{closure#0}, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>::{closure#0}>, core::task::poll::Poll<()>>
  17: __covrec_AA0894A7535443FB
  18: __covrec_F7D08DB9FAAECD07u
  19: std::panic::catch_unwind::<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<<curp::server::curp_node::CurpNode<xlineapi::command::Command, xline::server::command::CommandExecutor, xline::state::State<alloc::sync::Arc<dyn curp::client::ClientApi<Cmd = xlineapi::command::Command, Error = tonic::status::Status> + core::marker::Sync + core::marker::Send>>>>::sync_follower_task::{closure#0}, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>::{closure#0}>, core::task::poll::Poll<()>>
  20: __covrec_893B96BF6B55578Fu
  21: __covrec_1627240BFC6C4F95
  22: __covrec_620DAB19E6B3264F
  23: tokio::runtime::task::raw::poll::<<curp::server::curp_node::CurpNode<xlineapi::command::Command, xline::server::command::CommandExecutor, xline::state::State<alloc::sync::Arc<dyn curp::client::ClientApi<Cmd = xlineapi::command::Command, Error = tonic::status::Status> + core::marker::Sync + core::marker::Send>>>>::sync_follower_task::{closure#0}, alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>
  24: <tokio::runtime::task::raw::RawTask>::poll
  25: <tokio::runtime::task::LocalNotified<alloc::sync::Arc<tokio::runtime::scheduler::multi_thread::handle::Handle>>>::run
  26: <tokio::runtime::scheduler::multi_thread::worker::Context>::run_task::{closure#0}
  27: <tokio::runtime::scheduler::multi_thread::worker::Context>::run_task
  28: <tokio::runtime::scheduler::multi_thread::worker::Context>::run
  29: tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure#0}
  30: <tokio::runtime::context::scoped::Scoped<tokio::runtime::scheduler::Context>>::set::<tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure#0}, ()>
  31: tokio::runtime::context::set_scheduler::<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure#0}>::{closure#0}
  32: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::try_with::<tokio::runtime::context::set_scheduler<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure#0}>::{closure#0}, ()>
  33: <std::thread::local::LocalKey<tokio::runtime::context::Context>>::with::<tokio::runtime::context::set_scheduler<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure#0}>::{closure#0}, ()>
  34: tokio::runtime::context::set_scheduler::<(), tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}::{closure#0}>
  35: tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}
  36: tokio::runtime::context::runtime::enter_runtime::<tokio::runtime::scheduler::multi_thread::worker::run::{closure#0}, ()>
  37: tokio::runtime::scheduler::multi_thread::worker::run
  38: <tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}
  39: <tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}> as core::future::future::Future>::poll
  40: <tokio::runtime::task::core::Core<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>::poll::{closure#0}
  41: <tokio::runtime::task::core::Core<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>::poll
  42: tokio::runtime::task::harness::poll_future::<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>::{closure#0}
  43: <core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>::{closure#0}> as core::ops::function::FnOnce<()>>::call_once
  44: std::panicking::try::do_call::<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>::{closure#0}>, core::task::poll::Poll<()>>
  45: __rust_try
  46: std::panicking::try::<core::task::poll::Poll<()>, core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>::{closure#0}>>
  47: std::panic::catch_unwind::<core::panic::unwind_safe::AssertUnwindSafe<tokio::runtime::task::harness::poll_future<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>::{closure#0}>, core::task::poll::Poll<()>>
  48: tokio::runtime::task::harness::poll_future::<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
  49: <tokio::runtime::task::harness::Harness<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>::poll_inner
  50: <tokio::runtime::task::harness::Harness<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>>::poll
  51: tokio::runtime::task::raw::poll::<tokio::runtime::blocking::task::BlockingTask<<tokio::runtime::scheduler::multi_thread::worker::Launch>::launch::{closure#0}>, tokio::runtime::blocking::schedule::BlockingSchedule>
  52: <tokio::runtime::task::raw::RawTask>::poll
  53: <tokio::runtime::task::UnownedTask<tokio::runtime::blocking::schedule::BlockingSchedule>>::run
  54: <tokio::runtime::blocking::pool::Task>::run
  55: <tokio::runtime::blocking::pool::Inner>::run
  56: <tokio::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}
  57: std::sys_common::backtrace::__rust_begin_short_backtrace::<<tokio::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>
  58: <std::thread::Builder>::spawn_unchecked_::<<tokio::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>::{closure#1}::{closure#0}
  59: <core::panic::unwind_safe::AssertUnwindSafe<<std::thread::Builder>::spawn_unchecked_<<tokio::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>::{closure#1}::{closure#0}> as core::ops::function::FnOnce<()>>::call_once
  60: std::panicking::try::do_call::<core::panic::unwind_safe::AssertUnwindSafe<<std::thread::Builder>::spawn_unchecked_<<tokio::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>::{closure#1}::{closure#0}>, ()>
  61: __rust_try
  62: std::panicking::try::<(), core::panic::unwind_safe::AssertUnwindSafe<<std::thread::Builder>::spawn_unchecked_<<tokio::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>::{closure#1}::{closure#0}>>
  63: <std::thread::Builder>::spawn_unchecked_::<<tokio::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>::{closure#1}
  64: <<std::thread::Builder>::spawn_unchecked_<<tokio::runtime::blocking::pool::Spawner>::spawn_thread::{closure#0}, ()>::{closure#1} as core::ops::function::FnOnce<()>>::call_once::{shim:vtable#0}
  65: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/alloc/src/boxed.rs:2007:9
  66: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/alloc/src/boxed.rs:2007:9
  67: std::sys::unix::thread::Thread::new::thread_start
             at /rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/sys/unix/thread.rs:108:17
  68: <unknown>
  69: __clone

Code of Conduct

github-actions[bot] commented 2 weeks ago

👋 Thanks for opening this issue!

Reply with the following command on its own line to get help or engage: