search

databendlabs / databend

๐——๐—ฎ๐˜๐—ฎ, ๐—”๐—ป๐—ฎ๐—น๐˜†๐˜๐—ถ๐—ฐ๐˜€ & ๐—”๐—œ. Modern alternative to Snowflake. Cost-effective and simple for massive-scale analytics. https://databend.com
https://docs.databend.com
Other
7.85k stars 750 forks source link

bug: Unimplented clone SortSpillMeta #16660

Closed wubx closed 3 weeks ago

wubx commented 3 weeks ago

Search before asking

Version

main

What's Wrong?

 ba4cbb9a-b24a-4500-984a-3fccd1bb1074 2024-10-22T16:49:14.142959+08:00  INFO databend_query::pipelines::processors::transforms::aggregator::serde::transform_spill_reader: transform_spill_reader.rs:273 Read 0 aggregate spills successfully, total elapsed: 0ns
ba4cbb9a-b24a-4500-984a-3fccd1bb1074 2024-10-22T16:49:14.264044+08:00 ERROR databend_common_tracing::panic_hook: panic_hook.rs:53 panicked at src/query/pipeline/transforms/src/processors/transforms/sort/spill.rs:31:9:
not implemented: Unimplemented clone SortSpillMeta backtrace=   0: backtrace::backtrace::libunwind::trace
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/libunwind.rs:116:5
      backtrace::backtrace::trace_unsynchronized
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/mod.rs:66:5
      backtrace::backtrace::trace
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/mod.rs:53:14

   1: databend_common_tracing::panic_hook::captures_frames
             at /workspace/src/common/tracing/src/panic_hook.rs:66:5
      databend_common_tracing::panic_hook::backtrace
             at /workspace/src/common/tracing/src/panic_hook.rs:47:25
   2: databend_common_tracing::panic_hook::set_panic_hook::{{closure}}
             at /workspace/src/common/tracing/src/panic_hook.rs:33:9
   3: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2078:9
      std::panicking::rust_panic_with_hook
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:804:13
   4: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:663:13
   5: std::sys::backtrace::__rust_end_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:171:18
   6: rust_begin_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:661:5
   7: core::panicking::panic_fmt
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panicking.rs:74:14
   8: <databend_common_pipeline_transforms::processors::transforms::sort::spill::SortSpillMetaWithParams as databend_common_expression::block::BlockMetaInfo>::clone_self
             at /workspace/src/query/pipeline/transforms/src/processors/transforms/sort/spill.rs:31:9
   9: databend_common_expression::block::<impl core::clone::Clone for alloc::boxed::Box<dyn databend_common_expression::block::BlockMetaInfo>>::clone
             at /workspace/src/query/expression/src/block.rs:637:9
      <core::option::Option<T> as core::clone::Clone>::clone
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/option.rs:2038:29
      <databend_common_expression::block::DataBlock as core::clone::Clone>::clone
             at /workspace/src/query/expression/src/block.rs:46:5
      databend_common_expression::kernels::concat::<impl databend_common_expression::block::DataBlock>::concat
             at /workspace/src/query/expression/src/kernels/concat.rs:70:23
  10: databend_query::spillers::serialize::BlocksEncoder::add_blocks
             at /workspace/src/query/service/src/spillers/serialize.rs:74:25
  11: databend_query::spillers::spiller::Spiller::spill::{{closure}}
             at /workspace/src/query/service/src/spillers/spiller.rs:154:9
  12: databend_query::pipelines::processors::transforms::transform_sort_spill::TransformSortSpill<R>::spill::{{closure}}
             at /workspace/src/query/service/src/pipelines/processors/transforms/transform_sort_spill.rs:273:56
             at /workspace/src/query/service/src/pipelines/processors/transforms/transform_sort_spill.rs:209:35
      <async_backtrace::framed::Framed<F> as core::future::future::Future>::poll::{{closure}}
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/framed.rs:49:27
      async_backtrace::frame::Frame::in_scope::{{closure}}
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/frame.rs:184:17
      std::thread::local::LocalKey<T>::try_with
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/local.rs:283:12
      std::thread::local::LocalKey<T>::with
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/local.rs:260:9
  13: async_backtrace::frame::active_frame::with
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/frame.rs:81:9
      async_backtrace::frame::Frame::in_scope
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/frame.rs:180:13
      <async_backtrace::framed::Framed<F> as core::future::future::Future>::poll
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/framed.rs:49:9
             at /workspace/src/query/service/src/pipelines/processors/transforms/transform_sort_spill.rs:204:5
  14: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/future/future.rs:123:9
      <fastrace::future::InSpan<T> as core::future::future::Future>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/fastrace-0.7.2/src/future.rs:124:19
  15: databend_common_pipeline_core::processors::processor::ProcessorPtr::async_process::{{closure}}
             at /workspace/src/query/pipeline/core/src/processors/processor.rs:193:32
  16: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/future/future.rs:123:9
      <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/future/future.rs:123:9
      futures_util::future::future::FutureExt::poll_unpin
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/future/mod.rs:558:9
      <futures_util::future::select::Select<A,B> as core::future::future::Future>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/select.rs:114:35
  17: databend_query::pipelines::executor::processor_async_task::ProcessorAsyncTask::create::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/processor_async_task.rs:117:56
  18: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/future/future.rs:123:9
      futures_util::future::future::FutureExt::poll_unpin
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/future/mod.rs:558:9
      <futures_util::future::select::Select<A,B> as core::future::future::Future>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/future/select.rs:118:35
  19: databend_query::pipelines::executor::processor_async_task::ProcessorAsyncTask::create::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/processor_async_task.rs:136:64
  20: <databend_query::pipelines::executor::processor_async_task::ProcessorAsyncTask as core::future::future::Future>::poll::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/processor_async_task.rs:202:45
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
  21: databend_common_base::runtime::catch_unwind::catch_unwind
             at /workspace/src/common/base/src/runtime/catch_unwind.rs:47:11
  22: <databend_query::pipelines::executor::processor_async_task::ProcessorAsyncTask as core::future::future::Future>::poll
             at /workspace/src/query/service/src/pipelines/executor/processor_async_task.rs:202:24
  23: <fastrace::future::InSpan<T> as core::future::future::Future>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/fastrace-0.7.2/src/future.rs:124:19
  24: <databend_common_base::runtime::runtime_tracker::TrackingFuture<T> as core::future::future::Future>::poll
             at /workspace/src/common/base/src/runtime/runtime_tracker.rs:143:9
  25: <async_backtrace::framed::Framed<F> as core::future::future::Future>::poll::{{closure}}
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/framed.rs:49:27
      async_backtrace::frame::Frame::in_scope::{{closure}}
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/frame.rs:184:17
      std::thread::local::LocalKey<T>::try_with
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/local.rs:283:12
      std::thread::local::LocalKey<T>::with
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/local.rs:260:9
  26: async_backtrace::frame::active_frame::with
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/frame.rs:81:9
      async_backtrace::frame::Frame::in_scope
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/frame.rs:180:13
      <async_backtrace::framed::Framed<F> as core::future::future::Future>::poll
             at /opt/rust/cargo/git/checkouts/async-backtrace-9554bf0eaadc9fb8-shallow/dea4553/backtrace/src/framed.rs:49:9
      tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:328:17
      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/loom/std/unsafe_cell.rs:16:9
      tokio::runtime::task::core::Core<T,S>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:317:13
  27: tokio::runtime::task::harness::poll_future::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:485:19
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
  28: tokio::runtime::task::harness::poll_future
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:473:18
  29: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:208:27
  30: tokio::runtime::task::harness::Harness<T,S>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:153:15
  31: tokio::runtime::task::raw::RawTask::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/raw.rs:201:18
      tokio::runtime::task::LocalNotified<S>::run
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/mod.rs:427:9
      tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:594:13
      tokio::runtime::coop::with_budget
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/coop.rs:107:5
      tokio::runtime::coop::budget
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/coop.rs:73:5
      tokio::runtime::scheduler::multi_thread::worker::Context::run_task
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:593:9
  32: tokio::runtime::scheduler::multi_thread::worker::Context::run
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:544:24
  33: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:509:21
      tokio::runtime::context::scoped::Scoped<T>::set
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context/scoped.rs:40:9
  34: tokio::runtime::context::set_scheduler
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context.rs:180:9
      tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:504:9
      tokio::runtime::context::runtime::enter_runtime
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/context/runtime.rs:65:16
  35: tokio::runtime::scheduler::multi_thread::worker::run
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:496:5
  36: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/scheduler/multi_thread/worker.rs:462:45
      <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/task.rs:42:21
  37: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:328:17
      tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/loom/std/unsafe_cell.rs:16:9
      tokio::runtime::task::core::Core<T,S>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/core.rs:317:13
  38: tokio::runtime::task::harness::poll_future::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:485:19
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
  39: tokio::runtime::task::harness::poll_future
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:473:18
  40: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:208:27
  41: tokio::runtime::task::harness::Harness<T,S>::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/harness.rs:153:15
  42: tokio::runtime::task::raw::RawTask::poll
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/raw.rs:201:18
      tokio::runtime::task::UnownedTask<S>::run
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/task/mod.rs:464:9
  43: tokio::runtime::blocking::pool::Task::run
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:160:9
      tokio::runtime::blocking::pool::Inner::run
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:518:17
  44: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
             at /opt/rust/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.39.2/src/runtime/blocking/pool.rs:476:13
      std::sys::backtrace::__rust_begin_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:155:18
  45: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:542:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:541:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/ops/function.rs:250:5
  46: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/pal/unix/thread.rs:108:17
  47: start_thread
             at ./nptl/pthread_create.c:442:8
  48: __GI___clone3
             at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
 panic.file=src/query/pipeline/transforms/src/processors/transforms/sort/spill.rs panic.line=31 panic.column=9
ba4cbb9a-b24a-4500-984a-3fccd1bb1074 2024-10-22T16:49:14.264593+08:00 ERROR databend_common_tracing::panic_hook: panic_hook.rs:53 panicked at src/query/pipeline/transforms/src/processors/transforms/sort/spill.rs:31:9:

How to Reproduce?

TPCH-300 with small memory(32G) WH

set global spill_file_format='arrow';
Run Q9

Are you willing to submit PR?

wubx commented 3 weeks ago 
spilling_file_format='parquet'

Run Q9 image

Below is the detailed log databend-query-c100.2024-10-22-18.0.log