search

GreptimeTeam / docs

Document for GreptimeDB
https://docs.greptime.com/
Apache License 2.0
37 stars 37 forks source link

Bug: Crash after `select count(<column>) from <table>` #1173

Closed mjf closed 1 month ago

mjf commented 1 month ago

Everything in default after git pull and make combo (cargo 1.80.1 (376290515 2024-07-16)):

RUST_BACKTRACE=full cargo run -- standalone start

Created database test and used Telegraf to ingest some data via [[outputs.influxdb2]] and using just [[inputs.cpu]], collecting every 100ms and flushing data into GreptimeDB every 100ms from 10 threads (10x the same output target in urls), effectively causing many duplicates.

Then run from psql:

postgres://greptime_user@127.0.0.1:4003/test
test> select count(1) from cpu;
┌─────────────────┐
│ COUNT(Int64(1)) │
╞═════════════════╡
│           21085 │
└─────────────────┘
(1 row)

This works.

This causes a crash:

postgres://greptime_user@127.0.0.1:4003/test
test> select count(host) from cpu;
ERROR:  XX000:
Time: 54.499 ms
3678490 XX000

The same holds for select count(*) from cpu.

When I stop the Telegraf, the crashes no longer occure. If I start the Telegraf again, the described crashes reappear.

The backtrace:

2024-09-05T08:36:31.323850Z ERROR read_from_region: common_telemetry::panic_hook: panicked at src/mito2/src/read/dedup.rs:327:9:
assertion failed: self.initialized backtrace=   0: common_telemetry::panic_hook::set_panic_hook::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/common/telemetry/src/panic_hook.rs:37:25
   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2077:9
      std::panicking::rust_panic_with_hook
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:799:13
   2: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:656:13
   3: std::sys_common::backtrace::__rust_end_short_backtrace
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:171:18
   4: rust_begin_unwind
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:652:5
   5: core::panicking::panic_fmt
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panicking.rs:72:14
   6: core::panicking::panic
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panicking.rs:146:5
   7: mito2::read::dedup::LastFieldsBuilder::merge_last_non_null
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/dedup.rs:327:9
   8: <mito2::read::dedup::LastNonNull as mito2::read::dedup::DedupStrategy>::push_batch
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/dedup.rs:425:26
   9: mito2::read::dedup::LastNonNullIter<I>::next_batch
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/dedup.rs:549:34
  10: <mito2::read::dedup::LastNonNullIter<I> as core::iter::traits::iterator::Iterator>::next
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/dedup.rs:562:9
  11: <alloc::boxed::Box<I,A> as core::iter::traits::iterator::Iterator>::next
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:1997:9
  12: mito2::read::Source::next_batch::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read.rs:700:35
  13: mito2::read::merge::Node::new::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/merge.rs:316:49
  14: mito2::read::merge::MergeReader::new::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/merge.rs:104:56
  15: mito2::read::merge::MergeReaderBuilder::build::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/merge.rs:278:35
  16: mito2::read::seq_scan::SeqScan::build_reader_from_sources::{{closure}}::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/seq_scan.rs:266:38
  17: mito2::read::seq_scan::SeqScan::build_reader_from_sources::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/seq_scan.rs:251:5
  18: mito2::read::seq_scan::SeqScan::build_merge_reader::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/seq_scan.rs:238:92
  19: mito2::read::seq_scan::SeqScan::uncached_scan_part_impl::{{closure}}
             at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/seq_scan.rs:422:18
  20: <async_stream::async_stream::AsyncStream<T,U> as futures_core::stream::Stream>::poll_next
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stream-0.3.5/src/async_stream.rs:56:13
  21: <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  22: <common_recordbatch::RecordBatchStreamWrapper<S> as futures_core::stream::Stream>::poll_next
             at /home/user/tmp/20240905-092710/greptimedb/src/common/recordbatch/src/lib.rs:273:9
  23: <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  24: futures_util::stream::stream::StreamExt::poll_next_unpin
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  25: <table::table::scan::StreamWithMetricWrapper as futures_core::stream::Stream>::poll_next
             at /home/user/tmp/20240905-092710/greptimedb/src/table/src/table/scan.rs:231:27
  26: <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  27: futures_util::stream::stream::StreamExt::poll_next_unpin
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  28: <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  29: datafusion_physical_plan::repartition::RepartitionExec::pull_from_input::{{closure}}
             at /home/user/.cargo/git/checkouts/arrow-datafusion-8f64698f94ffb3e6/7823ef2f/datafusion/physical-plan/src/repartition/mod.rs:779:40
  30: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17
  31: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
      tokio::runtime::task::core::Core<T,S>::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:13
  32: tokio::runtime::task::harness::poll_future::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19
  33: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
  34: std::panicking::try::do_call
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
  35: __rust_try
  36: std::panicking::try
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
  37: std::panic::catch_unwind
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
  38: tokio::runtime::task::harness::poll_future
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18
  39: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27
  40: tokio::runtime::task::harness::Harness<T,S>::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15
  41: tokio::runtime::task::raw::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:271:5
  42: tokio::runtime::task::raw::RawTask::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18
  43: tokio::runtime::task::LocalNotified<S>::run
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:427:9
  44: tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:585:13
  45: tokio::runtime::coop::with_budget
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:107:5
      tokio::runtime::coop::budget
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:73:5
      tokio::runtime::scheduler::multi_thread::worker::Context::run_task
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:584:9
  46: tokio::runtime::scheduler::multi_thread::worker::Context::run
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:535:24
  47: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:500:21
  48: tokio::runtime::context::scoped::Scoped<T>::set
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/scoped.rs:40:9
  49: tokio::runtime::context::set_scheduler::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:26
  50: std::thread::local::LocalKey<T>::try_with
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/local.rs:283:12
  51: std::thread::local::LocalKey<T>::with
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/local.rs:260:9
  52: tokio::runtime::context::set_scheduler
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:9
  53: tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:495:9
  54: tokio::runtime::context::runtime::enter_runtime
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/runtime.rs:65:16
  55: tokio::runtime::scheduler::multi_thread::worker::run
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:487:5
  56: tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:455:45
  57: <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/task.rs:42:21
  58: tokio::runtime::task::core::Core<T,S>::poll::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17
  59: tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
      tokio::runtime::task::core::Core<T,S>::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:13
  60: tokio::runtime::task::harness::poll_future::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19
  61: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
  62: std::panicking::try::do_call
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
  63: __rust_try
  64: std::panicking::try
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
  65: std::panic::catch_unwind
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
  66: tokio::runtime::task::harness::poll_future
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18
  67: tokio::runtime::task::harness::Harness<T,S>::poll_inner
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27
  68: tokio::runtime::task::harness::Harness<T,S>::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15
  69: tokio::runtime::task::raw::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:271:5
  70: tokio::runtime::task::raw::RawTask::poll
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18
  71: tokio::runtime::task::UnownedTask<S>::run
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:464:9
  72: tokio::runtime::blocking::pool::Task::run
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:159:9
  73: tokio::runtime::blocking::pool::Inner::run
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:513:17
  74: tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}
             at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:471:13
  75: std::sys_common::backtrace::__rust_begin_short_backtrace
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:155:18
  76: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/mod.rs:542:17
  77: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
  78: std::panicking::try::do_call
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
  79: __rust_try
  80: std::panicking::try
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
  81: std::panic::catch_unwind
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/mod.rs:541:30
  82: core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/ops/function.rs:250:5
  83: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2063:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2063:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys/pal/unix/thread.rs:108:17
  84: start_thread
  85: __clone3
 panic.file="src/mito2/src/read/dedup.rs" panic.line=327 panic.column=9
thread 'global-worker' panicked at src/mito2/src/read/dedup.rs:327:9:
assertion failed: self.initialized
stack backtrace:
   0:     0x55ccdf617865 - std::backtrace_rs::backtrace::libunwind::trace::hb2aae96f7a63ad27
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/../../backtrace/src/backtrace/libunwind.rs:116:5
   1:     0x55ccdf617865 - std::backtrace_rs::backtrace::trace_unsynchronized::h21f79cd140dff600
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x55ccdf617865 - std::sys_common::backtrace::_print_fmt::hc8ebdc1b64ae52ee
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:68:5
   3:     0x55ccdf617865 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h7ba491a968dfae1d
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x55ccdf64a17b - core::fmt::rt::Argument::fmt::he322040f856802e1
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/fmt/rt.rs:165:63
   5:     0x55ccdf64a17b - core::fmt::write::hb4f7da2a314ef133
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/fmt/mod.rs:1168:21
   6:     0x55ccdf612d3f - std::io::Write::write_fmt::h9e79d11af9b4686b
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/io/mod.rs:1835:15
   7:     0x55ccdf61763e - std::sys_common::backtrace::_print::h9276d72631ea2ebd
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:47:5
   8:     0x55ccdf61763e - std::sys_common::backtrace::print::h301edb80f46c58f8
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:34:9
   9:     0x55ccdf618d79 - std::panicking::default_hook::{{closure}}::h08c68fc043d12d43
  10:     0x55ccdf618b1a - std::panicking::default_hook::hdf65558cafad5039
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:298:9
  11:     0x55ccde0cd323 - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h4166977facef78e5
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2077:9
  12:     0x55ccde0c2e8f - common_telemetry::panic_hook::set_panic_hook::{{closure}}::h7b1aafb40d11f72c
                               at /home/user/tmp/20240905-092710/greptimedb/src/common/telemetry/src/panic_hook.rs:51:9
  13:     0x55ccdf6193eb - <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call::h69f90a497fd344b0
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2077:9
  14:     0x55ccdf6193eb - std::panicking::rust_panic_with_hook::h9576f09a7a058c94
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:799:13
  15:     0x55ccdf61912b - std::panicking::begin_panic_handler::{{closure}}::hd8f56243f630c203
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:656:13
  16:     0x55ccdf617d29 - std::sys_common::backtrace::__rust_end_short_backtrace::h5bed6f770a09acf8
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:171:18
  17:     0x55ccdf618e97 - rust_begin_unwind
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:652:5
  18:     0x55ccdf646bd3 - core::panicking::panic_fmt::h088f9f84f0369284
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panicking.rs:72:14
  19:     0x55ccdf646c7c - core::panicking::panic::h41df2c6d8f6b057e
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panicking.rs:146:5
  20:     0x55ccd5654508 - mito2::read::dedup::LastFieldsBuilder::merge_last_non_null::h8b8f8809dd3c415f
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/dedup.rs:327:9
  21:     0x55ccd5655bfe - <mito2::read::dedup::LastNonNull as mito2::read::dedup::DedupStrategy>::push_batch::h52f14241cad38b61
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/dedup.rs:425:26
  22:     0x55ccd52e1a2d - mito2::read::dedup::LastNonNullIter<I>::next_batch::h0ea4b7ffc81063f6
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/dedup.rs:549:34
  23:     0x55ccd52e1ea3 - <mito2::read::dedup::LastNonNullIter<I> as core::iter::traits::iterator::Iterator>::next::h86b9225f03f0ac14
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/dedup.rs:562:9
  24:     0x55ccd5237c7e - <alloc::boxed::Box<I,A> as core::iter::traits::iterator::Iterator>::next::hab983dca5494a4b0
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:1997:9
  25:     0x55ccd59d5e15 - mito2::read::Source::next_batch::{{closure}}::h369bc8a4eac60530
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read.rs:700:35
  26:     0x55ccd5a6089f - mito2::read::merge::Node::new::{{closure}}::h11158c240d556d56
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/merge.rs:316:49
  27:     0x55ccd5a5e8d0 - mito2::read::merge::MergeReader::new::{{closure}}::h513a48916aba0f89
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/merge.rs:104:56
  28:     0x55ccd5a605c6 - mito2::read::merge::MergeReaderBuilder::build::{{closure}}::h5e8d172ca67a4cf8
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/merge.rs:278:35
  29:     0x55ccd5534f31 - mito2::read::seq_scan::SeqScan::build_reader_from_sources::{{closure}}::{{closure}}::h48e18af53650f6ba
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/seq_scan.rs:266:38
  30:     0x55ccd55348c0 - mito2::read::seq_scan::SeqScan::build_reader_from_sources::{{closure}}::h1286e4864bb253af
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/seq_scan.rs:251:5
  31:     0x55ccd552f86c - mito2::read::seq_scan::SeqScan::build_merge_reader::{{closure}}::h59a704c000bff707
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/seq_scan.rs:238:92
  32:     0x55ccd5538044 - mito2::read::seq_scan::SeqScan::uncached_scan_part_impl::{{closure}}::h072a81286f59fe5c
                               at /home/user/tmp/20240905-092710/greptimedb/src/mito2/src/read/seq_scan.rs:422:18
  33:     0x55ccd59c0610 - <async_stream::async_stream::AsyncStream<T,U> as futures_core::stream::Stream>::poll_next::h39ee6aeed8550778
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/async-stream-0.3.5/src/async_stream.rs:56:13
  34:     0x55ccd56903c0 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h38fd53bf3a37251b
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  35:     0x55ccd53fef34 - <common_recordbatch::RecordBatchStreamWrapper<S> as futures_core::stream::Stream>::poll_next::hcf466eecf9d80df9
                               at /home/user/tmp/20240905-092710/greptimedb/src/common/recordbatch/src/lib.rs:273:9
  36:     0x55ccda117bd7 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h4e3da011055ace58
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  37:     0x55ccda117a31 - futures_util::stream::stream::StreamExt::poll_next_unpin::h5282886010607f30
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  38:     0x55ccd90638bf - <table::table::scan::StreamWithMetricWrapper as futures_core::stream::Stream>::poll_next::h474ddf35e910f410
                               at /home/user/tmp/20240905-092710/greptimedb/src/table/src/table/scan.rs:231:27
  39:     0x55ccdb4652d7 - <core::pin::Pin<P> as futures_core::stream::Stream>::poll_next::h49d9233f750106ca
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-core-0.3.30/src/stream.rs:120:9
  40:     0x55ccdb4605a1 - futures_util::stream::stream::StreamExt::poll_next_unpin::hf84ace14ba70bf3f
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/mod.rs:1638:9
  41:     0x55ccdb4de119 - <futures_util::stream::stream::next::Next<St> as core::future::future::Future>::poll::h383d299b9d3dc9c7
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.30/src/stream/stream/next.rs:32:9
  42:     0x55ccdb2e8232 - datafusion_physical_plan::repartition::RepartitionExec::pull_from_input::{{closure}}::h9fc6943703bb76fe
                               at /home/user/.cargo/git/checkouts/arrow-datafusion-8f64698f94ffb3e6/7823ef2f/datafusion/physical-plan/src/repartition/mod.rs:779:40
  43:     0x55ccdb3fb54d - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::h16399b4c4f346024
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17
  44:     0x55ccdb3f9deb - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h5d5def209a91df68
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
  45:     0x55ccdb3f9deb - tokio::runtime::task::core::Core<T,S>::poll::h1114a078c3f70955
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:13
  46:     0x55ccdb644748 - tokio::runtime::task::harness::poll_future::{{closure}}::hb5e69e8425b73651
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19
  47:     0x55ccdb5ca50d - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h9b9ed4323602e2ff
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
  48:     0x55ccdb3ebd01 - std::panicking::try::do_call::h3422948ef2a8c985
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
  49:     0x55ccdb42949b - __rust_try
  50:     0x55ccdb3e8969 - std::panicking::try::h4ff1d183616b1b71
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
  51:     0x55ccdb3c27fe - std::panic::catch_unwind::h69fe1542a38b71aa
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
  52:     0x55ccdb640c52 - tokio::runtime::task::harness::poll_future::h309235cc321ce63e
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18
  53:     0x55ccdb64549e - tokio::runtime::task::harness::Harness<T,S>::poll_inner::h013e7d1cdbd0a24f
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27
  54:     0x55ccdb64a0b7 - tokio::runtime::task::harness::Harness<T,S>::poll::h7ce01e9c4982818b
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15
  55:     0x55ccdb5811fd - tokio::runtime::task::raw::poll::had02a6c9aeca2c62
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:271:5
  56:     0x55ccdf2186b7 - tokio::runtime::task::raw::RawTask::poll::h6ac96dde8cbc8e3a
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18
  57:     0x55ccdf22fbf2 - tokio::runtime::task::LocalNotified<S>::run::h3f9c8f3495b99aba
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:427:9
  58:     0x55ccdf1f15cd - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::{{closure}}::hf5a2f7ed803f5997
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:585:13
  59:     0x55ccdf1f1424 - tokio::runtime::coop::with_budget::hb8fe063a36b9f933
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:107:5
  60:     0x55ccdf1f1424 - tokio::runtime::coop::budget::h12fa869ea11997a5
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:73:5
  61:     0x55ccdf1f1424 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::hfa37511330943c60
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:584:9
  62:     0x55ccdf1f0af3 - tokio::runtime::scheduler::multi_thread::worker::Context::run::h20aad7133dbbeb6b
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:535:24
  63:     0x55ccdf1f0759 - tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::{{closure}}::hdc76dd627c97f816
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:500:21
  64:     0x55ccdf20bb20 - tokio::runtime::context::scoped::Scoped<T>::set::h1ccc129e76d5535e
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/scoped.rs:40:9
  65:     0x55ccdf1cf6cb - tokio::runtime::context::set_scheduler::{{closure}}::h83725b585b762072
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:26
  66:     0x55ccdf1caf30 - std::thread::local::LocalKey<T>::try_with::h041786fd15739917
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/local.rs:283:12
  67:     0x55ccdf1cacdb - std::thread::local::LocalKey<T>::with::h350fb7a9f3594567
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/local.rs:260:9
  68:     0x55ccdf1cf694 - tokio::runtime::context::set_scheduler::h94eec0c261c08c23
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context.rs:180:9
  69:     0x55ccdf1f0664 - tokio::runtime::scheduler::multi_thread::worker::run::{{closure}}::h2073612d184a8ade
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:495:9
  70:     0x55ccdf24f2f0 - tokio::runtime::context::runtime::enter_runtime::h1a15c3e85f7c9bc2
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/runtime.rs:65:16
  71:     0x55ccdf1f040f - tokio::runtime::scheduler::multi_thread::worker::run::hfaa8fb9dcb0f9410
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:487:5
  72:     0x55ccdf1f029b - tokio::runtime::scheduler::multi_thread::worker::Launch::launch::{{closure}}::h11da15974e083f3c
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/scheduler/multi_thread/worker.rs:455:45
  73:     0x55ccdf230bae - <tokio::runtime::blocking::task::BlockingTask<T> as core::future::future::Future>::poll::hd0e25f68a93c3117
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/task.rs:42:21
  74:     0x55ccdf1e3939 - tokio::runtime::task::core::Core<T,S>::poll::{{closure}}::he22ca20a643c65d4
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:328:17
  75:     0x55ccdf1e2541 - tokio::loom::std::unsafe_cell::UnsafeCell<T>::with_mut::h0472b236224b5ff7
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/loom/std/unsafe_cell.rs:16:9
  76:     0x55ccdf1e2541 - tokio::runtime::task::core::Core<T,S>::poll::h7b50ca87b711f9c8
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/core.rs:317:13
  77:     0x55ccdf1c9a5e - tokio::runtime::task::harness::poll_future::{{closure}}::h623b05b51c17fcb1
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:485:19
  78:     0x55ccdf1d5f31 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h23ac88d4b37d3eb8
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
  79:     0x55ccdf20f7d9 - std::panicking::try::do_call::hd15ab1ee08c57538
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
  80:     0x55ccdf21190b - __rust_try
  81:     0x55ccdf20d016 - std::panicking::try::h806236856b299902
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
  82:     0x55ccdf2030fe - std::panic::catch_unwind::h90011236ed5248c0
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
  83:     0x55ccdf1c9060 - tokio::runtime::task::harness::poll_future::ha8e6edcd372e15e6
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:473:18
  84:     0x55ccdf1c2d9e - tokio::runtime::task::harness::Harness<T,S>::poll_inner::hb0d35b16529132ef
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:208:27
  85:     0x55ccdf1c23c7 - tokio::runtime::task::harness::Harness<T,S>::poll::hfb32b12448f0c72c
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/harness.rs:153:15
  86:     0x55ccdf21896d - tokio::runtime::task::raw::poll::h2488da27bb38ee43
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:271:5
  87:     0x55ccdf2186b7 - tokio::runtime::task::raw::RawTask::poll::h6ac96dde8cbc8e3a
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/raw.rs:201:18
  88:     0x55ccdf22fcb7 - tokio::runtime::task::UnownedTask<S>::run::hbb0dc3afb45ce547
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/task/mod.rs:464:9
  89:     0x55ccdf1f45d7 - tokio::runtime::blocking::pool::Task::run::h7e7fa5d50b41d6a1
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:159:9
  90:     0x55ccdf1f9d7e - tokio::runtime::blocking::pool::Inner::run::hf313338b7b7712b6
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:513:17
  91:     0x55ccdf1f9a94 - tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}::hc6ca27fbac11cbbc
                               at /home/user/.cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/blocking/pool.rs:471:13
  92:     0x55ccdf1e9c46 - std::sys_common::backtrace::__rust_begin_short_backtrace::hf251a6657e203ea5
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys_common/backtrace.rs:155:18
  93:     0x55ccdf25e4f2 - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h3f4283f4ac9b31ee
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/mod.rs:542:17
  94:     0x55ccdf1d6562 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::hb2b815ce8500bd55
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/panic/unwind_safe.rs:272:9
  95:     0x55ccdf20fa33 - std::panicking::try::do_call::hec2ccd588c41ed00
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:559:40
  96:     0x55ccdf21190b - __rust_try
  97:     0x55ccdf20c491 - std::panicking::try::h221bee9c07f7ff86
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panicking.rs:523:19
  98:     0x55ccdf25e12b - std::panic::catch_unwind::h72584583fa0cb58e
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/panic.rs:149:14
  99:     0x55ccdf25e12b - std::thread::Builder::spawn_unchecked_::{{closure}}::h236c0fb2403ea465
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/thread/mod.rs:541:30
 100:     0x55ccdf25e9cf - core::ops::function::FnOnce::call_once{{vtable.shim}}::h2f176d4ebfc161fd
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/core/src/ops/function.rs:250:5
 101:     0x55ccdf6201ab - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hd9d8a12a4cec8653
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2063:9
 102:     0x55ccdf6201ab - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hc87e6ebe32d18c90
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/alloc/src/boxed.rs:2063:9
 103:     0x55ccdf6201ab - std::sys::pal::unix::thread::Thread::new::thread_start::h68df5163d04f9130
                               at /rustc/72fdf913c53dd0e75313ba83e4aa80df3f6e2871/library/std/src/sys/pal/unix/thread.rs:108:17
 104:     0x7f6062d02947 - start_thread
 105:     0x7f6062d88970 - __clone3
 106:                0x0 - <unknown>
mjf commented 1 month ago

Ah, I just clicked here from docs, so wrong repo. Closing, sorry for that.