bug: TPCDS Q67-Io error: LZ4 error: ERROR_blockChecksum_invalid.

[wubx]

Search before asking

• [X] I had searched in the issues and found no similar issues.

Version

main

What's Wrong?

1b148b53-9fa6-49d3-9b6f-6fdeb944dcd9 2024-10-21T21:06:09.627781+08:00 ERROR databend_common_tracing::panic_hook: panic_hook.rs:53 panicked at src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs:311:91:
called `Result::unwrap()` on an `Err` value: FromStdError. Code: 1001, Text = Io error: LZ4 error: ERROR_blockChecksum_invalid.

  0: databend_common_exception::exception_backtrace::capture
             at /workspace/src/common/exception/src/exception_backtrace.rs:82:17
   1: databend_common_exception::exception::ErrorCode<C>::from_std_error
             at /workspace/src/common/exception/src/exception.rs:291:24
   2: databend_common_exception::exception_into::<impl core::convert::From<databend_common_arrow::arrow::error::Error> for databend_common_exception::exception::ErrorCode>::from
             at /workspace/src/common/exception/src/exception_into.rs:116:18
   3: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/result.rs:1989:27
      databend_common_expression::utils::arrow::read_column
             at /workspace/src/query/expression/src/utils/arrow.rs:99:15
   4: databend_common_expression::utils::arrow::deserialize_column
             at /workspace/src/query/expression/src/utils/arrow.rs:90:5
      databend_query::pipelines::processors::transforms::aggregator::serde::transform_spill_reader::TransformSpillReader<Method,V>::deserialize
             at /workspace/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs:311:26
   5: <databend_query::pipelines::processors::transforms::aggregator::serde::transform_spill_reader::TransformSpillReader<Method,V> as databend_common_pipeline_core::processors::processor::Processor>::process
             at /workspace/src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs:158:47
   6: databend_common_pipeline_core::processors::processor::ProcessorPtr::process
             at /workspace/src/query/pipeline/core/src/processors/processor.rs:169:9
   7: databend_query::pipelines::executor::executor_worker_context::ExecutorWorkerContext::execute_sync_task
             at /workspace/src/query/service/src/pipelines/executor/executor_worker_context.rs:169:9
      databend_query::pipelines::executor::executor_worker_context::ExecutorWorkerContext::execute_task
             at /workspace/src/query/service/src/pipelines/executor/executor_worker_context.rs:132:52
   8: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_single_thread
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:406:35
   9: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:50
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
  10: databend_common_base::runtime::catch_unwind::catch_unwind
             at /workspace/src/common/base/src/runtime/catch_unwind.rs:47:11
  11: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:34
      databend_common_base::runtime::runtime_tracker::ThreadTracker::tracking_function::{{closure}}::{{closure}}
             at /workspace/src/common/base/src/runtime/runtime_tracker.rs:208:17
      databend_common_base::runtime::thread::Thread::named_spawn::{{closure}}
             at /workspace/src/common/base/src/runtime/thread.rs:78:21
      std::sys::backtrace::__rust_begin_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:155:18
  12: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:542:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:541:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/ops/function.rs:250:5
  13: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/pal/unix/thread.rs:108:17
  14: start_thread
             at ./nptl/pthread_create.c:442:8
  15: __GI___clone3
             at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
 backtrace=   0: backtrace::backtrace::libunwind::trace
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/libunwind.rs:116:5
      backtrace::backtrace::trace_unsynchronized
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/mod.rs:66:5
      backtrace::backtrace::trace
             at /opt/rust/cargo/git/checkouts/backtrace-rs-fb1f822361417489-shallow/72265be/src/backtrace/mod.rs:53:14
   1: databend_common_tracing::panic_hook::captures_frames
             at /workspace/src/common/tracing/src/panic_hook.rs:66:5
      databend_common_tracing::panic_hook::backtrace
             at /workspace/src/common/tracing/src/panic_hook.rs:75:9
      databend_common_tracing::panic_hook::log_panic
             at /workspace/src/common/tracing/src/panic_hook.rs:47:25
   2: databend_common_tracing::panic_hook::set_panic_hook::{{closure}}
             at /workspace/src/common/tracing/src/panic_hook.rs:33:9
   3: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2078:9
      std::panicking::rust_panic_with_hook
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:804:13
   4: std::panicking::begin_panic_handler::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:670:13
   5: std::sys::backtrace::__rust_end_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:171:18
   6: rust_begin_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:661:5
   7: core::panicking::panic_fmt
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panicking.rs:74:14
   8: core::result::unwrap_failed
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/result.rs:1679:5
   9: core::result::Result<T,E>::unwrap
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/result.rs:1102:23
      databend_query::pipelines::processors::transforms::aggregator::serde::transform_spill_reader::TransformSpillReader<Method,V>::deserialize
             at /workspace/src/query/service/src/pipelines/executor/executor_worker_context.rs:169:9
      databend_query::pipelines::executor::executor_worker_context::ExecutorWorkerContext::execute_task
             at /workspace/src/query/service/src/pipelines/executor/executor_worker_context.rs:132:52
  13: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_single_thread
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:406:35
  14: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:50
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
  15: databend_common_base::runtime::catch_unwind::catch_unwind
             at /workspace/src/common/base/src/runtime/catch_unwind.rs:47:11
  16: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}
             at /workspace/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:34
      databend_common_base::runtime::runtime_tracker::ThreadTracker::tracking_function::{{closure}}::{{closure}}
             at /workspace/src/common/base/src/runtime/runtime_tracker.rs:208:17
      databend_common_base::runtime::thread::Thread::named_spawn::{{closure}}
             at /workspace/src/common/base/src/runtime/thread.rs:78:21
      std::sys::backtrace::__rust_begin_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:155:18
  17: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:542:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:541:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/ops/function.rs:250:5
  18: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/pal/unix/thread.rs:108:17
  19: start_thread
             at ./nptl/pthread_create.c:442:8
  20: __GI___clone3
             at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
 panic.file=src/query/service/src/pipelines/processors/transforms/aggregator/serde/transform_spill_reader.rs panic.line=311 panic.column=91
1b148b53-9fa6-49d3-9b6f-6fdeb944dcd9 2024-10-21T21:06:09.822540+08:00  INFO databend_enterprise_query::storages::fuse::operations::vacuum_temporary_files: vacuum_temporary_files.rs:122 vacuum removed 16 temp files in "_query_spill/d0626/1b148b53-9fa6-49d3-9b6f-6fdeb944dcd9/"(elapsed: 0 seconds), batch size: 3282325504 bytes

settings:

spilling_file_format      parquet 

How to Reproduce?

No response

Are you willing to submit PR?

• [ ] Yes I am willing to submit a PR!

[wubx]

main

Error: error happens after fetched 0 rows: APIError: ResponseError with 1104: called `Result::unwrap()` on an `Err` value: FromStdError. Code: 1001, Text = Ipc error: Expected compressed length of 1144251 got 1144259.

   0: databend_common_exception::exception_backtrace::capture
             at /data1/wubx/databend/src/common/exception/src/exception_backtrace.rs:82:17
   1: databend_common_exception::exception::ErrorCode<C>::from_std_error
             at /data1/wubx/databend/src/common/exception/src/exception.rs:291:24
   2: databend_common_exception::exception_into::<impl core::convert::From<arrow_schema::error::ArrowError> for databend_common_exception::exception::ErrorCode>::from
             at /data1/wubx/databend/src/common/exception/src/exception_into.rs:133:18
   3: <core::result::Result<T,F> as core::ops::try_trait::FromResidual<core::result::Result<core::convert::Infallible,E>>>::from_residual
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/result.rs:1989:27
      databend_common_expression::utils::arrow::read_column
             at /data1/wubx/databend/src/query/expression/src/utils/arrow.rs:108:15

             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
  10: databend_common_base::runtime::catch_unwind::catch_unwind
             at /data1/wubx/databend/src/common/base/src/runtime/catch_unwind.rs:47:11
  11: databend_query::pipelines::executor::query_pipeline_executor::QueryPipelineExecutor::execute_threads::{{closure}}
             at /data1/wubx/databend/src/query/service/src/pipelines/executor/query_pipeline_executor.rs:378:34
      databend_common_base::runtime::runtime_tracker::ThreadTracker::tracking_function::{{closure}}::{{closure}}
             at /data1/wubx/databend/src/common/base/src/runtime/runtime_tracker.rs:208:17
      databend_common_base::runtime::thread::Thread::named_spawn::{{closure}}
             at /data1/wubx/databend/src/common/base/src/runtime/thread.rs:78:21
      std::sys::backtrace::__rust_begin_short_backtrace
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/backtrace.rs:155:18
  12: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:542:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:553:40
      std::panicking::try
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panicking.rs:517:19
      std::panic::catch_unwind
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/panic.rs:350:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/thread/mod.rs:541:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/core/src/ops/function.rs:250:5
  13: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/alloc/src/boxed.rs:2064:9
      std::sys::pal::unix::thread::Thread::new::thread_start
             at /rustc/cf2df68d1f5e56803c97d91e2b1a9f1c9923c533/library/std/src/sys/pal/unix/thread.rs:108:17
  14: start_thread
             at ./nptl/pthread_create.c:442:8
  15: __GI___clone3
             at ./misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

[wubx]

https://github.com/databendlabs/databend/issues/16605

Test this issue after it complete.

[wubx]

The new painic seems to be a different problem