search

sharkdp / fd

A simple, fast and user-friendly alternative to 'find'
Apache License 2.0
34.07k stars 818 forks source link

a race between spawn_receiver() and spawn_sender() #1060

Closed euncharm1ng closed 2 years ago

euncharm1ng commented 2 years ago

Hi, I found a race bug that makes fd panic by reaching an unreachable code at Line 1176 in std/sync/mpsc/mod.rs.

I suspect that fd crashes when the receiver thread spawned by spawn_receiver() does not proceed to execute line 352 and the sender threads spawned by spawn_senders() race to send messages to the receiver at line 548.

How to reproduce the crash

I built it with cargo build and observed this error as I ran ./fd in /fd/target/debug/ under stress testing workload (to better explore concurrency errors). Attached the stack backtrace below.

To make this race happen more deterministically, I recommend you to insert sleep operation at the following three locations (with the sleep operations, I could reproduce this crash out of seven test runs on average)

Line 350 in src/walk.rs

348 thread::spawn(move || {
349    // This will be set to `Some` if the `--exec` argument was supplied.
350 +  /* SLEEP HERE */
351 +  std::thread::sleep(std::time::Duration::from_millis(15));
352    if let Some(ref cmd) = config.command {

Line 406 in src/walk.rs

405 parallel_walker.run(|| {
406 +   /* SLEEP HERE */     
407 +   std::thread::sleep(std::time::Duration::from_millis(7));
408     let config = Arc::clone(config);

Line line 547 in src/walk.rs

547 +  /* SLEEP HERE*/
548 +  std::thread::sleep(std::time::Duration::from_millis(5));
549    let send_result = tx_thread.send(WorkerResult::Entry(entry));

Stack backtrace

$ ./fd
build/
build/memchr-5991ad5b883ec7ba/
build/memchr-5991ad5b883ec7ba/stderr
build/memchr-5991ad5b883ec7ba/output
build/memchr-5991ad5b883ec7ba/root-output
build/memchr-5991ad5b883ec7ba/invoked.timestamp
build/memchr-5991ad5b883ec7ba/out/
build/libc-fc149cee1b11b1a6/
build/libc-0f1f7ee0722358c0/
build/libc-0f1f7ee0722358c0/stderr
build/num-traits-85034bd872b0575f/
build/libc-fc149cee1b11b1a6/build_script_build-fc149cee1b11b1a6
build/fd-find-90e8afdb18057a1d/
build/num-traits-85034bd872b0575f/output
build/libc-0f1f7ee0722358c0/output
build/fd-find-90e8afdb18057a1d/output
build/fd-find-90e8afdb18057a1d/invoked.timestamp
build/num-traits-85034bd872b0575f/stderr
build/fd-find-90e8afdb18057a1d/stderr
build/num-traits-85034bd872b0575f/invoked.timestamp
build/fd-find-90e8afdb18057a1d/root-output
build/fd-find-90e8afdb18057a1d/out/
build/libc-0f1f7ee0722358c0/invoked.timestamp
build/num-traits-85034bd872b0575f/out/
build/num-traits-85034bd872b0575f/root-output
build/fd-find-90e8afdb18057a1d/out/_fd.ps1
build/fd-find-90e8afdb18057a1d/out/fd.elv
build/num-traits-85034bd872b0575f/out/probe1.ll
build/fd-find-90e8afdb18057a1d/out/fd.fish
build/num-traits-85034bd872b0575f/out/probe2.ll
build/num-traits-85034bd872b0575f/out/probe4.ll
build/num-traits-85034bd872b0575f/out/probe5.ll
build/num-traits-85034bd872b0575f/out/probe6.ll
build/libc-0f1f7ee0722358c0/out/
build/num-traits-85034bd872b0575f/out/probe0.ll
build/num-traits-85034bd872b0575f/out/probe3.ll
build/libc-0f1f7ee0722358c0/root-output
build/fd-find-90e8afdb18057a1d/out/fd.bash
build/libc-fc149cee1b11b1a6/build-script-build
build/libc-fc149cee1b11b1a6/build_script_build-fc149cee1b11b1a6.d
build/memoffset-851c5101ea5ade3a/
build/libc-11347ee5380a2f37/
build/crossbeam-utils-f4458ffcd6e1ce7a/
build/indexmap-09aa4e44ab385d6e/
build/memoffset-851c5101ea5ade3a/stderr
build/libc-11347ee5380a2f37/build_script_build-11347ee5380a2f37.d
build/indexmap-09aa4e44ab385d6e/output
build/indexmap-09aa4e44ab385d6e/invoked.timestamp
build/crossbeam-utils-f4458ffcd6e1ce7a/stderr
build/indexmap-09aa4e44ab385d6e/root-output
build/crossbeam-utils-f4458ffcd6e1ce7a/invoked.timestamp
build/indexmap-09aa4e44ab385d6e/stderr
build/crossbeam-utils-f4458ffcd6e1ce7a/root-output
build/indexmap-09aa4e44ab385d6e/out/
build/crossbeam-utils-f4458ffcd6e1ce7a/output
build/crossbeam-utils-f4458ffcd6e1ce7a/out/
build/libc-11347ee5380a2f37/build-script-build
build/indexmap-09aa4e44ab385d6e/out/probe0.ll
build/memoffset-851c5101ea5ade3a/output
thread '<unnamed>' panicked at 'internal error: entered unreachable code', /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sync/mpsc/mod.rs:1176:43
stack backtrace:
   0:     0x5608864b308d - std::backtrace_rs::backtrace::libunwind::trace::hc3e38a8606722936
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0x5608864b308d - std::backtrace_rs::backtrace::trace_unsynchronized::h2f19a289e03c0b8f
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x5608864b308d - std::sys_common::backtrace::_print_fmt::h9676fce66147eca3
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:66:5
   3:     0x5608864b308d - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h10611c82bfaad0e8
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:45:22
   4:     0x5608864d43fc - core::fmt::write::h5c45f38d408cd7ee
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/fmt/mod.rs:1197:17
   5:     0x5608864aedd1 - std::io::Write::write_fmt::heafbb9227d102eb7
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/io/mod.rs:1672:15
   6:     0x5608864b46b5 - std::sys_common::backtrace::_print::hc1a4cd4741da9c7d
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:48:5
   7:     0x5608864b46b5 - std::sys_common::backtrace::print::hda87bae524f5a31a
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:35:9
   8:     0x5608864b46b5 - std::panicking::default_hook::{{closure}}::h0f4b9dcdb5def95b
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:295:22
   9:     0x5608864b43d6 - std::panicking::default_hook::h3c12320e36084ccc
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:314:9
  10:     0x5608864b4c46 - std::panicking::rust_panic_with_hook::h4c2a8da8e5208878
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:698:17
  11:     0x5608864b4af9 - std::panicking::begin_panic_handler::{{closure}}::hf28ddb4373b067d3
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:586:13
  12:     0x5608864b3544 - std::sys_common::backtrace::__rust_end_short_backtrace::h92654a076df72a48
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:138:18
  13:     0x5608864b4869 - rust_begin_unwind
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:584:5
  14:     0x560885c272e3 - core::panicking::panic_fmt::hbe07bf5951d9d17b
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/panicking.rs:142:14
  15:     0x560885c271ad - core::panicking::panic::h1efed56daac899ac
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/panicking.rs:48:5
  16:     0x560885c53d15 - std::sync::mpsc::Receiver<T>::recv::h7432302886bdc4ef
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sync/mpsc/mod.rs:1176:43
  17:     0x560885c856f9 - fd::walk::ReceiverBuffer<W>::recv::hdcbca17e52206b85
                               at /home/uja/capstone/fd/src/walk.rs:237:20
  18:     0x560885c85869 - fd::walk::ReceiverBuffer<W>::poll::h78c679ef201312ac
                               at /home/uja/capstone/fd/src/walk.rs:244:15
  19:     0x560885c85628 - fd::walk::ReceiverBuffer<W>::process::h0ccd7531903a2faf
                               at /home/uja/capstone/fd/src/walk.rs:217:30
  20:     0x560885c8655f - fd::walk::spawn_receiver::{{closure}}::h29f60ac12a6ba482
                               at /home/uja/capstone/fd/src/walk.rs:396:13
  21:     0x560885c9dfc0 - std::sys_common::backtrace::__rust_begin_short_backtrace::h0fa9d6e35655910b
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:122:18
  22:     0x560885c44880 - std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}::h2f4a18329aac724c
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/thread/mod.rs:501:17
  23:     0x560885c60050 - <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once::h56424cb37ce1c46f
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/panic/unwind_safe.rs:271:9
  24:     0x560885c9645f - std::panicking::try::do_call::h5ec589e0a6c548cd
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:492:40
  25:     0x560885c9683b - __rust_try
  26:     0x560885c96388 - std::panicking::try::he8af7c9c1d7e5023
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:456:19
  27:     0x560885c3879b - std::panic::catch_unwind::h6010672670f55fbd
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panic.rs:137:14
  28:     0x560885c442de - std::thread::Builder::spawn_unchecked_::{{closure}}::hea74d13c99608c35
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/thread/mod.rs:500:30
  29:     0x560885c8d74e - core::ops::function::FnOnce::call_once{{vtable.shim}}::head8a7dacbee8c2c
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/ops/function.rs:248:5
  30:     0x5608864b9e83 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h857142660bb60205
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/alloc/src/boxed.rs:1951:9
  31:     0x5608864b9e83 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h907c82d772aa1821
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/alloc/src/boxed.rs:1951:9
  32:     0x5608864b9e83 - std::sys::unix::thread::Thread::new::thread_start::hb34581143c4c2465
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys/unix/thread.rs:108:17
  33:     0x7f9b459a5609 - start_thread
                               at /build/glibc-eX1tMB/glibc-2.31/nptl/pthread_create.c:477:8
  34:     0x7f9b45777293 - clone
  35:                0x0 - <unknown>
thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: Any { .. }', src/walk.rs:158:44
stack backtrace:
   0:     0x5608864b308d - std::backtrace_rs::backtrace::libunwind::trace::hc3e38a8606722936
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0x5608864b308d - std::backtrace_rs::backtrace::trace_unsynchronized::h2f19a289e03c0b8f
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x5608864b308d - std::sys_common::backtrace::_print_fmt::h9676fce66147eca3
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:66:5
   3:     0x5608864b308d - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h10611c82bfaad0e8
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:45:22
   4:     0x5608864d43fc - core::fmt::write::h5c45f38d408cd7ee
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/fmt/mod.rs:1197:17
   5:     0x5608864aedd1 - std::io::Write::write_fmt::heafbb9227d102eb7
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/io/mod.rs:1672:15
   6:     0x5608864b46b5 - std::sys_common::backtrace::_print::hc1a4cd4741da9c7d
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:48:5
   7:     0x5608864b46b5 - std::sys_common::backtrace::print::hda87bae524f5a31a
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:35:9
   8:     0x5608864b46b5 - std::panicking::default_hook::{{closure}}::h0f4b9dcdb5def95b
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:295:22
   9:     0x5608864b43d6 - std::panicking::default_hook::h3c12320e36084ccc
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:314:9
  10:     0x5608864b4c46 - std::panicking::rust_panic_with_hook::h4c2a8da8e5208878
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:698:17
  11:     0x5608864b4b37 - std::panicking::begin_panic_handler::{{closure}}::hf28ddb4373b067d3
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:588:13
  12:     0x5608864b3544 - std::sys_common::backtrace::__rust_end_short_backtrace::h92654a076df72a48
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:138:18
  13:     0x5608864b4869 - rust_begin_unwind
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:584:5
  14:     0x560885c272e3 - core::panicking::panic_fmt::hbe07bf5951d9d17b
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/panicking.rs:142:14
  15:     0x560885c27373 - core::result::unwrap_failed::h26ea43169558ec2c
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/result.rs:1805:5
  16:     0x560885c7e046 - core::result::Result<T,E>::unwrap::h6515768cce6d3f18
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/result.rs:1098:23
  17:     0x560885c986cb - fd::walk::scan::h9090d884565046f3
                               at /home/uja/capstone/fd/src/walk.rs:158:21
  18:     0x560885c70593 - fd::run::h0ad070e9d8d18ae3
                               at /home/uja/capstone/fd/src/main.rs:84:5
  19:     0x560885c6f971 - fd::main::h1c822a1cd5fcd68f
                               at /home/uja/capstone/fd/src/main.rs:57:18
  20:     0x560885c8d8ab - core::ops::function::FnOnce::call_once::h40e0fb98c9f6b3cf
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/ops/function.rs:248:5
  21:     0x560885c9e05e - std::sys_common::backtrace::__rust_begin_short_backtrace::h98999cee00243c71
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/sys_common/backtrace.rs:122:18
  22:     0x560885c5a921 - std::rt::lang_start::{{closure}}::h8fd5054fd6a2b7c6
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/rt.rs:145:18
  23:     0x5608864a90ae - core::ops::function::impls::<impl core::ops::function::FnOnce<A> for &F>::call_once::h56c5caa7b45fba43
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/core/src/ops/function.rs:280:13
  24:     0x5608864a90ae - std::panicking::try::do_call::h1498f2b7a1e774a5
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:492:40
  25:     0x5608864a90ae - std::panicking::try::h4b973efd55be7933
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:456:19
  26:     0x5608864a90ae - std::panic::catch_unwind::h9d79e9672f0a6494
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panic.rs:137:14
  27:     0x5608864a90ae - std::rt::lang_start_internal::{{closure}}::h3a943360205874e9
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/rt.rs:128:48
  28:     0x5608864a90ae - std::panicking::try::do_call::he83dac6bf807a6e8
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:492:40
  29:     0x5608864a90ae - std::panicking::try::h7793b8c5aee78b34
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panicking.rs:456:19
  30:     0x5608864a90ae - std::panic::catch_unwind::hdd8d52cfb7a2b9bf
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/panic.rs:137:14
  31:     0x5608864a90ae - std::rt::lang_start_internal::h8ae113c8a55bf0d4
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/rt.rs:128:20
  32:     0x560885c5a8f0 - std::rt::lang_start::h46cc5c715e8add3d
                               at /rustc/43347397f7c5ca9a670a3bb3890c7187e24a52ab/library/std/src/rt.rs:144:17
  33:     0x560885c74e5c - main
  34:     0x7f9b4567c0b3 - __libc_start_main
  35:     0x560885c2757e - _start
  36:                0x0 - <unknown>

Hope this helps. Please let me know if you need more information.

tavianator commented 2 years ago

Looks like https://github.com/rust-lang/rust/issues/39364. Cool that you found a consistent reproducer! #933 tracks replacing the stdlib's mpsc with crossbeam-channels

euncharm1ng commented 2 years ago

Ohh, I see. Thanks for your reply!

sharkdp commented 2 years ago

Thank you for reporting this. I think it's a good idea to keep this open.

sharkdp commented 2 years ago

This can also fail in CI: https://github.com/sharkdp/fd/actions/runs/3255706132/jobs/5345297564#step:10:319