search

grafana / pyroscope-rs

Pyroscope Profiler for Rust. Profile your Rust applications.
Apache License 2.0
132 stars 22 forks source link

Segmentation Fault with Jemalloc #165

Open zenixan opened 1 month ago

zenixan commented 1 month ago

Describe the bug you encountered:

We got segmentation fault when enabling a pyroscope agent:

pub fn start_pyroscope(pyroscope_config: PyroscopeConfig) {
    if pyroscope_config.enabled {
        let agent = PyroscopeAgent::builder(pyroscope_config.server_url, pyroscope_config.app_name)
            .backend(pprof_backend(PprofConfig::new().sample_rate(100)))
            .tags(vec![("hostname", get_short_host_name().as_str())])
            .build();
        if let Err(err) = agent.map(|e| e.start()) {
            error!("Failed to start pyroscope agent: {}", err);
        }
    }
}

Backtrace from core dump:

#0  0x0000564c02398304 in cache_bin_alloc_impl (adjust_low_water=false, success=<synthetic pointer>, bin=<optimized out>) at include/jemalloc/internal/cache_bin.h:399
#1  cache_bin_alloc_easy (success=<synthetic pointer>, bin=<optimized out>) at include/jemalloc/internal/cache_bin.h:399
#2  imalloc_fastpath (fallback_alloc=0x564c02396eb0 <_rjem_je_malloc_default>, size=32) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:326
#3  _rjem_malloc (size=32) at src/jemalloc.c:2746
#4  0x0000564c030e6538 in alloc::raw_vec::finish_grow (new_layout=..., current_memory=<error reading variable: Cannot access memory at address 0xfffffffffffff100>, alloc=<optimized out>)
    at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/core/src/result.rs:829
#5  0x0000564c030e6a32 in alloc::raw_vec::RawVec<T,A>::grow_amortized (len=<optimized out>, additional=1, self=<optimized out>) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/alloc/src/raw_vec.rs:433
#6  alloc::raw_vec::RawVec<T,A>::reserve_for_push (self=0x7f759ebe7570, len=<optimized out>) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/alloc/src/raw_vec.rs:318
#7  0x0000564c030df476 in alloc::vec::Vec<T,A>::push (self=0x20, value=94884477706408) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/alloc/src/vec/mod.rs:1922
#8  parking_lot_core::parking_lot::deadlock_impl::acquire_resource::{{closure}} (thread_data=0x7f759ebe7540) at src/parking_lot.rs:1226
#9  parking_lot_core::parking_lot::with_thread_data (f=...) at src/parking_lot.rs:207
#10 parking_lot_core::parking_lot::deadlock_impl::acquire_resource (key=94884477706408) at src/parking_lot.rs:1225
#11 0x0000564c0230d48c in parking_lot_core::parking_lot::deadlock::acquire_resource (_key=<optimized out>, _key=<optimized out>) at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot_core-0.9.8/src/parking_lot.rs:1114
#12 parking_lot::raw_rwlock::RawRwLock::deadlock_acquire (self=<optimized out>) at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot-0.12.1/src/raw_rwlock.rs:1140
#13 <parking_lot::raw_rwlock::RawRwLock as lock_api::rwlock::RawRwLock>::try_lock_exclusive (self=<optimized out>) at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot-0.12.1/src/raw_rwlock.rs:86
#14 lock_api::rwlock::RwLock<R,T>::try_write (self=<optimized out>) at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/lock_api-0.4.10/src/rwlock.rs:494
#15 pprof::profiler::perf_signal_handler (_signal=<optimized out>, _siginfo=<optimized out>, ucontext=0x7f759ebe4e40) at src/profiler.rs:291
#16 <signal handler called>
#17 rtree_leaf_elm_state_update (state=extent_state_active, elm2=0x7f76fb7ffe78, elm1=<optimized out>, rtree=<optimized out>, tsdn=0x7f76bffb0000) at include/jemalloc/internal/rtree.h:315
#18 _rjem_je_emap_update_edata_state (tsdn=tsdn@entry=0x7f759ebe6800, emap=<optimized out>, edata=edata@entry=0x7f770262bf80, state=state@entry=extent_state_active) at src/emap.c:36
#19 0x0000564c023f1365 in extent_activate_locked (pac=0x7f7702a03a70, ecache=0x7f7702a0d288, edata=0x7f770262bf80, eset=<optimized out>, tsdn=0x7f759ebe6800) at src/extent.c:281
#20 extent_recycle_extract (ehooks=0x7f7702a000c0, guarded=<optimized out>, alignment=4096, size=32768, expand_edata=0x7f770262bf80, ecache=0x7f7702a0d288, pac=0x7f7702a03a70, tsdn=0x7f759ebe6800) at src/extent.c:444
#21 extent_recycle (tsdn=tsdn@entry=0x7f759ebe6800, pac=pac@entry=0x7f7702a03a70, ehooks=ehooks@entry=0x7f7702a000c0, ecache=ecache@entry=0x7f7702a0d288, expand_edata=expand_edata@entry=0x0, size=size@entry=32768, alignment=<optimized out>, 
    zero=<optimized out>, commit=<optimized out>, growing_retained=<optimized out>, guarded=<optimized out>) at src/extent.c:606
#22 0x0000564c023f1765 in extent_alloc_retained (guarded=false, commit=0x7f759ebe56dd, zero=<optimized out>, alignment=<optimized out>, size=140143151179776, expand_edata=0x0, ehooks=<optimized out>, pac=0x7f7702a03a70, tsdn=0x7f759ebe6800)
    at src/extent.c:782
#23 _rjem_je_ecache_alloc_grow (tsdn=tsdn@entry=0x7f759ebe6800, pac=pac@entry=0x7f7702a03a70, ehooks=<optimized out>, ecache=ecache@entry=0x7f7702a0d288, expand_edata=expand_edata@entry=0x0, size=size@entry=32768, alignment=<optimized out>, 
    zero=<optimized out>, guarded=<optimized out>) at src/extent.c:104
#24 0x0000564c023fc445 in pac_alloc_real (tsdn=tsdn@entry=0x7f759ebe6800, pac=pac@entry=0x7f7702a03a70, ehooks=ehooks@entry=0x7f7702a000c0, size=size@entry=32768, alignment=alignment@entry=4096, zero=zero@entry=true, guarded=false)
    at src/pac.c:124
#25 0x0000564c023fc580 in pac_alloc_impl (tsdn=0x7f759ebe6800, self=0x7f7702a03a70, size=32768, alignment=4096, zero=<optimized out>, guarded=<optimized out>, frequent_reuse=false, deferred_work_generated=0x7f759ebe58df) at src/pac.c:178
#26 0x0000564c023fb529 in pai_alloc (deferred_work_generated=0x7f759ebe58df, frequent_reuse=false, guarded=false, zero=true, alignment=<optimized out>, size=32768, self=0x7f7702a03a70, tsdn=0x7f759ebe6800) at include/jemalloc/internal/pai.h:43
#27 _rjem_je_pa_alloc (tsdn=tsdn@entry=0x7f759ebe6800, shard=shard@entry=0x7f7702a03a58, size=32768, alignment=alignment@entry=4096, slab=slab@entry=false, szind=39, zero=true, guarded=false, deferred_work_generated=0x7f759ebe58df)
    at src/pa.c:139
#28 0x0000564c0239f9e9 in _rjem_je_arena_extent_alloc_large (tsdn=tsdn@entry=0x7f759ebe6800, arena=arena@entry=0x7f7702a010c0, usize=usize@entry=28672, alignment=alignment@entry=4096, zero=<optimized out>) at src/arena.c:338
#29 0x0000564c023f94f5 in _rjem_je_large_palloc (tsdn=0x7f759ebe6800, arena=0x7f7702a010c0, usize=28672, alignment=4096, zero=<optimized out>) at include/jemalloc/internal/arena_inlines_b.h:22
#30 0x0000564c023a3625 in _rjem_je_arena_palloc (tsdn=tsdn@entry=0x7f759ebe6800, arena=0x7f7702a010c0, usize=28672, alignment=<optimized out>, zero=zero@entry=true, tcache=tcache@entry=0x0) at src/arena.c:1230
#31 0x0000564c024079bb in ipallocztm (arena=<optimized out>, is_internal=true, tcache=0x0, zero=true, alignment=<optimized out>, usize=<optimized out>, tsdn=0x7f759ebe6800) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:80
#32 _rjem_je_tsd_tcache_data_init (tsd=tsd@entry=0x7f759ebe6800) at src/tcache.c:717
#33 0x0000564c02408130 in _rjem_je_tsd_tcache_enabled_data_init (tsd=tsd@entry=0x7f759ebe6800) at src/tcache.c:644
#34 0x0000564c02409cdb in tsd_data_init (tsd=0x7f759ebe6800) at src/tsd.c:244
#35 _rjem_je_tsd_fetch_slow (tsd=tsd@entry=0x7f759ebe6800, minimal=minimal@entry=false) at src/tsd.c:311
#36 0x0000564c02397331 in tsd_fetch_impl (minimal=false, init=true) at include/jemalloc/internal/tsd.h:422
#37 tsd_fetch () at include/jemalloc/internal/tsd.h:448
#38 imalloc (dopts=<synthetic pointer>, sopts=<synthetic pointer>) at src/jemalloc.c:2681
#39 _rjem_je_malloc_default (size=32) at src/jemalloc.c:2722
#40 0x0000564c030e6538 in alloc::raw_vec::finish_grow (new_layout=..., current_memory=..., alloc=<optimized out>) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/core/src/result.rs:829
#41 0x0000564c030e6a32 in alloc::raw_vec::RawVec<T,A>::grow_amortized (len=<optimized out>, additional=1, self=<optimized out>) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/alloc/src/raw_vec.rs:433
#42 alloc::raw_vec::RawVec<T,A>::reserve_for_push (self=0x7f759ebe7570, len=<optimized out>) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/alloc/src/raw_vec.rs:318
#43 0x0000564c030df476 in alloc::vec::Vec<T,A>::push (self=0x7f7680000000, value=140146223906880) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/alloc/src/vec/mod.rs:1922
#44 parking_lot_core::parking_lot::deadlock_impl::acquire_resource::{{closure}} (thread_data=0x7f759ebe7540) at src/parking_lot.rs:1226
#45 parking_lot_core::parking_lot::with_thread_data (f=...) at src/parking_lot.rs:207
#46 parking_lot_core::parking_lot::deadlock_impl::acquire_resource (key=140146223906880) at src/parking_lot.rs:1225
#47 0x0000564c030ba27c in parking_lot_core::parking_lot::deadlock::acquire_resource (_key=<optimized out>) at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot_core-0.9.8/src/parking_lot.rs:1114
#48 <parking_lot::raw_mutex::RawMutex as lock_api::mutex::RawMutex>::lock (self=0x7f7655e48040) at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/parking_lot-0.12.1/src/raw_mutex.rs:74
#49 lock_api::mutex::Mutex<R,T>::lock (self=<optimized out>, self=<optimized out>, self=<optimized out>) at /usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/lock_api-0.4.10/src/mutex.rs:214
#50 tokio::loom::std::parking_lot::Mutex<T>::lock (self=<optimized out>, self=<optimized out>, self=<optimized out>) at src/loom/std/parking_lot.rs:62
#51 tokio::runtime::blocking::pool::Inner::run (self=0x7f7655e48010, worker_thread_id=<optimized out>) at src/runtime/blocking/pool.rs:505
#52 0x0000564c030c385b in tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} () at src/runtime/blocking/pool.rs:471
#53 std::sys_common::backtrace::__rust_begin_short_backtrace (f=...) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/std/src/sys_common/backtrace.rs:155
#54 0x0000564c030c0859 in std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}} () at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/std/src/thread/mod.rs:529
#55 <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once (self=...) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/core/src/panic/unwind_safe.rs:272
#56 std::panicking::try::do_call (data=<optimized out>) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/std/src/panicking.rs:554
#57 std::panicking::try (f=...) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/std/src/panicking.rs:518
#58 std::panic::catch_unwind (f=...) at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/std/src/panic.rs:142
#59 std::thread::Builder::spawn_unchecked_::{{closure}} () at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/std/src/thread/mod.rs:528
#60 core::ops::function::FnOnce::call_once{{vtable-shim}} () at /rustc/aedd173a2c086e558c2b66d3743b344f977621a7/library/core/src/ops/function.rs:250
#61 0x0000564c031683f5 in <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once () at library/alloc/src/boxed.rs:2015
#62 <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once () at library/alloc/src/boxed.rs:2015
#63 std::sys::pal::unix::thread::Thread::new::thread_start () at library/std/src/sys/pal/unix/thread.rs:108
#64 0x00007f7703199609 in start_thread (arg=<optimized out>) at pthread_create.c:477
#65 0x00007f7702f69353 in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

pyroscope-rs version and environment

OS: Linux 5.15.0-94-generic #104-Ubuntu SMP Tue Jan 9 15:25:40 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

Cargo.toml:

[dependencies]
pyroscope = "0.5.4"
pyroscope_pprofrs = "0.2"
tikv-jemallocator = "0.5.0"
tikv-jemalloc-ctl = "0.5.0"
korniltsev commented 1 month ago

well, this sucks.

pprof::profiler::perf_signal_handler should not call to jemalloc (transitively)

I hope this issue is somehow addressed in pprof and we just need to bump the dependency.

korniltsev commented 1 month ago

Thanks for the report!