Closed raghu999 closed 1 year ago
We upgraded to 0.30.0 the error did not happen for 3 consecutive days and the same error started to happen again this morning even with 0.30.0 version.
thread 'vector-worker' panicked at 'internal error: entered unreachable code: malformed path. This is a bug.', /cargo/git/checkouts/vrl-e892d04501659efd/113005b/lib/value/src/value/crud/mod.rs:192:26
stack backtrace:
0: 0x55c1eed4a060 - std::backtrace_rs::backtrace::libunwind::trace::ha9053a9a07ca49cb
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
1: 0x55c1eed4a060 - std::backtrace_rs::backtrace::trace_unsynchronized::h9c2852a457ad564e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x55c1eed4a060 - std::sys_common::backtrace::_print_fmt::h457936fbfaa0070f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:65:5
3: 0x55c1eed4a060 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h5779d7bf7f70cb0c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:44:22
4: 0x55c1ee2ff07e - core::fmt::write::h5a4baaff1bcd3eb5
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/fmt/mod.rs:1232:17
5: 0x55c1eed112a4 - std::io::Write::write_fmt::h4bc1f301cb9e9cce
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/io/mod.rs:1684:15
6: 0x55c1eed4f0af - std::sys_common::backtrace::_print::h5fcdc36060f177e8
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:47:5
7: 0x55c1eed4f0af - std::sys_common::backtrace::print::h54ca9458b876c8bf
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:34:9
8: 0x55c1eed4eb8d - std::panicking::default_hook::{{closure}}::hbe471161c7664ed6
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:271:22
9: 0x55c1eed4fe0a - std::panicking::default_hook::ha3500da57aa4ac4f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:290:9
10: 0x55c1eed4fe0a - std::panicking::rust_panic_with_hook::h50c09d000dc561d2
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:692:13
11: 0x55c1eed4f784 - std::panicking::begin_panic_handler::{{closure}}::h9e2b2176e00e0d9c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:583:13
12: 0x55c1eed4f6e6 - std::sys_common::backtrace::__rust_end_short_backtrace::h5739b8e512c09d02
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:150:18
13: 0x55c1eed4f6d1 - rust_begin_unwind
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
14: 0x55c1edc95722 - core::panicking::panic_fmt::hf33a1475b4dc5c3e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
15: 0x55c1ef0eaea7 - value::value::crud::get_matching_coalesce_key::h8616604a90b20b0e
16: 0x55c1ef0ea12f - value::value::Value::insert::h1b65e10e512c86f9
17: 0x55c1efa8b9b0 - vector_core::event::log_event::LogEvent::insert::h77bd427b6e9eb76f
18: 0x55c1f0097633 - vector::sources::splunk_hec::SplunkSource::event_service::{{closure}}::{{closure}}::h0317cc6e0e53029b
19: 0x55c1f00910e7 - <futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll::h0c68f5cad7cdb69c
20: 0x55c1ef478466 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hca242ba6c6316134
21: 0x55c1ef477ea9 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hbbe405f17eb08249
22: 0x55c1f005db6a - <warp::filter::service::FilteredFuture<F> as core::future::future::Future>::poll::h1fff50b02834de06
23: 0x55c1f005683c - hyper::proto::h1::dispatch::Dispatcher<D,Bs,I,T>::poll_inner::ha61e30b5ef4d4f60
24: 0x55c1f004f548 - <hyper::server::server::new_svc::NewSvcTask<I,N,S,E,W> as core::future::future::Future>::poll::hca4652c1536f8209
25: 0x55c1f00650a6 - tokio::runtime::task::raw::poll::h6f9dce34652f0558
26: 0x55c1eed99950 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::hc24e71da8373f516
27: 0x55c1eeda26b5 - tokio::runtime::task::raw::poll::h33675ffb32f6f519
28: 0x55c1eed80ba3 - std::sys_common::backtrace::__rust_begin_short_backtrace::h0e019256e1308fea
29: 0x55c1eed80913 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h258f70a52f08f6e0
30: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h39990b24eedef2ab
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
31: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h01a027258444143b
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
32: 0x55c1eed52513 - std::sys::unix::thread::Thread::new::thread_start::ha4f1cdd9c25884ba
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys/unix/thread.rs:108:17
33: 0x7f0856f94ea7 - start_thread
34: 0x7f0856d68a2f - clone
35: 0x0 - <unknown>
thread 'vector-worker' panicked at 'internal error: entered unreachable code: malformed path. This is a bug.', /cargo/git/checkouts/vrl-e892d04501659efd/113005b/lib/value/src/value/crud/mod.rs:192:26
stack backtrace:
0: 0x55c1eed4a060 - std::backtrace_rs::backtrace::libunwind::trace::ha9053a9a07ca49cb
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
1: 0x55c1eed4a060 - std::backtrace_rs::backtrace::trace_unsynchronized::h9c2852a457ad564e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x55c1eed4a060 - std::sys_common::backtrace::_print_fmt::h457936fbfaa0070f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:65:5
3: 0x55c1eed4a060 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h5779d7bf7f70cb0c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:44:22
4: 0x55c1ee2ff07e - core::fmt::write::h5a4baaff1bcd3eb5
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/fmt/mod.rs:1232:17
5: 0x55c1eed112a4 - std::io::Write::write_fmt::h4bc1f301cb9e9cce
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/io/mod.rs:1684:15
6: 0x55c1eed4f0af - std::sys_common::backtrace::_print::h5fcdc36060f177e8
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:47:5
7: 0x55c1eed4f0af - std::sys_common::backtrace::print::h54ca9458b876c8bf
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:34:9
8: 0x55c1eed4eb8d - std::panicking::default_hook::{{closure}}::hbe471161c7664ed6
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:271:22
9: 0x55c1eed4fe0a - std::panicking::default_hook::ha3500da57aa4ac4f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:290:9
10: 0x55c1eed4fe0a - std::panicking::rust_panic_with_hook::h50c09d000dc561d2
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:692:13
11: 0x55c1eed4f784 - std::panicking::begin_panic_handler::{{closure}}::h9e2b2176e00e0d9c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:583:13
12: 0x55c1eed4f6e6 - std::sys_common::backtrace::__rust_end_short_backtrace::h5739b8e512c09d02
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:150:18
13: 0x55c1eed4f6d1 - rust_begin_unwind
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
14: 0x55c1edc95722 - core::panicking::panic_fmt::hf33a1475b4dc5c3e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
15: 0x55c1ef0eaea7 - value::value::crud::get_matching_coalesce_key::h8616604a90b20b0e
16: 0x55c1ef0ea12f - value::value::Value::insert::h1b65e10e512c86f9
17: 0x55c1efa8b9b0 - vector_core::event::log_event::LogEvent::insert::h77bd427b6e9eb76f
18: 0x55c1f0097633 - vector::sources::splunk_hec::SplunkSource::event_service::{{closure}}::{{closure}}::h0317cc6e0e53029b
19: 0x55c1f00910e7 - <futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll::h0c68f5cad7cdb69c
20: 0x55c1ef478466 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hca242ba6c6316134
21: 0x55c1ef477ea9 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hbbe405f17eb08249
22: 0x55c1f005db6a - <warp::filter::service::FilteredFuture<F> as core::future::future::Future>::poll::h1fff50b02834de06
23: 0x55c1f005683c - hyper::proto::h1::dispatch::Dispatcher<D,Bs,I,T>::poll_inner::ha61e30b5ef4d4f60
24: 0x55c1f004f548 - <hyper::server::server::new_svc::NewSvcTask<I,N,S,E,W> as core::future::future::Future>::poll::hca4652c1536f8209
25: 0x55c1f00650a6 - tokio::runtime::task::raw::poll::h6f9dce34652f0558
26: 0x55c1eed99950 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::hc24e71da8373f516
27: 0x55c1eeda26b5 - tokio::runtime::task::raw::poll::h33675ffb32f6f519
28: 0x55c1eed80ba3 - std::sys_common::backtrace::__rust_begin_short_backtrace::h0e019256e1308fea
29: 0x55c1eed80913 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h258f70a52f08f6e0
30: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h39990b24eedef2ab
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
31: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h01a027258444143b
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
32: 0x55c1eed52513 - std::sys::unix::thread::Thread::new::thread_start::ha4f1cdd9c25884ba
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys/unix/thread.rs:108:17
33: 0x7f0856f94ea7 - start_thread
34: 0x7f0856d68a2f - clone
35: 0x0 - <unknown>
thread 'vector-worker' panicked at 'internal error: entered unreachable code: malformed path. This is a bug.', /cargo/git/checkouts/vrl-e892d04501659efd/113005b/lib/value/src/value/crud/mod.rs:192:26
stack backtrace:
0: 0x55c1eed4a060 - std::backtrace_rs::backtrace::libunwind::trace::ha9053a9a07ca49cb
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
1: 0x55c1eed4a060 - std::backtrace_rs::backtrace::trace_unsynchronized::h9c2852a457ad564e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x55c1eed4a060 - std::sys_common::backtrace::_print_fmt::h457936fbfaa0070f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:65:5
3: 0x55c1eed4a060 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h5779d7bf7f70cb0c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:44:22
4: 0x55c1ee2ff07e - core::fmt::write::h5a4baaff1bcd3eb5
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/fmt/mod.rs:1232:17
5: 0x55c1eed112a4 - std::io::Write::write_fmt::h4bc1f301cb9e9cce
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/io/mod.rs:1684:15
6: 0x55c1eed4f0af - std::sys_common::backtrace::_print::h5fcdc36060f177e8
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:47:5
7: 0x55c1eed4f0af - std::sys_common::backtrace::print::h54ca9458b876c8bf
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:34:9
8: 0x55c1eed4eb8d - std::panicking::default_hook::{{closure}}::hbe471161c7664ed6
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:271:22
9: 0x55c1eed4fe0a - std::panicking::default_hook::ha3500da57aa4ac4f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:290:9
10: 0x55c1eed4fe0a - std::panicking::rust_panic_with_hook::h50c09d000dc561d2
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:692:13
11: 0x55c1eed4f784 - std::panicking::begin_panic_handler::{{closure}}::h9e2b2176e00e0d9c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:583:13
12: 0x55c1eed4f6e6 - std::sys_common::backtrace::__rust_end_short_backtrace::h5739b8e512c09d02
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:150:18
13: 0x55c1eed4f6d1 - rust_begin_unwind
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
14: 0x55c1edc95722 - core::panicking::panic_fmt::hf33a1475b4dc5c3e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
15: 0x55c1ef0eaea7 - value::value::crud::get_matching_coalesce_key::h8616604a90b20b0e
16: 0x55c1ef0ea12f - value::value::Value::insert::h1b65e10e512c86f9
17: 0x55c1efa8b9b0 - vector_core::event::log_event::LogEvent::insert::h77bd427b6e9eb76f
18: 0x55c1f0097633 - vector::sources::splunk_hec::SplunkSource::event_service::{{closure}}::{{closure}}::h0317cc6e0e53029b
19: 0x55c1f00910e7 - <futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll::h0c68f5cad7cdb69c
20: 0x55c1ef478466 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hca242ba6c6316134
21: 0x55c1ef477ea9 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hbbe405f17eb08249
22: 0x55c1f005db6a - <warp::filter::service::FilteredFuture<F> as core::future::future::Future>::poll::h1fff50b02834de06
23: 0x55c1f005683c - hyper::proto::h1::dispatch::Dispatcher<D,Bs,I,T>::poll_inner::ha61e30b5ef4d4f60
24: 0x55c1f004f548 - <hyper::server::server::new_svc::NewSvcTask<I,N,S,E,W> as core::future::future::Future>::poll::hca4652c1536f8209
25: 0x55c1f00650a6 - tokio::runtime::task::raw::poll::h6f9dce34652f0558
26: 0x55c1eed99950 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::hc24e71da8373f516
27: 0x55c1eeda26b5 - tokio::runtime::task::raw::poll::h33675ffb32f6f519
28: 0x55c1eed80ba3 - std::sys_common::backtrace::__rust_begin_short_backtrace::h0e019256e1308fea
29: 0x55c1eed80913 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h258f70a52f08f6e0
30: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h39990b24eedef2ab
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
31: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h01a027258444143b
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
32: 0x55c1eed52513 - std::sys::unix::thread::Thread::new::thread_start::ha4f1cdd9c25884ba
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys/unix/thread.rs:108:17
33: 0x7f0856f94ea7 - start_thread
34: 0x7f0856d68a2f - clone
35: 0x0 - <unknown>
thread 'vector-worker' panicked at 'internal error: entered unreachable code: malformed path. This is a bug.', /cargo/git/checkouts/vrl-e892d04501659efd/113005b/lib/value/src/value/crud/mod.rs:192:26
stack backtrace:
0: 0x55c1eed4a060 - std::backtrace_rs::backtrace::libunwind::trace::ha9053a9a07ca49cb
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
1: 0x55c1eed4a060 - std::backtrace_rs::backtrace::trace_unsynchronized::h9c2852a457ad564e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x55c1eed4a060 - std::sys_common::backtrace::_print_fmt::h457936fbfaa0070f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:65:5
3: 0x55c1eed4a060 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h5779d7bf7f70cb0c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:44:22
4: 0x55c1ee2ff07e - core::fmt::write::h5a4baaff1bcd3eb5
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/fmt/mod.rs:1232:17
5: 0x55c1eed112a4 - std::io::Write::write_fmt::h4bc1f301cb9e9cce
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/io/mod.rs:1684:15
6: 0x55c1eed4f0af - std::sys_common::backtrace::_print::h5fcdc36060f177e8
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:47:5
7: 0x55c1eed4f0af - std::sys_common::backtrace::print::h54ca9458b876c8bf
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:34:9
8: 0x55c1eed4eb8d - std::panicking::default_hook::{{closure}}::hbe471161c7664ed6
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:271:22
9: 0x55c1eed4fe0a - std::panicking::default_hook::ha3500da57aa4ac4f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:290:9
10: 0x55c1eed4fe0a - std::panicking::rust_panic_with_hook::h50c09d000dc561d2
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:692:13
11: 0x55c1eed4f784 - std::panicking::begin_panic_handler::{{closure}}::h9e2b2176e00e0d9c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:583:13
12: 0x55c1eed4f6e6 - std::sys_common::backtrace::__rust_end_short_backtrace::h5739b8e512c09d02
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:150:18
13: 0x55c1eed4f6d1 - rust_begin_unwind
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
14: 0x55c1edc95722 - core::panicking::panic_fmt::hf33a1475b4dc5c3e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
15: 0x55c1ef0eaea7 - value::value::crud::get_matching_coalesce_key::h8616604a90b20b0e
16: 0x55c1ef0ea12f - value::value::Value::insert::h1b65e10e512c86f9
17: 0x55c1efa8b9b0 - vector_core::event::log_event::LogEvent::insert::h77bd427b6e9eb76f
18: 0x55c1f0097633 - vector::sources::splunk_hec::SplunkSource::event_service::{{closure}}::{{closure}}::h0317cc6e0e53029b
19: 0x55c1f00910e7 - <futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll::h0c68f5cad7cdb69c
20: 0x55c1ef478466 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hca242ba6c6316134
21: 0x55c1ef477ea9 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hbbe405f17eb08249
22: 0x55c1f005db6a - <warp::filter::service::FilteredFuture<F> as core::future::future::Future>::poll::h1fff50b02834de06
23: 0x55c1f005683c - hyper::proto::h1::dispatch::Dispatcher<D,Bs,I,T>::poll_inner::ha61e30b5ef4d4f60
24: 0x55c1f004f548 - <hyper::server::server::new_svc::NewSvcTask<I,N,S,E,W> as core::future::future::Future>::poll::hca4652c1536f8209
25: 0x55c1f00650a6 - tokio::runtime::task::raw::poll::h6f9dce34652f0558
26: 0x55c1eed99950 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::hc24e71da8373f516
27: 0x55c1eeda26b5 - tokio::runtime::task::raw::poll::h33675ffb32f6f519
28: 0x55c1eed80ba3 - std::sys_common::backtrace::__rust_begin_short_backtrace::h0e019256e1308fea
29: 0x55c1eed80913 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h258f70a52f08f6e0
30: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h39990b24eedef2ab
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
31: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h01a027258444143b
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
32: 0x55c1eed52513 - std::sys::unix::thread::Thread::new::thread_start::ha4f1cdd9c25884ba
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys/unix/thread.rs:108:17
33: 0x7f0856f94ea7 - start_thread
34: 0x7f0856d68a2f - clone
35: 0x0 - <unknown>
thread 'vector-worker' panicked at 'internal error: entered unreachable code: malformed path. This is a bug.', /cargo/git/checkouts/vrl-e892d04501659efd/113005b/lib/value/src/value/crud/mod.rs:192:26
stack backtrace:
0: 0x55c1eed4a060 - std::backtrace_rs::backtrace::libunwind::trace::ha9053a9a07ca49cb
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
1: 0x55c1eed4a060 - std::backtrace_rs::backtrace::trace_unsynchronized::h9c2852a457ad564e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
2: 0x55c1eed4a060 - std::sys_common::backtrace::_print_fmt::h457936fbfaa0070f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:65:5
3: 0x55c1eed4a060 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h5779d7bf7f70cb0c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:44:22
4: 0x55c1ee2ff07e - core::fmt::write::h5a4baaff1bcd3eb5
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/fmt/mod.rs:1232:17
5: 0x55c1eed112a4 - std::io::Write::write_fmt::h4bc1f301cb9e9cce
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/io/mod.rs:1684:15
6: 0x55c1eed4f0af - std::sys_common::backtrace::_print::h5fcdc36060f177e8
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:47:5
7: 0x55c1eed4f0af - std::sys_common::backtrace::print::h54ca9458b876c8bf
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:34:9
8: 0x55c1eed4eb8d - std::panicking::default_hook::{{closure}}::hbe471161c7664ed6
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:271:22
9: 0x55c1eed4fe0a - std::panicking::default_hook::ha3500da57aa4ac4f
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:290:9
10: 0x55c1eed4fe0a - std::panicking::rust_panic_with_hook::h50c09d000dc561d2
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:692:13
11: 0x55c1eed4f784 - std::panicking::begin_panic_handler::{{closure}}::h9e2b2176e00e0d9c
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:583:13
12: 0x55c1eed4f6e6 - std::sys_common::backtrace::__rust_end_short_backtrace::h5739b8e512c09d02
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys_common/backtrace.rs:150:18
13: 0x55c1eed4f6d1 - rust_begin_unwind
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/panicking.rs:579:5
14: 0x55c1edc95722 - core::panicking::panic_fmt::hf33a1475b4dc5c3e
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/core/src/panicking.rs:64:14
15: 0x55c1ef0eaea7 - value::value::crud::get_matching_coalesce_key::h8616604a90b20b0e
16: 0x55c1ef0ea12f - value::value::Value::insert::h1b65e10e512c86f9
17: 0x55c1efa8b9b0 - vector_core::event::log_event::LogEvent::insert::h77bd427b6e9eb76f
18: 0x55c1f0097633 - vector::sources::splunk_hec::SplunkSource::event_service::{{closure}}::{{closure}}::h0317cc6e0e53029b
19: 0x55c1f00910e7 - <futures_util::future::try_future::into_future::IntoFuture<Fut> as core::future::future::Future>::poll::h0c68f5cad7cdb69c
20: 0x55c1ef478466 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hca242ba6c6316134
21: 0x55c1ef477ea9 - <warp::filter::unify::UnifyFuture<F> as core::future::future::Future>::poll::hbbe405f17eb08249
22: 0x55c1f005db6a - <warp::filter::service::FilteredFuture<F> as core::future::future::Future>::poll::h1fff50b02834de06
23: 0x55c1f005683c - hyper::proto::h1::dispatch::Dispatcher<D,Bs,I,T>::poll_inner::ha61e30b5ef4d4f60
24: 0x55c1f004f548 - <hyper::server::server::new_svc::NewSvcTask<I,N,S,E,W> as core::future::future::Future>::poll::hca4652c1536f8209
25: 0x55c1f00650a6 - tokio::runtime::task::raw::poll::h6f9dce34652f0558
26: 0x55c1eed99950 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::hc24e71da8373f516
27: 0x55c1eeda26b5 - tokio::runtime::task::raw::poll::h33675ffb32f6f519
28: 0x55c1eed80ba3 - std::sys_common::backtrace::__rust_begin_short_backtrace::h0e019256e1308fea
29: 0x55c1eed80913 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h258f70a52f08f6e0
30: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h39990b24eedef2ab
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
31: 0x55c1eed52513 - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h01a027258444143b
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/alloc/src/boxed.rs:1987:9
32: 0x55c1eed52513 - std::sys::unix::thread::Thread::new::thread_start::ha4f1cdd9c25884ba
at /rustc/84c898d65adf2f39a5a98507f1fe0ce10a2b8dbc/library/std/src/sys/unix/thread.rs:108:17
33: 0x7f0856f94ea7 - start_thread
34: 0x7f0856d68a2f - clone
35: 0x0 - <unknown>
Hi @raghu999,
Taking a look at the provided stack trace, the panic looks to be occurring during the "remap" remap transform that is applied to events coming from the splunk_hec source. This is unexpected as that transform should be a noop.
As a workaround, you can try removing the "remap" transform from the config, as it is indeed a noop. Your configuration would then look like the following.
data_dir: /vector-data-dir
log_schema:
host_key: "host_name"
timestamp_key: "@timestamp"
api:
enabled: true
address: 127.0.0.1:8686
playground: false
sources:
internal_metrics:
type: internal_metrics
ingest:
address: 0.0.0.0:8080
valid_tokens: ["<token>"]
type: splunk_hec
internal_logs:
type: internal_logs
transforms:
metrics:
type: "remap"
inputs: ["internal_metrics"]
source: |2-
.tags.job = "team-ingest"
.tags.tenant_name = "team"
.tags.instance = "127.0.0.1:443"
.tags.environment = "prod"
.tags.dc = "ch2"
vector:
type: remap
inputs:
- internal_logs
source: |2-
.vector = del(.)
."@timestamp" = del(.vector.timestamp)
.message = del(.vector.message)
.log = {}
.log.logger = del(.vector.metadata.module_path)
.log.level = downcase(string!(del(.vector.metadata.level)))
.ecs = {}
.ecs.version = "1.7"
.cse = {}
.cse.version = "1.0"
.agent = {}
.agent.type = "vector"
.agent.version = "0.29.1"
.event = {}
.event.dataset = ["app"]
.host = {}
.host.hostname = get_env_var("ELK_HOSTNAME") ?? null
.cloud = {}
.cloud.provider = get_env_var("CLOUD_PROVIDER") ?? null
.cloud.region = get_env_var("CLOUD_REGION") ?? null
.tenant = {}
.tenant.name = "team_prod"
.tenant = {}
.service = {}
.service.name = "team_prod"
.host.hostname = get_hostname!()
sinks:
prom_exporter:
type: prometheus_exporter
inputs: [internal_metrics]
address: 0.0.0.0:8181
maas_remote_write:
type: prometheus_remote_write
inputs:
- metrics
default_namespace: team
endpoint: https://127.0.0.1:9090/api/v1/write
tenant_id: team_telemetry
elastic:
type: "elasticsearch"
inputs: [ingest]
endpoint: "${ELK_URL}"
compression: "gzip"
bulk:
index: "logz-team-app"
action: "create"
auth:
strategy: "basic"
user: "${ELK_USERNAME}"
password: "${ELK_PASSWORD}"
buffer:
type: "disk"
max_size: 400000000
when_full: "drop_newest"
batch:
max_bytes: 10490000
timeout_secs: 8
request:
concurrency: adaptive
timeout_secs: 120
tls:
verify_certificate: false
verify_hostname: false
suppress_type_name: true
elastic_internal:
type: "elasticsearch"
inputs: [vector]
compression: "gzip"
endpoint: "${ELK_URL}"
bulk:
index: "logz-forwarder"
action: "create"
auth:
strategy: "basic"
user: "${ELK_USERNAME}"
password: "${ELK_PASSWORD}"
buffer:
type: "disk"
max_size: 400000000
when_full: "drop_newest"
batch:
max_bytes: 10490000
request:
retry_attempts: 10
tls:
verify_certificate: false
verify_hostname: false
suppress_type_name: true
I believe I have reproduced and fixed this issue in #17943. This will close when that PR gets merged.
Since this is an intermittent issue, it is possible that there are other causes, so feel free to re-open with more context if the issue still crops up after the next release.
A note for the community
Problem
We are seeing the below error on the vector. Copying below the Error and Backtrace.
Configuration
Version
0.29.1
Debug Output