search

vectordotdev / vector

A high-performance observability data pipeline.
https://vector.dev
Mozilla Public License 2.0
17.07k stars 1.48k forks source link

Vector panick due to onig regex search error #15199

Open massyah opened 1 year ago

massyah commented 1 year ago

A note for the community

Problem

Vector panics when applying a parse_groks transform, due to an unhandled error originating from Onig.

Regex search error: retry-limit-in-match over

It's quite difficult to precisely identify which of the grok rules or input log is causing the panic since this is happening once every ~ 10 minutes on one server (with a logging rate of approx 10k msg/minutes).

In the configuration indicated in this issue, you can see the definition of the transform phase causing the panic; and the debug output around the backtrace.

By trial and error, it seems that this grok rule is much more likely than other to trigger the panic : 

"%{TIMESTAMP_ISO8601:log_timestamp} \\| %{LOGLEVEL:log_level}\\s+\\| %{DATA:logger}\\.%{DATA:source}:\\d+ - : opid=%{DATA:opid} :%{GREEDYDATA:log}"

This is the structure of the log lines I'm trying to parse : 

2022-11-12 08:31:52.197 | DEBUG    | campaignDataFlow.models:perform:124 - : opid=unkown :SVC FOOBARLoader @ http://rtb-dataflow-FOOBAR-prod.traefik.service.consul/FOOBAR_loader?access_token=rtb-FOOBAR-token starting

(this is just an exemple, it is not the log line causing the panic)

I can bypass the problem by replacing the rule 

%{TIMESTAMP_ISO8601:log_timestamp} \| %{LOGLEVEL:log_level}\s+\| %{DATA:logger}\.%{DATA:source}:\d+ - : opid=%{DATA:opid} :%{GREEDYDATA:log}

with 

%{TIMESTAMP_ISO8601:log_timestamp} \| %{LOGLEVEL:log_level}\s+\| %{DATA:logger} - : opid=%{DATA:opid} :%{GREEDYDATA:log}

After applying the second rule (without the \.%{DATA:source}:\d+ part, the error did not happen once in 2h.

Configuration

[transforms.grokker]
  type = "remap" 
  inputs = ["well_known_formats"] 
  source = '''
  .raw = .message
  new_parsed,err = parse_groks(
    .message,
    patterns:[
      "\\[%{TIMESTAMP_ISO8601:log_timestamp}\\] %{WORD:source}.%{WORD:log_level}: Uncaught PHP Exception %{NOTSPACE:bundle}: %{GREEDYDATA:log}",
      "\\[%{TIMESTAMP_ISO8601:log_timestamp}\\] %{WORD:sourcePhp}.%{WORD:log_level}:%{GREEDYDATA:log}",
      "\\[%{TIMESTAMP_ISO8601:log_timestamp}\\] %{DATA:logger}.%{LOGLEVEL:log_level}: %{GREEDYDATA:log}",
      "\\[%{TIMESTAMP_ISO8601:log_timestamp}.*?\\] \\[\\d+\\] \\[%{LOGLEVEL:log_level}\\] %{GREEDYDATA:log}",
      "\\[%{TIMESTAMP_ISO8601:log_timestamp}.*?\\] \\[\\d+\\] \\[%{LOGLEVEL:log_level}\\] %{GREEDYDATA:log}",
      # python loguru
      "%{TIMESTAMP_ISO8601:log_timestamp} \\| %{LOGLEVEL:log_level}\\s+\\| %{DATA:logger}\\.%{DATA:source}:\\d+ - : opid=%{DATA:opid} :%{GREEDYDATA:log}",
      "%{TIMESTAMP_ISO8601:log_timestamp} \\| %{LOGLEVEL:log_level}\\s+\\| uvicorn:access-log - %{IPORHOST:client}:%{INT:port} - %{WORD:method} %{URIPATH:request}(?:%{URIPARAM:requestparam})? HTTP/%{NUMBER:http_version} %{NUMBER:status}%{DATA:log}",
      "%{TIMESTAMP_ISO8601:log_timestamp} \\| %{LOGLEVEL:log_level}\\s+\\| %{DATA:logger}\\.%{DATA:source}:\\d+ - %{GREEDYDATA:log}",

      # Access log 
      "%{TIMESTAMP_ISO8601:log_timestamp} \\| %{LOGLEVEL:log_level}\\s+\\| uvicorn:access-log - %{IPORHOST:client}:%{INT:port} - %{WORD:method} %{URIPATH:request}(?:%{URIPARAM:requestparam})? HTTP/%{NUMBER:http_version} %{NUMBER:status}",
      "%{TIMESTAMP_ISO8601:log_timestamp} \\| %{LOGLEVEL:log_level}\\s+\\| uvicorn:access-log - %{IPORHOST:client}:%{INT:port} - %{WORD:method} %{URIPATH:request}(?:%{URIPARAM:requestparam})? HTTP/%{NUMBER:http_version} %{NUMBER:status}%{DATA:log}",
      "(?<log_timestamp>\\d{4}-\\d{2}-\\d{2},\\d{2}:\\d{2}:\\d{2}.\\d{3}) \\| %{LOGLEVEL:log_level}\\s+\\| uvicorn:access-log - %{IPORHOST:client}:%{INT:port} - %{WORD:method} %{URIPATH:request}(?:%{URIPARAM:requestparam})? HTTP/%{NUMBER:http_version} %{NUMBER:status}",
      "(?<log_timestamp>\\d{4}-\\d{2}-\\d{2},\\d{2}:\\d{2}:\\d{2}.\\d{3}) \\| %{LOGLEVEL:log_level}\\s+\\| uvicorn:access-log - %{IPORHOST:client}:%{INT:port} - %{WORD:method} %{URIPATH:request}(?:%{URIPARAM:requestparam})? HTTP/%{NUMBER:http_version} %{NUMBER:status}%{DATA:log}",
      "(?<log_timestamp>\\d{4}-\\d{2}-\\d{2},\\d{2}:\\d{2}:\\d{2}.\\d{3}) \\| %{LOGLEVEL:log_level}%{SPACE}\\| uvicorn:access-log - %{IPV4:ip}:%{NUMBER} - %{GREEDYDATA:log}",
      "(?<log_timestamp>\\d{4}-\\d{2}-\\d{2},\\d{2}:\\d{2}:\\d{2}.\\d{3}) \\| %{LOGLEVEL:log_level}%{SPACE}\\| uvicorn:access-log - %{GREEDYDATA:log}",

      # structlog default
      "%{TIMESTAMP_ISO8601:log_timestamp} \\[%{LOGLEVEL:log_level}\\s+\\] %{DATA:log}%{WORD:app}=%{DATA:app} %{GREEDYDATA:kv}"
    ]
    )
  if err==null{
    .vector_parse="grok"
    .parsed=new_parsed
    .parsed.log,err = strip_whitespace(.parsed.log)
    if exists(.parsed.status){
      .parsed.status = to_int!(.parsed.status)
    }
    if exists(.parsed.port){
      .parsed.port = to_int!(.parsed.port)
    }
    if exists(.parsed.log_timestamp){
      del(.parsed.log_timestamp)
    }
    if contains(to_string(.message)??"","access-log"){
      .parsed.log_type = "access-log"
    }
    if exists(.parsed.opid) && (.parsed.opid=="unknown"||.parsed.opid=="unkown"){
      del(.parsed.opid)
    }
    if exists(.parsed.app) && is_array(.parsed.app) && is_array(.parsed.kv){
      push(array(.parsed.kv) ?? [],join(.parsed.app,"=")??{})
    }

  }
  '''

Version

vector 0.25.1 (x86_64-unknown-linux-gnu 9125a99 2022-11-07)

Debug Output

2022-11-12T06:28:16.112429Z TRACE transform{component_kind="transform" component_id=grokker component_type=remap component_name=grokker}: vector_common::internal_event::events_received: Events received. count=1 byte_size=2091
thread 'vector-worker' panicked at 'Onig: Regex search error: retry-limit-in-match over', /cargo/registry/src/github.com-1ecc6299db9ec823/onig-6.4.0/src/lib.rs:648:23
stack backtrace:
   0: rust_begin_unwind
             at ./rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at ./rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/core/src/panicking.rs:142:14
   2: onig::Regex::search_with_encoding
   3: <vrl_compiler::expression::function::FunctionExpressionAdapter<T> as vrl_compiler::expression::Expression>::resolve
2022-11-12T06:28:16.141994Z TRACE source{component_kind="source" component_id=logs component_type=docker_logs component_name=logs}: vector_common::internal_event::bytes_received: Bytes received. byte_size=115 protocol=http
2022-11-12T06:28:16.142023Z TRACE source{component_kind="source" component_id=logs component_type=docker_logs component_name=logs}: vector::internal_events::docker_logs: Events received. count=1 byte_size=1778 container_id=54b82a4a8b4d14aebde08dd392392ea9a430136a3620f0a7d86d69f898d8d1a7
   4: <vrl_compiler::expression::function_call::FunctionCall as vrl_compiler::expression::Expression>::resolve
   5: <vrl_compiler::2022-11-12T06:28:16.142047Z TRACE source{component_kind="source" component_id=logs component_type=docker_logs component_name=logs}: vector_buffers::topology::channel::limited_queue: Sent item.
expression::Expr as vrl_compiler::2022-11-12T06:28:16.142051Z TRACE source{component_kind="source" component_id=logs component_type=docker_logs component_name=logs}: vector_common::internal_event::events_sent: Events sent. count=1 byte_size=1890 output=_default
expression::Expression>::resolve
   6: <vrl_compiler::expression::Expr as vrl_compiler::expression::Expression>::resolve
2022-11-12T06:28:16.142066Z TRACE source{component_kind="source" component_id=logs component_type=docker_logs component_name=logs}: vector_buffers::topology::channel::limited_queue: Sent item.
2022-11-12T06:28:16.142071Z TRACE source{component_kind="source" component_id=logs component_type=docker_logs component_name=logs}: vector_core::fanout: Sent item to fanout.
2022-11-12T06:28:16.142078Z TRACE transform{component_kind="transform" component_id=stripper component_type=remap component_name=stripper}: vector_common::internal_event::events_received: Events received. count=1 byte_size=1890
2022-11-12T06:28:16.142116Z TRACE transform{component_kind="transform" component_id=stripper component_type=remap component_name=stripper}: vector_buffers::topology::channel::limited_queue: Sent item.
2022-11-12T06:28:16.142119Z TRACE transform{component_kind="transform" component_id=stripper component_type=remap component_name=stripper}: vector_core::fanout: Sent item to fanout.
2022-11-12T06:28:16.142121Z TRACE transform{component_kind="transform" component_id=stripper component_type=remap component_name=stripper}: vector_common::internal_event::events_sent: Events sent. count=1 byte_size=2064 output=_default
2022-11-12T06:28:16.142128Z TRACE transform{component_kind="transform" component_id=well_known_formats component_type=remap component_name=well_known_formats}: vector_common::internal_event::events_received: Events received. count=1 byte_size=2040
   7: <vrl_compiler::expression::block::Block as vrl_compiler::expression::Expression>::resolve
2022-11-12T06:28:16.142169Z TRACE transform{component_kind="transform" component_id=well_known_formats component_type=remap component_name=well_known_formats}: vector_buffers::topology::channel::limited_queue: Sent item.
2022-11-12T06:28:16.142173Z TRACE transform{component_kind="transform" component_id=well_known_formats component_type=remap component_name=well_known_formats}: vector_core::fanout: Sent item to fanout.
2022-11-12T06:28:16.142175Z TRACE transform{component_kind="transform" component_id=well_known_formats component_type=remap component_name=well_known_formats}: vector_common::internal_event::events_sent: Events sent. count=1 byte_size=2064 output=_default
2022-11-12T06:28:16.142181Z TRACE transform{component_kind="transform" component_id=grokker component_type=remap component_name=grokker}: vector_common::internal_event::events_received: Events received. count=1 byte_size=2040
   8: vrl::runtime::Runtime::resolve
   9: <vector::transforms::remap::Remap<Runner> as vector_core::transform::SyncTransform>::transform
  10: vector_core::transform::SyncTransform::transform_all
  11: tokio::runtime::task::core::CoreStage<T>::poll
  12: tokio::runtime::task::raw::poll
  13: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
  14: tokio::runtime::task::raw::poll
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
thread 'vector-worker' panicked at 'internal error: entered unreachable code: join error or bad poll', src/topology/builder.rs:764:30
stack backtrace:
   0: rust_begin_unwind
             at ./rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/std/src/panicking.rs:584:5
   1: core::panicking::panic_fmt
             at ./rustc/a55dd71d5fb0ec5a6a3a9e8c27b2127ba491ce52/library/core/src/panicking.rs:142:14
   2: vector::topology::builder::Runner::run_concurrently::{{closure}}
   3: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
   4: <core::future::from_generator::GenFuture<T> as core::future::future::Future>::poll
   5: tokio::runtime::task::core::CoreStage<T>::poll
   6: tokio::runtime::task::raw::poll
   7: tokio::runtime::scheduler::multi_thread::worker::Context::run_task
   8: tokio::runtime::task::raw::poll
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
2022-11-12T06:28:16.143264Z ERROR transform{component_kind="transform" component_id=grokker component_type=remap component_name=grokker}: vector::topology: An error occurred that Vector couldn't handle: the task panicked and was aborted.
2022-11-12T06:28:16.143280Z  INFO vector: Vector has stopped.
2022-11-12T06:28:16.143343Z  INFO source{component_kind="source" component_id=logs component_type=docker_logs component_name=logs}: vector::internal_events::docker_logs: Stopped watching for container logs. container_id=2e8406588c1f98a78be4ad7f73754d5411ec74f837952b3194469661ebe8dc67

Example Data

2022-11-12 08:31:52.197 | DEBUG | campaignDataFlow.models:perform:124 - : opid=unkown :SVC FOOBARLoader @ http://rtb-dataflow-FOOBAR-prod.traefik.service.consul/FOOBAR_loader?access_token=rtb-FOOBAR-token starting

Additional Context

Vector is running directly on the host machine.

References

jszwedko commented 11 months ago

Thanks for reporting this @massyah and apologies for the delay in response. This appears to be an issue with the onig library we depend on; that it panics instead of returning an error: https://github.com/rust-onig/rust-onig/issues/178

seluard commented 5 months ago

Hi ! I'm facing this problem also, I wil try to simplify the grok filter. I will request help here if I'm not able to get it working. Hope mantainers from this library can fix this or at least make it flexible to avoid this panics :)

andrey-mazo commented 3 months ago

I've hit the same issue with Vector version 0.34.1 (revision="86f1c22 2023-11-16 14:59:10.486846964"). Here's a full backtrace in case it's of any help:

thread 'vector-worker' panicked at 'Onig: Regex search error: retry-limit-in-match over', /cargo/registry/src/index.crates.io-6f17d22bba15001f/onig-6.4.0/src/lib.rs:648:23
stack backtrace:
   0:     0xaaaabda1c190 - std::backtrace_rs::backtrace::libunwind::trace::hc366d13e3075cfeb
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0xaaaabda1c190 - std::backtrace_rs::backtrace::trace_unsynchronized::hb9a35e1f782586fe
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0xaaaabda1c190 - std::sys_common::backtrace::_print_fmt::hae0d6c377f414e8d
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:65:5
   3:     0xaaaabda1c190 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h0f8916ea7cac138e
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:44:22
   4:     0xaaaabd1d860c - core::fmt::rt::Argument::fmt::hf491ec0da00c9b40
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/fmt/rt.rs:138:9
   5:     0xaaaabd1d860c - core::fmt::write::h075f235dea7f2ae2
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/fmt/mod.rs:1094:21
   6:     0xaaaabd9e4048 - std::io::Write::write_fmt::hed461f96db027bb0
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/io/mod.rs:1714:15
   7:     0xaaaabda1edf4 - std::sys_common::backtrace::_print::h9f6cc87d40ba87f7
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:47:5
   8:     0xaaaabda1edf4 - std::sys_common::backtrace::print::hc36324e2f472d029
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:34:9
   9:     0xaaaabda1e9f4 - std::panicking::default_hook::{{closure}}::h8d8f8741b904046c
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:269:22
  10:     0xaaaabda1f924 - std::panicking::default_hook::h6fd0090c1af86ae9
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:288:9
  11:     0xaaaabda1f924 - std::panicking::rust_panic_with_hook::h25082e1f49b04873
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:705:13
  12:     0xaaaabda1f458 - std::panicking::begin_panic_handler::{{closure}}::hd64c14bd1869359d
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:597:13
  13:     0xaaaabda1f3cc - std::sys_common::backtrace::__rust_end_short_backtrace::h8613664e493d4a0e
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:151:18
  14:     0xaaaabda1f3c0 - rust_begin_unwind
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:593:5
  15:     0xaaaabcbfada4 - core::panicking::panic_fmt::h354dad60a2e185d5
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/panicking.rs:67:14
  16:     0xaaaabd65c3bc - onig::Regex::search_with_encoding::h35c6e8639a336fda
  17:     0xaaaabfe65dd4 - <vrl::compiler::expression::function::FunctionExpressionAdapter<T> as vrl::compiler::expression::Expression>::resolve::hb114431c2d8de9ac
  18:     0xaaaabfd564d8 - <vrl::compiler::expression::function_call::FunctionCall as vrl::compiler::expression::Expression>::resolve::hf7c372825b22c986
  19:     0xaaaabfd53eb0 - <vrl::compiler::expression::Expr as vrl::compiler::expression::Expression>::resolve::h3fa7493716323bcd
  20:     0xaaaabfd55548 - <vrl::compiler::expression::if_statement::IfStatement as vrl::compiler::expression::Expression>::resolve::h85d129daceee49b1
  21:     0xaaaabfd89530 - vrl::compiler::runtime::Runtime::resolve::hb68b28fd526cedab
  22:     0xaaaabf3cf34c - <vector::transforms::remap::Remap<Runner> as vector_core::transform::SyncTransform>::transform::h237ae68b7a8f78fe
  23:     0xaaaabf3d19fc - vector_core::transform::SyncTransform::transform_all::h862daaa044af83cd
  24:     0xaaaabfa3ffb0 - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::hf96a09481a1907f8
  25:     0xaaaabfa3ec90 - tokio::runtime::task::raw::poll::h51d2677f3c51ad38
  26:     0xaaaabda62530 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h9e1163c3d3d3b4ae
  27:     0xaaaabda67414 - tokio::runtime::task::raw::poll::hbe336b0a84fb5d6e
  28:     0xaaaabda4cc1c - std::sys_common::backtrace::__rust_begin_short_backtrace::h53f28b04ee1f0e34
  29:     0xaaaabda4c908 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h627863dbf5535ef5
  30:     0xaaaabda221cc - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hd6a6c972c6220bcb
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/alloc/src/boxed.rs:1993:9
  31:     0xaaaabda221cc - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h3b759ee01954e7b0
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/alloc/src/boxed.rs:1993:9
  32:     0xaaaabda221cc - std::sys::unix::thread::Thread::new::thread_start::hd2c1311433b59ac2
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys/unix/thread.rs:108:17
  33:     0xffff99ceee18 - <unknown>
  34:     0xffff99d57e9c - <unknown>
  35:                0x0 - <unknown>
thread 'vector-worker' panicked at 'internal error: entered unreachable code: join error or bad poll', src/topology/builder.rs:901:30
stack backtrace:
   0:     0xaaaabda1c190 - std::backtrace_rs::backtrace::libunwind::trace::hc366d13e3075cfeb
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0xaaaabda1c190 - std::backtrace_rs::backtrace::trace_unsynchronized::hb9a35e1f782586fe
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0xaaaabda1c190 - std::sys_common::backtrace::_print_fmt::hae0d6c377f414e8d
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:65:5
   3:     0xaaaabda1c190 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h0f8916ea7cac138e
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:44:22
   4:     0xaaaabd1d860c - core::fmt::rt::Argument::fmt::hf491ec0da00c9b40
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/fmt/rt.rs:138:9
   5:     0xaaaabd1d860c - core::fmt::write::h075f235dea7f2ae2
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/fmt/mod.rs:1094:21
   6:     0xaaaabd9e4048 - std::io::Write::write_fmt::hed461f96db027bb0
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/io/mod.rs:1714:15
   7:     0xaaaabda1edf4 - std::sys_common::backtrace::_print::h9f6cc87d40ba87f7
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:47:5
   8:     0xaaaabda1edf4 - std::sys_common::backtrace::print::hc36324e2f472d029
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:34:9
   9:     0xaaaabda1e9f4 - std::panicking::default_hook::{{closure}}::h8d8f8741b904046c
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:269:22
  10:     0xaaaabda1f924 - std::panicking::default_hook::h6fd0090c1af86ae9
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:288:9
  11:     0xaaaabda1f924 - std::panicking::rust_panic_with_hook::h25082e1f49b04873
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:705:13
  12:     0xaaaabda1f434 - std::panicking::begin_panic_handler::{{closure}}::hd64c14bd1869359d
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:595:13
  13:     0xaaaabda1f3cc - std::sys_common::backtrace::__rust_end_short_backtrace::h8613664e493d4a0e
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys_common/backtrace.rs:151:18
  14:     0xaaaabda1f3c0 - rust_begin_unwind
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/panicking.rs:593:5
  15:     0xaaaabcbfada4 - core::panicking::panic_fmt::h354dad60a2e185d5
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/core/src/panicking.rs:67:14
  16:     0xaaaabfa3e074 - vector::topology::builder::Runner::run_concurrently::{{closure}}::h249ca7fa73d2b017
  17:     0xaaaabfa38abc - vector::topology::builder::build_sync_transform::{{closure}}::h6238fa298cf2cea2
  18:     0xaaaabf9e34f4 - <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll::h5afd4fb766976306
  19:     0xaaaabf9e2c2c - tokio::runtime::task::raw::poll::h6ed805ee8684ea68
  20:     0xaaaabda62450 - tokio::runtime::scheduler::multi_thread::worker::Context::run_task::h9e1163c3d3d3b4ae
  21:     0xaaaabda67414 - tokio::runtime::task::raw::poll::hbe336b0a84fb5d6e
  22:     0xaaaabda4cc1c - std::sys_common::backtrace::__rust_begin_short_backtrace::h53f28b04ee1f0e34
  23:     0xaaaabda4c908 - core::ops::function::FnOnce::call_once{{vtable.shim}}::h627863dbf5535ef5
  24:     0xaaaabda221cc - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::hd6a6c972c6220bcb
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/alloc/src/boxed.rs:1993:9
  25:     0xaaaabda221cc - <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once::h3b759ee01954e7b0
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/alloc/src/boxed.rs:1993:9
  26:     0xaaaabda221cc - std::sys::unix::thread::Thread::new::thread_start::hd2c1311433b59ac2
                               at /rustc/d5c2e9c342b358556da91d61ed4133f6f50fc0c3/library/std/src/sys/unix/thread.rs:108:17
  27:     0xffff99ceee18 - <unknown>
  28:     0xffff99d57e9c - <unknown>
  29:                0x0 - <unknown>