Fluentd stuck with high cpu and not passing logs to Elasticsearch

Describe the bug

Fluentd stucks with 100% cpu and not passing logs to elasticsearch. Found it is stuck in event_loop.

To Reproduce

Run a fluentd with attached configuration

Expected behavior

Fluentd should not stuck and pass logs to Elasticsearch

Your Environment

- Fluentd version: v1.15.0
- Operating system:Ubuntu 18.04.6 LTS
- Kernel version:5.4.0-1078-azure

Your Configuration

<source>
  @type tail
  path /data/logs/*/app/*.log
  exclude_path ["/data/logs/*/app/*.gz", "/data/logs/*/app/*.zip", "/data/logs/*/app/*-%Y-%m-%d.log"]    
  pos_file /data/logs/app.log.pos
  pos_file_compaction_interval 168h
  read_bytes_limit_per_second 200000     
  path_key tailed_path
  tag ms-logs-application
  read_from_head true
  follow_inodes true
  refresh_interval 40s
  <parse>
    @type none
  </parse>
  time_format %Y-%m-%dT%H:%M:%S.%NZ      
</source>
<filter ms-logs-application>
  @type concat
  key message
  multiline_start_regexp /\d{4}-\d{1,2}-\d{1,2}/
  flush_interval 10
  timeout_label @NORMAL
</filter>
<match ms-logs-application>
  @type relabel
  num_threads 4
  @label @NORMAL
</match>
<label @NORMAL>
  <filter ms-logs-application>
   @type parser
   key_name message
   reserve_data true
    <parse>
      @type grok
    grok_failure_key grokfailure
    <grok>
      pattern (?<message>[^\]]+ (?<timestamp>%{HOUR}:%{MINUTE}:%{SECOND}.%{NONNEGINT})\|\[(?<thread>[^\]]+)?\]\|%{IPORHOST:pod_instance}\|(?<severity>.*?)\|(?<log_type>.*?)\|(?<application>.*?)\|(?<microservice>.*?)\|(?<uuid>.*?)\|(?<message_type>.*?)\|(?<fullmessage>(.|\r|\n)*))
      </grok>       
    </parse>
  </filter>  

  <filter ms-logs-application>
    @type record_transformer
    remove_keys fullmessage
    enable_ruby
    <record>
      host.name ${hostname}
      remote_ip "#{(Socket.ip_address_list.detect do |intf| intf.ipv4_private? end).ip_address}"
      log.file.path "${record['tailed_path']}"
    #remote_ip "%#{@metadata.ip_address}"
    </record>
  </filter>

  <match ms-logs-application>
    @type rewrite_tag_filter
    num_threads 4
    <rule>
      key grokfailure
      pattern /.*/
      tag grokfailure_log_app
    </rule>
    <rule>
      key application
      pattern /.*/
      tag ms-logs-app-matched
    </rule>     
  </match>

  <match ms-logs-app-matched>
    @type elasticsearch_dynamic
    num_threads 4
    @log_level info
    suppress_type_name true
    include_tag_key true
    reload_connections true
    hosts https://10.8.x.x:9200,https://10.8.x.x:9200,https://10.8.x.x:9200
    scheme https
    client_cert /fluentd/etc/elk.crt
    ca_file /fluentd/etc/elk.crt
    client_key /fluentd/etc/elk.key
    user elastic
    password <some password>
    logstash_format true
    slow_flush_log_threshold 45.0
    logstash_prefix spp-application-${record['application']}
    <buffer>
       @type file
       path /data/logs/*/app/*.log
       flush_mode interval
       retry_type exponential_backoff
       flush_thread_count 4
       flush_interval 10s
       retry_forever true
       retry_max_interval 30
       chunk_limit_size 12M
       queue_limit_length 32
       overflow_action throw_exception
      </buffer>
  </match>  

  <match grokfailure_log_app>
    @type elasticsearch_dynamic
    num_threads 4
    @log_level info
    suppress_type_name true
    include_tag_key true
    reload_connections true
    hosts https://10.8.x.x:9200,https://10.8.x.x:9200,https://10.8.x.x:9200
    scheme https
    client_cert /fluentd/etc/elk.crt
    ca_file /fluentd/etc/elk.crt
    client_key /fluentd/etc/elk.key
    user elastic
    password <some password>
    logstash_format true
    slow_flush_log_threshold 45.0
    logstash_prefix spp-nonematch
  </match>    
</label>   
<filter ms-logs-application>
 @type parser
 key_name message
 reserve_data true
  <parse>
    @type grok
    grok_failure_key grokfailure
    <grok>
    pattern (?<message>[^\]]+ (?<timestamp>%{HOUR}:%{MINUTE}:%{SECOND}.%{NONNEGINT})\|\[(?<thread>[^\]]+)?\]\|%{IPORHOST:pod_instance}\|(?<severity>.*?)\|(?<log_type>.*?)\|(?<application>.*?)\|(?<microservice>.*?)\|(?<uuid>.*?)\|(?<message_type>.*?)\|(?<fullmessage>(.|\r|\n)*))
    </grok>     
  </parse>
</filter>  

<filter ms-logs-application>
  @type record_transformer
  remove_keys fullmessage
  enable_ruby     
  <record>
    host.name ${hostname}
    remote_ip "#{(Socket.ip_address_list.detect do |intf| intf.ipv4_private? end).ip_address}"
    log.file.path "${record['tailed_path']}"
    #remote_ip "%#{@metadata.ip_address}"
  </record>
</filter>

<match ms-logs-application>
  @type rewrite_tag_filter
  num_threads 4
  <rule>
    key grokfailure
    pattern /.*/
    tag grokfailure_log_app
  </rule>
  <rule>
    key application
    pattern /.*/
    tag ms-logs-app-matched
  </rule>     
</match>

<match ms-logs-app-matched>
  @type elasticsearch_dynamic
  num_threads 4
  @log_level info
  suppress_type_name true
  include_tag_key true
  reload_connections true
  hosts https://10.8.x.x:9200,https://10.8.x.x:9200,https://10.8.x.x:9200
  scheme https
  client_cert /fluentd/etc/elk.crt
  ca_file /fluentd/etc/elk.crt
  client_key /fluentd/etc/elk.key
  user elastic
  password <some password>
  logstash_format true
  slow_flush_log_threshold 45.0
  logstash_prefix spp-application-${record['application']}
</match>   

<match grokfailure_log_app>
  @type elasticsearch_dynamic
  num_threads 4
  @log_level info
  #host 10.8.22.5
  suppress_type_name true
  include_tag_key true
  reload_connections true
  hosts https://10.8.x.x:9200,https://10.8.x.x:9200,https://10.8.x.x:9200
  scheme https
  client_cert /fluentd/etc/elk.crt
  ca_file /fluentd/etc/elk.crt
  client_key /fluentd/etc/elk.key
  user elastic
  password <some password>
  logstash_format true
  slow_flush_log_threshold 45.0
  logstash_prefix spp-nonematch
</match>

Your Error Log

Fluend log:

2022-07-14 10:07:36 +0800 [info]: #0 detected rotation of /data/logs/shop-detailquery-ms/app/detailquery-ms-10.35.64.113.log; waiting 5 seconds
2022-07-14 10:07:36 +0800 [info]: #0 following tail of /data/logs/shop-detailquery-ms/app/detailquery-ms-10.35.64.113.log
2022-07-14 10:07:52 +0800 [info]: #0 detected rotation of /data/logs/spp-data-upload-ms/app/data-upload-ms-10.35.64.124.log; waiting 5 seconds
2022-07-14 10:07:52 +0800 [info]: #0 following tail of /data/logs/spp-data-upload-ms/app/data-upload-ms-10.35.64.124.log
2022-07-14 10:08:00 +0800 [info]: #0 following tail of /data/logs/shop-detailquery-ms/app/detailquery-ms-10.35.64.113-2022-07-13.log
2022-07-14 10:08:00 +0800 [info]: #0 following tail of /data/logs/spp-data-upload-ms/app/data-upload-ms-10.35.64.124-2022-07-13.log
2022-07-14 10:08:52 +0800 [info]: #0 Timeout flush: ms-logs-application:default

sigdump:

Sigdump at 2022-07-14 11:06:26 +0800 process 19 (/usr/local/bundle/bin/fluentd)
  Thread #<Thread:0x00007fa27b67fc58 run> status=run priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:52:in `backtrace'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:52:in `dump_backtrace'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:34:in `block in dump_all_thread_backtrace'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:33:in `each'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:33:in `dump_all_thread_backtrace'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:16:in `block in dump'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:136:in `open'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:136:in `_open_dump_path'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:14:in `dump'
      /usr/local/lib/ruby/gems/3.1.0/gems/sigdump-0.2.4/lib/sigdump.rb:7:in `block in setup'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/engine.rb:152:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/engine.rb:152:in `run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/supervisor.rb:760:in `block in run_worker'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/supervisor.rb:1032:in `main_process'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/supervisor.rb:751:in `run_worker'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/command/fluentd.rb:386:in `<top (required)>'
      <internal:/usr/local/lib/ruby/3.1.0/rubygems/core_ext/kernel_require.rb>:85:in `require'
      <internal:/usr/local/lib/ruby/3.1.0/rubygems/core_ext/kernel_require.rb>:85:in `require'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/bin/fluentd:15:in `<top (required)>'
      /usr/local/bundle/bin/fluentd:25:in `load'
      /usr/local/bundle/bin/fluentd:25:in `<main>'
  Thread #<Thread:0x00007fa274f9d788@event_loop /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run_once'
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/event_loop.rb:93:in `block in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa3458@flush_thread_0 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa3160@flush_thread_1 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa2eb8@flush_thread_2 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa2c10@flush_thread_3 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa2990@enqueue_thread /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1463:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1463:in `enqueue_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa2828@event_loop /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run_once'
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/event_loop.rb:93:in `block in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa0c58@flush_thread_0 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa09d8@flush_thread_1 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa0780@flush_thread_2 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa0528@flush_thread_3 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa02a8@enqueue_thread /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1463:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1463:in `enqueue_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa0140@event_loop /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run_once'
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/event_loop.rb:93:in `block in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa57a8@flush_thread_0 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa52a8@flush_thread_1 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa5028@flush_thread_2 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa4d30@flush_thread_3 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa48f8@enqueue_thread /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1463:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1463:in `enqueue_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa4740@event_loop /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run_once'
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/event_loop.rb:93:in `block in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274faac80@flush_thread_0 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274faa8e8@flush_thread_1 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274faa690@flush_thread_2 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274faa410@flush_thread_3 /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `wait'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1524:in `flush_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:501:in `block (2 levels) in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274faa190@enqueue_thread /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1463:in `sleep'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:1463:in `enqueue_thread_run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274faa028@event_loop /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 sleep> status=sleep priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run_once'
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/event_loop.rb:93:in `block in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  Thread #<Thread:0x00007fa274fa92e0@event_loop /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:70 run> status=run priority=0
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/parser_regexp.rb:50:in `match'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/parser_regexp.rb:50:in `parse'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluent-plugin-grok-parser-2.6.2/lib/fluent/plugin/parser_grok.rb:65:in `block in parse'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluent-plugin-grok-parser-2.6.2/lib/fluent/plugin/parser_grok.rb:64:in `each'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluent-plugin-grok-parser-2.6.2/lib/fluent/plugin/parser_grok.rb:64:in `parse'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/filter_parser.rb:73:in `filter_with_time'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/filter.rb:87:in `block in filter_stream'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event.rb:196:in `block in each'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event.rb:195:in `each'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event.rb:195:in `each'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/filter.rb:85:in `filter_stream'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event_router.rb:218:in `block in filter_stream'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event_router.rb:217:in `each'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event_router.rb:217:in `reduce'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event_router.rb:217:in `filter_stream'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event_router.rb:193:in `emit_events'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event_router.rb:115:in `emit_stream'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/out_relabel.rb:29:in `process'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/output.rb:840:in `emit_sync'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event_router.rb:196:in `emit_events'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/event_router.rb:115:in `emit_stream'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:619:in `receive_lines'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:1108:in `block in handle_notify'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:1144:in `with_io'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:1068:in `handle_notify'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:1010:in `block in on_notify'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:1010:in `synchronize'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:1010:in `on_notify'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:821:in `on_notify'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:399:in `block in setup_watcher'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin/in_tail.rb:753:in `on_timer'
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run_once'
      /usr/local/lib/ruby/gems/3.1.0/gems/cool.io-1.7.1/lib/cool.io/loop.rb:88:in `run'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/event_loop.rb:93:in `block in start'
      /usr/local/lib/ruby/gems/3.1.0/gems/fluentd-1.15.0/lib/fluent/plugin_helper/thread.rb:78:in `block in thread_create'
  GC stat:
      count: 6821
      time: 96835
      heap_allocated_pages: 836
      heap_sorted_length: 1372
      heap_allocatable_pages: 0
      heap_available_slots: 341588
      heap_live_slots: 340065
      heap_free_slots: 1523
      heap_final_slots: 0
      heap_marked_slots: 231286
      heap_eden_pages: 836
      heap_tomb_pages: 0
      total_allocated_pages: 4882
      total_freed_pages: 4046
      total_allocated_objects: 1057915410
      total_freed_objects: 1057575345
      malloc_increase_bytes: 72448
      malloc_increase_bytes_limit: 24396713
      minor_gc_count: 5973
      major_gc_count: 848
      compact_count: 0
      read_barrier_faults: 0
      total_moved_objects: 0
      remembered_wb_unprotected_objects: 1291
      remembered_wb_unprotected_objects_limit: 2394
      old_objects: 229923
      old_objects_limit: 245638
      oldmalloc_increase_bytes: 18464016
      oldmalloc_increase_bytes_limit: 107946353
  Built-in objects:
   341,588: TOTAL
    77,052: T_STRING
    56,589: T_ARRAY
    53,783: T_HASH
    49,955: T_IMEMO
    41,190: T_DATA
    30,066: T_OBJECT
    12,281: T_BIGNUM
    12,272: T_RATIONAL
     4,341: T_CLASS
     1,420: FREE
       754: T_ICLASS
       749: T_REGEXP
       472: T_MODULE
       433: T_STRUCT
       121: T_SYMBOL
       100: T_FILE
         9: T_FLOAT
         1: T_COMPLEX
  All objects:
    76,606: String
    50,470: Array
    45,096: Hash
    26,584: Fluent::EventTime
    25,881: Time
     8,307: Integer
     8,298: Rational
     2,524: Class
     1,323: Proc
       837: Gem::Requirement
       749: Regexp
       470: Module
       464: Gem::Dependency
       453: Method
       408: Thread::Mutex
       203: Gem::Version
       200: Fluent::Config::ConfigureProxy
       145: Fluent::Config::Section
       140: Gem::Specification
       140: Gem::StubSpecification::StubLine
       140: Fluent::Config::Element
       140: Gem::StubSpecification
       139: Monitor
       136: Fluent::Plugin::TailInput::StatWatcher
       136: Fluent::Plugin::TailInput::TimerTrigger
       136: Fluent::Plugin::TailInput::TailWatcher
       136: Fluent::Plugin::TailInput::TargetInfo
       134: Fluent::Plugin::TailInput::FilePositionEntry
       127: Fluent::Plugin::Base::State
       121: Symbol
       103: Encoding
       103: Fluent::Plugin::LocalMetrics
       100: Fluent::Plugin::TailInput::TailWatcher::RotateHandler
        97: File
        94: Range
        83: Fluent::PluginHelper::Timer::TimerWatcher
        59: Fluent::Plugin::TailInput::TailWatcher::FIFO
        59: Fluent::Plugin::TailInput::TailWatcher::IOHandler
        36: Thread::Backtrace
        31: Thread::ConditionVariable
        30: Fluent::Plugin::Buffer::Metadata
        27: Thread
        23: Fiber
        22: MatchData
        20: Set
        20: OptionParser::Switch::RequiredArgument
        18: Fluent::Plugin::Buffer::FileChunk
        17: OptionParser::Switch::NoArgument
        16: Fluent::Plugin::Output::FlushThreadState
        14: Object
        12: Fluent::GlobMatchPattern
        12: Fluent::EventRouter::Rule
        12: URI::HTTPS
        12: Faraday::RackBuilder
        12: Faraday::Utils::ParamsHash
        12: Faraday::Utils::Headers
        12: Faraday::SSLOptions
        12: Faraday::RequestOptions
        12: Faraday::Connection
        12: Elasticsearch::Transport::Redacted
        12: Elasticsearch::Transport::Transport::Connections::Connection
        12: Faraday::RackBuilder::Handler
        10: Fluent::Registry
        10: Strftime
         9: Float
         9: Fluent::Plugin::Buffer::MemoryChunk
         8: Fluent::PluginHelper::RecordAccessor::Accessor
         8: Faraday::Adapter::Excon
         7: LoadError
         6: OptionParser::OptionMap
         6: Concurrent::Event
         6: Coolio::Loop
         6: Fluent::PluginHelper::EventLoop::DefaultWatcher
         5: Gem::Platform
         5: BigDecimal
         5: Concurrent::Delay
         5: Fluent::Log
         5: Logger::Formatter
         5: ServerEngine::DaemonLogger
         5: Fluent::TimeParser
         4: Fluent::PluginLogger
         4: Fluent::Plugin::ElasticsearchOutputDynamic
         4: Elasticsearch::Transport::Transport::HTTP::Faraday
         4: Elasticsearch::Transport::Client
         4: Elasticsearch::Transport::Transport::Sniffer
         4: Elasticsearch::Transport::Transport::Serializer::MultiJson
         4: Elasticsearch::Transport::Transport::Connections::Selector::RoundRobin
         4: Elasticsearch::Transport::Transport::Connections::Collection
         4: MessagePack::Buffer
         4: MessagePack::Packer
         3: IO
         3: OptionParser::List
         3: JSON::Ext::Generator::State
         3: UnboundMethod
         3: Fluent::Plugin::MemoryBuffer
         3: Fluent::Plugin::NoneParser
         3: Fluent::MultiEventStream
         2: fatal
         2: Refinement
         2: MessagePack::Factory
         2: Fluent::Plugin::ParserFilter
         2: Fluent::Grok
         2: Fluent::Plugin::GrokParser
         2: Fluent::Plugin::RegexpParser
         2: Fluent::EventRouter::MatchCache
         2: Fluent::EventRouter
         2: Fluent::NoMatchMatch
         2: Fluent::Plugin::RecordTransformerFilter
         2: Fluent::Plugin::RecordTransformerFilter::RubyPlaceholderExpander::CleanroomExpander
         2: Fluent::Plugin::RecordTransformerFilter::RubyPlaceholderExpander
         2: Fluent::Plugin::RewriteTagFilterOutput
         2: OptionParser::Switch::OptionalArgument
         2: OptionParser::Switch::PlacedArgument
         2: Fluent::EventRouter::Pipeline::FilterOptimizer
         2: Fluent::EventRouter::Pipeline
         1: #<Class:0x00007fa27b6a86f8>
         1: #<Class:0x00007fa27b694ce8>
         1: Complex
         1: ThreadGroup
         1: IOError
         1: Binding
         1: RubyVM
         1: Ractor
         1: NoMemoryError
         1: SystemStackError
         1: Gem::PathSupport
         1: ARGF.class
         1: Process::Status
         1: DidYouMean::DeprecatedMapping
         1: Concurrent::Promises::InternalStates::Fulfilled
         1: Concurrent::Promises::InternalStates::Reserved
         1: Concurrent::Promises::InternalStates::Pending
         1: #<Class:0x00007fa277711008>::Config
         1: Gem::RequestSet::Lockfile::Tokenizer::Token
         1: Concurrent::LockFreeStack::Node
         1: OptionParser::CompletingHash
         1: URI::RFC2396_Parser
         1: URI::RFC3986_Parser
         1: Psych::Handler::DumperOptions
         1: TZInfo::UnaryMinusGlobalStringDeduper
         1: Concurrent::ImmediateExecutor
         1: Concurrent::AtomicReference
         1: Concurrent::Utility::ProcessorCounter
         1: Resolv::DNS::Config
         1: Resolv::DNS
         1: Resolv::Hosts
         1: Resolv
         1: #<Class:0x00007fa276a0b458>
         1: #<Class:0x00007fa276a03550>
         1: #<Class:0x00007fa2769a5018>
         1: OpenSSL::X509::Store
         1: OpenSSL::PKey::DH
         1: Fluent::Supervisor::LoggerInitializer
         1: Fluent::Compat::NullOutputChain
         1: Fluent::EngineClass
         1: Fluent::Label
         1: Fluent::RootAgent
         1: Fluent::SystemConfig
         1: Random
         1: Faraday::AdapterRegistry
         1: WEBrick::HTTPVersion
         1: OptionParser
         1: Fluent::Supervisor
         1: Fluent::Plugin::TailInput
         1: Fluent::Plugin::FileBuffer
         1: Fluent::NullFluentLogEventRouter
         1: ArgumentError
         1: Fluent::Plugin::TailInput::MetricsInfo
         1: Fluent::Capability
         1: Faraday::ConnectionOptions
         1: Fluent::Plugin::TailInput::PositionFile
         1: MultiJson::Adapters::Oj
         1: Fluent::Plugin::ConcatFilter
         1: Fluent::Plugin::RelabelOutput
         1: File::Stat
         1: EOFError
         1: Fluent::Plugin::ElasticsearchOutputDynamic::RequestInfo
  String 18,466,624 bytes
   Array 3 elements
    Hash 2 pairs

Additional context

No response

fluent / fluentd

Fluentd stuck with high cpu and not passing logs to Elasticsearch #3817