Describe the bug
Fluent bit hangs, any step to troubleshoot?
To Reproduce
Install and run Fluent Bit v3.1.7 on RHEL8.10
It is expected to run continuously
Expected behavior
It hangs randomly after 20min/2h
Your Environment
Version used: 3.1.7
Configuration:
[SERVICE]
# Flush
# =====
# set an interval of seconds before to flush records to a destination
flush 5
# Log_Level
# =========
# Set the verbosity level of the service, values can be:
#
# - error
# - warning
# - info
# - debug
# - trace
#
# by default 'info' is set, that means it includes 'error' and 'warning'.
log_level info
# Parsers File
# ============
# specify an optional 'Parsers' configuration file
parsers_file parsers.conf
# Plugins File
# ============
# specify an optional 'Plugins' configuration file to load external plugins.
# plugins_file plugins.conf
# HTTP Server
# ===========
# Enable/Disable the built-in HTTP Server for metrics
http_server Off
http_listen 0.0.0.0
http_port 2020
# Storage
# =======
# Fluent Bit can use memory and filesystem buffering based mechanisms
#
# - https://docs.fluentbit.io/manual/administration/buffering-and-storage
#
# storage metrics
# ---------------
# publish storage pipeline metrics in '/api/v1/storage'. The metrics are
# exported only if the 'http_server' option is enabled.
#
storage.metrics on
# storage.path
# ------------
# absolute file system path to store filesystem data buffers (chunks).
#
# storage.path /tmp/storage
# storage.sync
# ------------
# configure the synchronization mode used to store the data into the
# filesystem. It can take the values normal or full.
#
# storage.sync normal
# storage.checksum
# ----------------
# enable the data integrity check when writing and reading data from the
# filesystem. The storage layer uses the CRC32 algorithm.
#
# storage.checksum off
# storage.backlog.mem_limit
# -------------------------
# if storage.path is set, Fluent Bit will look for data chunks that were
# not delivered and are still in the storage layer, these are called
# backlog data. This option configure a hint of maximum value of memory
# to use when processing these records.
#
# storage.backlog.mem_limit 5M
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path /var/log/secure
Tag input.os.secure
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
path /var/log/audit/audit.log
Tag input.os.audit
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path /var/log/boot.log
Tag input.os.boot
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
multiline.parser python-multiline-regex
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
multiline.parser python-multiline-regex
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
multiline.parser python-multiline-regex
[INPUT]
Name tail
Path_Key filename
Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
################################
filters for
################################
[FILTER]
Name parser
Match
Key_Name log
Preserve_Key On
Reserve_Data On
Parser syslog-rfc3164
[FILTER]
Name lua
Match
script /etc/fluent-bit/scripts/append.lua
call append_tag
[FILTER]
Name lua
Match
script /etc/fluent-bit/scripts/append.lua
call append_unix_time_for_rfc3164_as_utc
[FILTER]
Name record_modifier
Match
Record category operations
############################
filters for input.os.audit
############################
[FILTER]
Name parser
Match input.os.audit
Key_Name log
Preserve_Key On
Reserve_Data On
Parser auditlog
[FILTER]
Name lua
Match input.os.audit
script /etc/fluent-bit/scripts/append.lua
call append_tag
[FILTER]
Name lua
Match input.os.audit
script /etc/fluent-bit/scripts/append.lua
call append_time_as_number
[FILTER]
Name record_modifier
Match input.os.audit
Record category security
###########################
filters for input.os.boot
###########################
[FILTER]
Name lua
Match input.os.boot
script /etc/fluent-bit/scripts/append.lua
call append_tag_and_time
[FILTER]
Name record_modifier
Match input.os.boot
Record category operations
#############################
filters for input.os.secure
#############################
[FILTER]
Name parser
Match input.os.secure
Key_Name log
Preserve_Key On
Reserve_Data On
Parser syslog-rfc3164
[FILTER]
Name lua
Match input.os.secure
script /etc/fluent-bit/scripts/append.lua
call append_tag
[FILTER]
Name lua
Match input.os.secure
script /etc/fluent-bit/scripts/append.lua
call append_unix_time_for_rfc3164
[FILTER]
Name record_modifier
Match input.os.secure
Record category security
######################################
filters for
######################################
[FILTER]
Name parser
Match
Key_Name log
Preserve_Key On
Reserve_Data On
Parser
[FILTER]
Name lua
Match
script /etc/fluent-bit/scripts/append.lua
call append_tag
[FILTER]
Name lua
Match
script /etc/fluent-bit/scripts/append.lua
call
[FILTER]
Name record_modifier
Match
Record category operations
[OUTPUT]
Name http
Match input.*
Host
Port 8081
Header
URI
format json
json_date_key false
Workers 2
Retry_Limit no_limits
compress gzip
[OUTPUT]
name stdout
match **
```
* Environment name and version: Baremetal installation on RHEL8.10
* Server type and version: N/A
* Operating System and version: RHEL8.10
* Filters and plugins: See above
**Additional context**
The fluent-bit was totally hanging, `service fluent-bit restart` times out:
```
Nov 20 11:43:27 fluent-bit[49564]: [2024/11/20 11:43:27] [engine] caught signal (SIGTERM)
Nov 20 11:44:57 systemd[1]: fluent-bit.service: State 'stop-sigterm' timed out. Killing.
Nov 20 11:44:57 systemd[1]: fluent-bit.service: Killing process 49564 (fluent-bit) with signal SIGKILL.
Nov 20 11:46:28 systemd[1]: fluent-bit.service: Processes still around after SIGKILL. Ignoring.
Nov 20 11:47:58 systemd[1]: fluent-bit.service: State 'stop-final-sigterm' timed out. Killing.
Nov 20 11:47:58 systemd[1]: fluent-bit.service: Killing process 49564 (fluent-bit) with signal SIGKILL.
Nov 20 11:49:28 systemd[1]: fluent-bit.service: Processes still around after final SIGKILL. Entering failed mode.
Nov 20 11:49:28 systemd[1]: fluent-bit.service: Failed with result 'timeout'.
Nov 20 11:49:28 systemd[1]: Stopped Fluent Bit.
```
fluent-bit dump at the time it was hanging:
```
Nov 20 15:13:27 fluent-bit[1551]: [2024/11/20 15:13:27] [engine] caught signal (SIGCONT)
Nov 20 15:13:27 fluent-bit[1551]: [2024/11/20 15:13:27] Fluent Bit Dump
Nov 20 15:13:27 fluent-bit[1551]: ===== Input =====
Nov 20 15:13:27 fluent-bit[1551]: tail.0 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.1 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.2 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.3 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.4 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.5 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 9.4K (9663 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 1
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 1
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.6 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.7 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 479b (479 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 1
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 1
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.8 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.9 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.10 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ down chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: └─ busy chunks: 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: └─ size err: 0
Nov 20 15:13:27 fluent-bit[1551]: tail.11 (tail)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ status
Nov 20 15:13:27 fluent-bit[1551]: │ └─ overlimit : no
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ mem size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │ └─ mem limit : 9.5M (10000000 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: ├─ tasks
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ total tasks : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ new : 0
Nov 20 15:13:27 fluent-bit[1551]: │ ├─ running : 0
Nov 20 15:13:27 fluent-bit[1551]: │ └─ size : 0b (0 bytes)
Nov 20 15:13:27 fluent-bit[1551]: │
Nov 20 15:13:27 fluent-bit[1551]: └─ chunks
Nov 20 15:13:27 fluent-bit[1551]: └─ total chunks : 0
Nov 20 15:13:27 fluent-bit[1551]: ├─ up chunks : 0
Nov 20 15:13:27
Bug Report
Describe the bug Fluent bit hangs, any step to troubleshoot?
To Reproduce Install and run Fluent Bit v3.1.7 on RHEL8.10 It is expected to run continuously
Expected behavior It hangs randomly after 20min/2h
Your Environment
Configuration:
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path /var/log/secure Tag input.os.secure Inotify_Watcher False DB /var/log/fluentbit_db_status.db Read_from_Head On Skip_Long_Lines On Skip_Empty_Lines On Buffer_Chunk_Size 64KB Buffer_Max_Size 256KB Mem_Buf_Limit 10MB Refresh_Interval 15
[INPUT] Name tail Path_Key filename path /var/log/audit/audit.log Tag input.os.audit Inotify_Watcher False DB /var/log/fluentbit_db_status.db Read_from_Head On Skip_Long_Lines On Skip_Empty_Lines On Buffer_Chunk_Size 64KB Buffer_Max_Size 256KB Mem_Buf_Limit 10MB Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path /var/log/boot.log Tag input.os.boot Inotify_Watcher False DB /var/log/fluentbit_db_status.db Read_from_Head On Skip_Long_Lines On Skip_Empty_Lines On Buffer_Chunk_Size 64KB Buffer_Max_Size 256KB Mem_Buf_Limit 10MB Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
multiline.parser python-multiline-regex
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
multiline.parser python-multiline-regex
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
multiline.parser python-multiline-regex
[INPUT] Name tail Path_Key filename Path
Tag
Inotify_Watcher False
DB /var/log/fluentbit_db_status.db
Read_from_Head On
Skip_Long_Lines On
Skip_Empty_Lines On
Buffer_Chunk_Size 64KB
Buffer_Max_Size 256KB
Mem_Buf_Limit 10MB
Refresh_Interval 15
################################
filters for
################################ [FILTER] Name parser Match
Key_Name log
Preserve_Key On
Reserve_Data On
Parser syslog-rfc3164
[FILTER] Name lua Match
script /etc/fluent-bit/scripts/append.lua
call append_tag
[FILTER] Name lua Match
script /etc/fluent-bit/scripts/append.lua
call append_unix_time_for_rfc3164_as_utc
[FILTER] Name record_modifier Match
Record category operations
############################
filters for input.os.audit
############################ [FILTER] Name parser Match input.os.audit Key_Name log Preserve_Key On Reserve_Data On Parser auditlog
[FILTER] Name lua Match input.os.audit script /etc/fluent-bit/scripts/append.lua call append_tag
[FILTER] Name lua Match input.os.audit script /etc/fluent-bit/scripts/append.lua call append_time_as_number
[FILTER] Name record_modifier Match input.os.audit Record category security
###########################
filters for input.os.boot
########################### [FILTER] Name lua Match input.os.boot script /etc/fluent-bit/scripts/append.lua call append_tag_and_time
[FILTER] Name record_modifier Match input.os.boot Record category operations
#############################
filters for input.os.secure
############################# [FILTER] Name parser Match input.os.secure Key_Name log Preserve_Key On Reserve_Data On Parser syslog-rfc3164
[FILTER] Name lua Match input.os.secure script /etc/fluent-bit/scripts/append.lua call append_tag
[FILTER] Name lua Match input.os.secure script /etc/fluent-bit/scripts/append.lua call append_unix_time_for_rfc3164
[FILTER] Name record_modifier Match input.os.secure Record category security
######################################
filters for
###################################### [FILTER] Name parser Match
Key_Name log
Preserve_Key On
Reserve_Data On
Parser
[FILTER] Name lua Match
script /etc/fluent-bit/scripts/append.lua
call append_tag
[FILTER] Name lua Match
script /etc/fluent-bit/scripts/append.lua
call
[FILTER] Name record_modifier Match
Record category operations