lueschem / edi

Embedded development infrastructure.
https://www.get-edi.io
GNU Lesser General Public License v3.0
38 stars 12 forks source link

Write blog post about monitoring with fluent-bit and InfluxDB #81

Closed lueschem closed 7 months ago

lueschem commented 10 months ago

Here is a first setup of fluent-bit writing data to InfluxDB:

compulab@iot-gate-imx8-0001c0282d92:~$ cat /etc/fluent-bit/fluent-bit.conf      
[SERVICE]
    # Flush
    # =====
    # set an interval of seconds before to flush records to a destination
    flush        1

    # Daemon
    # ======
    # instruct Fluent Bit to run in foreground or background mode.
    daemon       Off

    # Log_Level
    # =========
    # Set the verbosity level of the service, values can be:
    #
    # - error
    # - warning
    # - info
    # - debug
    # - trace
    #
    # by default 'info' is set, that means it includes 'error' and 'warning'.
    log_level    info

    # Parsers File
    # ============
    # specify an optional 'Parsers' configuration file
    parsers_file parsers.conf

    # Plugins File
    # ============
    # specify an optional 'Plugins' configuration file to load external plugins.
    plugins_file plugins.conf

    # HTTP Server
    # ===========
    # Enable/Disable the built-in HTTP Server for metrics
    http_server  Off
    http_listen  0.0.0.0
    http_port    2020

    # Storage
    # =======
    # Fluent Bit can use memory and filesystem buffering based mechanisms
    #
    # - https://docs.fluentbit.io/manual/administration/buffering-and-storage
    #
    # storage metrics
    # ---------------
    # publish storage pipeline metrics in '/api/v1/storage'. The metrics are
    # exported only if the 'http_server' option is enabled.
    #
    storage.metrics on

    # storage.path
    # ------------
    # absolute file system path to store filesystem data buffers (chunks).
    #
    # storage.path /tmp/storage

    # storage.sync
    # ------------
    # configure the synchronization mode used to store the data into the
    # filesystem. It can take the values normal or full.
    #
    # storage.sync normal

    # storage.checksum
    # ----------------
    # enable the data integrity check when writing and reading data from the
    # filesystem. The storage layer uses the CRC32 algorithm.
    #
    # storage.checksum off

    # storage.backlog.mem_limit
    # -------------------------
    # if storage.path is set, Fluent Bit will look for data chunks that were
    # not delivered and are still in the storage layer, these are called
    # backlog data. This option configure a hint of maximum value of memory
    # to use when processing these records.
    #
    # storage.backlog.mem_limit 5M

[input]
    name cpu
    tag  cpu.local2

    # Read interval (sec) Default: 1
    interval_sec 10

[input]
    name          netif
    tag           netif2
    interval_sec  10
    interface     eth0

[input]
    name          systemd
    path          /run/log/journal
    tag           host2.*
    systemd_filter PRIORITY=0
    systemd_filter PRIORITY=1
    systemd_filter PRIORITY=2
    systemd_filter PRIORITY=3
    systemd_filter_type or 

[filter]
    name          modify
    match         *
    add           hostname ${HOSTNAME}

[output]
    name          influxdb
    match         *
    port          443
    tls           on
    host          eu-central-1-1.aws.cloud2.influxdata.com
    bucket        demo-bucket
    http_token    XXXX
    org           iot-demo-fleet
    sequence_tag  off 
    tag_keys      hostname 

[output]
    name          stdout
    match         *
lueschem commented 7 months ago

https://www.get-edi.io/Monitoring-an-IoT-Fleet/