search

fluent / fluent-bit

Fast and Lightweight Logs and Metrics processor for Linux, BSD, OSX and Windows
https://fluentbit.io
Apache License 2.0
5.82k stars 1.58k forks source link

fluentbit -> fluentd causes "MessagePack::UnknownExtTypeError" #2643

Closed epcim closed 3 years ago

epcim commented 4 years ago

Bug Report

Details here, see @repeatedly comment on it: https://github.com/fluent/fluentd/issues/3136#issuecomment-701924565

Versions

edsiper commented 4 years ago

@epcim can you please share config for Fluent Bit and Fluentd that reproduces the issue ?

epcim commented 4 years ago

Simplified configs, I have kept some filters but removed most of the GREP/PARSERS...

Fluentbit:

apiVersion: v1
data:
  filter-custom.conf: |
    [FILTER]
        Name                modify
        Match               *
        Add                 cluster_name gc1-int-acme-us
        Add                 site gc1
        Add                 tenant acme.io
        Add                 hostname ${HOSTNAME}

    [FILTER]
        Name modify
        Match kube.*falco*
        Add rule DROPME
        Rename rule message
        Rename priority severity
    [FILTER]
        Name record_modifier
        Match kube.*falco*
        Remove_key log
        #Remove_key output
    [FILTER]
        Name grep
        Match kube.*falco*
        Exclude message ^$
        Exclude message DROPME
  filter-kubernetes.conf: |
    [FILTER]
        Name                kubernetes
        Match               kube.*
        Kube_URL            https://kubernetes.default.svc.cluster.local:443
        # process log key as json
        Merge_Log           On
        Merge_Log_Trim      On
        # allowed annotations
        K8S-Logging.Parser  On
        K8S-Logging.Exclude On
  fluent-bit.conf: |
    [SERVICE]
        Log_Level     debug
        Flush         5
        Daemon        off
        Parsers_File  parsers.conf
        HTTP_Server   On
        HTTP_Listen   0.0.0.0
        HTTP_Port     65012
    @INCLUDE input-forward.conf
    @INCLUDE input-systemd.conf
    @INCLUDE input-kubernetes.conf
    @INCLUDE filter-kubernetes.conf
    @INCLUDE filter-custom.conf
    @INCLUDE output-forward.conf
  input-forward.conf: |
    [INPUT]
        Name              forward
        Listen            0.0.0.0
        Port              65011
        Buffer_Chunk_Size 64KB
        Buffer_Max_Size   1024KB
  input-kubernetes.conf: |
    [INPUT]
        Name              tail
        Tag               kube.*
        Path              /var/log/containers/*.log
        Exclude_Path      /var/log/containers/*--*.log
        Parser            docker
        DB                /var/log/flb_kube.db
        Skip_Long_Lines   On
        Docker_Mode       On
        Rotate_Wait       10
        Refresh_Interval  10
        Buffer_Chunk_Size 32k
        # log lines bigger than 256k will cause monitored file to be removed from the monitored file list
        Buffer_Max_Size   256k
        Mem_Buf_Limit     1M
        Ignore_Older      4h
    [FILTER]
        Name     throttle
        Match    *
        Rate     40000
        Window   5
        Interval 60s
        Print_Status true
  input-systemd.conf: |
    [INPUT]
        Name              systemd
        Tag               systemd.*
        DB                /var/log/flb_systemd.db
        Strip_Underscores On
        Max_Entries       500
  output-forward.conf: |
    [OUTPUT]
        Name            forward
        Match           *
        #Time_as_Integer On
        Self_Hostname   fluentbit.gc1.int.acme.us
        Host            ${FLUENT_FORWARD_HOST}
        Port            ${FLUENT_FORWARD_PORT}
        # https://docs.fluentbit.io/manual/v/1.2/output/forward
        tls             On
        tls.verify      On
        tls.ca_file     /acme/secrets/identity/server_ca.crt
        tls.crt_file    /acme/secrets/identity/client.crt
        tls.key_file    /acme/secrets/identity/client.key
  parsers.conf: |
    [PARSER]
        # Attempt to make universal regex to match firstline of multiline
        # messages
        Name   firstline
        Format regex
        Regex  ^((Sun|Mon|Tue|Wed|Thu|Fri|Sat|Jan|Feb|Mar|Apr|May|June|July|Aug|Sep|Oct|Nov|Dec)[ ,]|[0-9]{4}\-|\[?(INFO|WARNING|DEBUG|ERROR|SEVERE|info|warning|debug|error|severe)[:\]])
    [PARSER]
        Name        acmeframework-json
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
        Time_Keep   On
    [PARSER]
        Name        acmeframework_stderr
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L%z
        Time_Keep   On
    [PARSER]
        Name      json
        Format    json
        Time_Keep On
    [PARSER]
        Name        docker
        Format      json
        Time_Key    time
        Time_Format %Y-%m-%dT%H:%M:%S.%L
        Time_Keep   On
        # Command      |  Decoder | Field | Optional Action
        # =============|==================|=================
        #Decode_Field_As   escaped_utf8    log    do_next
        #Decode_Field_As   escaped         log    do_next
    [PARSER]
        Name        syslog
        Format      regex
        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
        Time_Key    time
        Time_Format %b %d %H:%M:%S
        Types pri:integer pid:integer

    ## + 30x another parsers...

On fluentd side, corresponding sources.:


    @include source-forward.conf
    @include source-sqs.conf
    @include source-prometheus.conf

...
...

source-forward.conf: |
    <source>
      @type forward
      port 24224
      bind 0.0.0.0
      source_address_key _forward_source_address
      source_hostname_key _forward_source_hostname
      <transport tls>
        # https://docs.fluentd.org/input/forward#how-to-enable-tls-encryption
        version TLSv1_2
        cert_path /acme/secrets/identity/server.crt
        private_key_path /acme/secrets/identity/server.key
        ca_path /acme/secrets/identity/client_ca_with_fluent.crt
        client_cert_auth true
      </transport>
    </source>
  source-prometheus.conf: |
    <source>
      @type prometheus
      bind 0.0.0.0
      port 65021
      metrics_path /metrics
      aggregated_metrics_path /metrics_all
    </source>
    <source>
      @type prometheus_output_monitor
      interval 10
      <labels>
        hostname ${hostname}
      </labels>
    </source>
  source-sqs.conf: |
    <source>
      @type sqs
      sqs_url "https://sqs.us-east-2.amazonaws.com/xxxxxxxx/fluentd-staging"
      aws_key_id "#{ENV['AWS_ACCESS_KEY']}"
      aws_sec_key "#{ENV['AWS_SECRET_KEY']}"
      delete_message true
      source_queue_url_key _sqs_source_url
      tag sqs
      tag_key tag
      raw_message true
      compression true
    </source>

Let me know if you will need enable more debugging, but some details already are in linked fluend issue.

epcim commented 3 years ago

no more problems, closing