patrick-stephens
commented
2 years ago Diff output:
$ for i in ./dockerfiles/conf/*; do echo $i; file=${i##*/}; diff ./dockerfiles/conf/$file ./conf/$file; done
./dockerfiles/conf/fluent-bit.conf
4,5c4,5
< # Set an interval of seconds before to flush records to a destination
< Flush 5
---
> # set an interval of seconds before to flush records to a destination
> flush 1
9,10c9,10
< # Instruct Fluent Bit to run in foreground or background mode.
< Daemon Off
---
> # instruct Fluent Bit to run in foreground or background mode.
> daemon Off
22,23c22,23
< # By default 'info' is set, that means it includes 'error' and 'warning'.
< Log_Level info
---
> # by default 'info' is set, that means it includes 'error' and 'warning'.
> log_level info
25c25
< # Parsers_File
---
> # Parsers File
27,29c27,33
< # Specify an optional 'Parsers' configuration file
< Parsers_File parsers.conf
< Plugins_File plugins.conf
---
> # specify an optional 'Parsers' configuration file
> parsers_file parsers.conf
>
> # Plugins File
> # ============
> # specify an optional 'Plugins' configuration file to load external plugins.
> plugins_file plugins.conf
34,36c38,82
< HTTP_Server Off
< HTTP_Listen 0.0.0.0
< HTTP_Port 2020
---
> http_server Off
> http_listen 0.0.0.0
> http_port 2020
>
> # Storage
> # =======
> # Fluent Bit can use memory and filesystem buffering based mechanisms
> #
> # - https://docs.fluentbit.io/manual/administration/buffering-and-storage
> #
> # storage metrics
> # ---------------
> # publish storage pipeline metrics in '/api/v1/storage'. The metrics are
> # exported only if the 'http_server' option is enabled.
> #
> storage.metrics on
>
> # storage.path
> # ------------
> # absolute file system path to store filesystem data buffers (chunks).
> #
> # storage.path /tmp/storage
>
> # storage.sync
> # ------------
> # configure the synchronization mode used to store the data into the
> # filesystem. It can take the values normal or full.
> #
> # storage.sync normal
>
> # storage.checksum
> # ----------------
> # enable the data integrity check when writing and reading data from the
> # filesystem. The storage layer uses the CRC32 algorithm.
> #
> # storage.checksum off
>
> # storage.backlog.mem_limit
> # -------------------------
> # if storage.path is set, Fluent Bit will look for data chunks that were
> # not delivered and are still in the storage layer, these are called
> # backlog data. This option configure a hint of maximum value of memory
> # to use when processing these records.
> #
> # storage.backlog.mem_limit 5M
39,42c85,87
< Name cpu
< Tag cpu.local
< # Interval Sec
< # ====
---
> name cpu
> tag cpu.local
>
44c89
< Interval_Sec 1
---
> interval_sec 1
47,48c92,93
< Name stdout
< Match *
---
> name stdout
> match *
./dockerfiles/conf/parsers_ambassador.conf
./dockerfiles/conf/parsers_cinder.conf
./dockerfiles/conf/parsers.conf
28c28
< # https://rubular.com/r/P8zgLD5K73fp2n
---
> # https://rubular.com/r/IhIbCAIs7ImOkc
31c31
< Regex ^(?<host>[^ ]*) - - \[(?<time>[^\]]*)\] \\*"(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?\\*" (?<code>[^ ]*) (?<size>[^ ]*) \\*"(?<referer>[^\"]*)\\*" \\*"(?<agent>[^\"]*)\\*" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (\[(?<proxy_alternative_upstream_name>[^ ]*)\] )?(?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<reg_id>[^ ]*).*$
---
> Regex ^(?<host>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*) "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] (\[(?<proxy_alternative_upstream_name>[^ ]*)\] )?(?<upstream_addr>[^ ]*) (?<upstream_response_length>[^ ]*) (?<upstream_response_time>[^ ]*) (?<upstream_status>[^ ]*) (?<reg_id>[^ ]*).*$
67c67
< Regex ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*)\]|-)) (?<message>.+)$
---
> Regex ^\<(?<pri>[0-9]{1,5})\>1 (?<time>[^ ]+) (?<host>[^ ]+) (?<ident>[^ ]+) (?<pid>[-0-9]+) (?<msgid>[^ ]+) (?<extradata>(\[(.*?)\]|-)) (?<message>.+)$
86d85
< Time_Format %Y-%m-%dT%H:%M:%S.%L
./dockerfiles/conf/parsers_extra.conf
77a78,83
> # Parse IP Tables rules - this one regex should capture pretty much any IP Tables rule and split it into the various fields
> [PARSER]
> Name iptables
> Format regex
> Regex \[(?<rule_chain>\w*)-(?<rule_name>\w*)-(?<accept_or_drop>\w*)\]IN=(?<in_interface>[\w.]+)? OUT=(?<out_interface>[\w.]+)? MAC=(?<mac_address>[\w:]+)? SRC=(?<source>(?:[0-9]{1,3}\.){3}[0-9]{1,3}) DST=(?<dest>(?:[0-9]{1,3}\.){3}[0-9]{1,3}) LEN=(?<pkt_len>\d+) TOS=(?<pkt_tos>[\w\d]+) PREC=(?<pkt_prec>[\w\d]+) TTL=(?<pkt_ttl>\d+) ID=(?<pkt_id>\d+)\s?(?<pkg_frg>[A-Z\s].?)\s?PROTO=(?<protocol>[\w\d]+) (SPT=(?<source_port>.*) DPT=(?<dest_port>.*) (LEN=(?<proto_pkt_len>\w+)?)?(WINDOW=(?<proto_window_size>\d+) RES=(?<pkt_res>\w+)? (?<pkt_type>\w+)\s((?<pkt_flag>\w+)?)\s?URGP=(?<pkg_urgency>\d))? )?(TYPE=(?<pkt_icmp_type>\d+) CODE=(?<pkt_icmp_code>\d+) ID=(?<pkt_icmp_id>\d+) SEQ=(?<pkt_icmp_seq>\d+) )?$
> Types source_port:integer,dest_port:integer,pkt_ttl:integer,pkt_tos:integer,pkt_len:integer
78a85,172
> # Various parsers for Couchbase Server logs
>
> [PARSER]
> Name couchbase_json_log_nanoseconds
> Format json
> Time_Key timestamp
> Time_Format %Y-%m-%dT%H:%M:%S.%L
> Time_Keep On
> # Do not remove the time field from the output we ship
>
> [PARSER]
> Name couchbase_rebalance_report
> Format json
> Time_Key timestamp
> Time_Format %Y-%m-%dT%H:%M:%SZ
> Time_Keep On
>
> # The level may have optional brackets around it
> [PARSER]
> Name couchbase_simple_log
> Format regex
> Regex ^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+\[(?<level>\w+)\](?<message>.*)$
> Time_Key timestamp
> Time_Format %Y-%m-%dT%H:%M:%S.%L%z
> Time_Keep On
>
> [PARSER]
> Name couchbase_simple_log_space_separated
> Format regex
> Regex ^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+(?<level>\w+)\s+(?<message>.*)$
> Time_Key timestamp
> Time_Format %Y-%m-%dT%H:%M:%S.%L%z
> Time_Keep On
>
> # Slight change in time format to use Z at end instead of offset:
> # 2021-03-09T17:32:02.136Z INFO ...
> # https://rubular.com/r/EpG3M1dHb5AnTC
> [PARSER]
> Name couchbase_simple_log_utc
> Format regex
> Regex ^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z)\s+(?<level>\w+)(?<message>.*)$
> Time_Key timestamp
> Time_Format %Y-%m-%dT%H:%M:%S.%LZ
> Time_Keep On
>
> # Cope with two different log formats, e.g.:
> # 2021/03/09 17:32:15 cbauth: ...
> # 2021-03-09T17:32:15.303+00:00 [INFO] ...
> # https://rubular.com/r/XUt7xQqEJnrF2M
> [PARSER]
> Name couchbase_simple_log_mixed
> Format regex
> Regex ^(?<timestamp>\d+(-|/)\d+(-|/)\d+(T|\s+)\d+:\d+:\d+(\.\d+(\+|-)\d+:\d+|))\s+((\[)?(?<level>\w+)(\]|:))(?<message>.*)$
> Time_Key timestamp
> Time_Keep On
> # We cannot parse the time as different formats directly, it could be done downstream and/or left as current time
>
> [PARSER]
> Name couchbase_erlang_multiline
> Format regex
> # For some reason this cannot parse an ending close bracket ] followed by a new line immediately
> #Regex \[(?<logger>\w+):(?<level>\w+),(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),.*\](?<message>.*)$
> Regex \[(?<logger>\w+):(?<level>\w+),(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+.\d+Z),(?<message>.*)$
> Time_Key timestamp
> Time_Format %Y-%m-%dT%H:%M:%S.%L
> Time_Keep On
>
> # 2021-03-09T17:32:25.339+00:00 INFO CBAS.bootstrap.AnalyticsNCApplication [main] ...
> # https://rubular.com/r/9jh1oKtXBN5GEV
> # Can include an exception stack trace or a thread dump as well but ignoring these for now
> [PARSER]
> Name couchbase_java_multiline
> Format regex
> Regex ^(?<timestamp>\d+-\d+-\d+T\d+:\d+:\d+\.\d+(\+|-)\d+:\d+)\s+(?<level>\w+)\s+(?<class>.*)\s+\[(?<thread>.*)\]\s+(?<message>.*)$
> Time_Key timestamp
> Time_Format %Y-%m-%dT%H:%M:%S.%L%z
> Time_Keep On
>
> # A slight modification of the usual Apache/Apache2 parsers
> [PARSER]
> Name couchbase_http
> Format regex
> Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<timestamp>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*) - (?<client>.*)$
> Time_Key timestamp
> Time_Format %d/%b/%Y:%H:%M:%S %z
> Time_Keep On
>
> # End of Couchbase Server parsers
./dockerfiles/conf/parsers_java.conf
./dockerfiles/conf/parsers_mult.conf
diff: ./conf/parsers_mult.conf: No such file or directory
./dockerfiles/conf/parsers_openstack.conf
./dockerfiles/conf/plugins.conf
Bug Report
Describe the bug
We have duplication of config files:
We probably should make sure we only use the second one and remove the first. The default Windows configuration is only in the first one.