Closed hasesoft closed 9 years ago
Could you paste your config and entire log? With current information, I can't judge what is wrong.
I'm taking the form that inclusion is done, so there are other setting files, but only the following one file should be fine for the part multi_format concerns.
<match utmlog.**.tail.format.*.*>
type forest
subtype parser
<template>
### Parser parameter
key_name message
reserve_data true
replace_invalid_sequence false
suppress_parse_error_log false
time_parse true
#add_prefix
#remove_prefix
#inject_key_prefix
#hash_value_field
### Format parameter
#format syslog
format multi_format
## XTM-11. [SYSLOG-NG] ISODATE with facility and level
<pattern>
### HTTP-PROXY-LOG with URL pattern 1
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" op="[^ ]*" dstname="(?<site_addr>[^ ]*)" arg="(?<path_query>.*)" sent_bytes="[0-9]*" rcvd_bytes="[0-9]*" elapsed_time=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" reputation=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 2
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" op="[^ ]*" dstname="(?<site_addr>[^ ]*)" arg="(?<path_query>.*)" sent_bytes="[0-9]*" rcvd_bytes="[0-9]*" elapsed_time=".*" reputation=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 3
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" path="(?<path_query>.*)"$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 4
format /^(?<time>[^ ]*).*msg="ProxyAllow\:.*$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 5
format /^(?<time>[^ ]*).*ProxyDeny\:.*$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### TCP-LOG pattern 1
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) offset [0-9]* [A-Za-z]* [0-9]* win [0-9]* app_name=".*" cat_name=".*" app_beh_name=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" app_ctl_disp="[0-9]*" msg=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### TCP-LOG pattern 2
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) offset [0-9]* [A-Za-z]* [0-9]* win [0-9]* \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### UDP-LOG pattern 1
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) app_name=".*" cat_name=".*" app_beh_name=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" app_ctl_disp="[0-9]*" msg=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### UDP-LOG pattern 2
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
## XTM-23. [COMMON] CustomFileFormat without all
<pattern>
### HTTP-PROXY-LOG with URL pattern 1
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" op="[^ ]*" dstname="(?<site_addr>[^ ]*)" arg="(?<path_query>.*)" sent_bytes="[0-9]*" rcvd_bytes="[0-9]*" elapsed_time=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" reputation=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 2
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" op="[^ ]*" dstname="(?<site_addr>[^ ]*)" arg="(?<path_query>.*)" sent_bytes="[0-9]*" rcvd_bytes="[0-9]*" elapsed_time=".*" reputation=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 3
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" path="(?<path_query>.*)"$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 4
format /^(?<time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*).*msg="ProxyAllow\:.*$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 5
format /^(?<time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*).*ProxyDeny\:.*$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### TCP-LOG pattern 1
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) offset [0-9]* [A-Za-z]* [0-9]* win [0-9]* app_name=".*" cat_name=".*" app_beh_name=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" app_ctl_disp="[0-9]*" msg=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### TCP-LOG pattern 2
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) offset [0-9]* [A-Za-z]* [0-9]* win [0-9]* \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### UDP-LOG pattern 1
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) app_name=".*" cat_name=".*" app_beh_name=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" app_ctl_disp="[0-9]*" msg=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### UDP-LOG pattern 2
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
#<pattern>
# ### Other
# format none
# #time_format %b %d %H:%M:%S
# #types
#</pattern>
### Tagging
#tag utmlog.all.xtm33.abc-xtm33-01.tail.format.jsonbucket.redshift
tag ${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}.${tag_parts[3]}.${tag_parts[4]}.${tag_parts[6]}.${tag_parts[7]}
</template>
</match>
A chemistry problem with the other gem versions? A warning wasn't issued at least to the front where a rise dates.
Ah, okay. It's a v0.12 related problem. Maybe, using v0.10 doesn't have this problem.
I will fix it later.
v0.12/v0.10 means Fluentd version, not ruby.
I see. Thank you very much.
I'll try returning fluentd to 0.10.58.
Don't you have enough any procedures? It was returned to fluentd 0.10.58, but the same message is output.
Tried one is the next procedure.
% gem uninstall fluentd
% gem install fluentd -v "0.10.58"
Which of a related module, not itself fluentd is possibly may be the one vice is doing.
When it's so, the version before a renewal isn't remembered, so if it's returned, which version doesn't understand?
If it was returned to fluentd 0.10.55, it started to move. Some problems probably seem to occur already per 0.10.58, not 0.12 problems.
I released version 0.0.2 to fix this problem. If you have a time, please try this version.
As soon as it comes, confirmation will report a result.
"If fluentd was renewed, warning wasn't output any more." for, it was a mistake. It wasn't just output because the log level was being changed.
When this plugin was renewed in 0.0.2, warning wasn't output any more ! Thanks!
This case closes after I see how things are for a while.
Close
Good :)
When fluent-plugin-multi-format-parser is used, the next warning is issued. And it doesn't seem that plugin is used.
fluent.warn {"message":"parse failed undefined method `parse' for #Fluent::TextParser::MultiFormatParser:0x8db3f64"
How should it correspond?
About Environment
openSUSE 12.2 (i586) -----RPMS----------------------------------------------------------------------------------------------------- ruby19-1.9.3.p392-3.34.1.i586 ruby-common-2.1-3.1.noarch ruby19-devel-1.9.3.p392-3.34.1.i586 ruby20-2.0.0.p247-14.2.i586 ruby-devel-2.0-109.1.i586 ruby-2.0-59.1.i586 ruby20-devel-2.0.0.p247-12.1.i586 -----GEMS----------------------------------------------------------------------------------------------------- aws-sdk (1.59.1) aws-sdk-v1 (1.59.1) bigdecimal (1.2.5, 1.2.0) bson (2.3.0, 1.11.1) bson_ext (1.11.1) bundler (1.7.9) cool.io (1.2.4) diff-lcs (1.2.5) ffi (1.9.6) ffi-yajl (1.3.1) fluent-mixin-config-placeholders (0.3.0) fluent-mixin-plaintextformatter (0.2.6) fluent-plugin-config-expander (0.1.5) fluent-plugin-ec2-metadata (0.0.4) fluent-plugin-flume (0.1.1) fluent-plugin-forest (0.3.0) fluent-plugin-jsonbucket (0.0.2) fluent-plugin-multi-format-parser (0.0.1) fluent-plugin-parser (0.4.0) fluent-plugin-record-reformer (0.4.0) fluent-plugin-redshift (0.0.4) fluent-plugin-rewrite-tag-filter (1.5.1) fluent-plugin-s3 (0.5.1) fluent-plugin-tail-ex (0.1.1) fluent-plugin-tail-multiline (0.1.5) fluent-plugin-td (0.10.23) fluent-plugin-td-monitoring (0.2.0) fluent-plugin-webhdfs (0.4.1) fluentd (0.12.1) http_parser.rb (0.6.0) httpclient (2.4.0) io-console (0.4.2) iobuffer (1.1.2) ipaddress (0.8.0) json (1.8.1, 1.7.7) libyajl2 (1.2.0) ltsv (0.1.0) mime-types (2.4.3, 1.25.1) mini_portile (0.6.1) minitest (5.5.0, 4.3.2) mixlib-cli (1.5.0) mixlib-config (2.1.0) mixlib-log (1.6.0) mixlib-shellout (2.0.0) msgpack (0.5.9) net-dhcp (1.3.2) nokogiri (1.6.5) ohai (8.0.1, 6.20.0) parallel (1.3.3) pg (0.17.1) power_assert (0.2.2) psych (2.0.8, 2.0.0) rake (10.4.2, 0.9.6) rdoc (4.2.0, 4.0.0) rspec (3.1.0) rspec-core (3.1.7) rspec-expectations (3.1.2) rspec-mocks (3.1.3) rspec-support (3.1.2) rubyzip (1.1.6) sigdump (0.2.2) string-scrub (0.0.5, 0.0.3) systemu (2.6.4, 2.5.2) td-client (0.8.67) test-unit (3.0.8, 2.0.0.0) thread_safe (0.3.4) thrift (0.8.0) tzinfo (1.2.2) tzinfo-data (1.2014.10) uuidtools (2.1.5) webhdfs (0.6.0) wmi-lite (1.0.0)
yajl-ruby (1.2.1)