repeatedly
commented
9 years ago Could you paste your config and entire log? With current information, I can't judge what is wrong.
Closed hasesoft closed 9 years ago
repeatedly
commented
9 years ago Could you paste your config and entire log? With current information, I can't judge what is wrong.
hasesoft
commented
9 years ago I'm taking the form that inclusion is done, so there are other setting files, but only the following one file should be fine for the part multi_format concerns.
<match utmlog.**.tail.format.*.*>
type forest
subtype parser
<template>
### Parser parameter
key_name message
reserve_data true
replace_invalid_sequence false
suppress_parse_error_log false
time_parse true
#add_prefix
#remove_prefix
#inject_key_prefix
#hash_value_field
### Format parameter
#format syslog
format multi_format
## XTM-11. [SYSLOG-NG] ISODATE with facility and level
<pattern>
### HTTP-PROXY-LOG with URL pattern 1
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" op="[^ ]*" dstname="(?<site_addr>[^ ]*)" arg="(?<path_query>.*)" sent_bytes="[0-9]*" rcvd_bytes="[0-9]*" elapsed_time=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" reputation=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 2
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" op="[^ ]*" dstname="(?<site_addr>[^ ]*)" arg="(?<path_query>.*)" sent_bytes="[0-9]*" rcvd_bytes="[0-9]*" elapsed_time=".*" reputation=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 3
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" path="(?<path_query>.*)"$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 4
format /^(?<time>[^ ]*).*msg="ProxyAllow\:.*$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 5
format /^(?<time>[^ ]*).*ProxyDeny\:.*$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### TCP-LOG pattern 1
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) offset [0-9]* [A-Za-z]* [0-9]* win [0-9]* app_name=".*" cat_name=".*" app_beh_name=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" app_ctl_disp="[0-9]*" msg=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### TCP-LOG pattern 2
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) offset [0-9]* [A-Za-z]* [0-9]* win [0-9]* \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### UDP-LOG pattern 1
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) app_name=".*" cat_name=".*" app_beh_name=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" app_ctl_disp="[0-9]*" msg=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### UDP-LOG pattern 2
format /^(?<log_time>[^ ]*) (?<host>[^ ]*) (?<syslogtag>[^ ]*) (?<facility>[a-z0-9]*) (?<level>[a-z]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
## XTM-23. [COMMON] CustomFileFormat without all
<pattern>
### HTTP-PROXY-LOG with URL pattern 1
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" op="[^ ]*" dstname="(?<site_addr>[^ ]*)" arg="(?<path_query>.*)" sent_bytes="[0-9]*" rcvd_bytes="[0-9]*" elapsed_time=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" reputation=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 2
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" op="[^ ]*" dstname="(?<site_addr>[^ ]*)" arg="(?<path_query>.*)" sent_bytes="[0-9]*" rcvd_bytes="[0-9]*" elapsed_time=".*" reputation=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 3
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* (?<protocol>[^ ]*) (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[0-9]*) (?<dest_port>[0-9]*) msg="HTTP request" proxy_act=".*" path="(?<path_query>.*)"$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 4
format /^(?<time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*).*msg="ProxyAllow\:.*$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### HTTP-PROXY-LOG with URL pattern 5
format /^(?<time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*).*ProxyDeny\:.*$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### TCP-LOG pattern 1
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) offset [0-9]* [A-Za-z]* [0-9]* win [0-9]* app_name=".*" cat_name=".*" app_beh_name=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" app_ctl_disp="[0-9]*" msg=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### TCP-LOG pattern 2
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) offset [0-9]* [A-Za-z]* [0-9]* win [0-9]* \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### UDP-LOG pattern 1
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) app_name=".*" cat_name=".*" app_beh_name=".*" app_id="[0-9]*" app_cat_id="(?<cat_id>[0-9]*)" app_ctl_disp="[0-9]*" msg=".*" \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
<pattern>
### UDP-LOG pattern 2
format /^(?<log_time>[0-9]* [^ ]* [ 0-5][0-9] [^ ]*) (?<host>[^ ]*) msg_id="[^ ]*" (?<action>[^ ]*) [^ ]* [^ ]* [^ ]* (?<protocol>[^ ]*) [^ ]* [^ ]* (?<src_ip>[^ ]*) (?<dest_ip>[^ ]*) (?<src_port>[^ ]*) (?<dest_port>[^ ]*) \(.*\)$/
#time_format %b %d %H:%M:%S
#types
</pattern>
#<pattern>
# ### Other
# format none
# #time_format %b %d %H:%M:%S
# #types
#</pattern>
### Tagging
#tag utmlog.all.xtm33.abc-xtm33-01.tail.format.jsonbucket.redshift
tag ${tag_parts[0]}.${tag_parts[1]}.${tag_parts[2]}.${tag_parts[3]}.${tag_parts[4]}.${tag_parts[6]}.${tag_parts[7]}
</template>
</match>A chemistry problem with the other gem versions? A warning wasn't issued at least to the front where a rise dates.
repeatedly
commented
9 years ago Ah, okay. It's a v0.12 related problem. Maybe, using v0.10 doesn't have this problem.
I will fix it later.
repeatedly
commented
9 years ago v0.12/v0.10 means Fluentd version, not ruby.
hasesoft
commented
9 years ago I see. Thank you very much.
I'll try returning fluentd to 0.10.58.
hasesoft
commented
9 years ago Don't you have enough any procedures? It was returned to fluentd 0.10.58, but the same message is output.
Tried one is the next procedure.
% gem uninstall fluentd
% gem install fluentd -v "0.10.58"
hasesoft
commented
9 years ago Which of a related module, not itself fluentd is possibly may be the one vice is doing.
When it's so, the version before a renewal isn't remembered, so if it's returned, which version doesn't understand?
hasesoft
commented
9 years ago If it was returned to fluentd 0.10.55, it started to move. Some problems probably seem to occur already per 0.10.58, not 0.12 problems.
repeatedly
commented
9 years ago I released version 0.0.2 to fix this problem. If you have a time, please try this version.
hasesoft
commented
9 years ago As soon as it comes, confirmation will report a result.
hasesoft
commented
9 years ago "If fluentd was renewed, warning wasn't output any more." for, it was a mistake. It wasn't just output because the log level was being changed.
hasesoft
commented
9 years ago When this plugin was renewed in 0.0.2, warning wasn't output any more ! Thanks!
hasesoft
commented
9 years ago This case closes after I see how things are for a while.
hasesoft
commented
9 years ago Close
repeatedly
commented
9 years ago Good :)
When fluent-plugin-multi-format-parser is used, the next warning is issued. And it doesn't seem that plugin is used.
fluent.warn {"message":"parse failed undefined method `parse' for #Fluent::TextParser::MultiFormatParser:0x8db3f64"
How should it correspond?
About Environment
openSUSE 12.2 (i586) -----RPMS----------------------------------------------------------------------------------------------------- ruby19-1.9.3.p392-3.34.1.i586 ruby-common-2.1-3.1.noarch ruby19-devel-1.9.3.p392-3.34.1.i586 ruby20-2.0.0.p247-14.2.i586 ruby-devel-2.0-109.1.i586 ruby-2.0-59.1.i586 ruby20-devel-2.0.0.p247-12.1.i586 -----GEMS----------------------------------------------------------------------------------------------------- aws-sdk (1.59.1) aws-sdk-v1 (1.59.1) bigdecimal (1.2.5, 1.2.0) bson (2.3.0, 1.11.1) bson_ext (1.11.1) bundler (1.7.9) cool.io (1.2.4) diff-lcs (1.2.5) ffi (1.9.6) ffi-yajl (1.3.1) fluent-mixin-config-placeholders (0.3.0) fluent-mixin-plaintextformatter (0.2.6) fluent-plugin-config-expander (0.1.5) fluent-plugin-ec2-metadata (0.0.4) fluent-plugin-flume (0.1.1) fluent-plugin-forest (0.3.0) fluent-plugin-jsonbucket (0.0.2) fluent-plugin-multi-format-parser (0.0.1) fluent-plugin-parser (0.4.0) fluent-plugin-record-reformer (0.4.0) fluent-plugin-redshift (0.0.4) fluent-plugin-rewrite-tag-filter (1.5.1) fluent-plugin-s3 (0.5.1) fluent-plugin-tail-ex (0.1.1) fluent-plugin-tail-multiline (0.1.5) fluent-plugin-td (0.10.23) fluent-plugin-td-monitoring (0.2.0) fluent-plugin-webhdfs (0.4.1) fluentd (0.12.1) http_parser.rb (0.6.0) httpclient (2.4.0) io-console (0.4.2) iobuffer (1.1.2) ipaddress (0.8.0) json (1.8.1, 1.7.7) libyajl2 (1.2.0) ltsv (0.1.0) mime-types (2.4.3, 1.25.1) mini_portile (0.6.1) minitest (5.5.0, 4.3.2) mixlib-cli (1.5.0) mixlib-config (2.1.0) mixlib-log (1.6.0) mixlib-shellout (2.0.0) msgpack (0.5.9) net-dhcp (1.3.2) nokogiri (1.6.5) ohai (8.0.1, 6.20.0) parallel (1.3.3) pg (0.17.1) power_assert (0.2.2) psych (2.0.8, 2.0.0) rake (10.4.2, 0.9.6) rdoc (4.2.0, 4.0.0) rspec (3.1.0) rspec-core (3.1.7) rspec-expectations (3.1.2) rspec-mocks (3.1.3) rspec-support (3.1.2) rubyzip (1.1.6) sigdump (0.2.2) string-scrub (0.0.5, 0.0.3) systemu (2.6.4, 2.5.2) td-client (0.8.67) test-unit (3.0.8, 2.0.0.0) thread_safe (0.3.4) thrift (0.8.0) tzinfo (1.2.2) tzinfo-data (1.2014.10) uuidtools (2.1.5) webhdfs (0.6.0) wmi-lite (1.0.0)
yajl-ruby (1.2.1)