gestion des pipelines

[leadwire-apm]

Ajouter un menu admin permettant de gérer les pipelines

Une pipeline doit avoir:

• un nom (pipeline-id) (minuscule sans espace)
• un contenu (objet json)

Ajouter un bouton pour appliquer la pipeline, en exécutant cette commande:

PUT _ingest/pipeline/my-pipeline-id
{
    "on_failure" : [
      {
        "set" : {
          "field" : "error.message",
          "value" : "{{ _ingest.on_failure_message }}"
        }
      }
    ],
    "description" : "Pipeline for parsing apache error logs",
    "processors" : [
      {
        "grok" : {
          "field" : "message",
          "patterns" : [
            """\[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}""",
            """\[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\] \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}"""
          ],
          "pattern_definitions" : {
            "APACHE_TIME" : "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
          },
          "ignore_missing" : true
        }
      },
      {
        "date" : {
          "ignore_failure" : true,
          "field" : "apache.error.timestamp",
          "target_field" : "@timestamp",
          "formats" : [
            "EEE MMM dd H:m:s yyyy",
            "EEE MMM dd H:m:s.SSSSSS yyyy"
          ]
        }
      },
      {
        "remove" : {
          "field" : "apache.error.timestamp",
          "ignore_failure" : true
        }
      },
      {
        "grok" : {
          "field" : "source.address",
          "ignore_missing" : true,
          "patterns" : [
            "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$"
          ]
        }
      },
      {
        "geoip" : {
          "field" : "source.ip",
          "target_field" : "source.geo",
          "ignore_missing" : true
        }
      }
    ]

}

[leadwire-apm]

{ "filebeat-7.2.1-osquery-result-pipeline" : { "description" : "Pipeline for parsing osquery result logs", "processors" : [ { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "json.unixTime", "target_field" : "@timestamp", "formats" : [ "UNIX" ], "ignore_failure" : true } }, { "script" : { "source" : "def dict = ['result': new HashMap()]; for (entry in ctx['json'].entrySet()) { dict['result'][entry.getKey()] = entry.getValue(); } ctx['osquery'] = dict; ctx.remove('json');", "lang" : "painless" } }, { "rename" : { "field" : "osquery.result.hostIdentifier", "target_field" : "osquery.result.host_identifier", "ignore_missing" : true } }, { "rename" : { "field" : "osquery.result.unixTime", "target_field" : "osquery.result.unix_time", "ignore_missing" : true } }, { "rename" : { "field" : "osquery.result.calendarTime", "target_field" : "osquery.result.calendar_time", "ignore_missing" : true } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-nginx-error-pipeline" : { "description" : "Pipeline for parsing the Nginx error logs", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "%{DATA:nginx.error.time} \[%{DATA:log.level}\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYDATA:message}" ], "ignore_missing" : true } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "nginx.error.time", "target_field" : "@timestamp", "formats" : [ "yyyy/MM/dd H:m:s" ], "ignore_failure" : true } }, { "remove" : { "field" : "nginx.error.time" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-mongodb-log-pipeline" : { "description" : "Pipeline for parsing MongoDB logs", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}" ], "ignore_missing" : true } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "mongodb.log.timestamp", "target_field" : "@timestamp", "formats" : [ "yyyy-MM-dd'T'HH:mm:ss.SSSZZ" ] } }, { "remove" : { "field" : "mongodb.log.timestamp" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-elasticsearch-gc-pipeline" : { "description" : "Pipeline for parsing Elasticsearch JVM garbage collection logs", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "(?:%{JVM8HEADER}|%{JVM9HEADER}) Total time for which application threads were stopped: %{BASE10NUM:elasticsearch.gc.threads_total_stop_time_sec} seconds, Stopping threads took: %{BASE10NUM:elasticsearch.gc.stopping_threads_time_sec} seconds", "(?:%{JVM8HEADER}) \[GC \(%{DATA:elasticsearch.gc.phase.name}\) \[YG occupancy: %{BASE10NUM:elasticsearch.gc.young_gen.used_kb} K \(%{BASE10NUM:elasticsearch.gc.young_gen.size_kb} K\)\]%{BASE10NUM}: \[Rescan \(parallel\) , %{BASE10NUM:elasticsearch.gc.phase.parallel_rescan_time_sec} secs\]%{BASE10NUM}: \[weak refs processing, %{BASE10NUM:elasticsearch.gc.phase.weak_refs_processing_time_sec} secs\]%{BASE10NUM}: \[class unloading, %{BASE10NUM:elasticsearch.gc.phase.class_unload_time_sec} secs\]%{BASE10NUM}: \[scrub symbol table, %{BASE10NUM:elasticsearch.gc.phase.scrub_symbol_table_time_sec} secs\]%{BASE10NUM}: \[scrub string table, %{BASE10NUM:elasticsearch.gc.phase.scrub_string_table_time_sec} secs\]\[1 CMS-remark: %{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)\] %{BASE10NUM:elasticsearch.gc.heap.used_kb}K\(%{BASE10NUM:elasticsearch.gc.heap.size_kb}K\), %{BASE10NUM:elasticsearch.gc.phase.duration_sec} secs\] %{PROCTIME}", "(?:%{JVM8HEADER}) \[GC \(%{DATA:elasticsearch.gc.phase.name}\) \[%{BASE10NUM} CMS-initial-mark: %{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)\] %{BASE10NUM:elasticsearch.gc.heap.used_kb}K\(%{BASE10NUM:elasticsearch.gc.heap.size_kb}K\), %{BASE10NUM:elasticsearch.gc.phase.duration_sec} secs\] %{PROCTIME}", "%{JVM9HEADER} GC\(%{BASE10NUM}\) ParNew: %{BASE10NUM}K-\>%{BASE10NUM:elasticsearch.gc.young_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.young_gen.size_kb}K\)", "%{JVM9HEADER} GC\(%{BASE10NUM}\) Old: %{BASE10NUM}K-\>%{BASE10NUM:elasticsearch.gc.old_gen.used_kb}K\(%{BASE10NUM:elasticsearch.gc.old_gen.size_kb}K\)", "(?:%{JVM8HEADER}|%{JVM9HEADER}) %{GREEDYMULTILINE:message}" ], "pattern_definitions" : { "JVM8HEADER" : "%{TIMESTAMP_ISO8601:timestamp}: %{BASE10NUM:elasticsearch.gc.jvm_runtime_sec}:", "JVM9HEADER" : """[%{TIMESTAMP_ISO8601:timestamp}][%{POSINT:process.pid}][%{DATA:elasticsearch.gc.tags}%{SPACE}]""", "PROCTIME" : """[Times: user=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.user_sec} sys=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.sys_sec}, real=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.real_sec} secs]""", "GREEDYMULTILINE" : """(.| )""" } } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "timestamp", "target_field" : "@timestamp", "formats" : [ "ISO8601" ] } }, { "remove" : { "field" : "timestamp" } }, { "split" : { "field" : "elasticsearch.gc.tags", "separator" : ",", "ignore_missing" : true } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-mysql-error-pipeline" : { "description" : "Pipeline for parsing MySQL error logs", "processors" : [ { "grok" : { "ignore_missing" : true, "pattern_definitions" : { "LOCALDATETIME" : "[0-9]+ %{TIME}" }, "field" : "message", "patterns" : [ "%{LOCALDATETIME:mysql.error.timestamp} (\[%{DATA:log.level}\] )?%{GREEDYDATA:message}", "%{DATA:mysql.error.timestamp} %{NUMBER:mysql.thread_id:long} \[%{DATA:log.level}\] %{GREEDYDATA:message}", "%{GREEDYDATA:message}" ] } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "mysql.error.timestamp", "target_field" : "@timestamp", "formats" : [ "ISO8601", "yyMMdd H:m:s" ], "ignore_failure" : true } }, { "remove" : { "field" : "mysql.error.timestamp", "ignore_missing" : true } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-redis-slowlog-pipeline" : { "description" : "Pipeline for parsing redis slowlog logs", "processors" : [ ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-elasticsearch-audit-pipeline" : { "description" : "Pipeline for parsing elasticsearch audit logs", "processors" : [ { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "grok" : { "field" : "message", "patterns" : [ "^%{CHAR:first_char}" ], "pattern_definitions" : { "CHAR" : "." } } }, { "pipeline" : { "if" : "ctx.first_char != '{'", "name" : "filebeat-7.2.1-elasticsearch-audit-pipeline-plaintext" } }, { "pipeline" : { "if" : "ctx.first_char == '{'", "name" : "filebeat-7.2.1-elasticsearch-audit-pipeline-json" } }, { "date" : { "field" : "elasticsearch.audit.@timestamp", "target_field" : "@timestamp", "formats" : [ "ISO8601" ], "ignore_failure" : true } }, { "remove" : { "field" : "elasticsearch.audit.@timestamp" } }, { "remove" : { "field" : [ "first_char" ] } } ], "on_failure" : [ { "set" : { "value" : "{{ _ingest.on_failure_message }}", "field" : "error.message" } } ] }, "filebeat-7.2.1-postgresql-log-pipeline" : { "description" : "Pipeline for parsing PostgreSQL logs.", "processors" : [ { "grok" : { "ignore_missing" : true, "patterns" : [ "^%{LOCALDATETIME:postgresql.log.timestamp} %{WORD:event.timezone} \[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\] ((\[%{USERNAME:user.name}\]@\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}: (duration: %{NUMBER:temp.duration:float} ms statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})" ], "pattern_definitions" : { "POSTGRESQL_DBNAME" : """[a-zA-Z0-9]+[a-zA-Z0-9_\$]""", "LOCALDATETIME" : "[-0-9]+ %{TIME}", "GREEDYDATA" : """(.| | )""" }, "field" : "message" } }, { "date" : { "field" : "postgresql.log.timestamp", "target_field" : "@timestamp", "formats" : [ "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss" ], "ignore_failure" : true } }, { "script" : { "lang" : "painless", "source" : "ctx.event.duration = Math.round(ctx.temp.duration params.scale)", "params" : { "scale" : 1000000.0 }, "if" : "ctx.temp?.duration != null" } }, { "remove" : { "field" : "temp.duration", "ignore_missing" : true } } ], "on_failure" : [ { "set" : { "value" : "{{ _ingest.on_failure_message }}", "field" : "error.message" } } ] }, "filebeat-7.2.1-elasticsearch-server-pipeline" : { "description" : "Pipeline for parsing elasticsearch server logs", "processors" : [ { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "grok" : { "field" : "message", "patterns" : [ "^%{CHAR:first_char}" ], "pattern_definitions" : { "CHAR" : "." } } }, { "pipeline" : { "if" : "ctx.first_char != '{'", "name" : "filebeat-7.2.1-elasticsearch-server-pipeline-plaintext" } }, { "pipeline" : { "if" : "ctx.first_char == '{'", "name" : "filebeat-7.2.1-elasticsearch-server-pipeline-json" } }, { "script" : { "lang" : "painless", "source" : "if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.observation_duration != null) { if (ctx.elasticsearch.server.gc.observation_duration.unit == params.seconds_unit) { ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time params.ms_in_one_s;}if (ctx.elasticsearch.server.gc.observation_duration.unit == params.milliseconds_unit) { ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time; } if (ctx.elasticsearch.server.gc.observation_duration.unit == params.minutes_unit) { ctx.elasticsearch.server.gc.observation_duration.ms = ctx.elasticsearch.server.gc.observation_duration.time params.ms_in_one_m; }} if (ctx.elasticsearch.server.gc != null && ctx.elasticsearch.server.gc.collection_duration != null) { if (ctx.elasticsearch.server.gc.collection_duration.unit == params.seconds_unit) { ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time params.ms_in_one_s;} if (ctx.elasticsearch.server.gc.collection_duration.unit == params.milliseconds_unit) {ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time; } if (ctx.elasticsearch.server.gc.collection_duration.unit == params.minutes_unit) { ctx.elasticsearch.server.gc.collection_duration.ms = ctx.elasticsearch.server.gc.collection_duration.time params.ms_in_one_m; }}", "params" : { "minutes_unit" : "m", "seconds_unit" : "s", "milliseconds_unit" : "ms", "ms_in_one_s" : 1000, "ms_in_one_m" : 60000 } } }, { "remove" : { "field" : [ "elasticsearch.server.gc.collection_duration.time", "elasticsearch.server.gc.collection_duration.unit", "elasticsearch.server.gc.observation_duration.time", "elasticsearch.server.gc.observation_duration.unit" ], "ignore_missing" : true } }, { "date" : { "formats" : [ "ISO8601" ], "ignore_failure" : true, "field" : "elasticsearch.server.timestamp", "target_field" : "@timestamp" } }, { "remove" : { "field" : "elasticsearch.server.timestamp" } }, { "remove" : { "field" : [ "first_char" ] } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-nginx-access-default" : { "on_failure" : [ { "set" : { "value" : "{{ _ingest.on_failure_message }}", "field" : "error.message" } } ], "description" : "Pipeline for parsing Nginx access logs. Requires the geoip and user_agent plugins.", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "\"?(?:%{IP_LIST:nginx.access.remote_ip_list}|%{DATA:source.address}) - %{DATA:user.name} \[%{HTTPDATE:nginx.access.time}\] \"%{DATA:nginx.access.info}\" %{NUMBER:http.response.status_code:long} %{NUMBER:http.response.body.bytes:long} \"%{DATA:http.request.referrer}\" \"%{DATA:user_agent.original}\"" ], "pattern_definitions" : { "IP_LIST" : """%{IP}("?,?\s%{IP})""" }, "ignore_missing" : true } }, { "grok" : { "patterns" : [ "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}", "" ], "ignore_missing" : true, "field" : "nginx.access.info" } }, { "remove" : { "field" : "nginx.access.info" } }, { "split" : { "ignore_missing" : true, "field" : "nginx.access.remote_ip_list", "separator" : "\"?,?\s+" } }, { "split" : { "field" : "nginx.access.origin", "separator" : "\"?,?\s+", "ignore_missing" : true } }, { "set" : { "field" : "source.ip", "value" : "" } }, { "script" : { "params" : { "dot" : "." }, "lang" : "painless", "source" : "boolean isPrivate(def dot, def ip) { try { StringTokenizer tok = new StringTokenizer(ip, dot); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } try { ctx.source.ip = null; if (ctx.nginx.access.remote_ip_list == null) { return; } def found = false; for (def item : ctx.nginx.access.remote_ip_list) { if (!isPrivate(params.dot, item)) { ctx.source.ip = item; found = true; break; } } if (!found) { ctx.source.ip = ctx.nginx.access.remote_ip_list[0]; }} catch (Exception e) { ctx.source.ip = null; }" } }, { "remove" : { "field" : "source.ip", "if" : "ctx.source.ip == null" } }, { "convert" : { "field" : "source.ip", "target_field" : "source.address", "type" : "string", "ignore_missing" : true } }, { "remove" : { "field" : "message" } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "nginx.access.time", "target_field" : "@timestamp", "formats" : [ "dd/MMM/yyyy:H:m:s Z" ], "ignore_failure" : true } }, { "remove" : { "field" : "nginx.access.time" } }, { "user_agent" : { "field" : "user_agent.original" } }, { "geoip" : { "field" : "source.ip", "target_field" : "source.geo", "ignore_missing" : true } } ] }, "filebeat-7.2.1-iis-access-default" : { "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "description" : "Pipeline for parsing IIS access logs. Requires the geoip and user_agent plugins.", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATHWITHBRACKET:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}", "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}", "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} HTTP/%{NUMBER:http.version} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}", "%{TIMESTAMP_ISO8601:iis.access.time} \[%{IPORHOST:destination.address}\]\(http://%{IPORHOST:destination.address}\) %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} \[%{IPORHOST:source.address}\]\(http://%{IPORHOST:source.address}\) %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}", "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}" ], "patterndefinitions" : { "URIPATHWITHBRACKET" : """(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&-[]])+""" }, "ignore_missing" : true } }, { "remove" : { "field" : "message" } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "iis.access.time", "target_field" : "@timestamp", "formats" : [ "yyyy-MM-dd HH:mm:ss" ] } }, { "remove" : { "field" : "iis.access.time" } }, { "script" : { "lang" : "painless", "source" : "ctx.event.duration = Math.round(ctx.temp.duration params.scale)", "params" : { "scale" : 1000000.0 }, "if" : "ctx.temp?.duration != null" } }, { "remove" : { "field" : "temp.duration", "ignore_missing" : true } }, { "urldecode" : { "field" : "user_agent.original" } }, { "user_agent" : { "field" : "user_agent.original" } }, { "grok" : { "patterns" : [ "%{NOZONEIP:destination.ip}" ], "pattern_definitions" : { "NOZONEIP" : "[^%]" }, "field" : "destination.address", "ignore_failure" : true } }, { "grok" : { "field" : "source.address", "ignore_failure" : true, "patterns" : [ "%{NOZONEIP:source.ip}" ], "pattern_definitions" : { "NOZONEIP" : "[^%]" } } }, { "geoip" : { "field" : "source.ip", "target_field" : "source.geo", "ignore_missing" : true } } ] }, "filebeat-7.2.1-elasticsearch-deprecation-pipeline-plaintext" : { "processors" : [ { "grok" : { "field" : "message", "pattern_definitions" : { "GREEDYMULTILINE" : """(.| )""" }, "patterns" : [ "\[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\]\[%{LOGLEVEL:log.level}%{SPACE}\]\[%{DATA:elasticsearch.component}%{SPACE}\] %{GREEDYMULTILINE:message}" ] } } ], "description" : "Pipeline for parsing the Elasticsearch deprecation log file in plaintext format.", "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-nats-log-pipeline" : { "description" : "Pipeline for parsing nats log logs", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "\%{POSINT:process.pid}\? \[%{NATSLOGLEVEL:log.level}\] %{GREEDYDATA:nats.log.info}" ], "pattern_definitions" : { "NATSTIME" : "%{YEAR}/%{MONTHNUM}/%{MONTHDAY} %{TIME}", "NATSLOGLEVEL" : "(INF|DBG|WRN|ERR|FTL|TRC)" }, "ignore_missing" : true } }, { "grok" : { "field" : "nats.log.info", "patterns" : [ "%{IPV4:client.ip}:%{POSINT:client.port} - cid:%{POSINT:nats.log.client.id} - %{GREEDYDATA:nats.log.msg.info}", "%{GREEDYDATA:nats.log.msg.data}" ], "ignore_missing" : true } }, { "grok" : { "ignore_missing" : true, "field" : "nats.log.msg.info", "patterns" : [ "%{NATSDIRECTION:network.direction} %{NATSPAYLOAD:nats.log.msg.type}: \[%{GREEDYDATA:nats.log.msg.payload}\]", "%{NATSDIRECTION:network.direction} \[%{NATSNOINFO:nats.log.msg.type}\]", "%{NATSDIRECTION:network.direction} \[%{NATSUNSUB:nats.log.msg.type}\s+%{POSINT:nats.log.msg.sid}(\s+%{POSINT:nats.log.msg.max_messages})?\]", "%{NATSDIRECTION:network.direction} \[%{NATSPUB:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}(\s+%{NOTSPACE:nats.log.msg.reply_to})?\s+%{POSINT:nats.log.msg.bytes}\]", "%{NATSDIRECTION:network.direction} \[%{NATSSUB:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}(\s+%{NOTSPACE:nats.log.msg.queue_group})?\s+%{POSINT:nats.log.msg.sid}\]", "%{NATSDIRECTION:network.direction} \[%{NATSMSG:nats.log.msg.type}\s+%{NOTSPACE:nats.log.msg.subject}\s+%{POSINT:nats.log.msg.sid}(\s+%{NOTSPACE:nats.log.msg.reply_to})?\s+%{POSINT:nats.log.msg.bytes}\]", "%{NATSDIRECTION:network.direction} \[%{NATSCONNECTION:nats.log.msg.type}\s+%{GREEDYDATA:nats.log.msg.data}\]", "%{NATSDIRECTION:network.direction} \[%{NATSERROR:nats.log.msg.type}\s+%{GREEDYDATA:nats.log.msg.error\]", "%{GREEDYDATA:nats.log.msg.data}" ], "pattern_definitions" : { "NATSDIRECTION" : "(<<-|->>)", "NATSPUB" : "PUB", "NATSPONG" : "PONG", "NATSINFO" : "INFO", "NATSSUB" : "SUB", "NATSUNSUB" : "UNSUB", "NATSOK" : "OK", "NATSCONNECTION" : "(?:%{NATSCONNECT}|%{NATSINFO})", "NATSNOINFO" : "(?:%{NATSPING}|%{NATSPONG}|%{NATSOK})", "NATSCONNECT" : "CONNECT", "NATSMSG" : "MSG", "NATSPAYLOAD" : "MSG_PAYLOAD", "NATSERROR" : "-ERROR", "NATSPING" : "PING" } } }, { "remove" : { "field" : "nats.log.info" } }, { "remove" : { "field" : "nats.log.msg.info", "ignore_missing" : true } }, { "remove" : { "field" : "nats.log.msg.payload", "ignore_missing" : true } }, { "remove" : { "field" : "message" } }, { "rename" : { "field" : "nats.log.msg.data", "target_field" : "message", "ignore_missing" : true } }, { "script" : { "source" : """if (ctx.log.level == params.inf) { ctx.log.level = params.info; } else if (ctx.log.level == params.dbg) { ctx.log.level = params.debug; } else if (ctx.log.level == params.wrn) { ctx.log.level = params.warning; } else if (ctx.log.level == params.err) { ctx.log.level = params.error; } else if (ctx.log.level == params.ftl) { ctx.log.level = params.fatal; } else if (ctx.log.level == params.trc) { ctx.log.level = params.trace; }""", "params" : { "dbg" : "DBG", "debug" : "debug", "wrn" : "WRN", "warning" : "warning", "err" : "ERR", "error" : "error", "ftl" : "FTL", "inf" : "INF", "trc" : "TRC", "fatal" : "fatal", "trace" : "trace", "info" : "info" }, "lang" : "painless" } }, { "script" : { "lang" : "painless", "source" : """if (ctx.nats.log.msg.type == params.msg) { ctx.nats.log.msg.type = params.message; } else if (ctx.nats.log.msg.type == params.pub) { ctx.nats.log.msg.type = params.publish; } else if (ctx.nats.log.msg.type == params.sub) { ctx.nats.log.msg.type = params.subscribe; } else if (ctx.nats.log.msg.type == params.unsub) { ctx.nats.log.msg.type = params.unsubscribe; } else if (ctx.nats.log.msg.type == params.msg_payload) { ctx.nats.log.msg.type = params.payload; } else if (ctx.nats.log.msg.type == params.err) { ctx.nats.log.msg.type = params.error; } else if (ctx.nats.log.msg.type == params.pi) { ctx.nats.log.msg.type = params.ping; } else if (ctx.nats.log.msg.type == params.po) { ctx.nats.log.msg.type = params.pong; } else if (ctx.nats.log.msg.type == params.ok) { ctx.nats.log.msg.type = params.acknowledge; } else if (ctx.nats.log.msg.type == params.connect) { ctx.nats.log.msg.type = params.connection; } else if (ctx.nats.log.msg.type == params.info) { ctx.nats.log.msg.type = params.information; }""", "params" : { "po" : "PONG", "ok" : "OK", "connection" : "connection", "pub" : "PUB", "publish" : "publish", "sub" : "SUB", "subscribe" : "subscribe", "msg_payload" : "MSG_PAYLOAD", "acknowledge" : "acknowledge", "connect" : "CONNECT", "information" : "information", "message" : "message", "unsub" : "UNSUB", "unsubscribe" : "unsubscribe", "payload" : "payload", "err" : "-ERROR", "error" : "error", "info" : "INFO", "msg" : "MSG", "pi" : "PING", "ping" : "ping", "pong" : "pong" }, "if" : "ctx.nats.log.msg?.type != null" } }, { "script" : { "lang" : "painless", "source" : """if (ctx.network.direction == params.in) { ctx.network.direction = params.inbound; } else if (ctx.network.direction == params.out) { ctx.network.direction = params.outbound; }""", "params" : { "in" : "<<-", "inbound" : "inbound", "out" : "->>", "outbound" : "outbound" }, "if" : "ctx.network?.direction != null" } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "nats.log.timestamp", "target_field" : "@timestamp", "formats" : [ "yyyy/MM/dd HH:mm:ss.SSSSSS" ] } }, { "remove" : { "field" : "nats.log.timestamp" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-elasticsearch-slowlog-pipeline-json" : { "description" : "Pipeline for parsing the Elasticsearch slow logs in JSON format.", "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "processors" : [ { "json" : { "field" : "message", "target_field" : "elasticsearch.slowlog" } }, { "drop" : { "if" : "ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type != 'index_search_slowlog'" } }, { "remove" : { "field" : "elasticsearch.slowlog.type" } }, { "rename" : { "field" : "elasticsearch.slowlog.level", "target_field" : "log.level" } }, { "rename" : { "field" : "elasticsearch.slowlog.component", "target_field" : "elasticsearch.component" } }, { "dot_expander" : { "path" : "elasticsearch.slowlog", "field" : "cluster.name" } }, { "rename" : { "field" : "elasticsearch.slowlog.cluster.name", "target_field" : "elasticsearch.cluster.name" } }, { "dot_expander" : { "field" : "node.name", "path" : "elasticsearch.slowlog" } }, { "rename" : { "field" : "elasticsearch.slowlog.node.name", "target_field" : "elasticsearch.node.name" } }, { "dot_expander" : { "field" : "cluster.uuid", "path" : "elasticsearch.slowlog" } }, { "rename" : { "field" : "elasticsearch.slowlog.cluster.uuid", "target_field" : "elasticsearch.cluster.uuid", "ignore_missing" : true } }, { "dot_expander" : { "field" : "node.id", "path" : "elasticsearch.slowlog" } }, { "rename" : { "field" : "elasticsearch.slowlog.node.id", "target_field" : "elasticsearch.node.id", "ignore_missing" : true } }, { "grok" : { "field" : "elasticsearch.slowlog.message", "pattern_definitions" : { "GREEDYMULTILINE" : """(.| )""", "INDEXNAME" : "[a-zA-Z0-9_.-]" }, "patterns" : [ "(\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})?(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})?%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,?" ] } }, { "remove" : { "field" : "elasticsearch.slowlog.message" } } ] }, "filebeat-7.2.1-elasticsearch-server-pipeline-json" : { "description" : "Pipeline for parsing the Elasticsearch server log file in JSON format.", "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "processors" : [ { "json" : { "field" : "message", "target_field" : "elasticsearch.server" } }, { "drop" : { "if" : "ctx.elasticsearch.server.type != 'server'" } }, { "remove" : { "field" : "elasticsearch.server.type" } }, { "rename" : { "field" : "elasticsearch.server.level", "target_field" : "log.level" } }, { "rename" : { "field" : "elasticsearch.server.component", "target_field" : "elasticsearch.component" } }, { "dot_expander" : { "field" : "cluster.name", "path" : "elasticsearch.server" } }, { "rename" : { "field" : "elasticsearch.server.cluster.name", "target_field" : "elasticsearch.cluster.name" } }, { "dot_expander" : { "field" : "node.name", "path" : "elasticsearch.server" } }, { "rename" : { "field" : "elasticsearch.server.node.name", "target_field" : "elasticsearch.node.name" } }, { "dot_expander" : { "field" : "cluster.uuid", "path" : "elasticsearch.server" } }, { "rename" : { "field" : "elasticsearch.server.cluster.uuid", "target_field" : "elasticsearch.cluster.uuid", "ignore_missing" : true } }, { "dot_expander" : { "field" : "node.id", "path" : "elasticsearch.server" } }, { "rename" : { "field" : "elasticsearch.server.node.id", "target_field" : "elasticsearch.node.id", "ignore_missing" : true } }, { "grok" : { "field" : "elasticsearch.server.message", "pattern_definitions" : { "GREEDYMULTILINE" : """(.| )""", "INDEXNAME" : "[a-zA-Z0-9_.-]", "GC_ALL" : """[gc][%{NUMBER:elasticsearch.server.gc.overhead_seq}] overhead, spent [%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}] collecting in the last [%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}]""", "GC_YOUNG" : """[gc][young][%{NUMBER:elasticsearch.server.gc.young.one}][%{NUMBER:elasticsearch.server.gc.young.two}]%{SPACE}%{GREEDYMULTILINE:message}""" }, "patterns" : [ "%{GC_ALL}", "%{GC_YOUNG}", "((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message}" ] } }, { "remove" : { "field" : "elasticsearch.server.message" } } ] }, "filebeat-7.2.1-haproxy-log-pipeline" : { "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\)", "(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\{%{DATA:haproxy.http.request.captured_headers}\} \{%{DATA:haproxy.http.response.captured_headers}\} |\{%{DATA}\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", "(%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\[%{NUMBER:process.pid:long}\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \[%{NOTSPACE:haproxy.request_date}\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}" ], "ignore_missing" : false, "pattern_definitions" : { "HAPROXY_DATE" : "(%{MONTHDAY}[/-]%{MONTH}[/-]%{YEAR}:%{HOUR}:%{MINUTE}:%{SECOND})|%{SYSLOGTIMESTAMP}" } } }, { "date" : { "field" : "haproxy.request_date", "target_field" : "@timestamp", "formats" : [ "dd/MMM/yyyy:HH:mm:ss.SSS", "MMM dd HH:mm:ss" ] } }, { "remove" : { "field" : "haproxy.request_date" } }, { "remove" : { "field" : "message" } }, { "grok" : { "field" : "source.address", "ignore_failure" : true, "patterns" : [ "^%{IP:source.ip}$" ] } }, { "geoip" : { "ignore_missing" : true, "field" : "source.ip", "target_field" : "source.geo" } }, { "split" : { "separator" : """|""", "ignore_failure" : true, "field" : "haproxy.http.request.captured_headers" } }, { "split" : { "ignore_failure" : true, "field" : "haproxy.http.response.captured_headers", "separator" : """|""" } }, { "script" : { "lang" : "painless", "source" : "ctx.event.duration = Math.round(ctx.temp.duration params.scale)", "params" : { "scale" : 1000000.0 }, "if" : "ctx.temp?.duration != null" } }, { "remove" : { "ignore_missing" : true, "field" : "temp.duration" } }, { "convert" : { "field" : "haproxy.bytes_read", "target_field" : "http.response.bytes", "type" : "long", "if" : "ctx.containsKey('http')" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "description" : "Pipeline for parsing HAProxy http, tcp and default logs. Requires the geoip plugin." }, "filebeat-7.2.1-elasticsearch-slowlog-pipeline-plaintext" : { "description" : "Pipeline for parsing elasticsearch slow logs in plaintext format.", "processors" : [ { "grok" : { "field" : "message", "patterndefinitions" : { "GREEDYMULTILINE" : """(.| )*""", "INDEXNAME" : "[a-zA-Z0-9.-]" }, "patterns" : [ "\[%{TIMESTAMP_ISO8601:elasticsearch.slowlog.timestamp}\]\[%{WORD:log.level}(%{SPACE})?\]\[%{DATA:elasticsearch.slowlog.logger}\]%{SPACE}\%{WORD:elasticsearch.node.name}\?(\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})?(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})?%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,?" ] } }, { "split" : { "if" : "ctx.elasticsearch.slowlog?.stats != ''", "field" : "elasticsearch.slowlog.stats", "separator" : ",", "ignore_missing" : true } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-apache-access-default" : { "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] \"(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?\" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( \"%{DATA:http.request.referrer}\")?( \"%{DATA:user_agent.original}\")?", "%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] \"-\" %{NUMBER:http.response.status_code:long} -", "\[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} %{DATA:apache.access.ssl.cipher} \"%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}\" %{NUMBER:http.response.body.bytes:long}" ], "ignore_missing" : true } }, { "remove" : { "field" : "message" } }, { "grok" : { "field" : "source.address", "ignore_missing" : true, "patterns" : [ "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" ] } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "apache.access.time", "target_field" : "@timestamp", "formats" : [ "dd/MMM/yyyy:H:m:s Z" ], "ignore_failure" : true } }, { "remove" : { "field" : "apache.access.time", "ignore_failure" : true } }, { "user_agent" : { "ignore_failure" : true, "field" : "user_agent.original" } }, { "geoip" : { "field" : "source.ip", "target_field" : "source.geo", "ignore_missing" : true } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "description" : "Pipeline for parsing Apache HTTP Server access logs. Requires the geoip and user_agent plugins." }, "filebeat-7.2.1-elasticsearch-audit-pipeline-json" : { "description" : "Pipeline for parsing elasticsearch audit logs in JSON format", "processors" : [ { "json" : { "field" : "message", "target_field" : "elasticsearch.audit" } }, { "dot_expander" : { "field" : "event.action", "path" : "elasticsearch.audit" } }, { "rename" : { "field" : "elasticsearch.audit.event.action", "target_field" : "event.action" } }, { "dot_expander" : { "field" : "event.type", "path" : "elasticsearch.audit" } }, { "rename" : { "field" : "elasticsearch.audit.event.type", "target_field" : "elasticsearch.audit.layer" } }, { "dot_expander" : { "field" : "origin.address", "path" : "elasticsearch.audit" } }, { "grok" : { "field" : "elasticsearch.audit.origin.address", "patterns" : [ "\[%{IPORHOST:source.ip}\]:%{INT:source.port:int}", "%{IPORHOST:source.ip}:%{INT:source.port:int}" ] } }, { "rename" : { "field" : "elasticsearch.audit.origin.address", "target_field" : "source.address" } }, { "dot_expander" : { "field" : "url.path", "path" : "elasticsearch.audit" } }, { "dot_expander" : { "field" : "url.query", "path" : "elasticsearch.audit" } }, { "set" : { "if" : "ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query == null", "field" : "url.original", "value" : "{{elasticsearch.audit.url.path}}" } }, { "set" : { "value" : "{{elasticsearch.audit.url.path}}?{{elasticsearch.audit.url.query}}", "if" : "ctx.elasticsearch.audit?.url?.path != null && ctx.elasticsearch.audit?.url?.query != null", "field" : "url.original" } }, { "remove" : { "if" : "ctx.elasticsearch.audit?.url?.path != null", "field" : "elasticsearch.audit.url.path" } }, { "remove" : { "if" : "ctx.elasticsearch.audit?.url?.query != null", "field" : "elasticsearch.audit.url.query" } }, { "dot_expander" : { "field" : "node.id", "path" : "elasticsearch.audit" } }, { "dot_expander" : { "field" : "node.name", "path" : "elasticsearch.audit" } }, { "rename" : { "field" : "elasticsearch.audit.node", "target_field" : "elasticsearch.node" } }, { "dot_expander" : { "field" : "user.name", "path" : "elasticsearch.audit" } }, { "rename" : { "field" : "elasticsearch.audit.user.name", "target_field" : "user.name" } }, { "dot_expander" : { "field" : "request.method", "path" : "elasticsearch.audit" } }, { "rename" : { "ignore_missing" : true, "field" : "elasticsearch.audit.request.method", "target_field" : "http.request.method" } }, { "dot_expander" : { "field" : "request.body", "path" : "elasticsearch.audit" } }, { "rename" : { "ignore_missing" : true, "field" : "elasticsearch.audit.request.body", "target_field" : "http.request.body.content" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-kibana-log-pipeline" : { "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "processors" : [ { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "rename" : { "field" : "json", "target_field" : "kibana.log.meta" } }, { "date" : { "target_field" : "@timestamp", "field" : "kibana.log.meta.@timestamp", "formats" : [ "ISO8601" ] } }, { "remove" : { "field" : "kibana.log.meta.@timestamp" } }, { "rename" : { "field" : "kibana.log.meta.message", "target_field" : "message" } }, { "rename" : { "field" : "kibana.log.meta.state", "target_field" : "kibana.log.state", "ignore_missing" : true } }, { "rename" : { "field" : "kibana.log.meta.pid", "target_field" : "process.pid" } }, { "rename" : { "target_field" : "kibana.log.tags", "field" : "kibana.log.meta.tags" } }, { "rename" : { "field" : "kibana.log.meta.res.statusCode", "target_field" : "http.response.status_code", "ignore_missing" : true } }, { "rename" : { "field" : "kibana.log.meta.res.responseTime", "target_field" : "temp.duration", "ignore_missing" : true } }, { "script" : { "lang" : "painless", "source" : "ctx.event.duration = Math.round(ctx.temp.duration params.scale)", "params" : { "scale" : 1000000.0 }, "if" : "ctx.temp?.duration != null" } }, { "remove" : { "field" : "temp.duration", "ignore_missing" : true } }, { "rename" : { "field" : "kibana.log.meta.res.contentLength", "target_field" : "http.response.body.bytes", "ignore_missing" : true } }, { "rename" : { "ignore_missing" : true, "field" : "kibana.log.meta.req.method", "target_field" : "http.request.method" } }, { "rename" : { "target_field" : "http.request.referrer", "ignore_missing" : true, "field" : "kibana.log.meta.req.headers.referer" } }, { "rename" : { "field" : "kibana.log.meta.req.headers.user-agent", "target_field" : "user_agent.original", "ignore_missing" : true } }, { "rename" : { "field" : "kibana.log.meta.req.remoteAddress", "target_field" : "source.address", "ignore_missing" : true } }, { "set" : { "field" : "source.ip", "value" : "{{source.address}}", "if" : "ctx.source?.address != null" } }, { "rename" : { "field" : "kibana.log.meta.req.url", "target_field" : "url.original", "ignore_missing" : true } }, { "remove" : { "field" : "kibana.log.meta.req.referer", "ignore_missing" : true } }, { "remove" : { "field" : "kibana.log.meta.statusCode", "ignore_missing" : true } }, { "remove" : { "ignore_missing" : true, "field" : "kibana.log.meta.method" } }, { "append" : { "value" : "kibana", "field" : "service.name" } } ], "description" : "Pipeline for parsing Kibana logs" }, "filebeat-7.2.1-elasticsearch-audit-pipeline-plaintext" : { "description" : "Pipeline for parsing elasticsearch audit logs in plaintext format", "processors" : [ { "grok" : { "field" : "message", "pattern_definitions" : { "ES_TIMESTAMP" : """[%{TIMESTAMP_ISO8601:elasticsearch.audit.@timestamp}]""", "ES_AUDIT_ORIGIN_TYPE" : """(origin_type\=[%{WORD:elasticsearch.audit.origin.type}])?""", "ES_AUDIT_REALM" : """(realm\=[%{WORD:elasticsearch.audit.realm}])?""", "ES_AUDIT_URI_PARAMS" : """(params=[%{DATA:elasticsearch.audit.url.params}])?""", "ES_AUDIT_REQUEST" : """(request\=[%{WORD:elasticsearch.audit.request.name}])?""", "ES_AUDIT_REQUEST_BODY" : """(request_body\=[%{DATA:http.request.body.content}])?""", "ES_AUDIT_EVENT_TYPE" : """[%{WORD:elasticsearch.audit.event_type}]""", "ES_AUDIT_PRINCIPAL" : """(principal\=[%{DATA:user.name}])?""", "ES_AUDIT_ROLES" : """(roles\=[%{DATA:elasticsearch.audit.user.roles}])?""", "ES_AUDIT_ACTION" : """(action\=[%{DATA:elasticsearch.audit.action}([%{DATA:elasticsearch.audit.sub_action}])?])?""", "ES_NODE_NAME" : """([%{DATA:elasticsearch.node.name}])?""", "ES_AUDIT_LAYER" : """[%{WORD:elasticsearch.audit.layer}]""", "ES_AUDIT_ORIGIN_ADDRESS" : """(origin_address\=[%{IPORHOST:source.ip}])?""", "ES_AUDIT_URI" : """(uri=[%{DATA:url.original}])?""", "ES_AUDIT_INDICES" : """(indices\=[%{DATA:elasticsearch.audit.indices}])?""" }, "patterns" : [ "%{ES_TIMESTAMP}\s%{ES_NODE_NAME}\s%{ES_AUDIT_LAYER}\s%{ES_AUDIT_EVENT_TYPE}\s%{ES_AUDIT_ORIGIN_TYPE},?\s%{ES_AUDIT_ORIGIN_ADDRESS},?\s%{ES_AUDIT_PRINCIPAL},?\s%{ES_AUDIT_REALM},?\s%{ES_AUDIT_ROLES},?\s%{ES_AUDIT_ACTION},?\s%{ES_AUDIT_INDICES},?\s%{ES_AUDIT_URI},?\s%{ES_AUDIT_URI_PARAMS},?\s%{ES_AUDIT_REQUEST},?\s%{ES_AUDIT_REQUEST_BODY},?" ] } }, { "split" : { "field" : "elasticsearch.audit.user.roles", "separator" : ",", "ignore_missing" : true } }, { "split" : { "field" : "elasticsearch.audit.indices", "separator" : ",", "ignore_missing" : true } }, { "script" : { "lang" : "painless", "source" : "if (ctx.elasticsearch.audit.sub_action != null) { ctx.elasticsearch.audit.action += '[' + ctx.elasticsearch.audit.sub_action + ']' }" } }, { "remove" : { "field" : "elasticsearch.audit.sub_action", "ignore_missing" : true } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-santa-log-pipeline" : { "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "description" : "Pipeline for parsing Google Santa logs.", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "\[%{TIMESTAMP_ISO8601:process.start}\] I santad: action=%{NOT_SEPARATOR:santa.action}\|decision=%{NOT_SEPARATOR:santa.decision}\|reason=%{NOT_SEPARATOR:santa.reason}\|sha256=%{NOT_SEPARATOR:hash.sha256}\|path=%{NOT_SEPARATOR:process.executable}(\|args=%{NOT_SEPARATOR:process.args})?(\|cert_sha256=%{NOT_SEPARATOR:certificate.sha256})?(\|cert_cn=%{NOT_SEPARATOR:certificate.common_name})?\|pid=%{NUMBER:process.pid:long}\|ppid=%{NUMBER:process.ppid:long}\|uid=%{NUMBER:user.id}\|user=%{NOT_SEPARATOR:user.name}\|gid=%{NUMBER:group.id}\|group=%{NOT_SEPARATOR:group.name}\|mode=%{WORD:santa.mode}", "\[%{TIMESTAMP_ISO8601:timestamp}\] I santad: action=%{NOT_SEPARATOR:santa.action}\|mount=%{NOT_SEPARATOR:santa.disk.mount}\|volume=%{NOT_SEPARATOR:santa.disk.volume}\|bsdname=%{NOT_SEPARATOR:santa.disk.bsdname}\|fs=%{NOT_SEPARATOR:santa.disk.fs}\|model=%{NOT_SEPARATOR:santa.disk.model}\|serial=%{NOT_SEPARATOR:santa.disk.serial}\|bus=%{NOT_SEPARATOR:santa.disk.bus}\|dmgpath=%{NOT_SEPARATOR:santa.disk.dmgpath}?" ], "pattern_definitions" : { "NOT_SEPARATOR" : """[^|]+""" } } }, { "rename" : { "field" : "message", "target_field" : "log.original" } }, { "date" : { "formats" : [ "ISO8601" ], "ignore_failure" : true, "field" : "process.start", "target_field" : "process.start" } }, { "set" : { "field" : "@timestamp", "value" : "{{ process.start }}", "ignore_failure" : true } }, { "split" : { "field" : "process.args", "separator" : " ", "ignore_failure" : true } }, { "date" : { "field" : "timestamp", "target_field" : "@timestamp", "formats" : [ "ISO8601" ], "ignore_failure" : true } }, { "remove" : { "field" : "timestamp", "ignore_missing" : true } } ] }, "filebeat-7.2.1-elasticsearch-deprecation-pipeline" : { "description" : "Pipeline for parsing elasticsearch deprecation logs", "processors" : [ { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "grok" : { "pattern_definitions" : { "CHAR" : "." }, "field" : "message", "patterns" : [ "^%{CHAR:first_char}" ] } }, { "pipeline" : { "if" : "ctx.first_char != '{'", "name" : "filebeat-7.2.1-elasticsearch-deprecation-pipeline-plaintext" } }, { "pipeline" : { "if" : "ctx.first_char == '{'", "name" : "filebeat-7.2.1-elasticsearch-deprecation-pipeline-json" } }, { "date" : { "formats" : [ "ISO8601" ], "ignore_failure" : true, "field" : "elasticsearch.deprecation.timestamp", "target_field" : "@timestamp" } }, { "remove" : { "field" : "elasticsearch.deprecation.timestamp" } }, { "remove" : { "field" : [ "first_char" ] } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-icinga-main-pipeline" : { "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "description" : "Pipeline for parsing icinga main logs", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "\[%{TIMESTAMP:icinga.main.timestamp}\] %{WORD:log.level}/%{WORD:icinga.main.facility}: %{GREEDYMULTILINE:message}" ], "ignore_missing" : true, "pattern_definitions" : { "TIMESTAMP" : "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}", "GREEDYMULTILINE" : """(.| )""" } } }, { "date" : { "target_field" : "@timestamp", "formats" : [ "yyyy-MM-dd HH:mm:ss Z" ], "ignore_failure" : true, "field" : "icinga.main.timestamp" } }, { "remove" : { "field" : "icinga.main.timestamp" } } ] }, "apm" : { "description" : "Default enrichment for APM events", "processors" : [ { "pipeline" : { "name" : "apm_user_agent" } }, { "pipeline" : { "name" : "apm_user_geo" } } ] }, "filebeat-7.2.1-system-auth-pipeline" : { "processors" : [ { "grok" : { "field" : "message", "ignore_missing" : true, "pattern_definitions" : { "TIMESTAMP" : "(?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP})", "GREEDYMULTILINE" : """(.| )""" }, "patterns" : [ "%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long} ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?", "%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.ip}", "%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}", "%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: \s%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}", "%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}", "%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$", "%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{GREEDYMULTILINE:system.auth.message}" ] } }, { "remove" : { "field" : "message" } }, { "rename" : { "field" : "system.auth.message", "target_field" : "message", "ignore_missing" : true } }, { "set" : { "field" : "source.ip", "value" : "{{system.auth.ssh.dropped_ip}}", "if" : "ctx.containsKey('system') && ctx.system.containsKey('auth') && ctx.system.auth.containsKey('ssh') && ctx.system.auth.ssh.containsKey('dropped_ip')" } }, { "date" : { "field" : "system.auth.timestamp", "target_field" : "@timestamp", "formats" : [ "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ], "ignore_failure" : true } }, { "remove" : { "field" : "system.auth.timestamp" } }, { "geoip" : { "field" : "source.ip", "target_field" : "source.geo", "ignore_failure" : true } }, { "script" : { "ignore_failure" : true, "source" : """if (ctx.system.auth.ssh.event == "Accepted") { if (!ctx.containsKey("event")) { ctx.event = [:]; } ctx.event.type = "authentication_success"; ctx.event.category = "authentication"; ctx.event.action = "ssh_login"; ctx.event.outcome = "success"; } else if (ctx.system.auth.ssh.event == "Invalid" || ctx.system.auth.ssh.event == "Failed") { if (!ctx.containsKey("event")) { ctx.event = [:]; } ctx.event.type = "authentication_failure"; ctx.event.category = "authentication"; ctx.event.action = "ssh_login"; ctx.event.outcome = "failure"; }""", "lang" : "painless" } } ], "on_failure" : [ { "set" : { "value" : "{{ _ingest.on_failure_message }}", "field" : "error.message" } } ], "description" : "Pipeline for parsing system authorisation/secure logs" }, "filebeat-7.2.1-kafka-log-pipeline" : { "description" : "Pipeline for parsing Kafka log messages", "processors" : [ { "grok" : { "field" : "message", "trace_match" : true, "patterns" : [ "(?m)%{TIMESTAMP_ISO8601:kafka.log.timestamp}. %{LOGLEVEL:log.level} +%{JAVALOGMESSAGE:message} \(%{JAVACLASS:kafka.log.class}\)$[ \n](?'kafka.log.trace.full'.)" ] } }, { "grok" : { "pattern_definitions" : { "KAFKA_COMPONENT" : """[^]]""" }, "patterns" : [ "\[%{KAFKA_COMPONENT:kafka.log.component}\][,:.]? +%{JAVALOGMESSAGE:message}" ], "on_failure" : [ { "set" : { "field" : "kafka.log.component", "value" : "unknown" } } ], "field" : "message" } }, { "grok" : { "field" : "kafka.log.trace.full", "ignore_missing" : true, "patterns" : [ "%{JAVACLASS:kafka.log.trace.class}:\s%{JAVALOGMESSAGE:kafka.log.trace.message}" ], "on_failure" : [ { "remove" : { "field" : "kafka.log.trace" } } ] } }, { "remove" : { "field" : "kafka.log.trace.full", "ignore_missing" : true } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "kafka.log.timestamp", "target_field" : "@timestamp", "formats" : [ "yyyy-MM-dd HH:mm:ss,SSS" ], "ignore_failure" : true } }, { "remove" : { "field" : "kafka.log.timestamp" } } ], "on_failure" : [ { "set" : { "field" : "error.log", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-system-syslog-pipeline" : { "description" : "Pipeline for parsing Syslog messages.", "processors" : [ { "grok" : { "ignore_missing" : true, "field" : "message", "patterns" : [ "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{GREEDYMULTILINE:system.syslog.message}", "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}", "%{TIMESTAMP_ISO8601:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?: %{GREEDYMULTILINE:system.syslog.message}" ], "pattern_definitions" : { "GREEDYMULTILINE" : """(.| )""" } } }, { "remove" : { "field" : "message" } }, { "rename" : { "field" : "system.syslog.message", "target_field" : "message", "ignore_missing" : true } }, { "date" : { "field" : "system.syslog.timestamp", "target_field" : "@timestamp", "formats" : [ "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "ISO8601" ], "ignore_failure" : true } }, { "remove" : { "field" : "system.syslog.timestamp" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-icinga-debug-pipeline" : { "description" : "Pipeline for parsing icinga debug logs", "processors" : [ { "grok" : { "pattern_definitions" : { "TIMESTAMP" : "%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:%{MINUTE}:%{SECOND} %{ISO8601_TIMEZONE}", "GREEDYMULTILINE" : """(.| )""" }, "field" : "message", "patterns" : [ "\[%{TIMESTAMP:icinga.debug.timestamp}\] %{WORD:log.level}/%{WORD:icinga.debug.facility}: %{GREEDYMULTILINE:message}" ], "ignore_missing" : true } }, { "date" : { "field" : "icinga.debug.timestamp", "target_field" : "@timestamp", "formats" : [ "yyyy-MM-dd HH:mm:ss Z" ], "ignore_failure" : true } }, { "remove" : { "field" : "icinga.debug.timestamp" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-redis-log-pipeline" : { "description" : "Pipeline for parsing redis logs", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "(%{POSINT:process.pid:long}:%{CHAR:redis.log.role} )?(%{REDISTIMESTAMP1:redis.log.timestamp}||%{REDISTIMESTAMP2:redis.log.timestamp}) %{REDISLEVEL:log.level} %{GREEDYDATA:message}", "%{POSINT:process.pid:long}:signal-handler \(%{POSINT:redis.log.timestamp}\) %{GREEDYDATA:message}" ], "pattern_definitions" : { "REDISLEVEL" : """[.-#]""", "REDISTIMESTAMP1" : "%{MONTHDAY} %{MONTH} %{TIME}", "REDISTIMESTAMP2" : "%{MONTHDAY} %{MONTH} %{YEAR} %{TIME}", "CHAR" : "[a-zA-Z]" } } }, { "script" : { "lang" : "painless", "source" : """if (ctx.log.level == params.dot) { ctx.log.level = params.debug; } else if (ctx.log.level == params.dash) { ctx.log.level = params.verbose; } else if (ctx.log.level == params.asterisk) { ctx.log.level = params.notice; } else if (ctx.log.level == params.hash) { ctx.log.level = params.warning; }""", "params" : { "notice" : "notice", "hash" : "#", "warning" : "warning", "dot" : ".", "debug" : "debug", "dash" : "-", "verbose" : "verbose", "asterisk" : "*" } } }, { "script" : { "lang" : "painless", "source" : """if (ctx.redis.log.role == params.master_abbrev) { ctx.redis.log.role = params.master; } else if (ctx.redis.log.role == params.slave_abbrev) { ctx.redis.log.role = params.slave; } else if (ctx.redis.log.role == params.child_abbrev) { ctx.redis.log.role = params.child; } else if (ctx.redis.log.role == params.sentinel_abbrev) { ctx.redis.log.role = params.sentinel; } """, "params" : { "sentinel_abbrev" : "X", "sentinel" : "sentinel", "master_abbrev" : "M", "master" : "master", "slave_abbrev" : "S", "slave" : "slave", "child_abbrev" : "C", "child" : "child" } } }, { "rename" : { "target_field" : "event.created", "field" : "@timestamp" } }, { "date" : { "field" : "redis.log.timestamp", "target_field" : "@timestamp", "formats" : [ "dd MMM yyyy H:m:s.SSS", "dd MMM H:m:s.SSS", "dd MMM H:m:s", "UNIX" ], "ignore_failure" : true } }, { "remove" : { "field" : "redis.log.timestamp", "ignore_failure" : true } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-auditd-log-pipeline" : { "description" : "Pipeline for parsing Linux auditd logs", "processors" : [ { "grok" : { "patterns" : [ "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} old auid=%{NUMBER:auditd.log.old_auid} new auid=%{NUMBER:auditd.log.new_auid} old ses=%{NUMBER:auditd.log.old_ses} new ses=%{NUMBER:auditd.log.new_ses}", "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv} msg='\"?%{AUDIT_KEY_VALUES:auditd.log.sub_kv}['\"]", "%{AUDIT_PREFIX} %{AUDIT_KEY_VALUES:auditd.log.kv}", "%{AUDIT_PREFIX}", "%{AUDIT_TYPE} %{AUDIT_KEY_VALUES:auditd.log.kv}" ], "field" : "message", "pattern_definitions" : { "AUDIT_TYPE" : "^type=%{NOTSPACE:auditd.log.record_type}", "AUDIT_PREFIX" : """%{AUDIT_TYPE} msg=audit(%{NUMBER:auditd.log.epoch}:%{NUMBER:auditd.log.sequence}):(%{DATA})?""", "AUDIT_KEY_VALUES" : "%{WORD}=%{GREEDYDATA}" } } }, { "kv" : { "field_split" : """\s+""", "value_split" : "=", "target_field" : "auditd.log", "field" : "auditd.log.kv" } }, { "kv" : { "ignore_missing" : true, "field" : "auditd.log.sub_kv", "field_split" : """\s+""", "value_split" : "=", "target_field" : "auditd.log" } }, { "remove" : { "field" : "auditd.log.kv", "ignore_failure" : true } }, { "remove" : { "field" : "auditd.log.sub_kv", "ignore_failure" : true } }, { "remove" : { "ignore_failure" : true, "field" : "message" } }, { "date" : { "field" : "auditd.log.epoch", "target_field" : "@timestamp", "formats" : [ "UNIX" ], "ignore_failure" : true } }, { "remove" : { "ignore_failure" : true, "field" : "auditd.log.epoch" } }, { "convert" : { "ignore_missing" : true, "field" : "auditd.log.sequence", "type" : "integer" } }, { "script" : { "params" : { "single_quote" : "'", "double_quote" : "\"" }, "lang" : "painless", "source" : """ String trimQuotes(def singleQuote, def doubleQuote, def v) { if (v.startsWith(singleQuote) || v.startsWith(doubleQuote)) { v = v.substring(1, v.length()); } if (v.endsWith(singleQuote) || v.endsWith(doubleQuote)) { v = v.substring(0, v.length()-1); }
return v; }

boolean isHexAscii(String v) {
  def len = v.length();
  if (len == 0 || len % 2 != 0) {
    return false; 
  }

  for (int i = 0 ; i < len ; i++) {
    if (Character.digit(v.charAt(i), 16) == -1) {
      return false;
    }
  }

  return true;
}

String convertHexToString(String hex) {
    StringBuilder sb = new StringBuilder();

  for (int i=0; i < hex.length() - 1; i+=2) {
      String output = hex.substring(i, (i + 2));
      int decimal = Integer.parseInt(output, 16);
      sb.append((char)decimal);
  }

  return sb.toString();
}

def possibleHexKeys = ['exe', 'cmd'];

def audit = ctx.auditd.get("log");
Iterator entries = audit.entrySet().iterator();
while (entries.hasNext()) {
  def e = entries.next();
  def k = e.getKey();
  def v = e.getValue(); 

  // Remove entries whose value is ?
  if (v == "?" || v == "(null)" || v == "") {
    entries.remove();
    continue;
  }

  // Convert hex values to ASCII.
  if (possibleHexKeys.contains(k) && isHexAscii(v)) {
    v = convertHexToString(v);
    audit.put(k, v);
  }

  // Trim quotes.
  if (v instanceof String) {
    v = trimQuotes(params.single_quote, params.double_quote, v);
    audit.put(k, v);
  }

  // Convert arch.
  if (k == "arch" && v == "c000003e") {
    audit.put(k, "x86_64");
  }
}"""
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.arch",
      "target_field" : "host.architecture"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.acct",
      "target_field" : "user.name"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.uid",
      "target_field" : "user.id"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.gid",
      "target_field" : "user.group.id"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.agid",
      "target_field" : "user.audit.group.id"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.auid",
      "target_field" : "user.audit.id"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.fsgid",
      "target_field" : "user.filesystem.group.id"
    }
  },
  {
    "rename" : {
      "field" : "auditd.log.fsuid",
      "target_field" : "user.filesystem.id",
      "ignore_failure" : true
    }
  },
  {
    "rename" : {
      "target_field" : "user.effective.group.id",
      "ignore_failure" : true,
      "field" : "auditd.log.egid"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.euid",
      "target_field" : "user.effective.id"
    }
  },
  {
    "rename" : {
      "field" : "auditd.log.sgid",
      "target_field" : "user.saved.group.id",
      "ignore_failure" : true
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.suid",
      "target_field" : "user.saved.id"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.ogid",
      "target_field" : "user.owner.group.id"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.ouid",
      "target_field" : "user.owner.id"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.comm",
      "target_field" : "process.name"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.exe",
      "target_field" : "process.executable"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.pid",
      "target_field" : "process.pid"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.ppid",
      "target_field" : "process.ppid"
    }
  },
  {
    "convert" : {
      "ignore_missing" : true,
      "field" : "process.pid",
      "type" : "long"
    }
  },
  {
    "convert" : {
      "type" : "long",
      "ignore_missing" : true,
      "field" : "process.ppid"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.cmd",
      "target_field" : "process.args"
    }
  },
  {
    "split" : {
      "ignore_failure" : true,
      "field" : "process.args",
      "separator" : """\s+"""
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.terminal",
      "target_field" : "user.terminal"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.msg",
      "target_field" : "message"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.res",
      "target_field" : "event.outcome"
    }
  },
  {
    "rename" : {
      "field" : "auditd.log.record_type",
      "target_field" : "event.action",
      "ignore_failure" : true
    }
  },
  {
    "lowercase" : {
      "ignore_failure" : true,
      "field" : "event.action"
    }
  },
  {
    "rename" : {
      "ignore_failure" : true,
      "field" : "auditd.log.src",
      "target_field" : "source.address"
    }
  },
  {
    "rename" : {
      "field" : "auditd.log.dst",
      "target_field" : "destination.address",
      "ignore_failure" : true
    }
  },
  {
    "grok" : {
      "field" : "source.address",
      "patterns" : [
        "^%{IP:source.ip}$"
      ],
      "ignore_failure" : true
    }
  },
  {
    "geoip" : {
      "ignore_failure" : true,
      "field" : "source.ip",
      "target_field" : "source.geo"
    }
  }
],
"on_failure" : [
  {
    "set" : {
      "field" : "error.message",
      "value" : "{{ _ingest.on_failure_message }}"
    }
  }
]

}, "filebeat-7.2.1-icinga-startup-pipeline" : { "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "description" : "Pipeline for parsing icinga startup logs", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "%{WORD:log.level}/%{WORD:icinga.startup.facility}: %{GREEDYMULTILINE:message}" ], "ignore_missing" : true, "pattern_definitions" : { "GREEDYMULTILINE" : """(.| )""" } } } ] }, "filebeat-7.2.1-logstash-slowlog-pipeline-plain" : { "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "processors" : [ { "grok" : { "field" : "message", "pattern_definitions" : { "LOGSTASH_CLASS_MODULE" : """[\w.]+\s""", "LOGSTASH_LOGLEVEL" : "INFO|ERROR|DEBUG|FATAL|WARN|TRACE" }, "patterns" : [ "\[%{TIMESTAMP_ISO8601:logstash.slowlog.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.slowlog.module}\] %{GREEDYDATA:message}" ] } }, { "grok" : { "field" : "logstash.slowlog.module", "patterns" : [ "slowlog.logstash.%{WORD:logstash.slowlog.plugin_type}.%{WORD:logstash.slowlog.plugin_name}" ] } }, { "grok" : { "patterns" : [ "{:plugin_params=>%{GREEDYDATA:logstash.slowlog.plugin_params}, :took_in_nanos=>%{NUMBER:event.duration}, :took_in_millis=>%{NUMBER:logstash.slowlog.took_in_millis}, :event=>%{GREEDYDATA:logstash.slowlog.event}}" ], "field" : "message" } }, { "rename" : { "target_field" : "event.created", "field" : "@timestamp" } }, { "date" : { "target_field" : "@timestamp", "formats" : [ "ISO8601" ], "ignore_failure" : true, "field" : "logstash.slowlog.timestamp" } }, { "remove" : { "field" : [ "message", "logstash.slowlog.timestamp" ] } }, { "convert" : { "field" : "event.duration", "type" : "long" } }, { "convert" : { "field" : "logstash.slowlog.took_in_millis", "type" : "long" } } ], "description" : "Pipeline for parsing logstash slowlogs in the plain format" }, "apm_user_geo" : { "description" : "Add user geo information for APM events", "processors" : [ { "geoip" : { "database_file" : "GeoLite2-City.mmdb", "field" : "client.ip", "target_field" : "client.geo", "ignore_missing" : true } } ] }, "filebeat-7.2.1-mysql-slowlog-pipeline" : { "description" : "Pipeline for parsing MySQL slow logs.", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "^# User@Host: %{USER:user.name}(\[%{USER:mysql.slowlog.current_user}\])? @ %{HOSTNAME:source.domain}? \[%{IP:source.ip}?\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:temp.duration:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Thread_id: %{NUMBER:mysql.thread_id}%{METRICSPACE})?(Errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(Bytes_received: %{NUMBER:mysql.slowlog.bytes_received:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Read_first: %{NUMBER:mysql.slowlog.read_first:long}%{METRICSPACE})?(Read_last: %{NUMBER:mysql.slowlog.read_last:long}%{METRICSPACE})?(Read_key: %{NUMBER:mysql.slowlog.read_key:long}%{METRICSPACE})?(Read_next: %{NUMBER:mysql.slowlog.read_next:long}%{METRICSPACE})?(Read_prev: %{NUMBER:mysql.slowlog.read_prev:long}%{METRICSPACE})?(Read_rnd: %{NUMBER:mysql.slowlog.read_rnd:long}%{METRICSPACE})?(Read_rnd_next: %{NUMBER:mysql.slowlog.read_rnd_next:long}%{METRICSPACE})?(Sort_merge_passes: %{NUMBER:mysql.slowlog.sort_merge_passes:long}%{METRICSPACE})?(Sort_range_count: %{NUMBER:mysql.slowlog.sort_range_count:long}%{METRICSPACE})?(Sort_rows: %{NUMBER:mysql.slowlog.sort_rows:long}%{METRICSPACE})?(Sort_scan_count: %{NUMBER:mysql.slowlog.sort_scan_count:long}%{METRICSPACE})?(Created_tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables:long}%{METRICSPACE})?(Created_tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(Start: %{TIMESTAMP_ISO8601:event.start}%{METRICSPACE})?(End: %{TIMESTAMP_ISO8601:event.end}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}" ], "pattern_definitions" : { "GREEDYMULTILINE" : """(.| )""", "METRICSPACE" : """([ # ])""", "EXPLAIN" : """(# explain:. |#\s )""" }, "ignore_missing" : true } }, { "remove" : { "field" : "message" } }, { "script" : { "lang" : "painless", "source" : "for (field in params.fields) { def v = ctx.mysql.slowlog.get(field); if (v != null) { ctx.mysql.slowlog.put(field, params.mapping.get(v)) } }", "params" : { "fields" : [ "query_cache_hit", "tmp_table", "tmp_table_on_disk", "filesort", "filesort_on_disk", "priority_queue", "full_scan", "full_join" ], "mapping" : { "No" : false, "Yes" : true } } } }, { "script" : { "source" : "ctx.event.duration = Math.round(ctx.temp.duration 1000000) 1000", "if" : "ctx.temp?.duration != null", "lang" : "painless" } }, { "remove" : { "ignore_missing" : true, "field" : "temp.duration" } }, { "date" : { "field" : "mysql.slowlog.timestamp", "target_field" : "@timestamp", "formats" : [ "UNIX" ], "ignore_failure" : true } }, { "remove" : { "field" : "mysql.slowlog.timestamp", "ignore_missing" : true } } ], "on_failure" : [ { "set" : { "value" : "{{ _ingest.on_failure_message }}", "field" : "error.message" } } ] }, "filebeat-7.2.1-traefik-access-pipeline" : { "description" : "Pipeline for parsing Traefik access logs. Requires the geoip and user_agent plugins.", "processors" : [ { "dissect" : { "field" : "message", "pattern" : """%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] "%{http.request.method} %{url.original} HTTP/%{http.version}" %{http.response.status_code} %{traefik.access.message}""" } }, { "grok" : { "field" : "traefik.access.message", "patterns" : [ "(?:%{NUMBER:http.response.body.bytes:long}|-)( (?:\"%{DATA:http.request.referrer}\"|-)?( (?:\"%{DATA:user_agent.original}\"|-)?)?( (?:%{NUMBER:traefik.access.request_count:long}|-)?)?( (?:\"%{DATA:traefik.access.frontend_name}\"|-)?)?( \"%{DATA:traefik.access.backend_url}\")?( %{NUMBER:temp.duration:long}ms)?)?" ], "ignore_missing" : true } }, { "remove" : { "ignore_missing" : true, "field" : "message" } }, { "remove" : { "field" : "traefik.access.message", "ignore_missing" : true } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "formats" : [ "dd/MMM/yyyy:H:m:s Z" ], "field" : "traefik.access.time", "target_field" : "@timestamp" } }, { "remove" : { "field" : "traefik.access.time" } }, { "convert" : { "type" : "long", "field" : "http.response.status_code" } }, { "grok" : { "field" : "source.address", "patterns" : [ "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" ] } }, { "script" : { "lang" : "painless", "source" : "ctx.event.duration = Math.round(ctx.temp.duration params.scale)", "params" : { "scale" : 1000000.0 }, "if" : "ctx.temp?.duration != null" } }, { "remove" : { "ignore_missing" : true, "field" : "temp.duration" } }, { "user_agent" : { "field" : "user_agent.original", "ignore_failure" : true } }, { "geoip" : { "ignore_missing" : true, "field" : "source.ip", "target_field" : "source.geo" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-logstash-log-pipeline-plain" : { "description" : "Pipeline for parsing logstash logs in the plain format", "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "processors" : [ { "grok" : { "field" : "message", "pattern_definitions" : { "LOGSTASH_CLASS_MODULE" : """[\w.]+""", "LOGSTASH_LOGLEVEL" : "INFO|ERROR|DEBUG|FATAL|WARN|TRACE", "GREEDYMULTILINE" : """(.| )""" }, "patterns" : [ "\[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s\] %{GREEDYMULTILINE:message}" ] } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "logstash.log.timestamp", "target_field" : "@timestamp", "formats" : [ "ISO8601" ], "ignore_failure" : true } }, { "remove" : { "field" : "logstash.log.timestamp" } } ] }, "filebeat-7.2.1-iis-error-default" : { "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "%{TIMESTAMP_ISO8601:iis.error.time} %{IPORHOST:source.address} %{NUMBER:source.port:long} %{IPORHOST:destination.address} %{IPORHOST:destination.port:long} (?:HTTP/%{NUMBER:http.version}|-) (?:%{WORD:http.request.method}|-) (?:%{URIPATHPARAM:url.original}|-)(?: -)? (?:%{NUMBER:http.response.status_code:long}|-) (?:%{NUMBER}|-) (?:%{NOTSPACE:iis.error.reason_phrase}|-) (?:%{NOTSPACE:iis.error.queue_name}|-)" ], "ignore_missing" : true } }, { "remove" : { "field" : "message" } }, { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "date" : { "field" : "iis.error.time", "target_field" : "@timestamp", "formats" : [ "yyyy-MM-dd HH:mm:ss" ] } }, { "remove" : { "field" : "iis.error.time" } }, { "grok" : { "pattern_definitions" : { "NOZONEIP" : "[^%]" }, "field" : "destination.address", "patterns" : [ "%{NOZONEIP:destination.ip}" ] } }, { "grok" : { "patterns" : [ "%{NOZONEIP:source.ip}" ], "pattern_definitions" : { "NOZONEIP" : "[^%]" }, "field" : "source.address" } }, { "geoip" : { "field" : "source.ip", "target_field" : "source.geo", "ignore_failure" : true } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "description" : "Pipeline for parsing IIS error logs. Requires the geoip plugin." }, "filebeat-7.2.1-elasticsearch-deprecation-pipeline-json" : { "description" : "Pipeline for parsing the Elasticsearch deprecation log file in JSON format.", "on_failure" : [ { "set" : { "value" : "{{ _ingest.on_failure_message }}", "field" : "error.message" } } ], "processors" : [ { "json" : { "field" : "message", "target_field" : "elasticsearch.deprecation" } }, { "drop" : { "if" : "ctx.elasticsearch.deprecation.type != 'deprecation'" } }, { "remove" : { "field" : "elasticsearch.deprecation.type" } }, { "rename" : { "target_field" : "log.level", "field" : "elasticsearch.deprecation.level" } }, { "rename" : { "field" : "elasticsearch.deprecation.component", "target_field" : "elasticsearch.component" } }, { "dot_expander" : { "field" : "cluster.name", "path" : "elasticsearch.deprecation" } }, { "rename" : { "field" : "elasticsearch.deprecation.cluster.name", "target_field" : "elasticsearch.cluster.name" } }, { "dot_expander" : { "field" : "node.name", "path" : "elasticsearch.deprecation" } }, { "rename" : { "field" : "elasticsearch.deprecation.node.name", "target_field" : "elasticsearch.node.name" } }, { "dot_expander" : { "field" : "cluster.uuid", "path" : "elasticsearch.deprecation" } }, { "rename" : { "field" : "elasticsearch.deprecation.cluster.uuid", "target_field" : "elasticsearch.cluster.uuid", "ignore_missing" : true } }, { "dot_expander" : { "field" : "node.id", "path" : "elasticsearch.deprecation" } }, { "rename" : { "field" : "elasticsearch.deprecation.node.id", "target_field" : "elasticsearch.node.id", "ignore_missing" : true } }, { "remove" : { "field" : "message" } }, { "rename" : { "field" : "elasticsearch.deprecation.message", "target_field" : "message" } } ] }, "filebeat-7.2.1-elasticsearch-server-pipeline-plaintext" : { "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ], "processors" : [ { "grok" : { "field" : "message", "pattern_definitions" : { "GC_YOUNG" : """[gc][young][%{NUMBER:elasticsearch.server.gc.young.one}][%{NUMBER:elasticsearch.server.gc.young.two}]%{SPACE}%{GREEDYMULTILINE:message}""", "LOG_HEADER" : """[%{TIMESTAMPISO8601:elasticsearch.server.timestamp}][%{LOGLEVEL:log.level}%{SPACE}?][%{DATA:elasticsearch.component}%{SPACE}](%{SPACE})?([%{DATA:elasticsearch.node.name}])?(%{SPACE})?""", "GREEDYMULTILINE" : """(.| )*""", "INDEXNAME" : "[a-zA-Z0-9.-]", "GC_ALL" : """[gc][%{NUMBER:elasticsearch.server.gc.overhead_seq}] overhead, spent [%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}] collecting in the last [%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}]""" }, "patterns" : [ "%{LOG_HEADER}%{GC_ALL}", "%{LOG_HEADER}%{GC_YOUNG}", "%{LOG_HEADER}%{SPACE}((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message}" ] } } ], "description" : "Pipeline for parsing the Elasticsearch server log file in plaintext format." }, "filebeat-7.2.1-apache-error-pipeline" : { "description" : "Pipeline for parsing apache error logs", "processors" : [ { "grok" : { "field" : "message", "patterns" : [ "\[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}", "\[%{APACHE_TIME:apache.error.timestamp}\] \[%{DATA:apache.error.module}:%{LOGLEVEL:log.level}\] \[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message}" ], "pattern_definitions" : { "APACHE_TIME" : "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}" }, "ignore_missing" : true } }, { "date" : { "ignore_failure" : true, "field" : "apache.error.timestamp", "target_field" : "@timestamp", "formats" : [ "EEE MMM dd H:m:s yyyy", "EEE MMM dd H:m:s.SSSSSS yyyy" ] } }, { "remove" : { "field" : "apache.error.timestamp", "ignore_failure" : true } }, { "grok" : { "field" : "source.address", "ignore_missing" : true, "patterns" : [ "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" ] } }, { "geoip" : { "ignore_missing" : true, "field" : "source.ip", "target_field" : "source.geo" } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "filebeat-7.2.1-elasticsearch-slowlog-pipeline" : { "description" : "Pipeline for parsing elasticsearch slow logs.", "processors" : [ { "rename" : { "field" : "@timestamp", "target_field" : "event.created" } }, { "grok" : { "field" : "message", "patterns" : [ "^%{CHAR:first_char}" ], "pattern_definitions" : { "CHAR" : "." } } }, { "pipeline" : { "if" : "ctx.first_char != '{'", "name" : "filebeat-7.2.1-elasticsearch-slowlog-pipeline-plaintext" } }, { "pipeline" : { "if" : "ctx.first_char == '{'", "name" : "filebeat-7.2.1-elasticsearch-slowlog-pipeline-json" } }, { "date" : { "field" : "elasticsearch.slowlog.timestamp", "target_field" : "@timestamp", "formats" : [ "ISO8601" ], "ignore_failure" : true } }, { "remove" : { "field" : "elasticsearch.slowlog.timestamp" } }, { "script" : { "params" : { "scale" : 1000000.0 }, "if" : "ctx.elasticsearch.slowlog?.duration != null", "lang" : "painless", "source" : "ctx.event.duration = Math.round(ctx.elasticsearch.slowlog.duration params.scale)" } }, { "remove" : { "ignore_missing" : true, "field" : "elasticsearch.slowlog.duration" } }, { "remove" : { "field" : [ "first_char" ] } } ], "on_failure" : [ { "set" : { "field" : "error.message", "value" : "{{ _ingest.on_failure_message }}" } } ] }, "apm_user_agent" : { "description" : "Add user agent information for APM events", "processors" : [ { "user_agent" : { "field" : "user_agent.original", "target_field" : "user_agent", "ignore_missing" : true } } ] } }