Logtrail is not displaying data.

[Venkat49k]

Hi,

I think my issue was posted multiple times, still I am not able to figure out the mistake in my configuration. In Logtrail I am not seeing any data, please help me in resolving this issue. I am trying to pull the 'tomcat' logs from different server.

My logtrail conf file :

 {
  "index_patterns" : [
    {
      "es": {
        "default_index": "filebeat-*",
        "allow_url_parameter": false
      },
      "tail_interval_in_seconds": 10,
      "es_index_time_offset_in_seconds": 0,
      "display_timezone": "Etc/UTC",
      "display_timestamp_format": "MMM DD HH:mm:ss",
      "max_buckets": 500,
      "default_time_range_in_days" : 0,
      "max_hosts": 100,
      "max_events_to_keep_in_viewer": 5000,
      "fields" : {
        "mapping" : {
            "timestamp":"@timestamp",
            "display_timestamp":"@timestamp",
            "source" : "source",
            "hostname" : "host",
            "message": "message"
        },
         "message_format": "{{{message}}}"
      },
      "color_mapping" : {
        "field": "level",
        "mapping" : {
          "0": "#ff0000",
          "1": "#ff3232",
          "2": "#ff4c4c",
          "3": "#ff7f24",
          "4": "#ffb90f",
          "5": "#a2cd5a"
        }
      }
    }
  ]
}

[Venkat49k]

input {
beats {
port => 5044
ssl => true
ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt"
ssl_key => "/etc/pki/tls/private/logstash-forwarder.key"
}
}

filter {
if [type] == "syslog" {
grok {
match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?: %{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}" ]
add_field => [ "received_from", "%{host}" ]
}
syslog_pri { }
date {
match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
}
}
}

output {
elasticsearch {
hosts => ["localhost:9200"]
sniffing => true
manage_template => false
index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
document_type => "%{[@metadata][type]}"
}
}

Please help me in resolving this issue, as I was struck for multiple days :-(

[sivasamyk]

Can you share the output of curl <es-host>:9200/filebeat-*/_search?pretty and curl <es-host>:9200/filebeat-*/_mapping?pretty commands?

[Venkat49k]

[root@ip-10-0-1-245 elasticsearch]# curl 10.0.1.245:9200/filebeat-*/_search?pretty

{ "took" : 76, "timed_out" : false, "_shards" : { "total" : 45, "successful" : 45, "failed" : 0 }, "hits" : { "total" : 12185546, "max_score" : 1.0, "hits" : [ { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRS", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat org.springframework.beans.factory.support.ConstructorResolver.instantiate(ConstructorResolver.java:620)", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "fields" : null, "offset" : 1398677, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRV", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1127)", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "fields" : null, "offset" : 1399068, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRX", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:498)", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "offset" : 1399353, "fields" : null, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRY", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:320)", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "offset" : 1399490, "fields" : null, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRa", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:318)", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "offset" : 1399732, "fields" : null, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRf", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat org.springframework.context.annotation.AnnotationConfigApplicationContext.(AnnotationConfigApplicationContext.java:88)", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "offset" : 1400325, "fields" : null, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRl", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat org.springframework.boot.loader.MainMethodRunner.run(MainMethodRunner.java:48)", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "offset" : 1400799, "fields" : null, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRm", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat org.springframework.boot.loader.Launcher.launch(Launcher.java:87)", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "fields" : null, "offset" : 1400882, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRp", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "02 Apr 2019 01:30:21.092 [reactor-http-server-epoll-15] ERROR com.adaequare.taxilla.controller.advice.GlobalErrorAttributes.handleException", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.037Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "fields" : null, "offset" : 1401096, "type" : "log" } }, { "_index" : "filebeat-2019.04.02", "_type" : "log", "_id" : "AWneFpA7G1ov5HMGBkRu", "_score" : 1.0, "_source" : { "count" : 1, "input_type" : "log", "message" : "\tat com.adaequare.Application$$EnhancerBySpringCGLIB$$2e5d7262.CGLIB$webHandler$14()", "source" : "/opt/taxilla/logs/taxilla.log", "@timestamp" : "2019-04-02T12:46:53.038Z", "@version" : "1", "host" : "ip-10-0-1-79.us-west-2.compute.internal", "tags" : [ "beats_input_codec_plain_applied" ], "beat" : { "name" : "ip-10-0-1-79.us-west-2.compute.internal", "hostname" : "ip-10-0-1-79.us-west-2.compute.internal" }, "offset" : 1401586, "fields" : null, "type" : "log" } } ] } }

[Venkat49k]

[root@ip-10-0-1-245 elasticsearch]# curl 10.0.1.245:9200/filebeat-*/_mapping?pretty { "filebeat-2019.04.10" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "filebeat-2019.04.07" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "filebeat-2019.04.04" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "filebeat-2019.04.05" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "filebeat-2019.04.03" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "filebeat-2019.04.09" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "filebeat-2019.04.06" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "filebeat-2019.04.02" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } }, "filebeat-2019.04.08" : { "mappings" : { "log" : { "properties" : { "@timestamp" : { "type" : "date" }, "@version" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "beat" : { "properties" : { "hostname" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "count" : { "type" : "long" }, "host" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "input_type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "message" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "offset" : { "type" : "long" }, "source" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "tags" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "type" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } } } } }

[Venkat49k]

I have ran those commands form the ELK server, please find the output.