search

devasthali-os / lekhnath-paudyal

Writes logs in a proper format so that its easier to debug, stream to some NoSql indexing stores
0 stars 0 forks source link

logstash #1

Open prayagupa opened 7 years ago

prayagupa commented 7 years ago

https://www.elastic.co/downloads/logstash

wget https://artifacts.elastic.co/downloads/logstash/logstash-5.2.2.tar.gz
tar zxvf logstash-5.2.2.tar.gz 

wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.2.2.zip
unzip elasticsearch-5.2.2.zip

config/my_app_streaming.yml

input {
  file {
    path => "/tmp/access_log"
    start_position => "beginning"
  }
}

filter {
  if [path] =~ "access" {
    mutate { replace => { "type" => "apache_access" } }
    grok {
      match => { "message" => "%{COMBINEDAPACHELOG}" }
    }
  }
  date {
    match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
  }
}

output {
  elasticsearch {
    hosts => ["localhost:9200"]
  }
  stdout { codec => rubydebug }
}
curl -XGET http://localhost:9200/logstash-2017.03.28

{
  "took": 21,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "failed": 0
  },
  "hits": {
    "total": 3,
    "max_score": 1,
    "hits": [
      {
        "_index": "logstash-2017.03.28",
        "_type": "logs",
        "_id": "AVsTJtjIP8bL7zrxJypr",
        "_score": 1,
        "_source": {
          "path": "/tmp/my_app.log",
          "@timestamp": "2017-03-28T04:22:44.428Z",
          "@version": "1",
          "host": "M00974000.company.net",
          "message": "71.141.244.242 - kurt [18/May/2011:01:48:10 -0700] \"GET /admin HTTP/1.1\" 301 566 \"-\" \"Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3\""
        }
      },
      {
        "_index": "logstash-2017.03.28",
        "_type": "logs",
        "_id": "AVsTJtjIP8bL7zrxJyps",
        "_score": 1,
        "_source": {
          "path": "/tmp/my_app.log",
          "@timestamp": "2017-03-28T04:22:44.430Z",
          "@version": "1",
          "host": "M00974000.company.net",
          "message": "134.39.72.245 - - [18/May/2011:12:40:18 -0700] \"GET /favicon.ico HTTP/1.1\" 200 1189 \"-\" \"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; InfoPath.2; .NET4.0C; .NET4.0E)\""
        }
      },
      {
        "_index": "logstash-2017.03.28",
        "_type": "logs",
        "_id": "AVsTJtjIP8bL7zrxJypt",
        "_score": 1,
        "_source": {
          "path": "/tmp/my_app.log",
          "@timestamp": "2017-03-28T04:22:44.432Z",
          "@version": "1",
          "host": "M00974000.company.net",
          "message": "98.83.179.51 - - [18/May/2011:19:35:08 -0700] \"GET /css/main.css HTTP/1.1\" 200 1837 \"http://www.safesand.com/information.htm\" \"Mozilla/5.0 (Windows NT 6.0; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1\""
        }
      }
    ]
  }
}

cat /tmp/my_app.log

71.141.244.242 - kurt [18/May/2011:01:48:10 -0700] "GET /admin HTTP/1.1" 301 566 "-" "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2.3) Gecko/20100401 Firefox/3.6.3"
134.39.72.245 - - [18/May/2011:12:40:18 -0700] "GET /favicon.ico HTTP/1.1" 200 1189 "-" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; InfoPath.2; .NET4.0C; .NET4.0E)"
98.83.179.51 - - [18/May/2011:19:35:08 -0700] "GET /css/main.css HTTP/1.1" 200 1837 "http://www.safesand.com/information.htm" "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"

Refs

https://www.elastic.co/guide/en/logstash/current/config-examples.html

prayagupa commented 5 years ago

TODO

dockerize, AWSize