Open prayagupa opened 7 years ago
prayagupa
commented
7 years ago [2017-04-13T05:02:01,590][WARN ][o.e.b.BootstrapChecks ] [QCNIAhB] max file descriptors [4096] for elasticsearch process is too low, increase to at least [65536]
[2017-04-13T05:02:01,590][WARN ][o.e.b.BootstrapChecks ] [QCNIAhB] max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]# This is the folder that contains the rule yaml files
# Any .yaml file will be loaded as a rule
rules_folder: /etc/alert_rules/
# ElastAlert will buffer results from the most recent
# period of time, in case some log sources are not in real time
buffer_time:
minutes: 1
# The Elasticsearch hostname for metadata writeback
# Note that every rule can have its own Elasticsearch host
es_host: 172.21.3.9
# The Elasticsearch port
es_port: 9200
# The AWS region to use. Set this when using AWS-managed elasticsearch
#aws_region: us-west-1
# The AWS profile to use. Use this if you are using an aws-cli profile.
# See http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html
# for details
#profile: test
# Optional URL prefix for Elasticsearch
#es_url_prefix: elasticsearch
# Connect with TLS to Elasticsearch
#use_ssl: True
# Verify TLS certificates
#verify_certs: True
# GET request with body is the default option for Elasticsearch.
# If it fails for some reason, you can pass 'GET', 'POST' or 'source'.
# See http://elasticsearch-py.readthedocs.io/en/master/connection.html?highlight=send_get_body_as#transport
# for details
#es_send_get_body_as: GET
# Option basic-auth username and password for Elasticsearch
#es_username: someusername
#es_password: somepassword
# The index on es_host which is used for metadata storage
# This can be a unmapped index, but it is recommended that you run
# elastalert-create-index to set a mapping
# writeback_index: elastalert_status
writeback_index: alerting_metadata
# If an alert fails for some reason, ElastAlert will retry
# sending the alert until this time period has elapsed
alert_time_limit:
days: 1
1) orignal Internet protocol - 172.23.60.155
2) take note of the device name for the root volume (/dev/sda1 and volume vol-01efe3911d08d7724)
aws ec2 describe-instances --instance-ids i-01a9a3fae1642e228 --profile aws-federated --region us-west-2
{
"Reservations": [
{
"OwnerId": "500238854089",
"ReservationId": "r-02fdd288c62b559a7",
"Groups": [],
"Instances": [
{
"Monitoring": {
"State": "disabled"
},
"PublicDnsName": "",
"State": {
"Code": 16,
"Name": "running"
},
"EbsOptimized": false,
"LaunchTime": "2017-04-06T23:52:59.000Z",
"PrivateIpAddress": "172.23.60.155",
"ProductCodes": [],
"VpcId": "vpc-a77a82c2",
"StateTransitionReason": "",
"InstanceId": "i-01a9a3fae1642e228",
"ImageId": "ami-6f68cf0f",
"PrivateDnsName": "ip-172-23-60-155.us-west-2.compute.internal",
"KeyName": "streaming-server",
"SecurityGroups": [
{
"GroupName": "sgStreamingEndpoint",
"GroupId": "sg-1326eb68"
}
],
"ClientToken": "vmczt1491522778671",
"SubnetId": "subnet-bd67b2d8",
"InstanceType": "t2.micro",
"NetworkInterfaces": [
{
"Status": "in-use",
"MacAddress": "02:1b:7e:d4:ab:b7",
"SourceDestCheck": true,
"VpcId": "vpc-a77a82c2",
"Description": "Primary network interface",
"NetworkInterfaceId": "eni-8e00d5fc",
"PrivateIpAddresses": [
{
"PrivateDnsName": "ip-172-23-60-155.us-west-2.compute.internal",
"Primary": true,
"PrivateIpAddress": "172.23.60.155"
}
],
"PrivateDnsName": "ip-172-23-60-155.us-west-2.compute.internal",
"Attachment": {
"Status": "attached",
"DeviceIndex": 0,
"DeleteOnTermination": true,
"AttachmentId": "eni-attach-26516b43",
"AttachTime": "2017-04-06T23:52:59.000Z"
},
"Groups": [
{
"GroupName": "sgStreamingEndpoint",
"GroupId": "sg-1326eb68"
}
],
"Ipv6Addresses": [],
"SubnetId": "subnet-bd67b2d8",
"OwnerId": "500238854089",
"PrivateIpAddress": "172.23.60.155"
}
],
"SourceDestCheck": true,
"Placement": {
"Tenancy": "default",
"GroupName": "",
"AvailabilityZone": "us-west-2a"
},
"Hypervisor": "xen",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/sda1",
"Ebs": {
"Status": "attached",
"DeleteOnTermination": true,
"VolumeId": "vol-01efe3911d08d7724",
"AttachTime": "2017-04-06T23:53:00.000Z"
}
}
],
"Architecture": "x86_64",
"RootDeviceType": "ebs",
"RootDeviceName": "/dev/sda1",
"VirtualizationType": "hvm",
"Tags": [
{
"Value": "a0135-Gregor-Samsa",
"Key": "Name"
},
{
"Value": "",
"Key": "auto_terminate"
},
{
"Value": "",
"Key": "auto_start"
},
{
"Value": "",
"Key": "auto_stop"
}
],
"AmiLaunchIndex": 0
}
]
}
]
}
3 - stop the instance
aws ec2 stop-instances --instance-ids i-01a9a3fae1642e228 --profile aws-federated --region us-west-2
{
"StoppingInstances": [
{
"InstanceId": "i-01a9a3fae1642e228",
"CurrentState": {
"Code": 64,
"Name": "stopping"
},
"PreviousState": {
"Code": 16,
"Name": "running"
}
}
]
}4 -
aws ec2 describe-volumes --volume-id vol-01efe3911d08d7724 --profile aws-federated --region us-west-2
{
"Volumes": [
{
"AvailabilityZone": "us-west-2a",
"Attachments": [
{
"AttachTime": "2017-04-06T23:53:00.000Z",
"InstanceId": "i-01a9a3fae1642e228",
"VolumeId": "vol-01efe3911d08d7724",
"State": "attaching",
"DeleteOnTermination": true,
"Device": "/dev/sda1"
}
],
"Tags": [
{
"Value": "a0135-Gregor-Samsa",
"Key": "Name"
}
],
"Encrypted": false,
"VolumeType": "gp2",
"VolumeId": "vol-01efe3911d08d7724",
"State": "in-use",
"Iops": 100,
"SnapshotId": "snap-5ab96e76",
"CreateTime": "2017-04-06T23:53:00.192Z",
"Size": 10
}
]
}
#detach
aws ec2 detach-volume --volume-id vol-01efe3911d08d7724 --profile aws-federated --region us-west-2
{
"AttachTime": "2017-04-06T23:53:00.000Z",
"InstanceId": "i-01a9a3fae1642e228",
"VolumeId": "vol-01efe3911d08d7724",
"State": "detaching",
"Device": "/dev/sda1"
}
5 - Attach a volume
aws ec2 attach-volume --volume-id vol-01efe3911d08d7724 --instance-id i-01a9a3fae1642e228 --device /dev/sda1 --region us-west-2 --profile aws-federated
{
"AttachTime": "2017-04-17T09:17:14.489Z",
"InstanceId": "i-01a9a3fae1642e228",
"VolumeId": "vol-01efe3911d08d7724",
"State": "attaching",
"Device": "/dev/sda1"
}https://www.elastic.co/guide/en/beats/filebeat/1.2/filebeat-template.html
curl -XPUT 'http://172.21.3.9:9200/_template/filebeat' -d@/usr/local/filebeat-5.3.0-linux-x86_64/filebeat.template.json
curl -XPUT 'http://172.21.3.9:9200/_template/filebeat' -d '
{
"mappings": {
"_default_": {
"_all": {
"norms": false
},
"_meta": {
"version": "5.3.0"
},
"date_detection": false,
"dynamic_templates": [
{
"strings_as_keyword": {
"mapping": {
"ignore_above": 1024,
"type": "keyword"
},
"match_mapping_type": "string"
}
}
],
"properties": {
"@timestamp": {
"type": "date"
},
"apache2": {
"properties": {
"access": {
"properties": {
"agent": {
"norms": false,
"type": "text"
},
"body_sent": {
"properties": {
"bytes": {
"type": "long"
}
}
},
"geoip": {
"properties": {
"continent_name": {
"ignore_above": 1024,
"type": "keyword"
},
"country_iso_code": {
"ignore_above": 1024,
"type": "keyword"
},
"location": {
"type": "geo_point"
}
}
},
"http_version": {
"ignore_above": 1024,
"type": "keyword"
},
"method": {
"ignore_above": 1024,
"type": "keyword"
},
"referrer": {
"ignore_above": 1024,
"type": "keyword"
},
"remote_ip": {
"ignore_above": 1024,
"type": "keyword"
},
"response_code": {
"type": "long"
},
"url": {
"ignore_above": 1024,
"type": "keyword"
},
"user_agent": {
"properties": {
"device": {
"ignore_above": 1024,
"type": "keyword"
},
"major": {
"type": "long"
},
"minor": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"os": {
"ignore_above": 1024,
"type": "keyword"
},
"os_major": {
"type": "long"
},
"os_minor": {
"type": "long"
},
"os_name": {
"ignore_above": 1024,
"type": "keyword"
},
"patch": {
"type": "long"
}
}
},
"user_name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"properties": {
"client": {
"ignore_above": 1024,
"type": "keyword"
},
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"norms": false,
"type": "text"
},
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"pid": {
"type": "long"
},
"tid": {
"type": "long"
}
}
}
}
},
"beat": {
"properties": {
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"version": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"ignore_above": 1024,
"type": "keyword"
},
"fields": {
"properties": {}
},
"fileset": {
"properties": {
"module": {
"ignore_above": 1024,
"type": "keyword"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"input_type": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"norms": false,
"type": "text"
},
"meta": {
"properties": {
"cloud": {
"properties": {
"availability_zone": {
"ignore_above": 1024,
"type": "keyword"
},
"instance_id": {
"ignore_above": 1024,
"type": "keyword"
},
"machine_type": {
"ignore_above": 1024,
"type": "keyword"
},
"project_id": {
"ignore_above": 1024,
"type": "keyword"
},
"provider": {
"ignore_above": 1024,
"type": "keyword"
},
"region": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"mysql": {
"properties": {
"error": {
"properties": {
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"norms": false,
"type": "text"
},
"thread_id": {
"type": "long"
},
"timestamp": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"slowlog": {
"properties": {
"host": {
"ignore_above": 1024,
"type": "keyword"
},
"id": {
"type": "long"
},
"ip": {
"ignore_above": 1024,
"type": "keyword"
},
"lock_time": {
"properties": {
"sec": {
"type": "float"
}
}
},
"query": {
"ignore_above": 1024,
"type": "keyword"
},
"query_time": {
"properties": {
"sec": {
"type": "float"
}
}
},
"rows_examined": {
"type": "long"
},
"rows_sent": {
"type": "long"
},
"timestamp": {
"type": "long"
},
"user": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"nginx": {
"properties": {
"access": {
"properties": {
"agent": {
"norms": false,
"type": "text"
},
"body_sent": {
"properties": {
"bytes": {
"type": "long"
}
}
},
"geoip": {
"properties": {
"continent_name": {
"ignore_above": 1024,
"type": "keyword"
},
"country_iso_code": {
"ignore_above": 1024,
"type": "keyword"
},
"location": {
"type": "geo_point"
}
}
},
"http_version": {
"ignore_above": 1024,
"type": "keyword"
},
"method": {
"ignore_above": 1024,
"type": "keyword"
},
"referrer": {
"ignore_above": 1024,
"type": "keyword"
},
"remote_ip": {
"ignore_above": 1024,
"type": "keyword"
},
"response_code": {
"type": "long"
},
"url": {
"ignore_above": 1024,
"type": "keyword"
},
"user_agent": {
"properties": {
"device": {
"ignore_above": 1024,
"type": "keyword"
},
"major": {
"type": "long"
},
"minor": {
"type": "long"
},
"name": {
"ignore_above": 1024,
"type": "keyword"
},
"os": {
"ignore_above": 1024,
"type": "keyword"
},
"os_major": {
"type": "long"
},
"os_minor": {
"type": "long"
},
"os_name": {
"ignore_above": 1024,
"type": "keyword"
},
"patch": {
"type": "long"
}
}
},
"user_name": {
"ignore_above": 1024,
"type": "keyword"
}
}
},
"error": {
"properties": {
"connection_id": {
"type": "long"
},
"level": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"norms": false,
"type": "text"
},
"pid": {
"type": "long"
},
"tid": {
"type": "long"
}
}
}
}
},
"offset": {
"type": "long"
},
"read_timestamp": {
"ignore_above": 1024,
"type": "keyword"
},
"source": {
"ignore_above": 1024,
"type": "keyword"
},
"system": {
"properties": {
"syslog": {
"properties": {
"hostname": {
"ignore_above": 1024,
"type": "keyword"
},
"message": {
"ignore_above": 1024,
"type": "keyword"
},
"pid": {
"ignore_above": 1024,
"type": "keyword"
},
"program": {
"ignore_above": 1024,
"type": "keyword"
},
"timestamp": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
}
},
"tags": {
"ignore_above": 1024,
"type": "keyword"
},
"type": {
"ignore_above": 1024,
"type": "keyword"
}
}
}
},
"order": 0,
"settings": {
"index.mapping.total_fields.limit": 10000,
"index.refresh_interval": "5s"
},
"template": "filebeat-*"
}'
java
elasticsearch
indexstorage.service
forwarder-logging
https://www.elastic.co/guide/en/beats/filebeat/current/configuration-logging.html