search

nxhack / logstash

Configurations of my logstash: logstash, filebeat, grok patterns: sshd, postfix, apache, sysdig, zimbra mailbox.log, zimbra zimbra.log, Datadog Dogstatsd, fail2ban
91 stars 33 forks source link

Steps to use in existing logstash-5.x and elasticsearch-5.x #2

Open premhunt opened 7 years ago

premhunt commented 7 years ago

Hi Nx,

It is a very great script but missing the how to use part as I am still struggling to setup of your files on two different system. Looking for a small how to doc to help me out. Thanks in advance

nxhack commented 7 years ago

Hi @premhunt I wrote a simple script. I have not verified it. Please use it for understanding the overview.

#!/bin/bash
#
# My environment:
#  Ubuntu 16.04.2 LTS
#   systemd

sudo mkdir -p /etc/logstash/settings/indexer
sudo mkdir -p /etc/logstash/settings/shipper
sudo mkdir -p /usr/share/logstash/data/indexer/queue
sudo mkdir -p /usr/share/logstash/data/shipper/queue
sudo mkdir -p /etc/logstash/indexer.d
sudo mkdir -p /etc/logstash/shipper.d
sudo mkdir -p /var/log/logstash/indexer
sudo mkdir -p /var/log/logstash/shipper
sudo mkdir -p /etc/logstash/geoip

sudo cp /etc/logstash/jvm.options /etc/logstash/settings/indexer/
sudo cp /etc/logstash/jvm.options /etc/logstash/settings/shipper/
sudo cp /etc/logstash/log4j2.properties /etc/logstash/settings/indexer/
sudo cp /etc/logstash/log4j2.properties /etc/logstash/settings/shipper/
sudo cp /etc/logstash/logstash.yml /etc/logstash/settings/indexer/
sudo cp /etc/logstash/logstash.yml /etc/logstash/settings/shipper/

cd /etc/logstash/settings/indexer
cat <<EOF | sudo patch
--- logstash.yml.orig
+++ logstash.yml
@@ -25,8 +25,7 @@
 # Which directory should be used by logstash and its plugins
 # for any persistent needs. Defaults to LOGSTASH_HOME/data
 #
-path.data: /var/lib/logstash
-path.data: /var/lib/logstash
+path.data: /usr/share/logstash/data/indexer
 #
 # ------------ Pipeline Settings --------------
 #
@@ -98,6 +96,7 @@
 # Default is path.data/queue
 #
 # path.queue:
+path.queue: /usr/share/logstash/data/indexer/queue
 #
 # If using queue.type: persisted, the page data files size. The queue data consists of
 # append-only data files separated into pages. Default is 250mb
EOF

cd /etc/logstash/settings/shipper
cat <<EOF | sudo patch
--- logstash.yml.orig
+++ logstash.yml
@@ -25,8 +25,7 @@
 # Which directory should be used by logstash and its plugins
 # for any persistent needs. Defaults to LOGSTASH_HOME/data
 #
-path.data: /var/lib/logstash
-path.data: /var/lib/logstash
+path.data: /usr/share/logstash/data/shipper
 #
 # ------------ Pipeline Settings --------------
 #
@@ -98,6 +96,7 @@
 # Default is path.data/queue
 #
 # path.queue:
+path.queue: /usr/share/logstash/data/shipper/queue
 #
 # If using queue.type: persisted, the page data files size. The queue data consists of
 # append-only data files separated into pages. Default is 250mb
EOF

sudo cp /etc/default/logstash /etc/default/logstash-indexer
sudo cp /etc/default/logstash /etc/default/logstash-shipper

cd /etc/default
cat <<EOF | sudo patch
--- logstash-indexer.orig
+++ logstash-indexer
@@ -1,11 +1,11 @@
 JAVACMD="/usr/bin/java"
-LS_HOME="/usr/share/logstash"
-LS_SETTINGS_DIR="/etc/logstash"
-LS_PIDFILE="/var/run/logstash.pid"
+LS_HOME="/usr/share/logstash/indexer"
+LS_SETTINGS_DIR="/etc/logstash/indexer.d"
+LS_PIDFILE="/var/run/logstash-indexer.pid"
 LS_USER="logstash"
 LS_GROUP="logstash"
-LS_GC_LOG_FILE="/var/log/logstash/gc.log"
+LS_GC_LOG_FILE="/var/log/logstash/indexer/gc.log"
 LS_OPEN_FILES="16384"
 LS_NICE="19"
-SERVICE_NAME="logstash"
-SERVICE_DESCRIPTION="logstash"
+SERVICE_NAME="logstash-indexer"
+SERVICE_DESCRIPTION="logstash-indexer"
EOF

cat <<EOF | sudo patch
--- logstash-shipper.orig
+++ logstash-shipper
@@ -1,11 +1,11 @@
 JAVACMD="/usr/bin/java"
-LS_HOME="/usr/share/logstash"
-LS_SETTINGS_DIR="/etc/logstash"
-LS_PIDFILE="/var/run/logstash.pid"
+LS_HOME="/usr/share/logstash/shipper"
+LS_SETTINGS_DIR="/etc/logstash/shipper.d"
+LS_PIDFILE="/var/run/logstash-shipper.pid"
 LS_USER="logstash"
 LS_GROUP="logstash"
-LS_GC_LOG_FILE="/var/log/logstash/gc.log"
+LS_GC_LOG_FILE="/var/log/logstash/shipper/gc.log"
 LS_OPEN_FILES="16384"
 LS_NICE="19"
-SERVICE_NAME="logstash"
-SERVICE_DESCRIPTION="logstash"
+SERVICE_NAME="logstash-shipper"
+SERVICE_DESCRIPTION="logstash-shipper"
EOF

cd /etc/logstash/geoip
sudo wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz
sudo wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.mmdb.gz
sudo wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz
sudo gunzip GeoLite2-City.mmdb.gz
sudo gunzip GeoLite2-Country.mmdb.gz
sudo tar xfz GeoLite2-ASN.tar.gz --strip=1 --wildcards */GeoLite2-ASN.mmdb

cd /etc/logstash
sudo wget https://raw.githubusercontent.com/ua-parser/uap-core/master/regexes.yaml

sudo cp /etc/systemd/system/logstash.service /etc/systemd/system/logstash-indexer.service
sudo cp /etc/systemd/system/logstash.service /etc/systemd/system/logstash-shipper.service

cd /etc/systemd/system/
cat <<EOF | sudo patch
--- logstash-indexer.service.orig
+++ logstash-indexer.service
@@ -1,5 +1,5 @@
 [Unit]
-Description=logstash
+Description=logstash-indexer

 [Service]
 Type=simple
@@ -8,9 +8,9 @@
 # Load env vars from /etc/default/ and /etc/sysconfig/ if they exist.
 # Prefixing the path with '-' makes it try to load, but if the file doesn't
 # exist, it continues onward.
-EnvironmentFile=-/etc/default/logstash
-EnvironmentFile=-/etc/sysconfig/logstash
-ExecStart=/usr/share/logstash/bin/logstash "--path.settings" "/etc/logstash"
+EnvironmentFile=-/etc/default/logstash-indexer
+EnvironmentFile=-/etc/sysconfig/logstash-indexer
+ExecStart=/usr/share/logstash/bin/logstash "--path.settings" "/etc/logstash/settings/indexer" "--http.port" "9601" "-f" "/etc/logstash/indexer.d" "-l" "/var/log/logstash/indexer"
 Restart=always
 WorkingDirectory=/
 Nice=19
EOF

cat <<EOF | sudo patch
--- logstash-shipper.orig
+++ logstash-shipper.service
@@ -1,5 +1,5 @@
 [Unit]
-Description=logstash
+Description=logstash-shipper

 [Service]
 Type=simple
@@ -8,9 +8,9 @@
 # Load env vars from /etc/default/ and /etc/sysconfig/ if they exist.
 # Prefixing the path with '-' makes it try to load, but if the file doesn't
 # exist, it continues onward.
-EnvironmentFile=-/etc/default/logstash
-EnvironmentFile=-/etc/sysconfig/logstash
-ExecStart=/usr/share/logstash/bin/logstash "--path.settings" "/etc/logstash"
+EnvironmentFile=-/etc/default/logstash-shipper
+EnvironmentFile=-/etc/sysconfig/logstash-shipper
+ExecStart=/usr/share/logstash/bin/logstash "--path.settings" "/etc/logstash/settings/shipper" "--http.port" "9600" "-f" "/etc/logstash/shipper.d" "-l" "/var/log/logstash/shipper"
 Restart=always
 WorkingDirectory=/
 Nice=19
EOF

sudo systemctl daemon-reload
sudo systemctl stop logstash
sudo systemctl disable logstash
sudo systemctl enable logstash-indexer
sudo systemctl start logstash-indexer
sudo systemctl enable logstash-shipper
sudo systemctl start logstash-shipper
premhunt commented 7 years ago

Thanks for quick response. I will make changes and will update you as document