Closed Jymit closed 6 years ago
DO Droplet - Centos 7.5 x64
$ yum update && yum install docker epel-release net-tools git python-pip wget java nmap fail2ban zip unzip -y
$ wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.3.1.rpm
$ rpm --install elasticsearch-6.3.1.rpm
$ find / -name elasticsearch
$ sudo systemctl start elasticsearch.service
$ journalctl -f
$ ifconfig -a |grep inet
$ tailf /var/log/elasticsearch/moshimoshi.log
$ free -h && sync && echo 3 > /proc/sys/vm/drop_caches && free -h
$ egrep '^[^#]+' elasticsearch.yml
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
cluster.name: moshimoshi
http.port: 9200
network.host: centos-elk
$ egrep '^[^#]+' /etc/hosts
127.0.0.1 centos-elk centos-elk
127.0.0.1 localhost.localdomain localhost
127.0.0.1 localhost4.localdomain4 localhost4
178.128.38.xxx centos-elk centos-elk
::1 centos-elk centos-elk
::1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
$ egrep '^[^#]+' jvm.options
-Xms300m
-Xmx300m
$ /usr/share/elasticsearch/bin/elasticsearch-certutil ca --pem
$ /usr/share/elasticsearch/bin/elasticsearch-certutil cert --ca-cert ca/ca.crt --ca-key ca/ca.key --out certs.zip -pem
$ egrep '^[^#]+' /etc/elasticsearch/elasticsearch.yml
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
cluster.name: moshimoshi
network.host: centos-elk
http.port: 9200
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.key: /etc/elasticsearch/instance/instance.key
xpack.security.transport.ssl.certificate: /etc/elasticsearch/instance/instance.crt
xpack.security.transport.ssl.certificate_authorities: [ "/etc/elasticsearch/ca/ca.crt" ]
xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.verification_mode: certificate
xpack.security.http.ssl.key: /etc/elasticsearch/instance/instance.key
xpack.security.http.ssl.certificate: /etc/elasticsearch/instance/instance.crt
xpack.security.http.ssl.certificate_authorities: [ "/etc/elasticsearch/ca/ca.crt" ]
Will build this on Darwin (Mac) and Linux (Centos 7)
$ vi osChecker.sh
#!/bin/sh
who=$(whoami) os=$(uname)
function osCheck(){ if [[ $os = "Darwin" ]]; then echo "Darwin" #Mac elif [[ $os = "Linux" ]]; then echo "Linux" #Linux else echo " Unknown OS, please raise this with your support contact" fi }
echo $(osCheck)
function whoAreYou(){
if [[ $who = "root" ]]; then
echo "Root! - Be careful!"
else
echo "You are running as user $who"
fi
}
echo $(whoAreYou)
- On my droplet
Linux Root! - Be careful!
- Laptop
Darwin You are running as user khondhu
- Prompt for ES IP and Prt
- Next check is ES up
$ vi prompter.sh
read -p "Enter Elasticsearch cluster IP: " cluster_ip read -p "Enter Elasticsearch cluster Port: " cluster_port read -p "HTTP OR HTTPS: " es_protocol
echo You entered an Elasticsearch cluster $es_protocol://$cluster_ip:$cluster_port sleep 1 echo Lets curl to check. sleep 1 curl -s $es_protocol://$cluster_ip:$cluster_port sleep 1
#!/bin/sh
who=$(whoami)
os=$(uname)
function osCheck(){
if [[ $os = *"Darwin"* ]]; then
echo "The OS is Darwin"
elif [[ $os = *"Linux"* ]]; then
echo "The OS is Linux"
else
echo " Unknown OS, we currently support Darwin (MacOS) and Linux, please raise this with your support contact. "
fi
}
function whoAreYou(){
if [[ $who = *"root"* ]]; then
echo "You are running as user Root"
else
echo "You are running as user $who"
fi
}
echo $(osCheck)
sleep 1
echo $(whoAreYou)
sleep 1
read -p "Enter the Elasticsearch cluster IP or hostname: " cluster_ip
read -p "Enter the Elasticsearch cluster port number: " cluster_port
read -p "Enter whether you are using 'http' OR 'https': " es_protocol
echo "You entered Elasticsearch cluster $es_protocol://$cluster_ip:$cluster_port "
sleep 1
echo "Lets curl to verify"
sleep 1
curl -s $es_protocol://$cluster_ip:$cluster_port
# add function, if curl response empty or missing then ask to try again
sleep 1
echo "."
read -p " Please enter the full path location to your elasticsearch.yml, for example /etc/elasticsearch/elasticsearch.yml:" es_yaml_locate
sleep 2
echo "."
echo "Thank you, you entered $es_yaml_locate"
sleep 1
echo "."
echo "Lets see what you have configured"
#echo "Lets have a look if you have HTTP and or TRANSPORT security in place"
sleep 1
egrep '^[^#]+' $es_yaml_locate
sleep 1
printf "What SSL/TLS bundle is in use here: ";
curl -s -XGET localhost:9200/_xpack/ssl/certificates?pretty |grep format |uniq -c |awk '{print $4}'| cut -f2 -d\"
sleep 2;
echo "The path to the bundle location on node xxx is: "
curl -s -XGET localhost:9200/_xpack/ssl/certificates?pretty |grep path |uniq |awk '{print $3}'| cut -f2 -d\"
sleep 2;
#This needs improvement for multinode enviroments however it gives an indication on where the CA/Cert bundles could reside anyhowi
echo .
sleep 1;
sleep 1;
echo "Please have your ca/cert passphrase at the ready"
echo "You are using a PKCS#12 bundle"
echo "So we shall go for: "
curl -s -XGET localhost:9200/_xpack/ssl/certificates?pretty |grep path |uniq |awk '{print $3}'| cut -f2 -d\" |xargs openssl pkcs12 -info -in
echo "bye $who"
#!/bin/sh
read -p "Enter the Elasticsearch cluster IP or hostname: " cluster_ip
read -p "Enter the Elasticsearch cluster port number: " cluster_port
read -p "Enter whether you are using 'http' OR 'https': " es_protocol
echo ""
sleep 2
echo "You entered $es_protocol://$cluster_ip:$cluster_port "
echo ""
read -p "Enter the full path location to the elasticsearch.yml: " yaml_locate
sleep 2
echo "You entered $yaml_locate"
echo ""
sleep 2
echo "We will use the elasticsearch.yml as the source of truth and use the ssl/tls settings from here for this script"
echo ""
sleep 3
echo "You have configured: "
egrep '^[^#]+' $yaml_locate --color
echo ""
moreyaml=`egrep '^[^#]+' $yaml_locate`
if [[ "$moreyaml" == *"xpack.security.http"* ]]; then
echo "HTTP/S TLS enabled: "
fi
egrep '^[^#]+' $yaml_locate |grep 'xpack.security.http'
sleep 1
echo ""
if [[ "$moreyaml" == *"xpack.security.transport"* ]]; then
echo "TRANSPORT TLS enabled: "
fi
egrep '^[^#]+' $yaml_locate |grep 'xpack.security.transport'
sleep 1
echo ""
if [[ "$moreyaml" == *"ssl.certificate_authorities"* ]]; then
echo "The CA is located at: "
fi
sleep 1
egrep '^[^#]+' $yaml_locate |grep 'ssl.certificate_authorities'
echo ""
echo "The CA path we are going to use here is "
moreCA=`grep 'xpack.security.http.ssl.certificate_authorities' $yaml_locate |awk '{print $3}'| cut -f2 -d\"`
echo $moreCA
echo ""
sleep 2
read -p "Enter an admin (elastic) user for this clustername: " admin_user
read -p "Enter the admin (elastic) users credentials: " admin_pass
#let admin_pass $(read -s)
sleep 2
echo ""
echo "You entered $admin_user"
echo "You entered $admin_pass"
sleep 2
echo ""
echo "Lets curl to verify "
sleep 2
echo ""
curl -s --cacert $moreCA -u $admin_user:$admin_pass -XGET $es_protocol://$cluster_ip:$cluster_port
#!/bin/sh
#title :ElasticCertTool.sh
#description :Script for Elasticsearch cluster SSL/TLS bundle discovery, troubleshooting.
#author :Jymit Singh Khondhu
#date :7/7/2018
#init
whoami=$(whoami)
os=$(uname)
datetime=$(date -u)
touch ElasticCertToolOutput.log
#preamble
echo "This script has been created by Elastic Support and is for troubleshooting Elaticsearch SSL/TLS "
sleep 3
echo "Only run this script if you already have SSL/TLS setup in your Elasticsearch Cluster "
echo "It is to be ran locally where the Elasticsearch service is up and running "
sleep 3
echo "If not then please Ctrl+C now "
sleep 5
echo "For the latest (at the time of writing) documentation for securing the Elastic Stack "
sleep 1
echo "Navigate to https://www.elastic.co/guide/en/elastic-stack-overview/current/xpack-security.html "
sleep 3
echo ""
echo "Please have the a resolvable hostname/IP, port number, HTTP/S, elasticsearch.yml full path and admin user credentials at the ready "
sleep 2
echo ""
#os and date
function osCheck(){
if [[ $os = *"Darwin"* ]]; then
echo " > The OS is Darwin "
elif [[ $os = *"Linux"* ]]; then
echo " > The OS is Linux "
else
echo "Unknown OS, we currently support Darwin (MacOS) and Linux, please raise this with your support contact. "
fi
}
function whoAreYou(){
if [[ $whoami = *"root"* ]]; then
echo " > You are running as user Root "
else
echo " > You are running as user $whoami "
fi
}
echo $(osCheck)
echo $(whoAreYou)
echo "> Date time of script execution is $datetime "
echo ""
sleep 2
#cluster read inputs
read -p "Enter the Elasticsearch cluster IP or hostname: " cluster_ip
read -p "Enter the Elasticsearch cluster port number (9200): " cluster_port
read -p "Enter whether you are using 'http' OR 'https': " es_protocol
echo ""
sleep 1
echo " > You entered $es_protocol://$cluster_ip:$cluster_port "
echo ""
sleep 1
#cluster config inputs
read -p "Enter the full path to the elasticsearch.yml: " yaml_locate
sleep 1
echo ""
echo " > You entered $yaml_locate "
echo ""
sleep 2
echo "We will use this elasticsearch.yml and it contents going forward "
echo ""
sleep 2
echo "You have configured: "
egrep '^[^#]+' $yaml_locate --color
echo ""
moreyaml=`egrep '^[^#]+' $yaml_locate`
sleep 2
if [[ "$moreyaml" == *"xpack.security.http"* ]]; then
echo " > HTTP/S TLS IS ENABLED "
else
echo " > HTTP/S TLS IS NOT ENABLED "
fi
egrep '^[^#]+' $yaml_locate |grep 'xpack.security.http'
sleep 2
echo ""
if [[ "$moreyaml" == *"xpack.security.transport"* ]]; then
echo " > TRANSPORT TLS IS ENABLED "
else
echo " > TRANSPORT TLS IS NOT ENABLED "
fi
egrep '^[^#]+' $yaml_locate |grep 'xpack.security.transport'
sleep 2
echo ""
echo "Lets check for a CA (Certificate Authority) in the elasticsearch yaml. "
echo ""
sleep 2
if [[ "$moreyaml" == *"ssl.certificate_authorities"* ]]; then
echo "As we have found a CA in the elasticsearch yaml, we will continue knowing that PEM is in use here. "
else
echo "No CA found, PEM is not in use so we will presume PKCS#12 is. "
fi
sleep 2
#egrep '^[^#]+' $yaml_locate |grep 'ssl.certificate_authorities'
#echo ""
if [[ "$moreyaml" == *"ssl.certificate_authorities"* ]]; then
echo " > The CA path we are going to use going forward is: "
else
echo "Okay PEM is not in use so we will presume PKCS#12 is ergo no CA path to output. "
fi
moreCA=`grep 'xpack.security.http.ssl.certificate_authorities' $yaml_locate |awk '{print $3}'| cut -f2 -d\"`
echo " $moreCA "
#Credentials read inputs
echo ""
sleep 2
echo "We are now going to prompt for the Elasticsearch Admin users credentials "
sleep 3
read -p "Enter an admin (elastic) user for this cluster: " admin_user
read -p "Enter the admin (elastic) users password: " admin_pass
#let admin_pass $(read -s)
sleep 1
echo ""
echo "You entered $admin_user and $admin_pass "
sleep 2
echo ""
echo "Lets curl to verify "
sleep 2
echo ""
curl -s --cacert $moreCA -u $admin_user:$admin_pass -XGET $es_protocol://$cluster_ip:$cluster_port
sleep 2
echo ""
echo ""
echo "Now lets use the Elasticsearch Certificate API to start querying the SSL/TLS bundle "
sleep 3
echo " > Elasticsearch Certificate API output: "
curl -s --cacert $moreCA -u $admin_user:$admin_pass -XGET $es_protocol://$cluster_ip:$cluster_port/_xpack/ssl/certificates?pretty
sleep 3
echo ""
echo ""
echo "Confirmation that the bundle is use is PKCS#12 / PEM: "
moreBundle=`curl -s --cacert $moreCA -u $admin_user:$admin_pass -XGET $es_protocol://$cluster_ip:$cluster_port/_xpack/ssl/certificates?pretty |grep format |uniq -c |awk '{print $4}'| cut -f2 -d\"`
echo " > $moreBundle "
sleep 2
echo ""
if [[ "$moreBundle" == *"PKCS12"* ]]; then
pkcs12=pkcs12
echo " > $pkcs12 CONFIRMED going forward "
elif [[ "$moreBundle" == *"PEM"* ]]; then
pem=pem
echo " > $pem CONFIRMED going forward "
else
echo "Something went wrong. "
fi
echo ""
sleep 2
echo ""
echo "The path to the bundle location we will use is "
morePath=`curl -s --cacert $moreCA -u $admin_user:$admin_pass -XGET $es_protocol://$cluster_ip:$cluster_port/_xpack/ssl/certificates?pretty |grep path |uniq |awk '{print $3}'| cut -f2 -d\"`
echo $morePath
echo ""
sleep 2
echo "If your SSL/TLS bundle has a passphrase please have it at the ready "
echo ""
sleep 2
moreOpenSSL=`curl -s --cacert $moreCA -u $admin_user:$admin_pass -XGET $es_protocol://$cluster_ip:$cluster_port/_xpack/ssl/certificates?pretty | grep path |uniq |awk '{print $3}'| cut -f2 -d\" |xargs openssl pkcs12 -info -in`
echo "$moreOpenSSL"
echo
openBAG=`echo "$moreOpenSSL" |grep 'Bag' `
openOID=`echo "$moreOpenSSL" |grep '2.16.840.1.113894.746875.1.1' `
openFRIEND=`echo "$moreOpenSSL" |grep 'friendlyName' `
openSUBJECT=`echo "$moreOpenSSL" |grep 'subject' `
openISSUER=`echo "$moreOpenSSL" |grep 'issuer' `
echo "Bag Attributes to be aware of here "
echo
echo "$openOID "
echo
if [[ "$openOID" == *"2.16.840.1.113894.746875.1.1"* ]]; then
echo " > This bundle contains Oracle's OID for a trusted CA. This bundle contains a CA "
else
echo " > This bundle does not contain a CA "
fi
echo
echo "$openFRIEND "
echo
echo "$openSUBJECT "
echo
echo "$openISSUER "
echo "Now we have checked the bundle, as a final check let us check the elasticsearch.log "
echo "All of the needful can be sent back to the Elastic Support team to provide you with best practices and next steps "
sleep 1
read -p "Enter the full path to your elasticsearch log file (path.logs): " cluster_logfile
echo ""
sleep 1
echo " > You entered $cluster_logfile "
sleep 1
echo ""
echo "Lets check for any SSL/TLS issues in the logfile $cluster_logfile "
sleep 1
echo ""
SSLexcep=`grep 'SSLException' $cluster_logfile |sort |uniq |sort -nr`
echo "Any SSLExceptions: $SSLexcep "
sleep 1
echo ""
CERTexcep=`grep 'CertificateException' $cluster_logfile |sort |uniq |sort -nr`
echo "Any CertificateExceptions: $CERTexcep "
sleep 1
echo ""
SSLHANDSHAKEexcep=`grep 'SSLHandshakeException' $cluster_logfile |sort |uniq |sort -nr`
echo "Any Handshake Exceptions: $SSLHANDSHAKEexcep "
sleep 1
echo ""
SUNPATHexcep=`grep 'SunCertPathBuilderException' $cluster_logfile |sort |uniq |sort -nr`
echo "Any SunCertPathBuilderExceptions: $SUNPATHexcep "
sleep 1
echo ""
echo " THE END "
SSL/TLS troubleshooting script
Elastic Stack v6.3.0
Create and test on single node with P12 (PKCS#12) bundle.
Stop Elasticsearch, Kibana.
Update yaml.
SSL Certificate API docs here.
Iteration 1.
. .. . .. . .. . .. . Other things to note and work on.
Troubleshooting things to add. PEM single node cluster v624 SSL/TLS for HTTP & TRANSPORT:
http.
https.
with ca.
ca and auth
Add openssl clauses into the script
Check a PKCS#12 file (.pfx or .p12): $ openssl pkcs12 -info -in elastic-certificates.p12
The key part here is the 2.16.840.1.113894.746875.1.1 This is Oracle's OID for a trusted CA entry in a PKCS#12.
We assume that the PKCS#12 contains the certificate chain. That's usually true, but not always. If the
openssl pkcs12 -info
command outputs multiple---- BEGIN CERTIFICATE
blocks, then the PKCS#12 contains the full chain, and this should all work fine. If it has only a single certificate block, then the customer will need to get their CA cert from elsewhere.