search

vieyahn2017 / shellv

shell command test and study
4 stars 1 forks source link

7.29 precheck && aftercheck #53

Closed vieyahn2017 closed 5 years ago

vieyahn2017 commented 5 years ago

beforecheck && aftercheck

vieyahn2017 commented 5 years ago

之前的check，基于sshpass
https://github.com/vieyahn2017/shellV/issues/49

目前已经重构，基于paramiko了

vieyahn2017 commented 5 years ago

check_ssh_connect.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import paramiko
import warnings

warnings.filterwarnings('ignore')

def main():
    # noinspection PyBroadException
    try:
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        #入参数，依次是ip，端口，用户名，密码
        ssh.connect(hostname=sys.argv[1], port=int(sys.argv[2]), username='root', password=sys.argv[3])
        ssh.close()
        print("normal")

    except:
        print("exception")

main()

ssh_execute_command.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys
import paramiko
import time
import warnings

warnings.filterwarnings('ignore')

def main():
    # noinspection PyBroadException
    try:
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        #参数，依次是ip,端口(22),用户名,密码
        ssh.connect(hostname=sys.argv[1], port=int(sys.argv[2]), username=sys.argv[3], password=sys.argv[4])
        stdin, stdout, stderr = ssh.exec_command(sys.argv[5])
        print(stdout.read().decode())
        time.sleep(3)
        ssh.close()
        print("vbdp_normal")

    except:
        print("exception")

main()
vieyahn2017 commented 5 years ago

precheck.sh

#! /bin/bash

# 服务包路径
SERVER_PATH=/opt
# 服务包最小磁盘（单位G）
SERVER_SPACE=50
# 因为测试环境限制，测试取值可以适当减小。比如20

# 安装包路径
PACKAGE_PATH=/tmp
# 安装包最小磁盘（单位G）
PACKAGE_SPACE=10

# gauss100安装目标路径
GAUSS_PATH=/home
# gauss100安装最小磁盘（单位G）
GAUSS_SPACE=50
# 因为测试环境限制，测试取值可以适当减小。比如25

check_disk_space () {
    log "================check disk space=================="
    local check_server_ip=$1
    local check_types=$2
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $check_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))
    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $check_server_ip | awk -F ',' '{print $'$count'}'`
        log "check at ${service_ip}:"
        for check_type in `echo $check_types | tr ',' ' '`; do
            if [ "$check_type" == "server" ]; then
                sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  'df -h "${SERVER_PATH}"'`
                actual_value=`echo "$sshout" | awk 'NR==2{print $4}' | sed s/G//g`
                # 防止路径不存在，获取空值
                if [ -z "$actual_value" ]; then
                    sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "df -h /"`
                    actual_value=`echo "$sshout" | awk 'NR==2{print $4}' | sed s/G//g`
                fi
                if [ "$actual_value" -ge "$SERVER_SPACE" ]; then
                    log ${SERVER_PATH} : expected ${SERVER_SPACE}G, actual: ${actual_value}G
                else
                    log  ${SERVER_PATH} : expected ${SERVER_SPACE}G, actual: ${actual_value}G ----- checked disk failed at ${service_ip}.
                fi
            elif [ "$check_type" == "package" ]; then
                sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  'df -h "${PACKAGE_PATH}"'`
                actual_value=`echo "$sshout" | awk 'NR==2{print $4}' | sed s/G//g`
                # 防止路径不存在，获取空值
                if [ -z "$actual_value" ]; then
                    sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "df -h /"`
                    actual_value=`echo "$sshout" | awk 'NR==2{print $4}' | sed s/G//g`
                fi
                if [ "$actual_value" -ge "$PACKAGE_SPACE" ]; then
                    log ${PACKAGE_PATH} : expected ${PACKAGE_SPACE}G, actual: ${actual_value}G
                else
                    log  ${PACKAGE_PATH} : expected ${PACKAGE_SPACE}G, actual: ${actual_value}G ----- checked disk failed at ${service_ip}.
                fi
            elif [ "$check_type" == "space" ]; then
                sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  'df -h "${GAUSS_PATH}"'`
                actual_value=`echo "$sshout" | awk 'NR==2{print $4}' | sed s/G//g`
                # 防止路径不存在，获取空值
                if [ -z "$actual_value" ]; then
                    sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "df -h /"`
                    actual_value=`echo "$sshout" | awk 'NR==2{print $4}' | sed s/G//g`
                fi
                if [ "$actual_value" -ge "$GAUSS_SPACE" ]; then
                    log ${GAUSS_PATH} : expected ${GAUSS_SPACE}G, actual: ${actual_value}G
                else
                    log  ${GAUSS_PATH} : expected ${GAUSS_SPACE}G, actual: ${actual_value}G ----- checked disk failed at ${service_ip}.
                fi
            fi
        done
        log "--------------------------------------------------"
    done
}

check_port () {
    log "================check port in use=================="
    local check_server_ip=$1
    local check_port=$2
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $check_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))

    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $check_server_ip | awk -F ',' '{print $'$count'}'`
        log "check at ${service_ip}:"

        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "lsof -i:${check_port}"`
        result_lines=`echo "$sshout" | wc -l`
        if [ "$result_lines" -gt 2 ]; then
            log "checked port failed at ${service_ip}, ${check_port} is in use."
        else
            log "checked port success"
        fi
    done

    log "--------------------------------------------------"
}

check_firewall () {
    log "================check firewall=================="
    local check_server_ip=$1
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $check_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))

    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $check_server_ip | awk -F ',' '{print $'$count'}'`
        log "check at ${service_ip}:"

        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "firewall-cmd --state"`
        if [ `echo "$sshout" | wc -l` -le 2 ]; then
            log "checked firewall success"
        else
            log "checked firewall failed at ${service_ip}, try to stop it."
            ssh_execute $service_ip $SSH_PORT $ROOT_PWD "service firewalld stop" > /dev/null
        fi
    done

    log "--------------------------------------------------"
}

check_ida_component () {
    log "================check ida service=================="
    local check_server_ip=$1
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $check_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))

    components="nginx keepalived zookeeper gaussdb"
    Java_microservices_name=`declare | grep -E 'service[0-9]{1,2}_name' | cut -d "=" -f 2`
    check_items=`echo $Java_microservices_name $components | tr ' ' '|'`

    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $check_server_ip | awk -F ',' '{print $'$count'}'`
        log "check at ${service_ip}:"

        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  'ps -ef | grep -E "${check_items}" | grep -v "grep -E"'`
        for item in `echo $Java_microservices_name $components`; do
            if [ `echo "$sshout" | grep -c "$item"` -ne 0 ]; then
                log "checked service failed at ${service_ip}, $item is existed"
            fi
        done
    done

    log "--------------------------------------------------"
}

check_install_path () {
    log "================check install path=================="
    local check_server_ip=$1
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $check_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))

    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $check_server_ip | awk -F ',' '{print $'$count'}'`
        log "check at ${service_ip}:"

        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "ls /opt/fusion_viid/"`
        if [ `echo "$sshout" | wc -l` -le 2 ]; then
            log "checked install path success"
        else
            log "checked install path failed at ${service_ip}, the path (/opt/fusion_viid/) is existed."
        fi
    done

    log "--------------------------------------------------"
}

SRC_PATH=`cd "$(dirname "$0")"; pwd`

## 读入配置项
read_declare() {
    counter=1
    # ls_files=`ls config/install_conf*sh; ls install*sh | xargs -n 1`
    ls_files=`ls install*sh`
    echo "You will import variables from:"
    for filename in $ls_files;do
        echo "${counter}: ${filename}"
        counter=`expr $counter + 1`
    done
    echo "Please choose:"
    read choice
    judge_choice $counter $choice
    if [ $? -eq 0 ];then
        choose_file=`echo $ls_files | awk  '{print $'"$choice"'}'`
        echo "Now will import variables from: ${choose_file}"
        if [ `echo $choose_file | grep -c "/"` -eq 0 ];then
            source "./${choose_file}" declare
        else
            source $choose_file
        fi
    else
        echo "Your choice is invalid."
        exit 1
    fi
}

judge_choice() {
    local counter=$1
    local choice=$2
    echo `seq $counter` | grep -wq "$choice" &&  return 0 || return 1
}

read_pwd() {
    #输入待部署虚拟机的root账号的密码
    echo -e "\033[34mPlease enter the password of root account of the virtual machine to be deployed:\033[0m"
    read ROOT_PWD
}

func_check_ssh_connect()
{
    if [ "127.0.0.1"  == "$1" ]; then
        continue
    fi
    check_rslt=`python -x  tools/deploy/deploy_machine/prepare/check_ssh_connect.py $1 $2 $3`
    if [ "normal" != "$check_rslt" ]; then
       echo "ERROR:virtual machine ip :$1 can not be connected,please check and try again."
       return 0;
    fi
}

check_read_pwd_by_connection() {
    func_check_ssh_connect "$1" $SSH_PORT $ROOT_PWD
    if [ $? -ne 0 ]; then
        echo "connect failed, please check the pwd."
        exit 1
    fi
}

ssh_execute() {
    local service_ip=$1
    local port=$2
    local pwd=$3
    local pass_cmd=$4
    python -x tools/deploy/deploy_machine/execute/ssh_execute_command.py $service_ip $port root $pwd "$pass_cmd"
    # echo python -x tools/deploy/deploy_machine/execute/ssh_execute_command.py $service_ip $SSH_PORT root $ROOT_PWD $pass_cmd
}

LOG_FILE=precheck.log
rm -rf $LOG_FILE
touch $LOG_FILE

function log()
{
    # local timestamp=`date -d today +'%y-%m-%d %H:%M:%S'`
    # echo "[${timestamp}]: $1" >> $LOG_FILE
    echo "$@" >> $LOG_FILE
    echo "$@"
}

if [ "$1" == "include" ]; then
    # 内嵌在install执行
    ROOT_PWD=$root_pwd
else
    # "$1" == "import"
    ROOT_PWD="$2"
    if [ -z "$ROOT_PWD" ]; then
        read_pwd
    fi

    if [ -z "$3" ]; then
        read_declare
    else
        source "./$3" declare  > /dev/null
    fi

fi

if [ -z "$ssh_port" ];then
    SSH_PORT=22
else
    SSH_PORT=$ssh_port
fi

# 内嵌在install执行不检查
if [ "$1" != "include" ]; then
    # 以任意ip检查read_pwd用户输入的密码，这边以第一个ip为例。密码错误则退出
    check_read_pwd_by_connection `echo $Jdk_software_ip | awk -F ','  '{print $1}'`
fi

### 检查磁盘
check_disk_space $Jdk_software_ip "server,package"
check_disk_space $gauss100_ip "server,package,space"
#
### 检查端口
check_port $RestGateWay_ip 8080
check_port $RestGateWay_ip 8088
check_port $Nginx_sotfware_ip 8080
check_port $Nginx_sotfware_ip 8090

# 检查GaussDB 100和ZK的防火墙是否关闭
# 最新：检查所有节点的防火墙，并关闭
check_firewall $All_deploy_ips

# 检查是否存在旧的IDA组件: zookeeper，nginx，keepalived，所有微服务
# 每个节点全方位检查
check_ida_component $Jdk_software_ip

# 检查路径: 是否已存在旧的安装路径，存在则提示用户是否删除
check_install_path $Jdk_software_ip

echo "... more information at $LOG_FILE"
vieyahn2017 commented 5 years ago

主流程 

# 可以带上安装参数，uncheck，跳过安装检查
if [ "$1" != "uncheck" ];then
    function_step3_precheck_all_ips_step
fi

function_step3_precheck_all_ips_step ()
{
    echo -e "\033[36mStep $precheck_before_install_step/$total_step: precheck disk/firewall/port/services/folders:\033[0m"

    source ./precheck.sh include

    echo -e "\033[36mPrecheck finished. the results:\033[0m"
    if [ ! -f ${pwd_path} ]; then
        if [ `grep -c "checked disk failed" precheck.log` -ne 0 ]; then
            cat precheck.log | grep "checked disk failed"
            echo -e "\033[34mChecked disk failed, exit now.\033[0m"
            exit 1
        fi
        if [ `grep -c "checked port failed" precheck.log` -ne 0 ]; then
            cat precheck.log | grep "checked port failed"
            echo -e "\033[34mWhether to forcibly stop the process occupied by the port?:yes/no\033[0m"
            read delete_existed_port_yn
            if [[ "$delete_existed_port_yn" == "yes" ]] || [ "$delete_existed_port_yn" == "y" ];then
                echo "Then forcibly stop the process of the port, and continue."
            elif [[ "$delete_existed_port_yn" == "no" ]] || [ "$delete_existed_port_yn" == "n" ];then
                echo "you choose no, now will exit install."
                exit 1
            else
                echo "input error, exit install."
                exit 1
            fi
        fi

        if [ `grep -c "checked service failed" precheck.log` -ne 0 ]; then
            cat precheck.log | grep "checked service failed"
            echo -e "\033[34mWhether to forcibly stop these services?:yes/no\033[0m"
            read delete_existed_service_yn
            if [[ "$delete_existed_service_yn" == "yes" ]] || [ "$delete_existed_service_yn" == "y" ];then
                echo -e "Then forcibly stop theses services, and continue."
            elif [[ "$delete_existed_service_yn" == "no" ]] || [ "$delete_existed_service_yn" == "n" ];then
                echo -e "you choose no, now will exit install."
                exit 1
            else
                echo "input error, exit install."
                exit 1
            fi
        fi

        if [ `grep -c "checked install path failed" precheck.log` -ne 0 ]; then
            cat precheck.log | grep "checked install path failed"
            echo -e "\033[34mWhether to forcibly delete the folders?:yes/no\033[0m"
            read delete_existed_path_yn
            if [[ "$delete_existed_path_yn" == "yes" ]] || [ "$delete_existed_path_yn" == "y" ];then
                echo -e "Then forcibly delete the folders, and continue."
            elif [[ "$delete_existed_path_yn" == "no" ]] || [ "$delete_existed_path_yn" == "n" ];then
                echo -e "you choose no, now will exit install."
                exit 1
            else
                echo "input error, exit install."
                exit 1
            fi
        fi

    fi
}
vieyahn2017 commented 5 years ago

check.sh

#! /bin/bash

SRC_PATH=`cd "$(dirname "$0")"; pwd`

## 读入配置项
read_declare() {
    counter=1
    # ls_files=`ls config/install_conf*sh; ls install*sh | xargs -n 1`
    ls_files=`ls install*sh`
    echo "You will import variables from:"
    for filename in $ls_files;do
        echo "${counter}: ${filename}"
        counter=`expr $counter + 1`
    done
    echo "Please choose:"
    read choice
    judge_choice $counter $choice
    if [ $? -eq 0 ];then
        choose_file=`echo $ls_files | awk  '{print $'"$choice"'}'`
        echo "Now will import variables from: ${choose_file}"
        if [ `echo $choose_file | grep -c "/"` -eq 0 ];then
            source "./${choose_file}" declare
        else
            source $choose_file
        fi
    else
        echo "Your choice is invalid."
        exit 1
    fi
}

judge_choice() {
    local counter=$1
    local choice=$2
    echo `seq $counter` | grep -wq "$choice" &&  return 0 || return 1
}

judge_choice0() {
    local counter=$1
    local choice=$2
    echo `seq $counter` | grep -wq "$choice" &&  echo "yes" || echo "no"
}

read_pwd() {
    #输入待部署虚拟机的root账号的密码
    echo -e "\033[34mPlease enter the password of root account of the virtual machine to be deployed:\033[0m"
    read ROOT_PWD
}

func_check_ssh_connect()
{
    if [ "127.0.0.1"  == "$1" ]; then
        continue
    fi
    check_rslt=`python -x  tools/deploy/deploy_machine/prepare/check_ssh_connect.py $1 $2 $3`
    if [ "normal" != "$check_rslt" ]; then
       echo "ERROR:virtual machine ip :$1 can not be connected,please check and try again."
       return 0;
    fi
}

check_read_pwd_by_connection() {
    func_check_ssh_connect "$1" $SSH_PORT $ROOT_PWD
    if [ $? -ne 0 ]; then
        echo "connect failed, please check the pwd."
        exit 1
    fi
}

ssh_execute() {
    local service_ip=$1
    local port=$2
    local pwd=$3
    local pass_cmd=$4
    python -x tools/deploy/deploy_machine/execute/ssh_execute_command.py $service_ip $port root $pwd "$pass_cmd"
    # echo python -x tools/deploy/deploy_machine/execute/ssh_execute_command.py $service_ip $SSH_PORT root $ROOT_PWD $pass_cmd
}

check_gaussdb () {
    local gauss100_server_ip=$1
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $gauss100_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))
    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $gauss100_server_ip | awk -F ',' '{print $'$count'}'`
        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "ps -ef | grep gaussdb | grep -v grep"`
        echo -e "gaussdb at ${service_ip}:"  >> $LOG_FILE
        echo "$sshout"  >> $LOG_FILE
        echo -e "----------------------------------\n"  >> $LOG_FILE
        if [ `echo "$sshout" | wc -l` -ne 0 ]; then
            # echo -e "\033[34mgaussdb\033[0m at \033[35m${service_ip}\033[0m \033[32msuccess\033[0m"
            printf "%-30s%-15s%20s\n" "gaussdb" ${service_ip} "success"
        else
            # echo -e "\033[34mgaussdb\033[0m at \033[35m${service_ip}\033[0m \033[31mfailed\033[0m"
            printf "%-30s%-15s%20s\n" "gaussdb" ${service_ip} "failed"
        fi
    done
}

check_nginx () {
    local nginx_server_ip=$1
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $nginx_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))
    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $nginx_server_ip | awk -F ',' '{print $'$count'}'`
        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "ps -ef | grep nginx | grep viid | grep -v grep"`
        echo -e "nginx at ${service_ip}:"  >> $LOG_FILE
        echo "$sshout"  >> $LOG_FILE
        echo -e "----------------------------------\n"  >> $LOG_FILE
        if [ `echo "$sshout" | wc -l` -gt 3 ]; then
            # echo -e "\033[34mnginx\033[0m at \033[35m${service_ip}\033[0m \033[32msuccess\033[0m"
            printf "%-30s%-15s%20s\n" "nginx" ${service_ip} "success"
        else
            # echo -e "\033[34mnginx\033[0m at \033[35m${service_ip}\033[0m \033[31mfailed\033[0m"
            printf "%-30s%-15s%20s\n" "nginx" ${service_ip} "failed"
        fi
    done
}

check_keepalived () {
    local keepalived_server_ip=$1
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $keepalived_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))
    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $keepalived_server_ip | awk -F ',' '{print $'$count'}'`
        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "systemctl status keepalived"`

        echo -e "keepalived at ${service_ip}:\n"  >> $LOG_FILE
        echo "$sshout"  >> $LOG_FILE
        echo -e "----------------------------------\n"  >> $LOG_FILE

        keepalived_ps_count_rst=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "ps -ef | grep keepalived | grep -v grep | wc -l"`
        keepalived_ps_count=`echo "$keepalived_ps_count_rst" | awk 'NR==1{print $1}'`
        if [ "$keepalived_ps_count" == "2" -a `echo "$sshout" | grep -c ailed` -eq 0 ]; then
            # echo -e "\033[34mkeepalived\033[0m at \033[35m${service_ip}\033[0m \033[32msuccess\033[0m"
            printf "%-30s%-15s%20s\n" "keepalived" ${service_ip} "success"
        else
            # echo -e "\033[34mkeepalived\033[0m at \033[35m${service_ip}\033[0m \033[31mfailed\033[0m"
            printf "%-30s%-15s%20s\n" "keepalived"  ${service_ip} "failed"
        fi
    done
}

check_zookeeper () {
    local zookeeper_server_ip=$1
    local sshout
    local ps_count
    local service_ip
    local service_ip_num_begin=`echo $zookeeper_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))
    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $zookeeper_server_ip | awk -F ',' '{print $'$count'}'`
        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "ps -ef | grep zookeeper | grep viid | grep -v grep"`
        echo -e "zookeeper at ${service_ip}:"  >> $LOG_FILE
        echo "$sshout"  >> $LOG_FILE
        echo -e "----------------------------------\n"  >> $LOG_FILE
        ps_count_rst=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "ps -ef | grep zookeeper | grep viid | grep -v grep | wc -l"`
        ps_count=`echo "$ps_count_rst" | awk 'NR==1{print $1}'`
        if [ "$ps_count" == "1" ]; then
            # echo -e "\033[34zookeeper\033[0m at \033[35m${service_ip}\033[0m \033[32msuccess\033[0m"
            printf "%-30s%-15s%20s\n" "zookeeper" ${service_ip} "success"
        else
            # echo -e "\033[34mzookeeper\033[0m at \033[35m${service_ip}\033[0m \033[31mfailed\033[0m"
            printf "%-30s%-15s%20s\n" "zookeeper" ${service_ip} "failed"
        fi
    done
}

check_microservices () {
    local java_server_ip=$1
    local sshout
    local service_ip
    local service_ip_num_begin=`echo $java_server_ip | grep -o ','|wc -l`
    local service_ip_num=$(($service_ip_num_begin+1))
    for((i=0;i<$service_ip_num;i++)); do
        count=$(($i+1))
        service_ip=`echo $java_server_ip | awk -F ',' '{print $'$count'}'`
        sshout=`ssh_execute $service_ip $SSH_PORT $ROOT_PWD  "ps -ef | grep java | grep viid | grep -v grep"`
        echo -e "\java microservice at ${service_ip}:\n"  >> $LOG_FILE
        echo "$sshout"  >> $LOG_FILE
        echo -e "\----------------------------------\n"  >> $LOG_FILE
        for item in `echo "$sshout" | awk '{for(i=8;i<=NF;i++){if($i~/Service|RestGateWay/){print $i;break;}}}' | awk -F '/' '{for(i=1;i<=NF;i++){if($i~/Service|RestGateWay/){print $i;break;}}}' `; do
            # echo -e "\033[35m${service_ip}\033[0m \033[32msuccess\033[0m \033[34m${item}\033[0m"
            echo ${item} >> check_existed_java_servicename.log
            printf "%-30s%-15s%20s\n" ${item} ${service_ip} "success"
        done
    done
}

check_microservices_not_existed() {
    Java_microservices_name=`declare | grep -E 'service[0-9]{1,2}_name' | cut -d "=" -f 2`
    for item in `echo $Java_microservices_name`; do
        if [ `cat check_existed_java_servicename.log | grep -c "$item"` -eq 0 ]; then
            service_ip_name="${item}_ip"
            service_ip=`declare | grep "${service_ip_name}=" | cut -d "=" -f 2`
            printf "%-30s%-15s%20s\n" ${item} ${service_ip} "failed"
        fi
    done
    rm -rf check_existed_java_servicename.log
}

#current_time=`date +%y%m%d%k%M%S`
#LOG_FILE="check${current_time}.log"
LOG_FILE=check.log
rm -rf $LOG_FILE
touch $LOG_FILE

if [ "$1" == "include" ]; then
    # 内嵌在install执行
    ROOT_PWD=$root_pwd
else
    # "$1" == "import"
    ROOT_PWD="$2"
    if [ -z "$ROOT_PWD" ]; then
        read_pwd
    fi

    if [ -z "$3" ]; then
        read_declare
    else
        source "./$3" declare  > /dev/null
    fi

fi

if [ -z "$ssh_port" ];then
    SSH_PORT=22
else
    SSH_PORT=$ssh_port
fi

# 内嵌在install执行不检查
if [ "$1" != "include" ]; then
    # 以任意ip检查read_pwd用户输入的密码，这边以第一个ip为例。密码错误则退出
    check_read_pwd_by_connection `echo $Jdk_software_ip | awk -F ','  '{print $1}'`
fi

echo "----------------------------------"

## 检查gaussdb
check_gaussdb $gauss100_ip

## 检查nginx
check_nginx  $Nginx_sotfware_ip

## 检查keepalived
check_keepalived $Keepalived_software_ip

## 检查zookeeper
check_zookeeper $Zookeepr_software_ip

Java_microservices_ip=`declare | grep Service_ip | cut -d "=" -f 2 | sort | uniq | xargs | tr ' ' ','`
if [ `echo $Java_microservices_ip | grep -c $RestGateWay_ip` -eq 0 ];then
    Java_microservices_ip="${Java_microservices_ip},${RestGateWay_ip}"
fi

## 检查java微服务
check_microservices $Java_microservices_ip
check_microservices_not_existed

echo "----------------------------------"

echo "... more information at $LOG_FILE"