5.5 hbase shell - Githubissues

vieyahn2017 commented 5 years ago

hbase shell

vieyahn2017 commented 4 months ago

https://blog.51cto.com/u_16099326/6952811 https://www.cnblogs.com/zhengyan6/p/16148469.html

vieyahn2017 commented 4 months ago

落地 20190516


#!/bin/bash

# 名字空间，统一用obj
g_namespace="obj"
# 创建跨度 6个月，目前需求是一次性建好半年内所有表
g_period=6
# 清理周期 6个月
g_clean_period=6

# 分区
g_split_zones=20
# 其它Hbase建表参数。另外参数TTL默认值是"FOREVER", 但是不能手动设置为FOREVER
g_addition_params="COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'FAST_DIFF'"

## make_array.py生成分区数字序列
split_param=`python make_array.py $g_split_zones`
g_split_params="SPLITS => "$split_param

g_all_tables_all="
kafka_info
face
motorvehicle
nonmotorvehicle
person
image
wifi
detectivecode
subimageinfo
feature
caseinfo
videofragment
file
ddictionary
ape
viidserver
apestatus
aps
apsstatus
appplat
ias
tollgate
lane
data_check_record
dataclasstab
thing
keepalive_config
videolabel
colorarea
target
behavioranalysis
analysisrule
point
line
direction
scene
"

# 目前只做
g_all_tables="
face
person
"

#===================================================
# name: func_get_period_months
# author: yh
# desription: 生成年月（yyyymm）序列
# params: $1 起始年月 yyyymm(比如201905)
# params: $2 周期 比如6
## 调用get_period_months 201905 6
## 输出
#201905
#201906
#201907
#201908
#201909
#201910
#===================================================
func_get_period_months ()
{
    local yyyymm
    local year
    local month
    local mskipped
    local m
    local t
    yyyymm=$1
    year=`echo ${yyyymm:0:4}`
    month=`echo ${yyyymm:4:6}`
    mskipped=1

    for i in `seq 1 $2`;do
        m=`expr $month + $i - $mskipped`

        if [ $m -gt 12 ]; then
            year=`expr $year + 1`
            month=1
            mskipped=$i
            t=$year'01'

        else
            if [ $m -lt 10 ]; then
                t=$year'0'$m
            else
                t=$year''$m
            fi
        fi

        echo $t
    done
}

#===================================================
# name: func_check_item_in_array_str
# author: yh
# desription: 检查元素是否存在于数组字符串中
# params: $1 数组字符串 abc, efg, hij, jkl
# params: $2 字符串
# func_check_item_in_array_str "abc, efg, hij, jkl" abc 返回true
# func_check_item_in_array_str "abc, efg, hij, jkl" bcd 返回false
#===================================================
func_check_item_in_array_str ()
{
    local item
    local arr_str
    item=$2
    arr_str=$1

    #要将$a分割开，先存储旧的分隔符
    OLD_IFS="$IFS"
    IFS=","

    local arr
    arr=($arr_str)
    IFS="$OLD_IFS"  #恢复原来的分隔符

    local temp
    #遍历数组
    for temp in ${arr[@]}; do
        if [ "$item" == "$temp" ]; then
            echo true
            return
        fi
    done
    echo false
}

#===================================================
# name: func_list_create_objdata_by_month
# author: yh
# desription: objdata每月建表echo建表语句
# params: $1 年月 yyyymm(比如201905)
# using: 使用全局变量$g_split_params $g_addition_params $g_all_tables $g_namespace
# 目前的建表语句
# create 'obj:line_201801', {NAME => 'info', COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'FAST_DIFF'}, SPLITS => ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19']
#===================================================
func_list_create_objdata_by_month ()
{
    local table_params
    local yyyymm
    yyyymm=$1

    if [ -z "$g_addition_params" ]; then
        table_params="{NAME => 'info'}"
    else
        table_params="{NAME => 'info', ${g_addition_params}}"
    fi

    for item in $g_all_tables; do
        echo "create '${g_namespace}:${item}_${yyyymm}', ${table_params}, ${g_split_params}"
    done
}

#===================================================
# name: create
# author: yh
# desription: 建表
#===================================================
create ()
{
    local current_month
    current_month=`date "+%Y%m"`

    # 建表，每月一张表，一次性建好半年内所有表
    for month in `func_get_period_months $current_month $g_period`
    do
        func_list_create_objdata_by_month $month | hbase shell -n
    done
}

# create 2> /dev/null

#===================================================
# name: func_list_only_objdata_by_month
# author: yh
# desription: 列出objdata每月需要建的表
# params: $1 年月 yyyymm(比如201905)
# using: 使用全局变量$g_all_tables $g_namespace
#===================================================
func_list_only_objdata_by_month ()
{
    local yyyymm
    yyyymm=$1
    for item in $g_all_tables; do
        echo "${g_namespace}:${item}_${yyyymm}"
    done

}

#===================================================
# name: func_list_hbase_table_all_months
# author: yh
# desription: 列出objdata所有月份需要建的表，一次性建好半年内所有表
#===================================================
func_list_hbase_table_all_months ()
{
    local current_month
    current_month=`date "+%Y%m"`

    for month in `func_get_period_months $current_month $g_period`
    do
        func_list_only_objdata_by_month $month
    done

}

#===================================================
# name: func_create_hbase_table_by_name
# author: yh
# desription: 指定表名，建表，echo建表语句
# params: $1 表名(比如'obj:line_201801') 特别说明，$1需要带上名字空间obj:
# using: 使用全局变量$g_split_params $g_addition_params $g_all_tables
# 目前的建表语句
# create 'obj:line_201801', {NAME => 'info', COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'FAST_DIFF'}, SPLITS => ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19']
#===================================================
func_create_hbase_table_by_name ()
{
    local table_params
    local table_name
    table_name=$1
    if [ -z "$g_addition_params" ]; then
        table_params="{NAME => 'info'}"
    else
        table_params="{NAME => 'info', ${g_addition_params}}"
    fi

    echo "create '${table_name}', ${table_params}, ${g_split_params}"

}

#===================================================
# name: func_hbase_list
# author: yh
# desription: 列出所有的hbase表
#===================================================
func_hbase_list ()
{
    local hbase_list_result
    local hbase_list_arr_str
    local hbase_tables_arr
    hbase_list_result=`echo list | hbase shell 2>\dev\null`
    hbase_list_arr_str=${hbase_list_result##*[}
    hbase_tables_arr=`echo $hbase_list_arr_str | sed 's/\]//g' | sed 's/"//g'| sed 's/ //g'`
    echo $hbase_tables_arr
}

#===================================================
# name: func_check_update_objdata
# author: yh
# desription: 检查当前周期objdata需要建的表，不存在的表，echo输出表名，等待后续过程创建
#===================================================
func_check_update_tables ()
{
    local hbase_tables
    hbase_tables=`func_hbase_list`
    for item in `func_list_hbase_table_all_months`; do
        result=`func_check_item_in_array_str $hbase_tables $item`
        if [ "$result" == "false" ]; then
            echo "$item"
        fi
    done
}

#===================================================
# name: func_list_update
# author: yh
# desription: 调用func_check_update_tables检查当前周期objdata需要补建的表，按照名称创建
#===================================================
func_list_update ()
{
    for item in `func_check_update_tables`; do
        func_create_hbase_table_by_name $item
    done
}

#===================================================
# name: update
# author: yh
# desription: 更新
#===================================================
update ()
{
    func_list_update | hbase shell -n
}

# update 2> /dev/null

#===================================================
# name: func_list_clean
# author: yh
# desription: 根据更新周期，查找在清理日期之前建的表，生成清除语句(disbale/drop)
# using: 使用全局变量$g_clean_period $g_namespace
#===================================================
func_list_clean ()
{
    local yyyymm
    local year
    local month
    local clean_yyyymm
    # 先根据当前日期和清理周期，确定清理时间clean_yyyymm
    yyyymm=`date "+%Y%m"`
    year=`echo ${yyyymm:0:4}`
    month=`echo ${yyyymm:4:6}`
    if [ $month -le $g_clean_period ]; then
        year=`expr $year - 1`
        month=`expr $month + 12`
    fi
    yyyymm=$year$month
    clean_yyyymm=`expr $yyyymm - $g_clean_period`

    # 列出所有的表，过滤出符合obj:line_201801格式的表
    local hbase_tables
    hbase_tables=`func_hbase_list`
    for item in `echo $hbase_tables | xargs -d ',' -n 1 echo | grep -E "^${g_namespace}.*_[0-9]{6}$"`; do
        if [ ${item:0-6} -le $clean_yyyymm ]; then
            echo "disable '${item}'"
            echo "drop '${item}'"
        fi
    done

}

#===================================================
# name: clean
# author: yh
# desription: 清理
#===================================================
clean ()
{
    func_list_clean | hbase shell -n
}

#===================================================
# name: delete
# author: yh
# desription: delete
#===================================================
delete ()
{
    local table_name
    table_name=$1
    echo "disable '${table_name}'"
    echo "drop '${table_name}'"

}

#===================================================
# name: listgrep
# author: yh
# desription: list grep
#===================================================
listgrep ()
{
    local hbase_tables
    hbase_tables=`func_hbase_list`

    if [ -n "$1" ];then
        echo $hbase_tables | xargs -d ',' -n 1 echo | grep $1
    else
        echo $hbase_tables | xargs -d ',' -n 1 echo | grep $g_namespace
    fi
}

#===================================================
# name: main
# author: yh
# desription: main
#===================================================
main()
{
    methods="create/update/clean/list/grep/delete/help"
    if [ -z "$1" ];then
        echo -e "\033[34mneed an operation param ($methods).\033[0m"
        return 1
    fi
    action=$1
    case ${action} in
        "create")
            create
            ;;
        "update")
            update
            ;;
        "clean")
            clean
            ;;
        "list")
            echo -e "\033[36mhbase shell connecting, then list, please wait a moment.\033[0m"
            listgrep
            ;;
        "grep")
            echo -e "\033[36mhbase shell connecting, then list and grep, please wait a moment.\033[0m"
            listgrep $2
            ;;
        "delete")
            delete $2 | hbase shell -n
            ;;
        "help")
            echo -e "\033[36mopertaion supported: ($methods).\033[0m"
               ;;
        *)
            echo -e "\033[34munkown opertaion : ${action}. supported: ($methods).\033[0m"
            ;;
    esac
    return 0
}

SRC_PATH=`cd "$(dirname "$0")"; pwd`
cd ${SRC_PATH}
sh kinit.sh

main "$@"

vieyahn2017 commented 4 months ago

测试的 hbase_test.sh

check_in_array_str ()
{
    local item
    local arr_str
    item=$2
    arr_str=$1

    #要将$a分割开，先存储旧的分隔符
    OLD_IFS="$IFS"
    IFS=","

    local arr
    arr=($arr_str)
    IFS="$OLD_IFS"  #恢复原来的分隔符

    local temp
    #遍历数组
    for temp in ${arr[@]}; do
        if [ "$item" == "$temp" ]; then
            echo true
            return
        fi
    done
    echo false
}
check_in_array_str a,bb,cd,eee cd
check_in_array_str a,bb,cd,eee cde

# $1=201904 $2=6
get_period_months ()
{
    local yyyymm
    local year
    local month
    local mskipped
    local m
    local t
    yyyymm=$1
    year=`echo ${yyyymm:0:4}`
    month=`echo ${yyyymm:4:6}`
    mskipped=1

    for i in `seq 1 $2`;do
        m=`expr $month + $i - $mskipped`

        if [ $m -gt 12 ]; then
            year=`expr $year + 1`
            month=1
            mskipped=$i
            t=$year'01'

        else
            if [ $m -lt 10 ]; then
                t=$year'0'$m
            else
                t=$year''$m
            fi
        fi

        echo $t
    done
}

get_period_months 201904 6
201904
201905
201906
201907
201908
201909

get_period_months 201911 6
201911
201912
202001
202002
202003
202004

yyyymm=`date "+%Y%m"`
echo $yyyymm

echo "$IFS" | od -b
0000000 040 011 012 012
0000004

compare ()
{
    if [ $1 -gt $2 ] ; then
       echo $1 great than $2

    else if [ $1 -lt $2 ] ; then
            echo $1 less than $2

        else
            echo $1 equal $2
        fi
    fi
}

还有


yyyymm=`date "+%Y%m"`

hbase shell << EOF
create_namespace 'obj'
grant 'iuser', 'RWXCA', '@obj'
list_namespace

EOF

hbase shell << EOF

disable 'appdata'
drop 'appdata'
create 'appdata_$yyyymm', 'face', 'image', 'person' , 'point'

put 'appdata', 'key_column1', 'face:faceid', '1'
put 'appdata', 'key_column1', 'face:name', 'xuezhiqian'

EOF

# 判断表存在

function_create_appdata ()
{

hbase shell << EOF

    disable 'appdata'
    drop 'appdata'
    create 'appdata_$yyyymm', 'face', 'image', 'person' , 'point'

    put 'appdata', 'key_column1', 'face:faceid', '1'
    put 'appdata', 'key_column1', 'face:name', 'zhangsan'

EOF

}
# 层级： namespace table

vieyahn2017 commented 4 months ago

落地 20190611

#!/bin/bash

# 名字空间，统一用obj
# G_HBASE_NAMESPACE="obj"
# 最新更改：不建namespace，用默认的

# 创建跨度 6个月，目前需求是一次性建好半年内所有表
G_CREATE_PERIOD=6
# 安装默认清除原有的数据表。如果有需要保留原有数据，请设置为false
G_CREATE_OVERWRITE=true
# 清理周期 6个月
G_CLEAN_PERIOD=6

# 分区
G_SPLIT_ZONES=20
# 其它Hbase建表参数。另外参数TTL默认值是"FOREVER", 但是不能手动设置为FOREVER
G_ADDITION_PARAMS="COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'FAST_DIFF'"

## make_array.py生成分区数字序列
split_param=`python make_array.py $G_SPLIT_ZONES`
G_SPLIT_PARAMS="SPLITS => "$split_param

# 目前只做
G_ALL_TABLES=`cat objdata_list.dat`
if [ $? -ne 0 ]; then
    G_ALL_TABLES="
    face
    motorvehicle
    nonmotorvehicle
    person
    "
fi

#===================================================
# name: func_get_period_months
# author: yh
# desription: 生成年月（yyyymm）序列
# params: $1 起始年月 yyyymm(比如201905)
# params: $2 周期 比如6
## 调用get_period_months 201905 6
## 输出
#201905
#201906
#201907
#201908
#201909
#201910
#===================================================
func_get_period_months ()
{
    local yyyymm
    local year
    local month
    local mskipped
    local m
    local t
    yyyymm=$1
    year=`echo ${yyyymm:0:4}`
    month=`echo ${yyyymm:4:6}`
    mskipped=1

    for i in `seq 1 $2`;do
        m=`expr $month + $i - $mskipped`

        if [ $m -gt 12 ]; then
            year=`expr $year + 1`
            month=1
            mskipped=$i
            t=$year'01'

        else
            if [ $m -lt 10 ]; then
                t=$year'0'$m
            else
                t=$year''$m
            fi
        fi

        echo $t
    done
}

#===================================================
# name: func_check_item_in_array_str
# author: yh
# desription: 检查元素是否存在于数组字符串中
# params: $1 数组字符串 abc, efg, hij, jkl
# params: $2 字符串
# func_check_item_in_array_str "abc, efg, hij, jkl" abc 返回true
# func_check_item_in_array_str "abc, efg, hij, jkl" bcd 返回false
#===================================================
func_check_item_in_array_str ()
{
    local item
    local arr_str
    item=$2
    arr_str=$1

    #要将$a分割开，先存储旧的分隔符
    OLD_IFS="$IFS"
    IFS=","

    local arr
    arr=($arr_str)
    IFS="$OLD_IFS"  #恢复原来的分隔符

    local temp
    #遍历数组
    for temp in ${arr[@]}; do
        if [ "$item" == "$temp" ]; then
            echo true
            return
        fi
    done
    echo false
}

#===================================================
# name: func_list_create_objdata_by_month
# author: yh
# desription: objdata每月建表echo建表语句
# params: $1 年月 yyyymm(比如201905)
# using: 使用全局变量$G_SPLIT_PARAMS $G_ADDITION_PARAMS $G_ALL_TABLES $G_HBASE_NAMESPACE
# 目前的建表语句
# create 'obj:line_201801', {NAME => 'info', COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'FAST_DIFF'}, SPLITS => ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19']
#===================================================
func_list_create_objdata_by_month ()
{
    local table_params
    local yyyymm
    yyyymm=$1

    if [ -z "$G_ADDITION_PARAMS" ]; then
        table_params="{NAME => 'info'}"
    else
        table_params="{NAME => 'info', ${G_ADDITION_PARAMS}}"
    fi

    for item in $G_ALL_TABLES; do
        if [ -z "$G_HBASE_NAMESPACE" ] ;then
            echo "create '${item}_${yyyymm}', ${table_params}, ${G_SPLIT_PARAMS}"
        else
            echo "create '${G_HBASE_NAMESPACE}:${item}_${yyyymm}', ${table_params}, ${G_SPLIT_PARAMS}"
        fi

    done
}

#===================================================
# name: create
# author: yh
# desription: 建表
# using: 使用全局变量$G_CREATE_PERIOD
#===================================================
create ()
{
    local current_month
    current_month=`date "+%Y%m"`

    # 建表，每月一张表，一次性建好半年内所有表
    for month in `func_get_period_months $current_month $G_CREATE_PERIOD`
    do
        func_list_create_objdata_by_month $month | hbase shell -n
    done
}

# create 2> /dev/null

#===================================================
# name: func_list_only_objdata_by_month
# author: yh
# desription: 列出objdata每月需要建的表
# params: $1 年月 yyyymm(比如201905)
# using: 使用全局变量$G_ALL_TABLES $G_HBASE_NAMESPACE
#===================================================
func_list_only_objdata_by_month ()
{
    local yyyymm
    yyyymm=$1
    for item in $G_ALL_TABLES; do
        if [ -z "$G_HBASE_NAMESPACE" ] ;then
            echo "${item}_${yyyymm}"
        else
            echo "${G_HBASE_NAMESPACE}:${item}_${yyyymm}"
        fi
    done

}

#===================================================
# name: func_list_hbase_table_all_months
# author: yh
# desription: 列出objdata所有月份需要建的表，一次性建好半年内所有表
#===================================================
func_list_hbase_table_all_months ()
{
    local current_month
    current_month=`date "+%Y%m"`

    for month in `func_get_period_months $current_month $G_CREATE_PERIOD`
    do
        func_list_only_objdata_by_month $month
    done

}

#===================================================
# name: func_create_hbase_table_by_name
# author: yh
# desription: 指定表名，建表，echo建表语句
# params: $1 表名(比如'obj:line_201801') 特别说明，$1需要带上名字空间obj:
# using: 使用全局变量$G_SPLIT_PARAMS $G_ADDITION_PARAMS $G_ALL_TABLES
# 目前的建表语句
# create 'obj:line_201801', {NAME => 'info', COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'FAST_DIFF'}, SPLITS => ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19']
#===================================================
func_create_hbase_table_by_name ()
{
    local table_params
    local table_name
    table_name=$1
    if [ -z "$G_ADDITION_PARAMS" ]; then
        table_params="{NAME => 'info'}"
    else
        table_params="{NAME => 'info', ${G_ADDITION_PARAMS}}"
    fi

    echo "create '${table_name}', ${table_params}, ${G_SPLIT_PARAMS}"

}

#===================================================
# name: func_hbase_list
# author: yh
# desription: 列出所有的hbase表
#===================================================
func_hbase_list ()
{
    local hbase_list_result
    local hbase_list_arr_str
    local hbase_tables_arr
    hbase_list_result=`echo list | hbase shell 2>\dev\null`
    hbase_list_arr_str=${hbase_list_result##*[}
    hbase_tables_arr=`echo $hbase_list_arr_str | sed 's/\]//g' | sed 's/"//g'| sed 's/ //g'`
    echo $hbase_tables_arr
}

#===================================================
# name: func_check_update_objdata
# author: yh
# desription: 检查当前周期objdata需要建的表，不存在的表，echo输出表名，等待后续过程创建
#===================================================
func_check_update_tables ()
{
    local hbase_tables
    hbase_tables=`func_hbase_list`
    for item in `func_list_hbase_table_all_months`; do
        result=`func_check_item_in_array_str $hbase_tables $item`
        if [ "$result" == "false" ]; then
            echo "$item"
        fi
    done
}

#===================================================
# name: func_list_update
# author: yh
# desription: 调用func_check_update_tables检查当前周期objdata需要补建的表，按照名称创建
#===================================================
func_list_update ()
{
    for item in `func_check_update_tables`; do
        func_create_hbase_table_by_name $item
    done
}

#===================================================
# name: update
# author: yh
# desription: 更新
#===================================================
update ()
{
    func_list_update | hbase shell -n
}

# update 2> /dev/null

#===================================================
# name: func_list_clean
# author: yh
# desription: 根据更新周期，查找在清理日期之前建的表，生成清除语句(disbale/drop)
# using: 使用全局变量$G_HBASE_NAMESPACE
#===================================================
func_list_clean ()
{
    local yyyymm
    local year
    local month
    local clean_yyyymm
    local clean_period=$1
    # 先根据当前日期和清理周期，确定清理时间clean_yyyymm
    yyyymm=`date "+%Y%m"`
    year=`echo ${yyyymm:0:4}`
    month=`echo ${yyyymm:4:6}`
    if [ $month -le $clean_period ]; then
        year=`expr $year - 1`
        month=`expr $month + 12`
    fi
    yyyymm=$year$month
    clean_yyyymm=`expr $yyyymm - $clean_period`

    # 列出所有的表，过滤出符合obj:line_201801格式的表
    local hbase_tables
    hbase_tables=`func_hbase_list`
    for item in `echo $hbase_tables | xargs -d ',' -n 1 echo | grep -E "^${G_HBASE_NAMESPACE}.*_[0-9]{6}$"`; do
        if [ ${item:0-6} -le $clean_yyyymm ]; then
            echo "disable '${item}'"
            echo "drop '${item}'"
        fi
    done

}

#===================================================
# name: clean
# author: yh
# desription: 清理
# using: 使用全局变量$G_CLEAN_PERIOD
#===================================================
clean ()
{
    func_list_clean "$G_CLEAN_PERIOD" | hbase shell -n
}

#===================================================
# name: clean
# author: yh
# desription: 创建前的清理（删除之前的创建周期的建的新数据）
# using: 使用全局变量$G_CREATE_PERIOD
#===================================================
preclean ()
{
    func_list_clean "-$G_CREATE_PERIOD" | hbase shell -n
}

#===================================================
# name: delete
# author: yh
# desription: delete
#===================================================
delete ()
{
    local table_name
    table_name=$1
    echo "disable '${table_name}'"
    echo "drop '${table_name}'"

}

#===================================================
# name: listgrep
# author: yh
# desription: list grep
#===================================================
listgrep ()
{
    local hbase_tables
    hbase_tables=`func_hbase_list`

    if [ -n "$1" ];then
        echo $hbase_tables | xargs -d ',' -n 1 echo | grep $1
    else
        echo $hbase_tables | xargs -d ',' -n 1 echo
    fi
}

#===================================================
# name: main
# author: yh
# desription: main
#===================================================
main()
{
    methods="create/update/clean/list/grep/delete/help"
    if [ -z "$1" ];then
        echo -e "\033[34mneed an operation param ($methods).\033[0m"
        return 1
    fi
    action=$1
    case ${action} in
        "create")
            if [ "$G_CREATE_OVERWRITE" == "true" ]; then
                preclean
            fi
            create
            ;;
        "update")
            update
            ;;
        "clean")
            clean
            ;;
        "preclean")
            preclean
            ;;
        "list")
            echo -e "\033[36mhbase shell connecting, then list, please wait a moment.\033[0m"
            listgrep
            ;;
        "grep")
            echo -e "\033[36mhbase shell connecting, then list and grep, please wait a moment.\033[0m"
            listgrep $2
            ;;
        "delete")
            delete $2 | hbase shell -n
            ;;
        "help")
            echo -e "\033[36mopertaion supported: ($methods).\033[0m"
               ;;
        *)
            echo -e "\033[34munkown opertaion : ${action}. supported: ($methods).\033[0m"
            ;;
    esac
    return 0
}

SRC_PATH=`cd "$(dirname "$0")"; pwd`
source ${SRC_PATH}/kinit.sh
if [ $? -ne 0 ]; then
    exit 1
fi

main "$@"

vieyahn2017 commented 4 months ago

appdata.sh createnamespace.sh es_utils.sh hbase_utils.sh install.sh kinit.sh make_array.py objdata_list.dat readme.md

vieyahn2017 commented 4 months ago

es_utils.sh

#!/bin/bash

# 安装默认清除原有的数据表。如果有需要保留原有数据，请设置为false
G_CREATE_OVERWRITE=true
# 创建跨度 6个月，目前需求是一次性建好半年内所有表
G_CREATE_PERIOD=6
# 清理周期 6个月
G_CLEAN_PERIOD=6

G_HOST=8.6.1.1
G_PORT=24100

G_NUMBER_OF_SHARDS=3
G_NUMBER_OF_REPLICAS=2

check_connect ()
{
    echo "check connect"
    ping "$G_HOST" -c 5
}

check_connect
if [ $? -ne 0 ]; then
    echo "the host ${G_HOST} connection timeout. please check."
    exit 1
fi

settings='
    {
        "settings" : {
            "index" : {
                "number_of_shards" : g_number_of_shards,
                "number_of_replicas" : g_number_of_replicas
            }
        },
        "mappings": {
            "doc": {}
        }
    }
'
g_settings=`echo $settings | sed "s/g_number_of_shards/${G_NUMBER_OF_SHARDS}/g" | sed "s/g_number_of_replicas/${G_NUMBER_OF_REPLICAS}/g"`

# 目前只做
G_ALL_TABLES=`cat objdata_list.dat`
if [ $? -ne 0 ]; then
    G_ALL_TABLES="
    face
    motorvehicle
    nonmotorvehicle
    person
    "
fi

#===================================================
# name: func_get_period_months
# author: yh
# desription: 生成年月（yyyymm）序列
# params: $1 起始年月 yyyymm(比如201905)
# params: $2 周期 比如6
## 调用get_period_months 201905 6
## 输出
#201905
#201906
#201907
#201908
#201909
#201910
#===================================================
func_get_period_months ()
{
    local yyyymm
    local year
    local month
    local mskipped
    local m
    local t
    yyyymm=$1
    year=`echo ${yyyymm:0:4}`
    month=`echo ${yyyymm:4:6}`
    mskipped=1

    for i in `seq 1 $2`;do
        m=`expr $month + $i - $mskipped`

        if [ $m -gt 12 ]; then
            year=`expr $year + 1`
            month=1
            mskipped=$i
            t=$year'01'

        else
            if [ $m -lt 10 ]; then
                t=$year'0'$m
            else
                t=$year''$m
            fi
        fi

        echo $t
    done
}

create_index ()
{
    curl -XPUT --tlsv1.2 --negotiate -k -u : "https://${G_HOST}:${G_PORT}/$1"  -H 'Content-Type: application/json' -d "$g_settings"
}

create_index_raw ()
{
    curl -XPUT --tlsv1.2 --negotiate -k -u : "https://${G_HOST}:${G_PORT}/$1"  -H 'Content-Type: application/json' -d '
    {
        "settings": {
            "index": {
                "number_of_shards": 3,
                "number_of_replicas": 2
            }
        },
        "mappings": {
            "doc": {}
        }
    }'
}

#===================================================
# name: create
# author: yh
# desription: create
#===================================================
create ()
{
    local current_month
    current_month=`date "+%Y%m"`

    # 建表，每月一张表，一次性建好半年内所有表
    for month in `func_get_period_months $current_month $G_CREATE_PERIOD`
    do
        for item in $G_ALL_TABLES; do
            create_index "${item}_${month}"
            echo -e "\n"
        done
    done
}

#===================================================
# name: delete
# author: yh
# desription: delete
#===================================================
delete ()
{
    curl -XDELETE --negotiate -k -u:  "https://${G_HOST}:${G_PORT}/$1"
    echo -e "\n"
}

#===================================================
# name: clean
# author: yh
# desription: 清理
#===================================================
clean ()
{
    local yyyymm
    local year
    local month
    local clean_yyyymm
    local clean_period=$1
    # 先根据当前日期和清理周期，确定清理时间clean_yyyymm
    yyyymm=`date "+%Y%m"`
    year=`echo ${yyyymm:0:4}`
    month=`echo ${yyyymm:4:6}`
    if [ $month -le $clean_period ]; then
        year=`expr $year - 1`
        month=`expr $month + 12`
    fi
    yyyymm=$year$month
    clean_yyyymm=`expr $yyyymm - $clean_period`

    # 列出所有的表，过滤出符合face_201801格式的index
    for item in `curl -XGET --negotiate -k -u: "https://${G_HOST}:${G_PORT}/_cat/indices"   2>/dev/null  | awk '{print $3}' | grep -E "^.*_[0-9]{6}$"`; do
        if [ ${item:0-6} -le $clean_yyyymm ]; then
            delete $item
        fi
    done

}

#===================================================
# name: list
# author: yh
# desription: list
#
#===================================================
list ()
{
    curl -XGET --negotiate -k -u: "https://${G_HOST}:${G_PORT}/_cat/indices?v"
}

#===================================================
# name: get
# author: yh
# desription: get
#===================================================
get ()
{
    curl -XGET --negotiate -k -u: "https://${G_HOST}:${G_PORT}/$1/_mapping?pretty"
}

#===================================================
# name: main
# author: yh
# desription: main
#===================================================
main()
{
    methods="create/update/clean/list/delete/get/help"
    if [ -z "$1" ];then
        echo -e "\033[34mneed an operation param ($methods).\033[0m"
        return 1
    fi
    action=$1
    case ${action} in
        "create")
            if [ "$G_CREATE_OVERWRITE" == "true" ]; then
                clean "-$G_CREATE_PERIOD"
            fi
            create
            ;;
        "update")
            create
            ;;
        "clean")
            clean "$G_CLEAN_PERIOD"
            ;;
        "preclean")
            clean "-$G_CREATE_PERIOD"
            ;;
        "list")
            list
            ;;
        "createone")
            create_index $2
            ;;
        "get")
            get $2
            ;;
        "delete")
            delete $2
            ;;
        "help")
            echo -e "\033[36mopertaion supported: ($methods).\033[0m"
               ;;
        *)
            echo -e "\033[34munkown opertaion : ${action}. supported: ($methods).\033[0m"
            ;;
    esac
    return 0
}

SRC_PATH=`cd "$(dirname "$0")"; pwd`
source ${SRC_PATH}/kinit.sh
if [ $? -ne 0 ]; then
    exit 1
fi

main "$@"

vieyahn2017 commented 4 months ago

https://github.com/vieyahn2017/javaway/issues/147 重复了

vieyahn2017 / iBlog

5.5 hbase shell #283