search

arkdb / arkproxy

高性能、高可靠的数据库跨云中间件，完全兼容MySQL技术体系，由极数云舟出品开源
GNU Lesser General Public License v2.1
102 stars 30 forks source link

buffer overflow detected #5

Closed Jenvid closed 3 years ago

Jenvid commented 4 years ago

咨询下 下面遇到的这个问题是否是status里面的参数超出了？ 有没有哪个参数可以调整下

 show proxy status;
+------------------+--------------------+--------------------+------------------+------------------+-----------+
| Connection_Count | Min_Con_Buffer_Mem | Max_Con_Buffer_Mem | Total_Packet_Mem | Total_Buffer_Mem | Total_Mem |
+------------------+--------------------+--------------------+------------------+------------------+-----------+
|               60 |             137506 |            2794665 |         84672000 |        113333230 | 198005230 |
+------------------+--------------------+--------------------+------------------+------------------+-----------+
1 row in set (0.00 sec)
2020-10-13 19:30:49 140349055181568 [Note] assemble_kill_packet: KILL CONNECTION 2105942
2020-10-13 19:30:49 140349055181568 [Note] assemble_kill_packet: KILL CONNECTION 2105942
2020-10-13 19:30:54 140349055181568 [Note] assemble_kill_packet: KILL CONNECTION 2105942
2020-10-13 19:30:54 140349055181568 [Note] assemble_kill_packet: KILL CONNECTION 2105942
2020-10-13 19:31:03 140349055181568 [Note] assemble_kill_packet: KILL CONNECTION 2105942
*** buffer overflow detected ***: /usr/local/arkproxy/bin/arkproxy terminated
*** buffer overflow detected ***: /usr/local/arkproxy/bin/arkproxy terminated
======= Backtrace: =========
======= Backtrace: =========
/lib64/libc.so.6(__fortify_fail+0x37)[0x7fa696604577]
/lib64/libc.so.6(__fortify_fail+0x37)[0x7fa696604577]
/lib64/libc.so.6(+0x1166f2)[0x7fa6966026f2]
/usr/local/arkproxy/bin/arkproxy(_Z22init_backend_conn_infoP19backend_conn_structPcS1_S1_j+0x27)[0x561f1c833467]
/lib64/libc.so.6(+0x1166f2)[0x7fa6966026f2]
/usr/local/arkproxy/bin/arkproxy(_Z25mysql_fetch_server_statusP19backend_conn_struct+0x85)[0x561f1c833555]
/usr/local/arkproxy/bin/arkproxy(_Z22init_backend_conn_infoP19backend_conn_structPcS1_S1_j+0x27)[0x561f1c833467]
/usr/local/arkproxy/bin/arkproxy(_Z19proxy_timer_checkerP3THD+0x168)[0x561f1c837de8]
/usr/local/arkproxy/bin/arkproxy(_Z25mysql_fetch_server_statusP19backend_conn_struct+0x85)[0x561f1c833555]
/usr/local/arkproxy/bin/arkproxy(command_executer+0xab)[0x561f1c837f0b]
/usr/local/arkproxy/bin/arkproxy(_Z19proxy_timer_checkerP3THD+0x168)[0x561f1c837de8]
/lib64/libpthread.so.0(+0x7ea5)[0x7fa697e21ea5]
/usr/local/arkproxy/bin/arkproxy(command_executer+0xab)[0x561f1c837f0b]
/lib64/libc.so.6(clone+0x6d)[0x7fa6965ea8dd]
======= Memory map: ========
/lib64/libpthread.so.0(+0x7ea5)[0x7fa697e21ea5]
561f1c285000-561f1d0d1000 r-xp 00000000 fd:01 1319346                    /usr/local/arkproxy/bin/arkproxy
561f1d2d0000-561f1d3bd000 r--p 00e4b000 fd:01 1319346                    /usr/local/arkproxy/bin/arkproxy
561f1d3bd000-561f1d461000 rw-p 00f38000 fd:01 1319346                    /usr/local/arkproxy/bin/arkproxy
561f1d461000-561f1dc9e000 rw-p 00000000 00:00 0
561f1ef99000-561f202fe000 rw-p 00000000 00:00 0                          [heap]
7fa581a9a000-7fa581a9b000 ---p 00000000 00:00 0
superceix commented 4 years ago

你好，您那边这个问题能够复现吗，是否方便提供一下参数配置呢？

Jenvid commented 4 years ago

您好， 使用场景是测试同学在做压测，但是看数据库压力，几乎毛毛雨，看下面pmm截图记录，日志见附件 是否能复现，今天还在观察

# arkproxy负载也不高
%Cpu0  :  1.8 us,  1.8 sy,  0.0 ni, 96.4 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu1  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu2  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu3  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu4  :  1.7 us,  1.7 sy,  0.0 ni, 96.6 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu5  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu6  :  1.8 us,  0.0 sy,  0.0 ni, 98.2 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
%Cpu7  :  0.0 us,  0.0 sy,  0.0 ni,100.0 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem : 12136984 total,  3686752 free,  4097256 used,  4352976 buff/cache
KiB Swap: 16777212 total, 16777212 free,        0 used.  7631416 avail Mem

image error.log.20201013.txt

[arkproxy]
proxy_backend_passwd    =   4B4cxxxxx5OUsm
proxy_backend_user      =   arkproxy
proxy_port              =   3336
proxy_check_slave_lag_period  = 10
proxy_config_host =
proxy_config_passwd =
proxy_config_port = 0
proxy_config_user =
proxy_digest_trace = OFF
proxy_flush_thread_num = 5
proxy_format_fullpath = OFF
proxy_format_num_per_time = 10
proxy_format_parameterize = ON
proxy_format_thread_num  = 5
proxy_general_log = OFF
proxy_general_log_file = /usr/local/arkproxy/logs/proxy_general.log
proxy_log_error = /usr/local/arkproxy/logs/error.log
proxy_multi_write_mode = OFF
proxy_namespace =
proxy_non_encrypted_ips = 127.0.0.1
proxy_recv_queue_size = 20000
proxy_send_queue_size = 1000
proxy_server_address = %
proxy_server_heartbeat_period = 100
proxy_shell_listen_port = 3335
proxy_shell_password = xxx
proxy_shell_username = xxxx
proxy_sql_trace = OFF
proxy_sql_trace_thread_num = 5
proxy_thread_pool_idle_timeout  = 60
proxy_thread_pool_max_threads   = 65536
proxy_thread_pool_oversubscribe = 3
proxy_thread_pool_size          = 8
proxy_thread_pool_stall_limit   = 500
proxy_trace_busy_strategy = 0
proxy_trace_busy_wait_time = 500
proxy_trace_client_queue_size = 1000
proxy_trace_flush_thread_sleep_msec = 500
proxy_trace_format_thread_sleep_msec = 500
proxy_trace_hash_mem_length = 134217728
proxy_trace_hash_size = 200
proxy_trace_max_delay_time    = 3
proxy_trace_queue_array_length=500
proxy_trace_sql_bucket_length = 100
proxy_username_encypted = OFF
proxy_max_connections = 65535
core-file

[server1]
#proxy 类型，表示当前节点类型为 Server
proxy_type=server
#数据库地址
backend_host=192.168.xx1
#数据库端口
backend_port=3306
#路由权重
weight=1
#当前节点，如果延迟达到这个值，就会停止前其路由请求
#max_slave_lag= 200
#节点默认状态，中间层启动之后就是这个状态
server_status = ONLINE
#对这个节点的注释
config_comment = OMS readwriter master

[server2]
proxy_type=server
backend_host=192.168.xx2
backend_port=3306
max_slave_lag= 300
weight=30
config_comment = OMS readonly slave01

[server3]
proxy_type=server
backend_host=192.168.xx3
backend_port=3306
max_slave_lag= 300
weight=30
config_comment = OMS readonly slave02

#路由设置
[router1]
#proxy 类型，当前配置节点为路由节点
proxy_type=router
#路由类型:readwrite，读写节点
router_type=readwrite
#配置路由 db
router_servers=server1
#路由注释
config_comment = read_writer server

[router2]
proxy_type=router
#只读路由
router_type=readonly
#有多个路由目标时，以逗号分隔，写在一起
router_servers=server1,server2,server3
config_comment = readonly server

# 注： 同一类型的路由不能设置多次，比如 readonly，只能有一个配置区块，如果有多个 readonly 的 Server，请将其写到 router_servers 中以逗号分隔放在同一行