search

anhnongdan / bimax-counters

Bimax's counter, splitter and extractor. These are completely independent modules and can be deployed on different hosts. Used for both cbimax and sbimax.
0 stars 0 forks source link

[Redis] Verify Redis HA #23

Open anhnongdan opened 7 years ago

anhnongdan commented 7 years ago

Need more than 1 Sentinel. Set quorum = 2

ac4b47f164875195b5e31540fa2df3ed87d66102

anhnongdan commented 7 years ago 
[root@VNPT-HCM-BIMAX-02 redis]# vim /var/log/redis/sentinel.log
[root@VNPT-HCM-BIMAX-02 redis]# tailf /var/log/redis/sentinel.log
 |    `-._`-._        _.-'_.-'    |
  `-._    `-._`-.__.-'_.-'    _.-'
      `-._    `-.__.-'    _.-'
          `-._        _.-'
              `-.__.-'

23762:X 05 Oct 00:43:33.821 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128.
23762:X 05 Oct 00:43:33.826 # Sentinel ID is cec6a60836efe14294cf406865ea2bfcb519d83f
23762:X 05 Oct 00:43:33.826 # +monitor master mymaster 172.20.4.63 6379 quorum 1
23762:X 05 Oct 00:43:33.827 * +slave slave 172.20.4.52:6379 172.20.4.52 6379 @ mymaster 172.20.4.63 6379

23762:X 05 Oct 00:47:24.656 # +sdown master mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:24.656 # +odown master mymaster 172.20.4.63 6379 #quorum 1/1
23762:X 05 Oct 00:47:24.656 # +new-epoch 1
23762:X 05 Oct 00:47:24.656 # +try-failover master mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:24.661 # +vote-for-leader cec6a60836efe14294cf406865ea2bfcb519d83f 1
23762:X 05 Oct 00:47:24.661 # +elected-leader master mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:24.661 # +failover-state-select-slave master mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:24.751 # +selected-slave slave 172.20.4.52:6379 172.20.4.52 6379 @ mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:24.751 * +failover-state-send-slaveof-noone slave 172.20.4.52:6379 172.20.4.52 6379 @ mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:24.806 * +failover-state-wait-promotion slave 172.20.4.52:6379 172.20.4.52 6379 @ mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:25.616 # +promoted-slave slave 172.20.4.52:6379 172.20.4.52 6379 @ mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:25.616 # +failover-state-reconf-slaves master mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:25.694 # +failover-end master mymaster 172.20.4.63 6379
23762:X 05 Oct 00:47:25.694 # +switch-master mymaster 172.20.4.63 6379 172.20.4.52 6379
23762:X 05 Oct 00:47:25.694 * +slave slave 172.20.4.63:6379 172.20.4.63 6379 @ mymaster 172.20.4.52 6379
23762:X 05 Oct 00:47:55.745 # +sdown slave 172.20.4.63:6379 172.20.4.63 6379 @ mymaster 172.20.4.52 6379
23762:X 05 Oct 00:49:49.919 # -sdown slave 172.20.4.63:6379 172.20.4.63 6379 @ mymaster 172.20.4.52 6379

23762:X 05 Oct 00:50:00.062 * +convert-to-slave slave 172.20.4.63:6379 172.20.4.63 6379 @ mymaster 172.20.4.52 6379
23762:X 05 Oct 00:50:00.062 * +convert-to-slave slave 172.20.4.63:6379 172.20.4.63 6379 @ mymaster 172.20.4.52 6379

23762:X 05 Oct 00:50:10.066 * +convert-to-slave slave 172.20.4.63:6379 172.20.4.63 6379 @ mymaster 172.20.4.52 6379
anhnongdan commented 7 years ago 
[root@VNPT-HCM-ANALYTIC-04 ~]# redis-cli info replication
# Replication
role:slave
master_host:172.20.4.63
master_port:6379
master_link_status:down
master_last_io_seconds_ago:-1
master_sync_in_progress:0
slave_repl_offset:1634504
master_link_down_since_seconds:15
slave_priority:100
slave_read_only:1
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
[root@VNPT-HCM-ANALYTIC-04 ~]# redis-cli info replication
# Replication
role:slave
master_host:172.20.4.63
master_port:6379
master_link_status:down
master_last_io_seconds_ago:-1
master_sync_in_progress:0
slave_repl_offset:1634504
master_link_down_since_seconds:19
slave_priority:100
slave_read_only:1
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
[root@VNPT-HCM-ANALYTIC-04 ~]# redis-cli info replication
# Replication
role:master
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
[root@VNPT-HCM-ANALYTIC-04 ~]# redis-cli info replication
# Replication
role:master
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
You have mail in /var/spool/mail/root
[root@VNPT-HCM-ANALYTIC-04 ~]# redis-cli info replication
# Replication
role:master
connected_slaves:1
slave0:ip=172.20.4.63,port=6379,state=wait_bgsave,offset=0,lag=0
master_repl_offset:7341
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:2
repl_backlog_histlen:7340
anhnongdan commented 7 years ago 
[root@VNPT-HCM-BIMAX-01 redis]# redis-cli -p 6379 info replication
# Replication
role:slave
master_host:172.20.4.52
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:977324
slave_priority:100
slave_read_only:1
connected_slaves:0
master_repl_offset:0
repl_backlog_active:0
repl_backlog_size:1048576
repl_backlog_first_byte_offset:0
repl_backlog_histlen:0
anhnongdan commented 7 years ago

Sentinel is now running on separate host (so when redis down, sentinel is not). Review to have more than one Sentinel, though it works fine with only 1 as current.

=> Tested: supervisorctl stop to simulate redis down. (-> next: use segment fault)

anhnongdan commented 7 years ago

About Sentinel: https://github.com/anhnongdan/bimax-counters/wiki/About-Redis-Sentinel

https://discuss.pivotal.io/hc/en-us/articles/205309388-How-to-setup-HAProxy-and-Redis-Sentinel-for-automatic-failover-between-Redis-Master-and-Slave-servers