search

anhnongdan / bimax_ha_migration

all db backup, recovery and migration scenarios for bimax
0 stars 0 forks source link

[HA - Disk] NFS Failover for Extractor's /data/cdn #19

Open anhnongdan opened 6 years ago

anhnongdan commented 6 years ago

The dir is mounted from log receiver.

anhnongdan commented 6 years ago

Check mounted dirs:

[root@VNPT-HCM-BIMAX-03 ~]# cat /proc/mounts
rootfs / rootfs rw 0 0
sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
devtmpfs /dev devtmpfs rw,nosuid,size=28760240k,nr_inodes=7190060,mode=755 0 0
securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0
tmpfs /dev/shm tmpfs rw,nosuid,nodev 0 0
devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
tmpfs /run tmpfs rw,nosuid,nodev,mode=755 0 0
tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,mode=755 0 0
cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd 0 0
pstore /sys/fs/pstore pstore rw,nosuid,nodev,noexec,relatime 0 0
efivarfs /sys/firmware/efi/efivars efivarfs rw,nosuid,nodev,noexec,relatime 0 0
cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpuacct,cpu 0 0
cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0
cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event 0 0
cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0
cgroup /sys/fs/cgroup/memory cgroup rw,nosuid,nodev,noexec,relatime,memory 0 0
cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0
cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0
cgroup /sys/fs/cgroup/net_cls cgroup rw,nosuid,nodev,noexec,relatime,net_cls 0 0
cgroup /sys/fs/cgroup/hugetlb cgroup rw,nosuid,nodev,noexec,relatime,hugetlb 0 0
configfs /sys/kernel/config configfs rw,relatime 0 0
/dev/sda3 / xfs rw,relatime,attr2,inode64,noquota 0 0
systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=25,pgrp=1,timeout=300,minproto=5,maxproto=5,direct 0 0
mqueue /dev/mqueue mqueue rw,relatime 0 0
debugfs /sys/kernel/debug debugfs rw,relatime 0 0
hugetlbfs /dev/hugepages hugetlbfs rw,relatime 0 0
sunrpc /var/lib/nfs/rpc_pipefs rpc_pipefs rw,relatime 0 0
nfsd /proc/fs/nfsd nfsd rw,relatime 0 0
/dev/sda1 /boot/efi vfat rw,relatime,fmask=0077,dmask=0077,codepage=437,iocharset=ascii,shortname=winnt,errors=remount-ro 0 0
/dev/sdc /logtemp xfs rw,relatime,attr2,inode64,noquota 0 0
/dev/sdb1 /data ext4 rw,relatime,data=ordered 0 0
tmpfs /run/user/0 tmpfs rw,nosuid,nodev,relatime,size=5753884k,mode=700 0 0
172.20.4.63:/data/cdn /data/cdn nfs4 rw,relatime,vers=4.0,rsize=8192,wsize=8192,namlen=255,hard,proto=tcp,port=0,timeo=14,retrans=2,sec=sys,clientaddr=172.20.4.67,local_lock=none,addr=172.20.4.63 0 0
/dev/sda3 /var/lib/docker/devicemapper xfs rw,relatime,attr2,inode64,noquota 0 0
binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0
anhnongdan commented 6 years ago

Set up nfs: http://www.serverlab.ca/tutorials/linux/storage-file-systems-linux/how-to-setup-an-nfs-server-on-centos-6/

anhnongdan commented 6 years ago

Can't use ls command to check if the file is available . It hangs when NFS server down.

NFS info is still shown even when server die. 172.20.4.57:/data/nfs_test /data/nfs_test nfs4 rw,relatime,vers=4.0,rsize=1048576,wsize=1048576,namlen=255,hard,proto=tcp,port=0,timeo=600,retrans=2,sec=sys,clientaddr=172.20.4.67,local_lock=none,addr=172.20.4.57 0 0

This is the problem of HARD mount vs SOFT mount: https://www.centos.org/forums/viewtopic.php?t=8787

anhnongdan commented 6 years ago

If using soft mount is not the option.

Use timeout for hard mount.

Tested: Server up

[root@VNPT-HCM-BIMAX-03 test]# sh test_resolver.sh
name1: /data/nfs_test/abc/test1 -> /data/nfs_test/abc/test1
test1 on nfs
name2: /data/cdn/abc/test1 -> /data/cdn/abc/test1
test1 on /data/cdn
name3: /data/nfs_test/abc/test2 -> /data/cdn/abc/test2
test2 on /data/cdn

Server down

[root@VNPT-HCM-BIMAX-03 test]# sh test_resolver.sh
name1: /data/nfs_test/abc/test1 -> /data/cdn/abc/test1
test1 on /data/cdn
name2: /data/cdn/abc/test1 -> /data/cdn/abc/test1
test1 on /data/cdn
name3: /data/nfs_test/abc/test2 -> /data/cdn/abc/test2
test2 on /data/cdn