renzhengeek
commented
8 years ago Closed renzhengeek closed 8 years ago
renzhengeek
commented
8 years ago renzhengeek
commented
8 years ago m1:/opt/ocfs2-test/bin # ps aux | grep flock
root 16898 0.0 0.0 9240 936 pts/1 S+ 16:43 0:00 grep --color=auto flock
root 21127 0.0 0.4 66436 8704 pts/0 S+ Sep08 0:00 python /opt/ocfs2-test/bin/run_flock_unit_test.py -l /opt/ocfs2-test/log/flock_unit/fcntl_unit_test_2015-09-07-10-20.log -n m1,m2 -t fcntl -e /mnt/shared/flock_unit_test/flock_unit_test_file1 -f /mnt/shared/flock_unit_test/flock_unit_test_file2
root 21134 0.0 0.0 11808 1404 pts/0 S+ Sep08 0:00 bash -xc /usr/lib64/mpi/gcc/openmpi/bin/mpirun --allow-run-as-root -mca btl tcp,self -mca plm_rsh_agent ssh:rsh --allow-run-as-root -mca btl_tcp_if_include eth0 --allow-run-as-root -np 2 --host m1,m2 /opt/ocfs2-test/bin/flock_unit_test -t fcntl /mnt/shared/flock_unit_test/flock_unit_test_file1 /mnt/shared/flock_unit_test/flock_unit_test_file2 2>&1 | tee -a /opt/ocfs2-test/log/flock_unit/fcntl_unit_test_2015-09-07-10-20.log
root 21135 0.0 0.1 145668 3676 pts/0 Sl+ Sep08 0:05 /usr/lib64/mpi/gcc/openmpi/bin/mpirun --allow-run-as-root -mca btl tcp,self -mca plm_rsh_agent ssh:rsh --allow-run-as-root -mca btl_tcp_if_include eth0 --allow-run-as-root -np 2 --host m1,m2 /opt/ocfs2-test/bin/flock_unit_test -t fcntl /mnt/shared/flock_unit_test/flock_unit_test_file1 /mnt/shared/flock_unit_test/flock_unit_test_file2
root 21136 0.0 0.0 4496 672 pts/0 S+ Sep08 0:00 tee -a /opt/ocfs2-test/log/flock_unit/fcntl_unit_test_2015-09-07-10-20.log
root 21139 99.5 0.2 165684 4900 pts/0 Rl Sep08 1536:06 /opt/ocfs2-test/bin/flock_unit_test -t fcntl /mnt/shared/flock_unit_test/flock_unit_test_file1 /mnt/shared/flock_unit_test/flock_unit_test_file2
m1:/opt/ocfs2-test/bin # cat /proc/21139/stack
[<ffffffff8104150b>] kvm_clock_get_cycles+0x1b/0x20
[<ffffffff810b98f4>] __getnstimeofday+0x34/0xc0
[<ffffffff8104150b>] kvm_clock_get_cycles+0x1b/0x20
[<ffffffff810b9bb2>] ktime_get_ts+0x42/0xe0
[<ffffffff8105bbbe>] SyS_gettimeofday+0x2e/0x70
[<ffffffff81528d89>] system_call_fastpath+0x16/0x1b
[<ffffffffffffffff>] 0xffffffffffffffff
m2:/opt/ocfs2-test/log # ps aux|grep flock
root 19906 0.0 0.2 165680 4908 ? Dl Sep08 0:00 /opt/ocfs2-test/bin/flock_unit_test -t fcntl /mnt/shared/flock_unit_test/flock_unit_test_file1 /mnt/shared/flock_unit_test/flock_unit_test_file2
root 24449 0.0 0.0 9240 936 pts/0 S+ 16:31 0:00 grep --color=auto flock
m2:/opt/ocfs2-test/log # cat /proc/19906/stack
[<ffffffffa0556a65>] dlm_posix_lock+0x185/0x380 [dlm]
[<ffffffff811f2bae>] fcntl_setlk+0x12e/0x2d0
[<ffffffff811b7271>] SyS_fcntl+0x261/0x510
[<ffffffff81528d89>] system_call_fastpath+0x16/0x1b
[<00007ff690db3b42>] 0x7ff690db3b42
[<ffffffffffffffff>] 0xffffffffffffffff
m2:/opt/ocfs2-test/log # cat /proc/19906/stack
[<ffffffffa0556a65>] dlm_posix_lock+0x185/0x380 [dlm]
[<ffffffff811f2bae>] fcntl_setlk+0x12e/0x2d0
[<ffffffff811b7271>] SyS_fcntl+0x261/0x510
[<ffffffff81528d89>] system_call_fastpath+0x16/0x1b
[<00007ff690db3b42>] 0x7ff690db3b42
[<ffffffffffffffff>] 0xffffffffffffffffocfs2test@n1:/mnt/shared/flock_unit_test> lslocks -p 12588
COMMAND PID TYPE SIZE MODE M START END PATH
flock_unit_test 12588 POSIX 0B WRITE 0 0 0 /mnt/shared/flock_unit_test/flock_unit_test_file1and
n3:/mnt/shared/flock_unit_test # lslocks -p 1534n1:/mnt/shared/flock_unit_test # ls -i 471051 flock_unit_test_file1 471052 flock_unit_test_file2 n1:/mnt/shared/flock_unit_test # ls -i .. 471050 flock_unit_test 471049 lost+found
n1:~ # debugfs.ocfs2 /dev/sda
debugfs.ocfs2 1.8.2
debugfs: decode
renzhengeek
commented
8 years ago n3:~ # cat /proc/11153/stack
[
renzhengeek
commented
8 years ago 133 op->info.end = fl->fl_end;
134 WHERE_AM_I
135 if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
136 WHERE_AM_I
137 /* fl_owner is lockd which doesn't distinguish
138 processes on the nfs client */
139 op->info.owner = (__u64) fl->fl_pid;
140 xop->callback = fl->fl_lmops->lm_grant;
141 locks_init_lock(&xop->flc);
142 locks_copy_lock(&xop->flc, fl);
143 xop->fl = fl;
144 xop->file = file;
145 WHERE_AM_I
146 } else {
147 WHERE_AM_I
148 op->info.owner = (__u64)(long) fl->fl_owner;
149 xop->callback = NULL;
150 WHERE_AM_I
151 }
152
153 dump_stack();
154 send_op(op);
155
156 WHERE_AM_I
157 if (xop->callback == NULL) {
158 WHERE_AM_I
159 rv = wait_event_killable(recv_wq, (op->done != 0));
160 WHERE_AM_I
161 if (rv == -ERESTARTSYS) {
162 WHERE_AM_I
renzhengeek
commented
8 years ago -- Logs begin at Mon 2015-10-12 16:25:45 CST, end at Mon 2015-10-12 16:35:48 CST. --
Oct 12 16:35:48 m2 spice-vdagent[1273]: primary: property contains no data (zero length)
Oct 12 16:35:48 m2 spice-vdagent[1273]: clipboard: property contains no data (zero length)
Oct 12 16:35:01 m2 kernel: dlm_posix_lock:158
Oct 12 16:35:01 m2 kernel: dlm_posix_lock:156
Oct 12 16:35:01 m2 kernel: [<00007f1c653d7b42>] 0x7f1c653d7b41
Oct 12 16:35:01 m2 kernel: [
renzhengeek
commented
8 years ago https://www.redhat.com/archives/cluster-devel/2008-April/msg00126.html Move the code that handles cluster posix locks from gfs2 into the dlm so that it can be used by both gfs2 and ocfs2.
Signed-off-by: David Teigland
renzhengeek
commented
8 years ago ocfs2test@m1:~> ./a.out /mnt/shared/test
./a.out: waiting for ever
ocfs2test@m1:~/bin/ocfs2/bin> vim /mnt/shared/test
ocfs2test@m1:~/bin/ocfs2/bin> cd
ocfs2test@m1:~> ls
a.out b.out bin linux-kernel.tar.gz ocfs2-test plock-deadlock-a.c plock-deadlock-b.c public_html
ocfs2test@m1:~> ./b.out /mnt/shared/
open error -1: Is a directory
ocfs2test@m1:~> ./b.out /mnt/shared/test
./b.out: require look that has been granted
Alarm clock
ocfs2test@m1:~> ps aux|grep b.out
ocfs2te+ 20510 0.0 0.0 4064 356 pts/1 S+ 09:51 0:00 ./b.out /mnt/shared/test
ocfs2te+ 20601 0.0 0.0 9240 936 pts/3 S+ 09:52 0:00 grep --color=auto b.out
ocfs2test@m1:~> kill -l
1) SIGHUP 2) SIGINT 3) SIGQUIT 4) SIGILL 5) SIGTRAP
6) SIGABRT 7) SIGBUS 8) SIGFPE 9) SIGKILL 10) SIGUSR1
11) SIGSEGV 12) SIGUSR2 13) SIGPIPE 14) SIGALRM 15) SIGTERM
16) SIGSTKFLT 17) SIGCHLD 18) SIGCONT 19) SIGSTOP 20) SIGTSTP
21) SIGTTIN 22) SIGTTOU 23) SIGURG 24) SIGXCPU 25) SIGXFSZ
26) SIGVTALRM 27) SIGPROF 28) SIGWINCH 29) SIGIO 30) SIGPWR
31) SIGSYS 34) SIGRTMIN 35) SIGRTMIN+1 36) SIGRTMIN+2 37) SIGRTMIN+3
38) SIGRTMIN+4 39) SIGRTMIN+5 40) SIGRTMIN+6 41) SIGRTMIN+7 42) SIGRTMIN+8
43) SIGRTMIN+9 44) SIGRTMIN+10 45) SIGRTMIN+11 46) SIGRTMIN+12 47) SIGRTMIN+13
48) SIGRTMIN+14 49) SIGRTMIN+15 50) SIGRTMAX-14 51) SIGRTMAX-13 52) SIGRTMAX-12
53) SIGRTMAX-11 54) SIGRTMAX-10 55) SIGRTMAX-9 56) SIGRTMAX-8 57) SIGRTMAX-7
58) SIGRTMAX-6 59) SIGRTMAX-5 60) SIGRTMAX-4 61) SIGRTMAX-3 62) SIGRTMAX-2
63) SIGRTMAX-1 64) SIGRTMAX
ocfs2test@m1:~> kill -14 20510 ocfs2test@m1:~> ls /mnt/shared/flock_unit_test/
flock_unit_test_file1 flock_unit_test_file2
ocfs2test@m1:~> ./a.out /mnt/shared/flock_unit_test/flock_unit_test_file1
./a.out: waiting for ever
ocfs2test@m1:~> ls /mnt/shared/flock_unit_test/
flock_unit_test_file1 flock_unit_test_file2
ocfs2test@m1:~> ./a.out /mnt/shared/flock_unit_test/flock_unit_test_file1
./a.out: waiting for ever
ocfs2test@m1:~> ps aux|grep .out
ocfs2te+ 3479 99.9 0.0 4064 356 pts/0 R+ 10:20 7:50 ./a.out /mnt/shared/flock_unit_test/flock_unit_test_file1
ocfs2te+ 4069 0.0 0.0 4064 356 pts/2 D+ 10:28 0:00 ./b.out /mnt/shared/flock_unit_test/flock_unit_test_file1
ocfs2te+ 4071 0.0 0.0 9240 936 pts/1 S+ 10:28 0:00 grep --color=auto .out
ocfs2test@m1:~> kill -14 4069
ocfs2test@m1:~> ls /mnt/shared/flock_unit_test/
flock_unit_test_file1 flock_unit_test_file2
ocfs2test@m1:~> ./a.out /mnt/shared/flock_unit_test/flock_unit_test_file1
./a.out: waiting for ever
ocfs2test@m2:~> ./b.out /mnt/shared/flock_unit_test/flock_unit_test_file1
./b.out: require look that has been granted
Alarm clock
ocfs2test@m2:~>
ocfs2test@m2:~> ps aux|grep .out
ocfs2te+ 3766 0.0 0.0 4064 356 pts/0 D+ 10:35 0:00 ./b.out /mnt/shared/flock_unit_test/flock_unit_test_file1
ocfs2te+ 3828 0.0 0.0 9240 936 pts/1 S+ 10:35 0:00 grep --color=auto .out
ocfs2test@m2:~> kill -l
1) SIGHUP 2) SIGINT 3) SIGQUIT 4) SIGILL 5) SIGTRAP
6) SIGABRT 7) SIGBUS 8) SIGFPE 9) SIGKILL 10) SIGUSR1
11) SIGSEGV 12) SIGUSR2 13) SIGPIPE 14) SIGALRM 15) SIGTERM
16) SIGSTKFLT 17) SIGCHLD 18) SIGCONT 19) SIGSTOP 20) SIGTSTP
21) SIGTTIN 22) SIGTTOU 23) SIGURG 24) SIGXCPU 25) SIGXFSZ
26) SIGVTALRM 27) SIGPROF 28) SIGWINCH 29) SIGIO 30) SIGPWR
31) SIGSYS 34) SIGRTMIN 35) SIGRTMIN+1 36) SIGRTMIN+2 37) SIGRTMIN+3
38) SIGRTMIN+4 39) SIGRTMIN+5 40) SIGRTMIN+6 41) SIGRTMIN+7 42) SIGRTMIN+8
43) SIGRTMIN+9 44) SIGRTMIN+10 45) SIGRTMIN+11 46) SIGRTMIN+12 47) SIGRTMIN+13
48) SIGRTMIN+14 49) SIGRTMIN+15 50) SIGRTMAX-14 51) SIGRTMAX-13 52) SIGRTMAX-12
53) SIGRTMAX-11 54) SIGRTMAX-10 55) SIGRTMAX-9 56) SIGRTMAX-8 57) SIGRTMAX-7
58) SIGRTMAX-6 59) SIGRTMAX-5 60) SIGRTMAX-4 61) SIGRTMAX-3 62) SIGRTMAX-2
63) SIGRTMAX-1 64) SIGRTMAX
ocfs2test@m2:~> kill -14 3766
ocfs2test@m1:~> ./a.out /mnt/shared/flock_unit_test/flock_unit_test_file1
./a.out: Hold write lock for ever!
ocfs2test@m1:~> ./b.out /mnt/shared/flock_unit_test/flock_unit_test_file1
./b.out: requiring lock that has been granted!
ocfs2test@m1:~> vim test-sigalrm.c
ocfs2test@m1:~> gcc test-sigalrm.c -o c.out
ocfs2test@m1:~> ./c.out
./c.out: waiting SIGALRM after 3s!
signal 14 recieved
^C
Hi Dav,
Thanks for your help! I went through this problem again, and finally found
the *strange* different reaction between using "wait_event_killable" and "wait_event_interruptiable".
1. say we have an test file on shared disk;
/mnt/shared/test_file1
2. two process on one node case;
- process a to hold the write lock
- process b blocked to require write lock
3. each process from two nodes case;
- process a to hold the write lock from node1
- process b blocked to require write lock from node2
Two "wait_event_*" act the same as we expect,i.e. process b can be
interrupted by SIGARLM *when* we send signal by "kill -14 pid-for-b".
*BUT*, if we set alarm before "fcntl" in process b instead of "kill" cmd, the problem occurs.
That said, "wait_event_killable" works well but "wait_event_interruptiable" will block SIGALRM.
Could you please try my test codes(paste at bottom of this mail & also attached)in your environemnt?
I cannot see why the two ways make deference now, will make further investigation later;-)
Thanks,
Eric
1. process a
============
#include <stdlib.h>
#include <stdio.h>
#include <sys/file.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
int main(int argc, char *argv[])
{
int fd;
int ret;
struct flock fl;
if (argc < 2)
{
printf("usage: %s <filename>\n", argv[0]);
exit(1);
}
fd = open(argv[1], O_RDWR);
if (fd < 0) {
ret = errno;
printf("open error %d: %s\n", fd, strerror(ret));
exit(1);
}
memset(&fl, 0, sizeof(fl));
fl.l_type = F_WRLCK;
fl.l_whence = SEEK_SET;
fl.l_start = 0;
fl.l_len = 0;
ret = fcntl(fd, F_SETLKW, &fl);
if (ret == -1) {
ret == errno;
printf("fcntl %d: %s\n", fd, strerror(ret));
}
printf ("%s: Hold write lock for ever!\n", argv[0]);
while(1);
return ret;
}
2. process b
============
#include <stdlib.h>
#include <stdio.h>
#include <sys/file.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <signal.h>
static void signal_handler(int sig)
{
printf("signal %d recieved\n", sig);
}
static void set_alarm(int seconds)
{
int ret;
struct sigaction act;
memset(&act, 0, sizeof(act));
act.sa_handler = signal_handler;
act.sa_flags = SA_RESETHAND;
if (sigaction(SIGALRM, &act, NULL) == -1)
printf("Couldn't setup SIGALRM handler!\n");
ret = alarm(seconds);
if (ret)
printf("failure %d while setting up alarm: \"%s\"\n", ret,
strerror(ret));
}
int main(int argc, char *argv[])
{
int fd;
int ret;
struct flock fl;
if (argc < 2)
{
printf("usage: %s <filename>\n", argv[0]);
exit(1);
}
fd = open(argv[1], O_RDWR);
if (fd < 0) {
ret = errno;
printf("open error %d: %s\n", fd, strerror(ret));
exit(1);
}
memset(&fl, 0, sizeof(fl));
fl.l_type = F_WRLCK;
fl.l_whence = SEEK_SET;
fl.l_start = 0;
fl.l_len = 0;
printf ("%s: requiring lock that has been granted!\n", argv[0]);
/* wake me up after 10 seconds */
set_alarm(10);
printf ("%s: waiting SIGALRM after 10s!\n", argv[0]);
ret = fcntl(fd, F_SETLKW, &fl);
if (ret == -1) {
ret == errno;
printf("fcntl %d: %s\n", fd, strerror(ret));
}
return ret;
}
3. b process without fcntl to verify set alarm works well
=========================================================
#include <stdlib.h>
#include <stdio.h>
#include <sys/file.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <signal.h>
static void signal_handler(int sig)
{
printf("signal %d recieved\n", sig);
}
static void set_alarm(int seconds)
{
int ret;
struct sigaction act;
memset(&act, 0, sizeof(act));
act.sa_handler = signal_handler;
act.sa_flags = SA_RESETHAND;
if (sigaction(SIGALRM, &act, NULL) == -1)
printf("Couldn't setup SIGALRM handler!\n");
ret = alarm(seconds);
if (ret)
printf("failure %d while setting up alarm: \"%s\"\n", ret,
strerror(ret));
}
int main(int argc, char *argv[])
{
/*
int fd;
int ret;
struct flock fl;
if (argc < 2)
{
printf("usage: %s <filename>\n", argv[0]);
exit(1);
}
fd = open(argv[1], O_RDWR);
if (fd < 0) {
ret = errno;
printf("open error %d: %s\n", fd, strerror(ret));
exit(1);
}
memset(&fl, 0, sizeof(fl));
fl.l_type = F_WRLCK;
fl.l_whence = SEEK_SET;
fl.l_start = 0;
fl.l_len = 0;
printf ("%s: requiring lock that has been granted!\n", argv[0]);
*/
/* wake me up after 10 seconds */
set_alarm(3);
printf ("%s: waiting SIGALRM after 3s!\n", argv[0]);
/*
ret = fcntl(fd, F_SETLKW, &fl);
if (ret == -1) {
ret == errno;
printf("fcntl %d: %s\n", fd, strerror(ret));
}
*/
while(1);
return 0;
}ocfs2test@m1:~> ./a.out /mnt/shared/flock_unit_test/flock_unit_test_file1
./a.out: Hold write lock for ever!
ocfs2test@m1:~> ./b.out /mnt/shared/flock_unit_test/flock_unit_test_file1
./b.out: requiring lock that has been granted!
./b.out: waiting SIGALRM after 10s!
signal 14 recieved
fcntl 3: Unknown error -1commit a6b1533e9a57d76cd3d9b7649d29ac604b1874b8 Author: Eric Ren zren@suse.com Date: Wed Oct 14 23:28:26 2015 +0800
dlm: make posix locks interruptible
Replace wait_event_killable with wait_event_interruptible
so that a program waiting for a posix lock can be
interrupted by a signal. With the killable version,
a program was not interruptible by a signal if it
had a signal handler set for it, overriding the default
action of terminating the process.
Signed-off-by: Eric Ren <zren@suse.com>
Signed-off-by: David Teigland <teigland@redhat.com>