Closed xemul closed 2 years ago
======================== /tmp/zdtm-core-1575-futex.txt =========================
+ ps axf
PID TTY STAT TIME COMMAND
2 ? S 0:00 [kthreadd]
3 ? S 0:08 \_ [ksoftirqd/0]
4 ? S 0:00 \_ [kworker/0:0]
5 ? S< 0:00 \_ [kworker/0:0H]
7 ? R 0:13 \_ [rcu_sched]
8 ? S 0:00 \_ [rcu_bh]
9 ? S 0:00 \_ [migration/0]
10 ? S 0:00 \_ [watchdog/0]
11 ? S 0:00 \_ [watchdog/1]
12 ? S 0:00 \_ [migration/1]
13 ? S 0:09 \_ [ksoftirqd/1]
15 ? S< 0:00 \_ [kworker/1:0H]
16 ? S 0:00 \_ [kdevtmpfs]
17 ? S< 0:00 \_ [netns]
18 ? S< 0:00 \_ [perf]
19 ? S 0:00 \_ [khungtaskd]
20 ? S< 0:00 \_ [writeback]
21 ? SN 0:00 \_ [ksmd]
22 ? SN 0:00 \_ [khugepaged]
23 ? S< 0:00 \_ [crypto]
24 ? S< 0:00 \_ [kintegrityd]
25 ? S< 0:00 \_ [bioset]
26 ? S< 0:00 \_ [kblockd]
27 ? S< 0:00 \_ [ata_sff]
28 ? S< 0:00 \_ [md]
29 ? S< 0:00 \_ [devfreq_wq]
32 ? S 0:00 \_ [kworker/1:1]
34 ? S 0:00 \_ [kswapd0]
35 ? S< 0:00 \_ [vmstat]
36 ? S 0:00 \_ [fsnotify_mark]
37 ? S 0:00 \_ [ecryptfs-kthrea]
53 ? S< 0:00 \_ [kthrotld]
54 ? S< 0:00 \_ [acpi_thermal_pm]
55 ? S< 0:00 \_ [bioset]
56 ? S< 0:00 \_ [bioset]
57 ? S< 0:00 \_ [bioset]
58 ? S< 0:00 \_ [bioset]
59 ? S< 0:00 \_ [bioset]
60 ? S< 0:00 \_ [bioset]
61 ? S< 0:00 \_ [bioset]
62 ? S< 0:00 \_ [bioset]
63 ? S< 0:00 \_ [bioset]
64 ? S< 0:00 \_ [bioset]
65 ? S< 0:00 \_ [bioset]
66 ? S< 0:00 \_ [bioset]
67 ? S< 0:00 \_ [bioset]
68 ? S< 0:00 \_ [bioset]
69 ? S< 0:00 \_ [bioset]
70 ? S< 0:00 \_ [bioset]
71 ? S< 0:00 \_ [bioset]
72 ? S< 0:00 \_ [bioset]
73 ? S< 0:00 \_ [bioset]
74 ? S< 0:00 \_ [bioset]
75 ? S< 0:00 \_ [bioset]
76 ? S< 0:00 \_ [bioset]
77 ? S< 0:00 \_ [bioset]
78 ? S< 0:00 \_ [bioset]
82 ? S< 0:00 \_ [ipv6_addrconf]
95 ? S< 0:00 \_ [deferwq]
96 ? S< 0:00 \_ [charger_manager]
140 ? S< 0:00 \_ [kpsmoused]
141 ? S 0:00 \_ [scsi_eh_0]
143 ? S< 0:00 \_ [scsi_tmf_0]
155 ? S< 0:00 \_ [bioset]
471 ? S< 0:00 \_ [bioset]
481 ? S 0:00 \_ [jbd2/sda1-8]
482 ? S< 0:00 \_ [ext4-rsv-conver]
522 ? S< 0:00 \_ [kworker/0:1H]
523 ? S< 0:00 \_ [kworker/1:1H]
1633 ? S 0:00 \_ [kauditd]
8174 ? S 0:00 \_ [kworker/0:3]
11285 ? S 0:00 \_ [kworker/1:3]
18080 ? S 0:08 \_ [kworker/u4:1]
19647 ? S 0:01 \_ [kworker/u4:0]
23121 ? S 0:00 \_ [kworker/1:2]
6627 ? S 0:06 \_ [kworker/u4:3]
19152 ? S 0:00 \_ [kworker/1:0]
27254 ? S 0:00 \_ [kworker/u4:2]
28115 ? S 0:00 \_ [kworker/0:1]
1589 ? S 0:00 \_ /bin/bash -x /home/travis/build/criupatchwork/criu/test/abrt.sh 1575 26 7 futex
1591 ? R 0:00 \_ ps axf
1 ? Ss 0:06 /sbin/init
706 ? Ss 0:02 /lib/systemd/systemd-udevd --daemon
25667 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7209 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7210 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7211 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7212 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7213 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7214 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7215 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7216 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7217 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7218 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
7219 ? S 0:00 \_ /lib/systemd/systemd-udevd --daemon
941 ? Ss 0:00 dhclient -1 -v -pf /run/dhclient.eth0.pid -lf /var/lib/dhcp/dhclient.eth0.leases eth0
1147 ? Ssl 0:00 /usr/bin/dockerd -H tcp://127.0.0.1:2375 -H unix:///var/run/docker.sock --raw-logs
1233 ? Ssl 0:00 \_ docker-containerd -l unix:///var/run/docker/libcontainerd/docker-containerd.sock --shim docker-containerd-shim --metrics-interval=0 --start-timeout 2m --state-dir /var/run/docker/libcontainerd/containerd --runtime docker-runc
1153 ? Ss 0:00 dbus-daemon --system --fork
1184 ? Ss 0:00 /lib/systemd/systemd-logind
1189 ? Ssl 0:00 rsyslogd
1485 tty4 Ss+ 0:00 /sbin/getty -8 38400 tty4
1488 tty5 Ss+ 0:00 /sbin/getty -8 38400 tty5
1493 tty2 Ss+ 0:00 /sbin/getty -8 38400 tty2
1494 tty3 Ss+ 0:00 /sbin/getty -8 38400 tty3
1497 tty6 Ss+ 0:00 /sbin/getty -8 38400 tty6
1527 ? Ss 0:00 atd
1528 ? Ss 0:00 cron
1545 ? Ss 0:00 acpid -c /etc/acpi/events -s /var/run/acpid.socket
1554 ? Ssl 0:06 /usr/bin/searchd --nodetach
1612 ? Ssl 0:02 /usr/sbin/mysqld
1717 ? Ss 0:00 /usr/bin/python /usr/share/google/google_daemon/manage_accounts.py
775 ? S 0:00 \_ /usr/bin/python /usr/share/google/google_daemon/manage_accounts.py
1721 ? Ss 0:00 /usr/sbin/sshd -D
2264 ? Ss 0:00 \_ sshd: travis [priv]
2269 ? S 0:00 \_ sshd: travis@pts/0
2270 pts/0 Ss+ 0:00 \_ bash /home/travis/build.sh
3964 pts/0 S+ 0:00 \_ sudo make CCACHE=1 -C scripts/travis local
3965 pts/0 S+ 0:00 \_ make CCACHE=1 -C scripts/travis local
3966 pts/0 S+ 0:00 \_ /bin/sh ./travis-tests
20340 pts/0 S+ 0:00 \_ python2 ./test/zdtm.py run -a -p 2 --remote-lazy-pages -x maps007 -x fork -x fork2 -x uffd-events -x cgroupns -x socket_listen -x socket_listen6 -x cmdlinenv00 -x socket_close_data01 -x file_read
1495 pts/0 S+ 0:00 \_ ./zdtm_ct zdtm.py
1500 pts/0 S+ 0:00 | \_ python2 zdtm.py
1506 pts/0 S+ 0:00 | \_ python2 zdtm.py
1569 pts/0 D+ 0:00 | | \_ ../criu/criu page-server -o page-server.log -D dump/zdtm/static/futex/26/1 -v4 --pidfile ps.pid --port 12345 --lazy-pages --status-fd 4
1571 pts/0 Z+ 0:00 | | \_ [criu] <defunct>
1575 ? Ss 0:00 | \_ ./futex --pidfile=futex.pid --outfile=futex.out
1498 pts/0 S+ 0:00 \_ ./zdtm_ct zdtm.py
1501 pts/0 S+ 0:00 \_ python2 zdtm.py
1508 pts/0 S+ 0:00 \_ python2 zdtm.py
1570 pts/0 S+ 0:00 \_ ../criu/criu page-server -o page-server.log -D dump/zdtm/static/pid00/26/1 -v4 --pidfile ps.pid --port 12345 --lazy-pages --status-fd 4
1572 pts/0 S+ 0:00 \_ ../criu/criu lazy-pages -o lazy-pages.log -D dump/zdtm/static/pid00/26/1 -v4 --page-server --port 12345 --status-fd 4
1841 ? S 0:00 /usr/lib/postgresql/9.2/bin/postgres -D /var/ramfs/postgresql/9.2/main -c config_file=/etc/postgresql/9.2/main/postgresql.conf
1843 ? Ss 0:00 \_ postgres: checkpointer process
1844 ? Ss 0:00 \_ postgres: writer process
1845 ? Ss 0:00 \_ postgres: wal writer process
1846 ? Ss 0:00 \_ postgres: autovacuum launcher process
1847 ? Ss 0:00 \_ postgres: stats collector process
1908 ? Ss 0:03 /usr/sbin/haveged -w 1024
1915 ? Sl 0:00 /usr/bin/memcached -v -m 64 -p 11211 -U 11211 -u nobody -l 0.0.0.0 -c 1024 -I 1
1950 ? Sl 0:02 /usr/sbin/sshguard -i /var/run/sshguard.pid -l /var/log/auth.log -w /etc/sshguard/whitelist -a 40 -p 420 -s 1200
2017 tty1 Ss+ 0:00 /sbin/getty -8 38400 tty1
2037 ? Ss 0:00 /usr/bin/python /usr/share/google/google_daemon/manage_clock_sync.py
2040 ? Ss 0:00 /usr/bin/python /usr/share/google/google_daemon/manage_addresses.py
2059 ttyS0 Ss+ 0:00 /sbin/getty -L ttyS0 115200 vt102
2256 ? Ss 0:00 /usr/sbin/ntpd -p /var/run/ntpd.pid -g -c /var/lib/ntp/ntp.conf.dhcp -u 106:111
6395 ? S 0:01 upstart-udev-bridge --daemon
6398 ? S 0:00 upstart-file-bridge --daemon
6401 ? S 0:00 upstart-socket-bridge --daemon
+ ps -p 1575
PID TTY TIME CMD
1575 ? 00:00:00 futex
+ cat /proc/1575/status
Name: futex
State: S (sleeping)
Tgid: 1575
Ngid: 0
Pid: 1575
PPid: 1500
TracerPid: 0
Uid: 18943 18943 18943 18943
Gid: 58467 58467 58467 58467
FDSize: 1048576
Groups: 27495 48244
NStgid: 1575 26
NSpid: 1575 26
NSpgid: 1575 26
NSsid: 1575 26
VmPeak: 47900 kB
VmSize: 27196 kB
VmLck: 0 kB
VmPin: 0 kB
VmHWM: 4176 kB
VmRSS: 524 kB
VmData: 20728 kB
VmStk: 280 kB
VmExe: 24 kB
VmLib: 2024 kB
VmPTE: 76 kB
VmPMD: 12 kB
VmSwap: 0 kB
HugetlbPages: 0 kB
Threads: 1
SigQ: 0/29825
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: 0000000001001000
SigCgt: 0000000180004000
CapInh: 0000000000000000
CapPrm: 0000000000000000
CapEff: 0000000000000000
CapBnd: 0000003fffffffff
CapAmb: 0000000000000000
Seccomp: 0
Cpus_allowed: 3
Cpus_allowed_list: 0-1
Mems_allowed: 00000000,00000001
Mems_allowed_list: 0
voluntary_ctxt_switches: 34
nonvoluntary_ctxt_switches: 3
+ ls -l /proc/1575/fd
total 0
lrwx------ 1 18943 58467 64 Jun 28 21:50 0 -> /dev/null
l-wx------ 1 18943 58467 64 Jun 28 21:50 1 -> /home/travis/build/criupatchwork/criu/test/zdtm/static/futex.out.inprogress
l-wx------ 1 18943 58467 64 Jun 28 21:50 2 -> /home/travis/build/criupatchwork/criu/test/zdtm/static/futex.out.inprogress
lr-x------ 1 18943 58467 64 Jun 28 21:50 3 -> /home/travis/build/criupatchwork/criu/test/zdtm/static
+ cat /proc/1575/maps
00400000-00406000 r-xp 00000000 08:01 276766 /home/travis/build/criupatchwork/criu/test/zdtm/static/futex
00605000-00606000 r--p 00005000 08:01 276766 /home/travis/build/criupatchwork/criu/test/zdtm/static/futex
00606000-00607000 rw-p 00006000 08:01 276766 /home/travis/build/criupatchwork/criu/test/zdtm/static/futex
01382000-013a3000 rw-p 00000000 00:00 0 [heap]
2b4d0e706000-2b4d0e729000 r-xp 00000000 08:01 2792 /lib/x86_64-linux-gnu/ld-2.19.so
2b4d0e729000-2b4d0e72b000 rw-p 00000000 00:00 0
2b4d0e73e000-2b4d0e741000 rw-p 00000000 00:00 0
2b4d0e928000-2b4d0e929000 r--p 00022000 08:01 2792 /lib/x86_64-linux-gnu/ld-2.19.so
2b4d0e929000-2b4d0e92a000 rw-p 00023000 08:01 2792 /lib/x86_64-linux-gnu/ld-2.19.so
2b4d0e92a000-2b4d0e92b000 rw-p 00000000 00:00 0
2b4d0e92b000-2b4d0e944000 r-xp 00000000 08:01 2785 /lib/x86_64-linux-gnu/libpthread-2.19.so
2b4d0e944000-2b4d0eb43000 ---p 00019000 08:01 2785 /lib/x86_64-linux-gnu/libpthread-2.19.so
2b4d0eb43000-2b4d0eb44000 r--p 00018000 08:01 2785 /lib/x86_64-linux-gnu/libpthread-2.19.so
2b4d0eb44000-2b4d0eb45000 rw-p 00019000 08:01 2785 /lib/x86_64-linux-gnu/libpthread-2.19.so
2b4d0eb45000-2b4d0eb49000 rw-p 00000000 00:00 0
2b4d0eb49000-2b4d0ed07000 r-xp 00000000 08:01 2784 /lib/x86_64-linux-gnu/libc-2.19.so
2b4d0ed07000-2b4d0ef07000 ---p 001be000 08:01 2784 /lib/x86_64-linux-gnu/libc-2.19.so
2b4d0ef07000-2b4d0ef0b000 r--p 001be000 08:01 2784 /lib/x86_64-linux-gnu/libc-2.19.so
2b4d0ef0b000-2b4d0ef0d000 rw-p 001c2000 08:01 2784 /lib/x86_64-linux-gnu/libc-2.19.so
2b4d0ef0d000-2b4d0ef12000 rw-p 00000000 00:00 0
2b4d0ef12000-2b4d0ef13000 ---p 00000000 00:00 0
2b4d0ef13000-2b4d0f113000 rw-p 00000000 00:00 0
2b4d0f113000-2b4d0f114000 ---p 00000000 00:00 0
2b4d0f114000-2b4d0f314000 rw-p 00000000 00:00 0
2b4d0f314000-2b4d0f315000 ---p 00000000 00:00 0
2b4d0f315000-2b4d0f515000 rw-p 00000000 00:00 0
2b4d0f515000-2b4d0f516000 ---p 00000000 00:00 0
2b4d0f516000-2b4d0f716000 rw-p 00000000 00:00 0
2b4d0f716000-2b4d0f717000 ---p 00000000 00:00 0
2b4d0f717000-2b4d0f917000 rw-p 00000000 00:00 0
2b4d0f917000-2b4d0f918000 ---p 00000000 00:00 0
2b4d0f918000-2b4d0fb18000 rw-p 00000000 00:00 0
2b4d0fb18000-2b4d0fb19000 ---p 00000000 00:00 0
2b4d0fb19000-2b4d0fd19000 rw-p 00000000 00:00 0
2b4d0fd19000-2b4d0fd1a000 ---p 00000000 00:00 0
2b4d0fd1a000-2b4d0ff1a000 rw-p 00000000 00:00 0
2b4d0ff1a000-2b4d0ff1b000 ---p 00000000 00:00 0
2b4d0ff1b000-2b4d1011b000 rw-p 00000000 00:00 0
2b4d1011b000-2b4d1011c000 ---p 00000000 00:00 0
2b4d1011c000-2b4d1031c000 rw-p 00000000 00:00 0
7ffc4a6ad000-7ffc4a6f2000 rw-p 00000000 00:00 0 [stack]
7ffc4a784000-7ffc4a786000 r--p 00000000 00:00 0 [vvar]
7ffc4a786000-7ffc4a788000 r-xp 00000000 00:00 0 [vdso]
ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
+ exec
+ cat
+ echo 'bt
i r
disassemble $rip-0x10,$rip + 0x10
'
+ gdb -c /tmp/zdtm-core-1575-futex.core /proc/self/fd/33
GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1
Copyright (C) 2014 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /proc/self/fd/33...done.
[New LWP 26]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `./futex --pidfile=futex.pid --outfile=futex.out'.
Program terminated with signal SIGBUS, Bus error.
#0 0x00002b4d0ebc9755 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
(gdb) #0 0x00002b4d0ebc9755 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00002b4d0ebcbae0 in malloc () from /lib/x86_64-linux-gnu/libc.so.6
#2 0x00002b4d0ebbb142 in vasprintf () from /lib/x86_64-linux-gnu/libc.so.6
#3 0x00002b4d0eb9d597 in asprintf () from /lib/x86_64-linux-gnu/libc.so.6
#4 0x00002b4d0ebffb32 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#5 0x00002b4d0ebfeeb4 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#6 0x00002b4d0ebff863 in ?? () from /lib/x86_64-linux-gnu/libc.so.6
#7 0x0000000000402192 in test_msg (format=format@entry=0x404014 "PASS\n")
at msg.c:51
#8 0x0000000000401e8e in main (argc=<optimized out>, argv=<optimized out>)
at futex.c:78
(gdb) rax 0x0 0
rbx 0x2b4d0ef0b760 47609963132768
rcx 0x7ffc4a6f0ddd 140721557278173
rdx 0x7ffc4a6ee1e8 140721557266920
rsi 0x64 100
rdi 0x2b4d0ef0b760 47609963132768
rbp 0x64 0x64
rsp 0x7ffc4a6edfb0 0x7ffc4a6edfb0
r8 0x0 0
r9 0x2 2
r10 0x7ffc4a6ee1d0 140721557266896
r11 0x3 3
r12 0x7ffc4a6ee310 140721557267216
r13 0x2b4d0eccaa63 47609960770147
r14 0x7ffc4a6ee1e8 140721557266920
r15 0x0 0
rip 0x2b4d0ebc9755 0x2b4d0ebc9755
eflags 0x10217 [ CF PF AF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x6b 107
(gdb) Dump of assembler code from 0x2b4d0ebc9745 to 0x2b4d0ebc9765:
0x00002b4d0ebc9745: push %rbp
0x00002b4d0ebc9746: push %r12
0x00002b4d0ebc9748: push %rbp
0x00002b4d0ebc9749: push %rbx
0x00002b4d0ebc974a: sub $0xb8,%rsp
0x00002b4d0ebc9751: cmp $0xffffffffffffffbf,%rsi
=> 0x00002b4d0ebc9755: mov %rsi,0x8(%rsp)
0x00002b4d0ebc975a: ja 0x2b4d0ebc9ef9
0x00002b4d0ebc9760: mov %rsi,%rax
0x00002b4d0ebc9763: mov $0x20,%ebp
End of assembler dump.
(gdb) Dump of assembler code from 0x2b4d0ebc9745 to 0x2b4d0ebc9765:
0x00002b4d0ebc9745: push %rbp
0x00002b4d0ebc9746: push %r12
0x00002b4d0ebc9748: push %rbp
0x00002b4d0ebc9749: push %rbx
0x00002b4d0ebc974a: sub $0xb8,%rsp
0x00002b4d0ebc9751: cmp $0xffffffffffffffbf,%rsi
=> 0x00002b4d0ebc9755: mov %rsi,0x8(%rsp)
0x00002b4d0ebc975a: ja 0x2b4d0ebc9ef9
0x00002b4d0ebc9760: mov %rsi,%rax
0x00002b4d0ebc9763: mov $0x20,%ebp
End of assembler dump.
(gdb) quit
======================== /tmp/zdtm-core-1575-futex.txt =========================
Cc: @rppt
Apparently, we have a race between delivery of UFFDEVENT{PAGE_FAULT,REMOVE} and close(uffd). If close() wins, the event is never delivered to the lazy-pages daemon. With UFFD_EVENT_REMOVE, the calling thread is stuck in because there is nothing that will wake it up. With UFFD_EVENT_PAGE_FAULT, the faulting thread will get SIGBUS.
I'm still looking for an elegant solution.
A friendly reminder that this issue had no activity for 30 days.
https://travis-ci.org/xemul/criu/builds/250335435 https://travis-ci.org/criupatchwork/criu/builds/248123376
Cc @avagin