search

openzfs / zfs

OpenZFS on Linux and FreeBSD
https://openzfs.github.io/openzfs-docs
Other
10.69k stars 1.76k forks source link

zpool hangs #14426

Closed Mitmischer closed 1 year ago

Mitmischer commented 1 year ago

System information

Type | Version/Name Distribution Name | Ubuntu Distribution Version | 22.04.1 LTS Kernel Version | 5.15.0-50 Architecture | x86_64 OpenZFS Version | 2.1.4

Describe the problem you're observing

The command zpool status hangs. What additional info should I provide for debugging? How can I reboot the system safely? Below you can find the strace of zpool status where the command is sitting for several days now. From server logs I deduced that other incovations must have been wating for more than 30 days.

Describe how to reproduce the problem

Include any warning/errors/backtraces from the system logs

 ~> strace zpool status
execve("/usr/sbin/zpool", ["zpool", "status"], 0x7ffc4bb70ab8 /* 20 vars */) = 0
brk(NULL)                               = 0x55d8cc9c5000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffd987b8e60) = -1 EINVAL (Invalid argument)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f495156a000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/glibc-hwcaps/x86-64-v3/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/glibc-hwcaps/x86-64-v3", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/glibc-hwcaps/x86-64-v2/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/glibc-hwcaps/x86-64-v2", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/tls/haswell/x86_64/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/tls/haswell/x86_64", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/tls/haswell/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/tls/haswell", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/tls/x86_64/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/tls/x86_64", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/tls/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/tls", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/haswell/x86_64/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/haswell/x86_64", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/haswell/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/haswell", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/x86_64/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/x86_64", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib/libzfs.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
newfstatat(AT_FDCWD, "/usr/local/lib/R/site-library/ospsuite/lib", 0x7ffd987b8080, 0) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=40143, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 40143, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4951560000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libzfs.so.4", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=420624, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 435944, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f49514f5000
mmap(0x7f4951502000, 274432, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x7f4951502000
mmap(0x7f4951545000, 81920, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x50000) = 0x7f4951545000
mmap(0x7f4951559000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x63000) = 0x7f4951559000
mmap(0x7f495155d000, 9960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f495155d000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libzfs_core.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=118560, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 124776, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f49514d6000
mmap(0x7f49514dd000, 65536, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f49514dd000
mmap(0x7f49514ed000, 20480, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f49514ed000
mmap(0x7f49514f2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b000) = 0x7f49514f2000
mmap(0x7f49514f4000, 1896, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f49514f4000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libuutil.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=64136, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 70664, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f49514c4000
mmap(0x7f49514c9000, 24576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x7f49514c9000
mmap(0x7f49514cf000, 16384, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x7f49514cf000
mmap(0x7f49514d3000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe000) = 0x7f49514d3000
mmap(0x7f49514d5000, 1032, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f49514d5000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libnvpair.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=100496, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 102488, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f49514aa000
mmap(0x7f49514af000, 53248, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x7f49514af000
mmap(0x7f49514bc000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x12000) = 0x7f49514bc000
mmap(0x7f49514c2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f49514c2000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libm.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=940560, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 942344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f49513c3000
mmap(0x7f49513d1000, 507904, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe000) = 0x7f49513d1000
mmap(0x7f495144d000, 372736, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8a000) = 0x7f495144d000
mmap(0x7f49514a8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe4000) = 0x7f49514a8000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libblkid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=220192, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f49513c1000
mmap(NULL, 222136, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f495138a000
mprotect(0x7f4951391000, 172032, PROT_NONE) = 0
mmap(0x7f4951391000, 131072, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f4951391000
mmap(0x7f49513b1000, 36864, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x7f49513b1000
mmap(0x7f49513bb000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x30000) = 0x7f49513bb000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=30920, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 32808, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4951381000
mmap(0x7f4951383000, 16384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f4951383000
mmap(0x7f4951387000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f4951387000
mmap(0x7f4951388000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f4951388000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\237\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0 \0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0"..., 48, 848) = 48
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0i8\235HZ\227\223\333\350s\360\352,\223\340."..., 68, 896) = 68
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=2216304, ...}, AT_EMPTY_PATH) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
mmap(NULL, 2260560, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4951159000
mmap(0x7f4951181000, 1658880, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x28000) = 0x7f4951181000
mmap(0x7f4951316000, 360448, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1bd000) = 0x7f4951316000
mmap(0x7f495136e000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x214000) = 0x7f495136e000
mmap(0x7f4951374000, 52816, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4951374000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libcrypto.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=4447536, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 4461760, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950d17000
mmap(0x7f4950dc9000, 2478080, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb2000) = 0x7f4950dc9000
mmap(0x7f4951026000, 860160, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x30f000) = 0x7f4951026000
mmap(0x7f49510f8000, 385024, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3e0000) = 0x7f49510f8000
mmap(0x7f4951156000, 9408, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4951156000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libz.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=108936, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 110776, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950cfb000
mprotect(0x7f4950cfd000, 98304, PROT_NONE) = 0
mmap(0x7f4950cfd000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f4950cfd000
mmap(0x7f4950d0e000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13000) = 0x7f4950d0e000
mmap(0x7f4950d15000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19000) = 0x7f4950d15000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libudev.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=166240, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 170272, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950cd1000
mprotect(0x7f4950cd5000, 147456, PROT_NONE) = 0
mmap(0x7f4950cd5000, 106496, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x7f4950cd5000
mmap(0x7f4950cef000, 36864, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e000) = 0x7f4950cef000
mmap(0x7f4950cf9000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x7f4950cf9000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libtirpc.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=182912, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4950ccf000
mmap(NULL, 187256, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950ca1000
mprotect(0x7f4950ca8000, 151552, PROT_NONE) = 0
mmap(0x7f4950ca8000, 110592, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f4950ca8000
mmap(0x7f4950cc3000, 36864, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f4950cc3000
mmap(0x7f4950ccd000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2b000) = 0x7f4950ccd000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=338712, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 340960, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950c4d000
mprotect(0x7f4950c58000, 282624, PROT_NONE) = 0
mmap(0x7f4950c58000, 229376, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x7f4950c58000
mmap(0x7f4950c90000, 49152, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x43000) = 0x7f4950c90000
mmap(0x7f4950c9d000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4f000) = 0x7f4950c9d000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libkrb5.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=828000, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 830576, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950b82000
mprotect(0x7f4950ba3000, 634880, PROT_NONE) = 0
mmap(0x7f4950ba3000, 380928, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21000) = 0x7f4950ba3000
mmap(0x7f4950c00000, 249856, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7e000) = 0x7f4950c00000
mmap(0x7f4950c3e000, 61440, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xbb000) = 0x7f4950c3e000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libk5crypto.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=182928, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 188472, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950b53000
mprotect(0x7f4950b57000, 163840, PROT_NONE) = 0
mmap(0x7f4950b57000, 110592, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x7f4950b57000
mmap(0x7f4950b72000, 49152, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x7f4950b72000
mmap(0x7f4950b7f000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2b000) = 0x7f4950b7f000
mmap(0x7f4950b81000, 56, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4950b81000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libcom_err.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=18504, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 20552, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950b4d000
mmap(0x7f4950b4f000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f4950b4f000
mmap(0x7f4950b50000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f4950b50000
mmap(0x7f4950b51000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f4950b51000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libkrb5support.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=52080, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4950b4b000
mmap(NULL, 54224, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950b3d000
mprotect(0x7f4950b40000, 36864, PROT_NONE) = 0
mmap(0x7f4950b40000, 24576, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f4950b40000
mmap(0x7f4950b46000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x9000) = 0x7f4950b46000
mmap(0x7f4950b49000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x7f4950b49000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=22600, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 24592, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950b36000
mmap(0x7f4950b38000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7f4950b38000
mmap(0x7f4950b3a000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x7f4950b3a000
mmap(0x7f4950b3b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x7f4950b3b000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libresolv.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=68552, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 80456, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f4950b22000
mmap(0x7f4950b25000, 40960, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x7f4950b25000
mmap(0x7f4950b2f000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x7f4950b2f000
mmap(0x7f4950b32000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0x7f4950b32000
mmap(0x7f4950b34000, 6728, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f4950b34000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4950b20000
mmap(NULL, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f4950b1b000
arch_prctl(ARCH_SET_FS, 0x7f4950b1d7c0) = 0
set_tid_address(0x7f4950b1da90)         = 2179784
set_robust_list(0x7f4950b1daa0, 24)     = 0
rseq(0x7f4950b1e160, 0x20, 0, 0x53053053) = 0
mprotect(0x7f495136e000, 16384, PROT_READ) = 0
mprotect(0x7f4950b32000, 4096, PROT_READ) = 0
mprotect(0x7f4950b3b000, 4096, PROT_READ) = 0
mprotect(0x7f4950b49000, 4096, PROT_READ) = 0
mprotect(0x7f4950b51000, 4096, PROT_READ) = 0
mprotect(0x7f4950b7f000, 4096, PROT_READ) = 0
mprotect(0x7f4950c3e000, 53248, PROT_READ) = 0
mprotect(0x7f4950c9d000, 8192, PROT_READ) = 0
mprotect(0x7f4950ccd000, 4096, PROT_READ) = 0
mprotect(0x7f4950cf9000, 4096, PROT_READ) = 0
mprotect(0x7f4950d15000, 4096, PROT_READ) = 0
mprotect(0x7f49510f8000, 372736, PROT_READ) = 0
mprotect(0x7f4951388000, 4096, PROT_READ) = 0
mprotect(0x7f49513bb000, 20480, PROT_READ) = 0
mprotect(0x7f49514a8000, 4096, PROT_READ) = 0
mprotect(0x7f49514c2000, 4096, PROT_READ) = 0
mprotect(0x7f49514d3000, 4096, PROT_READ) = 0
mprotect(0x7f49514f2000, 4096, PROT_READ) = 0
mprotect(0x7f4951559000, 8192, PROT_READ) = 0
mprotect(0x55d8cbcfc000, 8192, PROT_READ) = 0
mprotect(0x7f49515a4000, 8192, PROT_READ) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
munmap(0x7f4951560000, 40143)           = 0
getrandom("\x5f\xb7\x70\xe0\x8f\xd8\xdc\x45", 8, GRND_NONBLOCK) = 8
brk(NULL)                               = 0x55d8cc9c5000
brk(0x55d8cc9e6000)                     = 0x55d8cc9e6000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=3048928, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 3048928, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f4950832000
close(3)                                = 0
access("/run/systemd/container", R_OK)  = -1 ENOENT (No such file or directory)
access("/sys/module/zfs", F_OK)         = 0
openat(AT_FDCWD, "/dev/zfs", O_RDWR|O_CLOEXEC) = 3
close(3)                                = 0
openat(AT_FDCWD, "/usr/lib/x86_64-linux-gnu/gconv/gconv-modules.cache", O_RDONLY) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=27002, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 27002, PROT_READ, MAP_SHARED, 3, 0) = 0x7f4951563000
close(3)                                = 0
futex(0x7f4951373a6c, FUTEX_WAKE_PRIVATE, 2147483647) = 0
openat(AT_FDCWD, "/dev/zfs", O_RDWR|O_EXCL|O_CLOEXEC) = 3
openat(AT_FDCWD, "/proc/self/mounts", O_RDONLY|O_CLOEXEC) = 4
openat(AT_FDCWD, "/dev/zfs", O_RDWR|O_CLOEXEC) = 5
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/redundant_metadata", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/sync", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/checksum", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/dedup", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/compression", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/snapdir", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/snapdev", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/aclmode", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/acltype", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/aclinherit", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/copies", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/primarycache", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/secondarycache", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/logbias", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/xattr", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/dnodesize", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/volmode", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/atime", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/relatime", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/devices", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/exec", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/setuid", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/readonly", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/zoned", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/vscan", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/nbmand", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/overlay", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/version", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/canmount", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/mounted", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/defer_destroy", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/keystatus", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/normalization", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/casesensitivity", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/keyformat", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/encryption", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/utf8only", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/origin", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/clones", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/mountpoint", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/sharenfs", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/type", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/sharesmb", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/mlslabel", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/context", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/fscontext", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/defcontext", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/rootcontext", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/receive_resume_token", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/encryptionroot", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/keylocation", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/redact_snaps", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/used", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/available", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/referenced", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/compressratio", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/refcompressratio", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/volblocksize", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/usedbysnapshots", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/usedbydataset", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/usedbychildren", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/usedbyrefreservation", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/userrefs", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/written", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/logicalused", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/logicalreferenced", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/filesystem_count", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/snapshot_count", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/guid", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/createtxg", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/pbkdf2iters", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/objsetid", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/quota", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/reservation", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/volsize", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/refquota", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/refreservation", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/filesystem_limit", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/snapshot_limit", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/recordsize", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/special_small_blocks", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/numclones", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/name", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/iscsioptions", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/stmf_sbd_lu", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/useraccounting", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/unique", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/inconsistent", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/ivsetguid", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/prevsnap", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/pbkdf2salt", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/keyguid", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/redacted", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/remaptxg", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.dataset/creation", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/altroot", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/bootfs", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/cachefile", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/comment", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/compatibility", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/size", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/free", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/freeing", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/checkpoint", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/leaked", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/allocated", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/expandsize", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/fragmentation", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/capacity", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/guid", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/load_guid", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/health", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/dedupratio", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/version", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/ashift", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/delegation", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/autoreplace", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/listsnapshots", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/autoexpand", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/readonly", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/multihost", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/failmode", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/autotrim", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/name", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/maxblocksize", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/tname", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/maxdnodesize", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/properties.pool/dedupditto", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:async_destroy", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:empty_bpobj", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.illumos:lz4_compress", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.joyent:multi_vdev_crash_dump", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:spacemap_histogram", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:enabled_txg", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:hole_birth", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:zpool_checkpoint", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:spacemap_v2", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:extensible_dataset", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:bookmarks", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.joyent:filesystem_limits", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:embedded_data", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:livelist", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:log_spacemap", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.open-zfs:large_blocks", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.zfsonlinux:large_dnode", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.illumos:sha512", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.illumos:skein", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.illumos:edonr", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:redaction_bookmarks", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:redacted_datasets", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:bookmark_written", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:device_removal", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.delphix:obsolete_counts", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.zfsonlinux:userobj_accounting", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.datto:bookmark_v2", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.datto:encryption", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.zfsonlinux:project_quota", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.zfsonlinux:allocation_classes", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/com.datto:resilver_defer", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.openzfs:device_rebuild", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.freebsd:zstd_compress", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
newfstatat(AT_FDCWD, "/sys/module/zfs/features.pool/org.openzfs:draid", {st_mode=S_IFDIR|0755, st_size=0, ...}, 0) = 0
mmap(NULL, 266240, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f49507f1000
ioctl(3, ZFS_IOC_POOL_CONFIGS, 0x7ffd987b2000) = 0
munmap(0x7f49507f1000, 266240)          = 0
ioctl(3, ZFS_IOC_POOL_STATS
mabod commented 1 year ago

Is the filesystem otherwise working correctly? So you can read/write to zfs datasets? Is it just the zpool status command that hangs?

Mitmischer commented 1 year ago

Thanks for the prompt reply! The filesystem itself seems to be working file, I can read and write files.

I tried some commands that I think are non-destructive and there might be a pattern to see here: zpool status, zpool list, zpool iostat, zpool history all hang. Interestingly, zpool status tank and zpool history tank do NOT hang! zpool iostat tank does hang though. zpool wait tank, zfs userspace tank, zfs get all tank, zfs wait tank and zfs list do not hang.

Is there anything else I should try? I'm afraid of losing data when scrubbing or rebooting the system, but I'm making a backup right now.

rincebrain commented 1 year ago

What does dmesg say right now? What are the last 100 lines or so in /proc/spl/kstat/zfs/dbgmsg?

mabod commented 1 year ago

I tried some commands that I think are non-destructive and there might be a pattern to see here: zpool status, zpool list, zpool iostat, zpool history all hang. Interestingly, zpool status tank and zpool history tank do NOT hang!

Have you other pools than tank? Did you delete any pool recently or did you unplug any device recently?

What does zpool status tank say?

Mitmischer commented 1 year ago

Thanks again :)

dmesg right now is not exciting:

[8489455.753434] vethc41bc27: renamed from eth0
[8489455.816659] br-3406282d8afe: port 3(veth6922e50) entered disabled state
[8489455.817532] device veth6922e50 left promiscuous mode
[8489455.817538] br-3406282d8afe: port 3(veth6922e50) entered disabled state
[8489515.714172] br-3406282d8afe: port 3(vethdbd0969) entered blocking state
[8489515.714176] br-3406282d8afe: port 3(vethdbd0969) entered disabled state
[8489515.714228] device vethdbd0969 entered promiscuous mode
[8489516.898691] eth0: renamed from vethcb648c0
[8489516.918664] IPv6: ADDRCONF(NETDEV_CHANGE): vethdbd0969: link becomes ready
[8489516.918713] br-3406282d8afe: port 3(vethdbd0969) entered blocking state
[8489516.918716] br-3406282d8afe: port 3(vethdbd0969) entered forwarding state
[8489517.829719] br-3406282d8afe: port 3(vethdbd0969) entered disabled state
[8489517.829799] vethcb648c0: renamed from eth0
[8489517.896646] br-3406282d8afe: port 3(vethdbd0969) entered disabled state
[8489517.897395] device vethdbd0969 left promiscuous mode
[8489517.897400] br-3406282d8afe: port 3(vethdbd0969) entered disabled state
[8489577.841912] br-3406282d8afe: port 3(veth5542d97) entered blocking state
[8489577.841918] br-3406282d8afe: port 3(veth5542d97) entered disabled state
[8489577.841991] device veth5542d97 entered promiscuous mode
[8489579.079104] eth0: renamed from vethf72681e
[8489579.101794] IPv6: ADDRCONF(NETDEV_CHANGE): veth5542d97: link becomes ready
[8489579.101843] br-3406282d8afe: port 3(veth5542d97) entered blocking state
[8489579.101845] br-3406282d8afe: port 3(veth5542d97) entered forwarding state
[8489580.135056] br-3406282d8afe: port 3(veth5542d97) entered disabled state
[8489580.135132] vethf72681e: renamed from eth0
[8489580.187481] br-3406282d8afe: port 3(veth5542d97) entered disabled state
[8489580.187976] device veth5542d97 left promiscuous mode
[8489580.187979] br-3406282d8afe: port 3(veth5542d97) entered disabled state
[8489640.166939] br-3406282d8afe: port 3(veth1573af9) entered blocking state
[8489640.166944] br-3406282d8afe: port 3(veth1573af9) entered disabled state
[8489640.167007] device veth1573af9 entered promiscuous mode
[8489641.398150] eth0: renamed from veth3386a08
[8489641.416973] IPv6: ADDRCONF(NETDEV_CHANGE): veth1573af9: link becomes ready
[8489641.417024] br-3406282d8afe: port 3(veth1573af9) entered blocking state
[8489641.417027] br-3406282d8afe: port 3(veth1573af9) entered forwarding state
[8489642.201908] br-3406282d8afe: port 3(veth1573af9) entered disabled state
[8489642.201986] veth3386a08: renamed from eth0
[8489642.249883] br-3406282d8afe: port 3(veth1573af9) entered disabled state
[8489642.250708] device veth1573af9 left promiscuous mode
[8489642.250712] br-3406282d8afe: port 3(veth1573af9) entered disabled state
[8489702.200708] br-3406282d8afe: port 3(vethc97aaa7) entered blocking state
[8489702.200712] br-3406282d8afe: port 3(vethc97aaa7) entered disabled state
[8489702.200911] device vethc97aaa7 entered promiscuous mode
[8489702.615238] eth0: renamed from vethb36bbf8
[8489702.656185] IPv6: ADDRCONF(NETDEV_CHANGE): vethc97aaa7: link becomes ready
[8489702.656234] br-3406282d8afe: port 3(vethc97aaa7) entered blocking state
[8489702.656237] br-3406282d8afe: port 3(vethc97aaa7) entered forwarding state
[8489703.366442] br-3406282d8afe: port 3(vethc97aaa7) entered disabled state
[8489703.366496] vethb36bbf8: renamed from eth0
[8489703.427369] br-3406282d8afe: port 3(vethc97aaa7) entered disabled state
[8489703.428164] device vethc97aaa7 left promiscuous mode
[8489703.428168] br-3406282d8afe: port 3(vethc97aaa7) entered disabled state
[8489763.360857] br-3406282d8afe: port 3(vethb69b9f3) entered blocking state
[8489763.360861] br-3406282d8afe: port 3(vethb69b9f3) entered disabled state
[8489763.362058] device vethb69b9f3 entered promiscuous mode
[8489764.655523] eth0: renamed from vethd977211
[8489764.679398] IPv6: ADDRCONF(NETDEV_CHANGE): vethb69b9f3: link becomes ready
[8489764.679448] br-3406282d8afe: port 3(vethb69b9f3) entered blocking state
[8489764.679451] br-3406282d8afe: port 3(vethb69b9f3) entered forwarding state
[8489765.413081] br-3406282d8afe: port 3(vethb69b9f3) entered disabled state
[8489765.413176] vethd977211: renamed from eth0
[8489765.482857] br-3406282d8afe: port 3(vethb69b9f3) entered disabled state
[8489765.486836] device vethb69b9f3 left promiscuous mode
[8489765.486841] br-3406282d8afe: port 3(vethb69b9f3) entered disabled state
[8489825.388007] br-3406282d8afe: port 3(vetha7caad9) entered blocking state
[8489825.388012] br-3406282d8afe: port 3(vetha7caad9) entered disabled state
[8489825.388229] device vetha7caad9 entered promiscuous mode
[8489826.946798] eth0: renamed from veth5eb6aea
[8489826.962591] IPv6: ADDRCONF(NETDEV_CHANGE): vetha7caad9: link becomes ready
[8489826.962649] br-3406282d8afe: port 3(vetha7caad9) entered blocking state
[8489826.962652] br-3406282d8afe: port 3(vetha7caad9) entered forwarding state
[8489827.982152] br-3406282d8afe: port 3(vetha7caad9) entered disabled state
[8489827.982239] veth5eb6aea: renamed from eth0
[8489828.056738] br-3406282d8afe: port 3(vetha7caad9) entered disabled state
[8489828.057601] device vetha7caad9 left promiscuous mode
[8489828.057605] br-3406282d8afe: port 3(vetha7caad9) entered disabled state
[8489887.979331] br-3406282d8afe: port 3(veth6244045) entered blocking state
[8489887.979336] br-3406282d8afe: port 3(veth6244045) entered disabled state
[8489887.979407] device veth6244045 entered promiscuous mode
[8489888.710020] eth0: renamed from vethc6f516d
[8489888.725656] IPv6: ADDRCONF(NETDEV_CHANGE): veth6244045: link becomes ready
[8489888.725719] br-3406282d8afe: port 3(veth6244045) entered blocking state
[8489888.725721] br-3406282d8afe: port 3(veth6244045) entered forwarding state
[8489889.502319] br-3406282d8afe: port 3(veth6244045) entered disabled state
[8489889.502382] vethc6f516d: renamed from eth0
[8489889.554411] br-3406282d8afe: port 3(veth6244045) entered disabled state
[8489889.554985] device veth6244045 left promiscuous mode
[8489889.554989] br-3406282d8afe: port 3(veth6244045) entered disabled state
[8489949.456260] br-3406282d8afe: port 3(vethabdd9bd) entered blocking state
[8489949.456264] br-3406282d8afe: port 3(vethabdd9bd) entered disabled state
[8489949.456332] device vethabdd9bd entered promiscuous mode
[8489950.562839] eth0: renamed from veth3ae6187
[8489950.593702] IPv6: ADDRCONF(NETDEV_CHANGE): vethabdd9bd: link becomes ready
[8489950.593758] br-3406282d8afe: port 3(vethabdd9bd) entered blocking state
[8489950.593761] br-3406282d8afe: port 3(vethabdd9bd) entered forwarding state
[8489951.764272] br-3406282d8afe: port 3(vethabdd9bd) entered disabled state
[8489951.764339] veth3ae6187: renamed from eth0
[8489951.829577] br-3406282d8afe: port 3(vethabdd9bd) entered disabled state
[8489951.830135] device vethabdd9bd left promiscuous mode
[8489951.830139] br-3406282d8afe: port 3(vethabdd9bd) entered disabled state
[8490011.725731] br-3406282d8afe: port 3(vethd13fdbe) entered blocking state
[8490011.725736] br-3406282d8afe: port 3(vethd13fdbe) entered disabled state
[8490011.725786] device vethd13fdbe entered promiscuous mode
[8490011.725849] br-3406282d8afe: port 3(vethd13fdbe) entered blocking state
[8490011.725851] br-3406282d8afe: port 3(vethd13fdbe) entered forwarding state
[8490011.726059] br-3406282d8afe: port 3(vethd13fdbe) entered disabled state
[8490013.088324] eth0: renamed from veth7152b4f
[8490013.104079] IPv6: ADDRCONF(NETDEV_CHANGE): vethd13fdbe: link becomes ready
[8490013.104132] br-3406282d8afe: port 3(vethd13fdbe) entered blocking state
[8490013.104135] br-3406282d8afe: port 3(vethd13fdbe) entered forwarding state
[8490014.086772] br-3406282d8afe: port 3(vethd13fdbe) entered disabled state
[8490014.086857] veth7152b4f: renamed from eth0
[8490014.153491] br-3406282d8afe: port 3(vethd13fdbe) entered disabled state
[8490014.154294] device vethd13fdbe left promiscuous mode
[8490014.154298] br-3406282d8afe: port 3(vethd13fdbe) entered disabled state
[8490074.103493] br-3406282d8afe: port 3(vethf1eda82) entered blocking state
[8490074.103497] br-3406282d8afe: port 3(vethf1eda82) entered disabled state
[8490074.103546] device vethf1eda82 entered promiscuous mode
[8490075.260024] eth0: renamed from veth7df9d2b
[8490075.295186] IPv6: ADDRCONF(NETDEV_CHANGE): vethf1eda82: link becomes ready
[8490075.295240] br-3406282d8afe: port 3(vethf1eda82) entered blocking state
[8490075.295242] br-3406282d8afe: port 3(vethf1eda82) entered forwarding state
[8490076.064905] br-3406282d8afe: port 3(vethf1eda82) entered disabled state
[8490076.065426] veth7df9d2b: renamed from eth0
[8490076.121705] br-3406282d8afe: port 3(vethf1eda82) entered disabled state
[8490076.122624] device vethf1eda82 left promiscuous mode
[8490076.122636] br-3406282d8afe: port 3(vethf1eda82) entered disabled state
[8490136.038812] br-3406282d8afe: port 3(veth85a6886) entered blocking state
[8490136.038816] br-3406282d8afe: port 3(veth85a6886) entered disabled state
[8490136.038932] device veth85a6886 entered promiscuous mode
[8490137.003469] eth0: renamed from vethcd06520
[8490137.034319] IPv6: ADDRCONF(NETDEV_CHANGE): veth85a6886: link becomes ready
[8490137.034366] br-3406282d8afe: port 3(veth85a6886) entered blocking state
[8490137.034368] br-3406282d8afe: port 3(veth85a6886) entered forwarding state
[8490137.782380] br-3406282d8afe: port 3(veth85a6886) entered disabled state
[8490137.782463] vethcd06520: renamed from eth0
[8490137.836806] br-3406282d8afe: port 3(veth85a6886) entered disabled state
[8490137.837605] device veth85a6886 left promiscuous mode
[8490137.837609] br-3406282d8afe: port 3(veth85a6886) entered disabled state
[8490197.796161] br-3406282d8afe: port 3(veth05ea919) entered blocking state
[8490197.796166] br-3406282d8afe: port 3(veth05ea919) entered disabled state
[8490197.796243] device veth05ea919 entered promiscuous mode
[8490197.796347] br-3406282d8afe: port 3(veth05ea919) entered blocking state
[8490197.796349] br-3406282d8afe: port 3(veth05ea919) entered forwarding state
[8490197.796553] br-3406282d8afe: port 3(veth05ea919) entered disabled state
[8490198.785525] eth0: renamed from veth7e9f60d
[8490198.813515] IPv6: ADDRCONF(NETDEV_CHANGE): veth05ea919: link becomes ready
[8490198.813572] br-3406282d8afe: port 3(veth05ea919) entered blocking state
[8490198.813575] br-3406282d8afe: port 3(veth05ea919) entered forwarding state
[8490199.614174] br-3406282d8afe: port 3(veth05ea919) entered disabled state
[8490199.614256] veth7e9f60d: renamed from eth0
[8490199.673595] br-3406282d8afe: port 3(veth05ea919) entered disabled state
[8490199.675221] device veth05ea919 left promiscuous mode
[8490199.675234] br-3406282d8afe: port 3(veth05ea919) entered disabled state

The output of dbgmsg (does not hang):

~> sudo tail -100 /proc/spl/kstat/zfs/dbgmsg
1674541104   metaslab.c:3646:metaslab_condense(): condensing: txg 2579025, msp[708] ffff8ce012dde800, vdev id 0, spa tank, smp size 663968, segments 11121, forcing condense=FALSE
1674546465   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2580072, spa tank, vdev_id 0, ms_id 709, smp_length 627192, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8285462319 ms, loading_time 84 ms, ms_max_size 4294877184, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674547069   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2580190, spa tank, vdev_id 0, ms_id 686, weight 780000000000001, selected txg 2580072 (604388 ms ago), alloc_txg 2580072, loaded 50732529 ms ago, max_size 2147475456
1674547207   metaslab.c:3646:metaslab_condense(): condensing: txg 2580217, msp[682] ffff8ce012c88000, vdev id 0, spa tank, smp size 545352, segments 25017, forcing condense=FALSE
1674550863   metaslab.c:3646:metaslab_condense(): condensing: txg 2580931, msp[709] ffff8ce012ddf800, vdev id 0, spa tank, smp size 627192, segments 20937, forcing condense=FALSE
1674552327   metaslab.c:3646:metaslab_condense(): condensing: txg 2581217, msp[688] ffff8ce012c99800, vdev id 0, spa tank, smp size 539496, segments 22947, forcing condense=FALSE
1674566151   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2583917, spa tank, vdev_id 0, ms_id 712, smp_length 418464, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3036237779 ms, loading_time 66 ms, ms_max_size 2154135552, max size error 6537216, old_weight 7c0000000000001, new_weight 7c0000000000001
1674566756   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2584035, spa tank, vdev_id 0, ms_id 689, weight 780000000000001, selected txg 2583917 (604377 ms ago), alloc_txg 2583917, loaded 63137794 ms ago, max_size 1073799168
1674572188   metaslab.c:3646:metaslab_condense(): condensing: txg 2585096, msp[690] ffff8ce012c9b000, vdev id 0, spa tank, smp size 552288, segments 28524, forcing condense=FALSE
1674572618   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2585180, spa tank, vdev_id 0, ms_id 713, smp_length 517304, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8373788327 ms, loading_time 64 ms, ms_max_size 4294950912, max size error 4291477504, old_weight 7c0000000000001, new_weight 7c0000000000001
1674572725   metaslab.c:3646:metaslab_condense(): condensing: txg 2585201, msp[692] ffff8ce012c99000, vdev id 0, spa tank, smp size 559288, segments 32797, forcing condense=FALSE
1674573222   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2585298, spa tank, vdev_id 0, ms_id 688, weight 740000000000001, selected txg 2585180 (604455 ms ago), alloc_txg 2585180, loaded 69604324 ms ago, max_size 1073725440
1674578854   metaslab.c:3646:metaslab_condense(): condensing: txg 2586398, msp[704] ffff8ce012dda800, vdev id 0, spa tank, smp size 555968, segments 20787, forcing condense=FALSE
1674581342   metaslab.c:3646:metaslab_condense(): condensing: txg 2586884, msp[683] ffff8ce012c89800, vdev id 0, spa tank, smp size 547464, segments 30910, forcing condense=FALSE
1674582331   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2587077, spa tank, vdev_id 0, ms_id 714, smp_length 343584, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8383500747 ms, loading_time 76 ms, ms_max_size 4294926336, max size error 4291682304, old_weight 7c0000000000001, new_weight 7c0000000000001
1674582331   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2587077, spa tank, vdev_id 0, ms_id 718, smp_length 310808, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8383500747 ms, loading_time 99 ms, ms_max_size 4300308480, max size error 4296704000, old_weight 7c0000000000001, new_weight 800000000000001
1674582930   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2587194, spa tank, vdev_id 0, ms_id 694, weight 780000000000001, selected txg 2587076 (604169 ms ago), alloc_txg 2587076, loaded 67275039 ms ago, max_size 1599619072
1674582935   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2587195, spa tank, vdev_id 0, ms_id 683, weight 780000000000001, selected txg 2587077 (604434 ms ago), alloc_txg 2587077, loaded 80417894 ms ago, max_size 1074012160
1674583385   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2587283, spa tank, vdev_id 0, ms_id 719, smp_length 387200, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3010008346 ms, loading_time 90 ms, ms_max_size 2147819520, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674583580   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2587321, spa tank, vdev_id 0, ms_id 720, smp_length 493888, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8384750066 ms, loading_time 68 ms, ms_max_size 4296499200, max size error 4292354048, old_weight 7c0000000000001, new_weight 800000000000001
1674583815   metaslab.c:3646:metaslab_condense(): condensing: txg 2587367, msp[629] ffff8ce012b9c000, vdev id 0, spa tank, smp size 871928, segments 52865, forcing condense=FALSE
1674583989   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2587401, spa tank, vdev_id 0, ms_id 692, weight 780000000000001, selected txg 2587283 (604364 ms ago), alloc_txg 2587283, loaded 71611589 ms ago, max_size 2147475456
1674584522   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2587505, spa tank, vdev_id 0, ms_id 719, weight 7c0000000000001, selected txg 2587387 (604114 ms ago), alloc_txg 1999968, loaded 1136579 ms ago, max_size 2147819520
1674584998   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2587598, spa tank, vdev_id 0, ms_id 719, smp_length 387200, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 476331 ms, loading_time 46 ms, ms_max_size 2147819520, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674585014   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2587601, spa tank, vdev_id 0, ms_id 722, smp_length 590088, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8386183638 ms, loading_time 67 ms, ms_max_size 4294959104, max size error 4289003520, old_weight 7c0000000000001, new_weight 7c0000000000001
1674585602   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2587716, spa tank, vdev_id 0, ms_id 682, weight 740000000000001, selected txg 2587598 (604534 ms ago), alloc_txg 2587598, loaded 92199062 ms ago, max_size 1073569792
1674585618   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2587719, spa tank, vdev_id 0, ms_id 693, weight 740000000000001, selected txg 2587601 (604464 ms ago), alloc_txg 2587601, loaded 69963025 ms ago, max_size 1073627136
1674585740   metaslab.c:3646:metaslab_condense(): condensing: txg 2587743, msp[722] ffff8ce012dea000, vdev id 0, spa tank, smp size 590088, segments 6759, forcing condense=FALSE
1674590399   metaslab.c:3646:metaslab_condense(): condensing: txg 2588653, msp[720] ffff8ce012dea800, vdev id 0, spa tank, smp size 553488, segments 18373, forcing condense=FALSE
1674592862   metaslab.c:3646:metaslab_condense(): condensing: txg 2589134, msp[701] ffff8ce012ddb000, vdev id 0, spa tank, smp size 528568, segments 18498, forcing condense=FALSE
1674597388   metaslab.c:3646:metaslab_condense(): condensing: txg 2590018, msp[697] ffff8ce012c9e800, vdev id 0, spa tank, smp size 529080, segments 22559, forcing condense=FALSE
1674603635   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2591238, spa tank, vdev_id 0, ms_id 725, smp_length 343328, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3015053860 ms, loading_time 88 ms, ms_max_size 2147672064, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674603635   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2591238, spa tank, vdev_id 0, ms_id 724, smp_length 914944, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2629918087 ms, loading_time 132 ms, ms_max_size 2148196352, max size error 696320, old_weight 7c0000000000001, new_weight 7c0000000000001
1674603881   metaslab.c:3646:metaslab_condense(): condensing: txg 2591286, msp[724] ffff8ce012dee800, vdev id 0, spa tank, smp size 914944, segments 25579, forcing condense=FALSE
1674604131   metaslab.c:3646:metaslab_condense(): condensing: txg 2591335, msp[706] ffff8ce012dda000, vdev id 0, spa tank, smp size 839272, segments 52435, forcing condense=FALSE
1674604234   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2591355, spa tank, vdev_id 0, ms_id 701, weight 780000000000001, selected txg 2591237 (604177 ms ago), alloc_txg 2591237, loaded 71284791 ms ago, max_size 1277517824
1674604239   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2591356, spa tank, vdev_id 0, ms_id 698, weight 740000000000001, selected txg 2591238 (604425 ms ago), alloc_txg 2591238, loaded 71289904 ms ago, max_size 1073684480
1674605740   spa_history.c:307:spa_history_log_sync(): txg 2591650 trim vdev=/dev/disk/by-id/nvme-KINGSTON_SA2000M8500G_50026B7684E0D1BE-part3 activated
1674605747   spa_history.c:307:spa_history_log_sync(): txg 2591683 trim vdev=/dev/disk/by-id/nvme-KINGSTON_SA2000M8500G_50026B7684E0D1BE-part3 complete
1674610412   metaslab.c:3646:metaslab_condense(): condensing: txg 2592596, msp[690] ffff8ce012c9b000, vdev id 0, spa tank, smp size 579672, segments 32431, forcing condense=FALSE
1674613960   metaslab.c:3646:metaslab_condense(): condensing: txg 2593289, msp[718] ffff8ce012def000, vdev id 0, spa tank, smp size 561264, segments 21899, forcing condense=FALSE
1674615015   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2593495, spa tank, vdev_id 0, ms_id 726, smp_length 429472, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3000337346 ms, loading_time 66 ms, ms_max_size 2153291776, max size error 5586944, old_weight 7c0000000000001, new_weight 7c0000000000001
1674615619   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2593613, spa tank, vdev_id 0, ms_id 704, weight 780000000000001, selected txg 2593495 (604447 ms ago), alloc_txg 2593495, loaded 78625037 ms ago, max_size 2147459072
1674617355   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2593952, spa tank, vdev_id 0, ms_id 727, smp_length 321360, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3041742199 ms, loading_time 43 ms, ms_max_size 4279222272, max size error 151379968, old_weight 7c0000000000001, new_weight 7c0000000000001
1674617580   metaslab.c:3646:metaslab_condense(): condensing: txg 2593996, msp[707] ffff8ce012dd8800, vdev id 0, spa tank, smp size 537144, segments 17216, forcing condense=FALSE
1674617959   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2594070, spa tank, vdev_id 0, ms_id 706, weight 780000000000001, selected txg 2593952 (604393 ms ago), alloc_txg 2593952, loaded 78911452 ms ago, max_size 1073946624
1674624364   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2595321, spa tank, vdev_id 0, ms_id 728, smp_length 647472, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2979455479 ms, loading_time 78 ms, ms_max_size 2148188160, max size error 385024, old_weight 7c0000000000001, new_weight 7c0000000000001
1674624553   metaslab.c:3646:metaslab_condense(): condensing: txg 2595358, msp[728] ffff8ce012dec000, vdev id 0, spa tank, smp size 647472, segments 20367, forcing condense=FALSE
1674624968   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2595439, spa tank, vdev_id 0, ms_id 695, weight 780000000000001, selected txg 2595321 (604606 ms ago), alloc_txg 2595321, loaded 105960477 ms ago, max_size 1073741824
1674625276   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2595499, spa tank, vdev_id 0, ms_id 730, smp_length 499984, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8373927882 ms, loading_time 91 ms, ms_max_size 4294787072, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674625276   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2595499, spa tank, vdev_id 0, ms_id 729, smp_length 761720, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3032167980 ms, loading_time 104 ms, ms_max_size 4294811648, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674625874   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2595616, spa tank, vdev_id 0, ms_id 712, weight 780000000000001, selected txg 2595498 (604125 ms ago), alloc_txg 2595498, loaded 59722121 ms ago, max_size 2102149120
1674625880   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2595617, spa tank, vdev_id 0, ms_id 720, weight 780000000000001, selected txg 2595499 (604427 ms ago), alloc_txg 2595499, loaded 42298973 ms ago, max_size 2147418112
1674627984   metaslab.c:3646:metaslab_condense(): condensing: txg 2596028, msp[713] ffff8ce012ddd800, vdev id 0, spa tank, smp size 547680, segments 11142, forcing condense=FALSE
1674629474   metaslab.c:3646:metaslab_condense(): condensing: txg 2596319, msp[729] ffff8ce012ded800, vdev id 0, spa tank, smp size 761720, segments 22900, forcing condense=FALSE
1674636427   metaslab.c:3646:metaslab_condense(): condensing: txg 2597677, msp[629] ffff8ce012b9c000, vdev id 0, spa tank, smp size 926248, segments 57261, forcing condense=FALSE
1674638219   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2598027, spa tank, vdev_id 0, ms_id 731, smp_length 851640, unflushed_allocs 0, unflushed_frees 16384, freed 0, defer 0 + 0, unloaded time 3022317218 ms, loading_time 97 ms, ms_max_size 2147680256, max size error 106496, old_weight 7c0000000000001, new_weight 7c0000000000001
1674638823   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2598145, spa tank, vdev_id 0, ms_id 709, weight 740000000000001, selected txg 2598027 (604458 ms ago), alloc_txg 2598027, loaded 92356635 ms ago, max_size 1073733632
1674639314   metaslab.c:3646:metaslab_condense(): condensing: txg 2598241, msp[731] ffff8ce012de9800, vdev id 0, spa tank, smp size 851640, segments 8287, forcing condense=FALSE
1674640405   metaslab.c:3646:metaslab_condense(): condensing: txg 2598454, msp[697] ffff8ce012c9e800, vdev id 0, spa tank, smp size 882520, segments 53325, forcing condense=FALSE
1674642991   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2598959, spa tank, vdev_id 0, ms_id 732, smp_length 137592, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3054414064 ms, loading_time 64 ms, ms_max_size 4247158784, max size error 210812928, old_weight 7c0000000000001, new_weight 7c0000000000001
1674642991   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2598959, spa tank, vdev_id 0, ms_id 735, smp_length 346576, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3007389799 ms, loading_time 64 ms, ms_max_size 2147868672, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674643590   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2599076, spa tank, vdev_id 0, ms_id 719, weight 780000000000001, selected txg 2598958 (604150 ms ago), alloc_txg 2598958, loaded 58590509 ms ago, max_size 1886724096
1674643595   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2599077, spa tank, vdev_id 0, ms_id 718, weight 780000000000001, selected txg 2598959 (604498 ms ago), alloc_txg 2598959, loaded 61263295 ms ago, max_size 2147336192
1674643994   metaslab.c:3646:metaslab_condense(): condensing: txg 2599155, msp[690] ffff8ce012c9b000, vdev id 0, spa tank, smp size 610752, segments 36217, forcing condense=FALSE
1674644885   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2599329, spa tank, vdev_id 0, ms_id 736, smp_length 173944, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3030212139 ms, loading_time 59 ms, ms_max_size 4295286784, max size error 32309248, old_weight 7c0000000000001, new_weight 800000000000001
1674644885   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2599329, spa tank, vdev_id 0, ms_id 737, smp_length 388304, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2982824340 ms, loading_time 82 ms, ms_max_size 2147590144, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674645484   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2599446, spa tank, vdev_id 0, ms_id 724, weight 780000000000001, selected txg 2599328 (604137 ms ago), alloc_txg 2599328, loaded 41848358 ms ago, max_size 2048573440
1674645489   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2599447, spa tank, vdev_id 0, ms_id 707, weight 780000000000001, selected txg 2599329 (604430 ms ago), alloc_txg 2599329, loaded 106441447 ms ago, max_size 1073995776
1674647389   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2599818, spa tank, vdev_id 0, ms_id 740, smp_length 523256, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8346148481 ms, loading_time 104 ms, ms_max_size 4294860800, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674647389   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2599818, spa tank, vdev_id 0, ms_id 738, smp_length 1302664, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8169911781 ms, loading_time 150 ms, ms_max_size 4295057408, max size error 98304, old_weight 7c0000000000001, new_weight 800000000000001
1674647988   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2599935, spa tank, vdev_id 0, ms_id 726, weight 780000000000001, selected txg 2599817 (604189 ms ago), alloc_txg 2599817, loaded 32972289 ms ago, max_size 2017918976
1674647993   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2599936, spa tank, vdev_id 0, ms_id 629, weight 780000000000001, selected txg 2599818 (604481 ms ago), alloc_txg 2599818, loaded 361963286 ms ago, max_size 2147418112
1674649246   metaslab.c:3646:metaslab_condense(): condensing: txg 2600183, msp[738] ffff8ce013201800, vdev id 0, spa tank, smp size 1302664, segments 10965, forcing condense=FALSE
1674652072   metaslab.c:3646:metaslab_condense(): condensing: txg 2600735, msp[714] ffff8ce012dd8000, vdev id 0, spa tank, smp size 539248, segments 19672, forcing condense=FALSE
1674653921   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2601096, spa tank, vdev_id 0, ms_id 742, smp_length 182872, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 3009017100 ms, loading_time 30 ms, ms_max_size 3941949440, max size error 96198656, old_weight 7c0000000000001, new_weight 7c0000000000001
1674654525   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2601214, spa tank, vdev_id 0, ms_id 725, weight 780000000000001, selected txg 2601096 (604417 ms ago), alloc_txg 2601096, loaded 50889411 ms ago, max_size 1073987584
1674657628   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2601820, spa tank, vdev_id 0, ms_id 743, smp_length 948424, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8334883934 ms, loading_time 95 ms, ms_max_size 4294901760, max size error 0, old_weight 7c0000000000001, new_weight 7c0000000000001
1674658232   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2601938, spa tank, vdev_id 0, ms_id 690, weight 780000000000001, selected txg 2601820 (604398 ms ago), alloc_txg 2601820, loaded 145853177 ms ago, max_size 2147467264
1674659174   metaslab.c:3646:metaslab_condense(): condensing: txg 2602122, msp[743] ffff8ce013202800, vdev id 0, spa tank, smp size 948424, segments 19354, forcing condense=FALSE
1674660869   metaslab.c:3646:metaslab_condense(): condensing: txg 2602453, msp[708] ffff8ce012dde800, vdev id 0, spa tank, smp size 584360, segments 36359, forcing condense=FALSE
1674663946   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2603054, spa tank, vdev_id 0, ms_id 746, smp_length 339912, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2980709058 ms, loading_time 89 ms, ms_max_size 2147737600, max size error 16384, old_weight 7c0000000000001, new_weight 7c0000000000001
1674663946   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2603054, spa tank, vdev_id 0, ms_id 745, smp_length 597704, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2844713184 ms, loading_time 92 ms, ms_max_size 4296146944, max size error 1204224, old_weight 7c0000000000001, new_weight 800000000000001
1674664069   metaslab.c:3646:metaslab_condense(): condensing: txg 2603078, msp[745] ffff8ce013202000, vdev id 0, spa tank, smp size 597704, segments 12004, forcing condense=FALSE
1674664545   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2603171, spa tank, vdev_id 0, ms_id 728, weight 780000000000001, selected txg 2603053 (604515 ms ago), alloc_txg 2603053, loaded 40180630 ms ago, max_size 1807925248
1674664550   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2603172, spa tank, vdev_id 0, ms_id 708, weight 740000000000001, selected txg 2603054 (604688 ms ago), alloc_txg 2603054, loaded 125046847 ms ago, max_size 1073700864
1674672763   metaslab.c:3646:metaslab_condense(): condensing: txg 2604776, msp[730] ffff8ce012de8000, vdev id 0, spa tank, smp size 531560, segments 18090, forcing condense=FALSE
1674673802   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2604979, spa tank, vdev_id 0, ms_id 750, smp_length 946232, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2950533962 ms, loading_time 118 ms, ms_max_size 2147581952, max size error 98304, old_weight 7c0000000000001, new_weight 7c0000000000001
1674673986   metaslab.c:3646:metaslab_condense(): condensing: txg 2605015, msp[750] ffff8ce013302000, vdev id 0, spa tank, smp size 946232, segments 8330, forcing condense=FALSE
1674674406   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2605097, spa tank, vdev_id 0, ms_id 714, weight 740000000000001, selected txg 2604979 (604431 ms ago), alloc_txg 2604979, loaded 92074490 ms ago, max_size 1073569792
1674680745   metaslab.c:3646:metaslab_condense(): condensing: txg 2606335, msp[713] ffff8ce012ddd800, vdev id 0, spa tank, smp size 932672, segments 58225, forcing condense=FALSE
1674682389   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2606656, spa tank, vdev_id 0, ms_id 751, smp_length 462648, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 8327163567 ms, loading_time 96 ms, ms_max_size 4295655424, max size error 819200, old_weight 7c0000000000001, new_weight 800000000000001
1674682993   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2606774, spa tank, vdev_id 0, ms_id 745, weight 780000000000001, selected txg 2606656 (604398 ms ago), alloc_txg 2606656, loaded 19046588 ms ago, max_size 2147450880
1674683198   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2606814, spa tank, vdev_id 0, ms_id 753, smp_length 119288, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2977889995 ms, loading_time 28 ms, ms_max_size 3891806208, max size error 33497088, old_weight 7c0000000000001, new_weight 7c0000000000001
1674683198   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2606814, spa tank, vdev_id 0, ms_id 754, smp_length 507512, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2985527371 ms, loading_time 72 ms, ms_max_size 4295491584, max size error 589824, old_weight 7c0000000000001, new_weight 800000000000001
1674683797   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2606931, spa tank, vdev_id 0, ms_id 731, weight 780000000000001, selected txg 2606813 (604156 ms ago), alloc_txg 2606813, loaded 45577299 ms ago, max_size 2104754176
1674683802   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2606932, spa tank, vdev_id 0, ms_id 729, weight 780000000000001, selected txg 2606814 (604383 ms ago), alloc_txg 2606814, loaded 58525511 ms ago, max_size 1073782784
1674685937   metaslab.c:2436:metaslab_load_impl(): metaslab_load: txg 2607349, spa tank, vdev_id 0, ms_id 756, smp_length 60024, unflushed_allocs 0, unflushed_frees 0, freed 0, defer 0 + 0, unloaded time 2978965222 ms, loading_time 19 ms, ms_max_size 4224565248, max size error 19783680, old_weight 7c0000000000001, new_weight 7c0000000000001
1674686541   metaslab.c:2549:metaslab_unload(): metaslab_unload: txg 2607467, spa tank, vdev_id 0, ms_id 713, weight 740000000000001, selected txg 2607349 (604391 ms ago), alloc_txg 2607349, loaded 113921369 ms ago, max_size 1073733632
1674686756   metaslab.c:3646:metaslab_condense(): condensing: txg 2607509, msp[727] ffff8ce012dec800, vdev id 0, spa tank, smp size 556344, segments 22595, forcing condense=FALSE
~> zpool status tank
  pool: tank
 state: ONLINE
  scan: scrub repaired 0B in 08:42:40 with 0 errors on Sun Jan  1 09:33:37 2023
config:

        NAME                                                 STATE     READ WRITE CKSUM
        tank                                                 ONLINE       0     0     0
          raidz1-0                                           ONLINE       0     0     0
            scsi-0ATA_TOSHIBA_HDWD240_51S1S1HZS5HH           ONLINE       0     0     0
            scsi-SATA_TOSHIBA_MD04ACA4_54V3K2GYFSAA          ONLINE       0     0     0
            scsi-0ATA_ST4000NC000-1FR1_Z306EQ57              ONLINE       0     0     0
            scsi-SATA_TOSHIBA_MG04ACA4_88N1KBYKFJKA          ONLINE       0     0     0
        logs
          nvme-KINGSTON_SA2000M8500G_50026B7684E0D1BE-part3  ONLINE       0     0     0
        cache
          nvme-KINGSTON_SA2000M8500G_50026B7684E0D1BE-part4  ONLINE       0     0     0

errors: No known data errors

tank is my only pool in the system. I didn't delete any pool recently and according to lsblk, all my devices are there and online.

Mitmischer commented 1 year ago

How can I find out whether running zpool scrub and rebooting the system is a safe thing to do?

rincebrain commented 1 year ago

If it's not safe, it's a bug.

Mitmischer commented 1 year ago

I had a cronjob for automatic scrubbing which ran and left the data unharmed. So, that was actually safe.

In the meantime, I rebooted the system, lost no data and the commands do not hang anymore. Maybe #14491 will give some more insight, as I can no longer debug this. I'll update the ticket if the problem occurs again and debugging is desired. Otherwise, I guess that rebooting does the trick.