Segmentation fault gluster volume status all clients

wonderful1984 commented 1 month ago

Description of problem: excute : gluster volume status all clients

Segmentation fault

version: 11.1

The exact command to reproduce the issue: gluster volume status all clients

The full output of the command that failed:

``` ~]# ldd /usr/local/sbin/gluster linux-vdso.so.1 (0x00007ffc223da000) libglusterfs.so.0 => /usr/local/lib/libglusterfs.so.0 (0x0000151cc9d3a000) libgfxdr.so.0 => /usr/local/lib/libgfxdr.so.0 (0x0000151cc9d2a000) libgfrpc.so.0 => /usr/local/lib/libgfrpc.so.0 (0x0000151cc9cf0000) libtirpc.so.3 => /usr/lib64/libtirpc.so.3 (0x0000151cc9cb2000) libz.so.1 => /usr/lib64/libz.so.1 (0x0000151cc9c98000) libm.so.6 => /usr/lib64/libm.so.6 (0x0000151cc9bbc000) libuuid.so.1 => /usr/lib64/libuuid.so.1 (0x0000151cc9bb1000) liburcu-bp.so.8 => /usr/lib64/liburcu-bp.so.8 (0x0000151cc9ba7000) liburcu-common.so.8 => /usr/lib64/liburcu-common.so.8 (0x0000151cc9ba0000) liburcu-cds.so.8 => /usr/lib64/liburcu-cds.so.8 (0x0000151cc9b94000) libreadline.so.8 => /usr/lib64/libreadline.so.8 (0x0000151cc9b3e000) libxml2.so.2 => /usr/lib64/libxml2.so.2 (0x0000151cc99d1000) libcrypto.so.1.1 => /usr/lib64/libcrypto.so.1.1 (0x0000151cc96e2000) libtcmalloc_minimal.so.4 => /lib64/libtcmalloc_minimal.so.4 (0x0000151cc9504000) libc.so.6 => /usr/lib64/libc.so.6 (0x0000151cc9303000) /lib64/ld-linux-x86-64.so.2 (0x0000151cc9e3a000) libgssapi_krb5.so.2 => /usr/lib64/libgssapi_krb5.so.2 (0x0000151cc92ae000) libkrb5.so.3 => /usr/lib64/libkrb5.so.3 (0x0000151cc91c7000) libk5crypto.so.3 => /usr/lib64/libk5crypto.so.3 (0x0000151cc91ad000) libcom_err.so.2 => /usr/lib64/libcom_err.so.2 (0x0000151cc91a7000) libtinfo.so.6 => /usr/lib64/libtinfo.so.6 (0x0000151cc9174000) liblzma.so.5 => /usr/lib64/liblzma.so.5 (0x0000151cc914b000) libstdc++.so.6 => /usr/lib64/libstdc++.so.6 (0x0000151cc8f65000) libgcc_s.so.1 => /usr/lib64/libgcc_s.so.1 (0x0000151cc8f49000) libkrb5support.so.0 => /usr/lib64/libkrb5support.so.0 (0x0000151cc8f37000) libkeyutils.so.1 => /usr/lib64/libkeyutils.so.1 (0x0000151cc8f30000) libresolv.so.2 => /usr/lib64/libresolv.so.2 (0x0000151cc8f1c000) libselinux.so.1 => /usr/lib64/libselinux.so.1 (0x0000151cc8ef0000) libpcre2-8.so.0 => /usr/lib64/libpcre2-8.so.0 (0x0000151cc8e54000) ``` ``` ~]# strace gluster volume status all clients execve("/usr/local/sbin/gluster", ["gluster", "volume", "status", "all", "clients"], 0x7ffc7c4ba9b0 /* 29 vars */) = 0 brk(NULL) = 0x1070000 arch_prctl(0x3001 /* ARCH_??? */, 0x7ffe3fa123d0) = -1 EINVAL (Invalid argument) mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x14f976c9e000 access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/glibc-hwcaps/x86-64-v4/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/glibc-hwcaps/x86-64-v4", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/glibc-hwcaps/x86-64-v3/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/glibc-hwcaps/x86-64-v3", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/glibc-hwcaps/x86-64-v2/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/glibc-hwcaps/x86-64-v2", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/tls/haswell/avx512_1/x86_64/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/tls/haswell/avx512_1/x86_64", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/tls/haswell/avx512_1/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/tls/haswell/avx512_1", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/tls/haswell/x86_64/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/tls/haswell/x86_64", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/tls/haswell/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/tls/haswell", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/tls/avx512_1/x86_64/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/tls/avx512_1/x86_64", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/tls/avx512_1/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/tls/avx512_1", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/tls/x86_64/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/tls/x86_64", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/tls/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/tls", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/haswell/avx512_1/x86_64/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/haswell/avx512_1/x86_64", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/haswell/avx512_1/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/haswell/avx512_1", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/haswell/x86_64/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/haswell/x86_64", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/haswell/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/haswell", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/avx512_1/x86_64/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/avx512_1/x86_64", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/avx512_1/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/avx512_1", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/x86_64/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/usr/local/lib/x86_64", 0x7ffe3fa11600, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/libglusterfs.so.0", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\246\1\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=1131200, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 1037040, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976ba0000 mprotect(0x14f976bb8000, 901120, PROT_NONE) = 0 mmap(0x14f976bb8000, 692224, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x18000) = 0x14f976bb8000 mmap(0x14f976c61000, 204800, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc1000) = 0x14f976c61000 mmap(0x14f976c94000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf3000) = 0x14f976c94000 mmap(0x14f976c99000, 17136, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f976c99000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libgfxdr.so.0", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@A\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=69968, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 61608, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976b90000 mmap(0x14f976b94000, 28672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x14f976b94000 mmap(0x14f976b9b000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xb000) = 0x14f976b9b000 mmap(0x14f976b9e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x14f976b9e000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libgfrpc.so.0", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 H\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=250440, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 235552, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976b56000 mmap(0x14f976b5a000, 57344, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x14f976b5a000 mmap(0x14f976b68000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x12000) = 0x14f976b68000 mmap(0x14f976b6e000, 139264, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x14f976b6e000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libtirpc.so.3", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3 newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=53395, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 53395, PROT_READ, MAP_PRIVATE, 3, 0) = 0x14f976b48000 close(3) = 0 openat(AT_FDCWD, "/usr/lib64/libtirpc.so.3", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\221\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=191136, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 196208, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976b18000 mmap(0x14f976b20000, 114688, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8000) = 0x14f976b20000 mmap(0x14f976b3c000, 36864, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x24000) = 0x14f976b3c000 mmap(0x14f976b45000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2c000) = 0x14f976b45000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libz.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libz.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0203\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=100552, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 102416, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976afe000 mprotect(0x14f976b01000, 86016, PROT_NONE) = 0 mmap(0x14f976b01000, 57344, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x14f976b01000 mmap(0x14f976b0f000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11000) = 0x14f976b0f000 mmap(0x14f976b16000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x14f976b16000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libm.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libm.so.6", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320\361\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=895272, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 897048, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976a22000 mmap(0x14f976a31000, 462848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0x14f976a31000 mmap(0x14f976aa2000, 368640, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x80000) = 0x14f976aa2000 mmap(0x14f976afc000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd9000) = 0x14f976afc000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libuuid.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libuuid.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0#\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=30984, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x14f976a20000 mmap(NULL, 32792, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976a17000 mmap(0x14f976a19000, 16384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x14f976a19000 mmap(0x14f976a1d000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x14f976a1d000 mmap(0x14f976a1e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x14f976a1e000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/liburcu-bp.so.8", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/liburcu-bp.so.8", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20#\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=35320, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 37960, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976a0d000 mmap(0x14f976a0f000, 16384, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x14f976a0f000 mmap(0x14f976a13000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x14f976a13000 mmap(0x14f976a15000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x14f976a15000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/liburcu-common.so.8", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/liburcu-common.so.8", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\320 \0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=22576, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 24744, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976a06000 mmap(0x14f976a08000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x14f976a08000 mmap(0x14f976a0a000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x14f976a0a000 mmap(0x14f976a0b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x14f976a0b000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/liburcu-cds.so.8", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/liburcu-cds.so.8", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20#\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=43192, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 45352, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f9769fa000 mprotect(0x14f9769fc000, 32768, PROT_NONE) = 0 mmap(0x14f9769fc000, 20480, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x14f9769fc000 mmap(0x14f976a01000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x14f976a01000 mmap(0x14f976a04000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x9000) = 0x14f976a04000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libreadline.so.8", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libreadline.so.8", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360y\1\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=343216, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 350856, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f9769a4000 mmap(0x14f9769ba000, 180224, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x14f9769ba000 mmap(0x14f9769e6000, 40960, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x42000) = 0x14f9769e6000 mmap(0x14f9769f0000, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4b000) = 0x14f9769f0000 mmap(0x14f9769f9000, 2696, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f9769f9000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libxml2.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libxml2.so.2", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\374\2\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=1488168, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 1494472, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976837000 mprotect(0x14f976866000, 1257472, PROT_NONE) = 0 mmap(0x14f976866000, 950272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2f000) = 0x14f976866000 mmap(0x14f97694e000, 303104, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x117000) = 0x14f97694e000 mmap(0x14f976999000, 40960, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x161000) = 0x14f976999000 mmap(0x14f9769a3000, 3528, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f9769a3000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libcrypto.so.1.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libcrypto.so.1.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\260\7\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=3047192, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x14f976835000 mmap(NULL, 3067600, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976548000 mmap(0x14f9765c2000, 1744896, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7a000) = 0x14f9765c2000 mmap(0x14f97676c000, 610304, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x224000) = 0x14f97676c000 mmap(0x14f976801000, 196608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2b8000) = 0x14f976801000 mmap(0x14f976831000, 16080, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f976831000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/glibc-hwcaps/x86-64-v4/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/glibc-hwcaps/x86-64-v4", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/glibc-hwcaps/x86-64-v3/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/glibc-hwcaps/x86-64-v3", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/glibc-hwcaps/x86-64-v2/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/glibc-hwcaps/x86-64-v2", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/tls/haswell/avx512_1/x86_64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/tls/haswell/avx512_1/x86_64", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/tls/haswell/avx512_1/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/tls/haswell/avx512_1", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/tls/haswell/x86_64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/tls/haswell/x86_64", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/tls/haswell/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/tls/haswell", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/tls/avx512_1/x86_64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/tls/avx512_1/x86_64", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/tls/avx512_1/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/tls/avx512_1", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/tls/x86_64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/tls/x86_64", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/tls/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/tls", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/haswell/avx512_1/x86_64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/haswell/avx512_1/x86_64", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/haswell/avx512_1/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/haswell/avx512_1", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/haswell/x86_64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/haswell/x86_64", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/haswell/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/haswell", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/avx512_1/x86_64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/avx512_1/x86_64", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/avx512_1/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/avx512_1", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/x86_64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) newfstatat(AT_FDCWD, "/lib64/x86_64", 0x7ffe3fa11460, 0) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/lib64/libtcmalloc_minimal.so.4", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\337\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=174848, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 1956928, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f97636a000 mprotect(0x14f976377000, 118784, PROT_NONE) = 0 mmap(0x14f976377000, 81920, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x14f976377000 mmap(0x14f97638b000, 32768, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x21000) = 0x14f97638b000 mmap(0x14f976394000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x29000) = 0x14f976394000 mmap(0x14f976396000, 1776704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f976396000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\324\2\0\0\0\0\0"..., 832) = 832 pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784 pread64(3, "\4\0\0\0000\0\0\0\5\0\0\0GNU\0\2\200\0\300\4\0\0\0\1\0\0\0\0\0\0\0"..., 64, 848) = 64 pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0Lu\217\332\326\261b\255\22g!\302\347\335|\363"..., 68, 912) = 68 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=2055336, ...}, AT_EMPTY_PATH) = 0 pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784 mmap(NULL, 2100272, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976169000 mmap(0x14f976195000, 1511424, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2c000) = 0x14f976195000 mmap(0x14f976306000, 331776, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19d000) = 0x14f976306000 mmap(0x14f976357000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1ed000) = 0x14f976357000 mmap(0x14f97635d000, 52272, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f97635d000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libgssapi_krb5.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libgssapi_krb5.so.2", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000\325\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=344552, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 346976, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976114000 mprotect(0x14f976120000, 286720, PROT_NONE) = 0 mmap(0x14f976120000, 229376, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xc000) = 0x14f976120000 mmap(0x14f976158000, 53248, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x44000) = 0x14f976158000 mmap(0x14f976166000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x51000) = 0x14f976166000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libkrb5.so.3", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libkrb5.so.3", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0PX\2\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=940776, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 943360, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f97602d000 mmap(0x14f976050000, 434176, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x23000) = 0x14f976050000 mmap(0x14f9760ba000, 299008, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x8d000) = 0x14f9760ba000 mmap(0x14f976103000, 69632, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd5000) = 0x14f976103000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libk5crypto.so.3", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libk5crypto.so.3", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0W\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=92560, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x14f97602b000 mmap(NULL, 94424, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f976013000 mmap(0x14f976018000, 49152, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x5000) = 0x14f976018000 mmap(0x14f976024000, 16384, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x11000) = 0x14f976024000 mmap(0x14f976028000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x14f976028000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libcom_err.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libcom_err.so.2", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\"\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=18560, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 20552, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f97600d000 mmap(0x14f97600f000, 4096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x14f97600f000 mmap(0x14f976010000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x14f976010000 mmap(0x14f976011000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x14f976011000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libtinfo.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libtinfo.so.6", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\352\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=204048, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 207168, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975fda000 mprotect(0x14f975fe8000, 131072, PROT_NONE) = 0 mmap(0x14f975fe8000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xe000) = 0x14f975fe8000 mmap(0x14f975ff9000, 57344, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1f000) = 0x14f975ff9000 mmap(0x14f976008000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2d000) = 0x14f976008000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/liblzma.so.5", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/liblzma.so.5", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0205\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=161992, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 163856, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975fb1000 mprotect(0x14f975fb4000, 147456, PROT_NONE) = 0 mmap(0x14f975fb4000, 98304, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x14f975fb4000 mmap(0x14f975fcc000, 45056, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b000) = 0x14f975fcc000 mmap(0x14f975fd8000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x26000) = 0x14f975fd8000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libstdc++.so.6", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \240\t\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=1973368, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 1988544, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975dcb000 mprotect(0x14f975e61000, 1306624, PROT_NONE) = 0 mmap(0x14f975e61000, 974848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x96000) = 0x14f975e61000 mmap(0x14f975f4f000, 327680, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x184000) = 0x14f975f4f000 mmap(0x14f975fa0000, 57344, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1d4000) = 0x14f975fa0000 mmap(0x14f975fae000, 10176, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f975fae000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libgcc_s.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0203\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=100720, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x14f975dc9000 mmap(NULL, 103112, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975daf000 mmap(0x14f975db2000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x14f975db2000 mmap(0x14f975dc3000, 16384, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x14f975dc3000 mmap(0x14f975dc7000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x14f975dc7000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libkrb5support.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libkrb5support.so.0", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240G\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=68016, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 70344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975d9d000 mmap(0x14f975da1000, 36864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x14f975da1000 mmap(0x14f975daa000, 12288, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x14f975daa000 mmap(0x14f975dad000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0x14f975dad000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libkeyutils.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\"\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=22656, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 24592, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975d96000 mmap(0x14f975d98000, 8192, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x14f975d98000 mmap(0x14f975d9a000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x14f975d9a000 mmap(0x14f975d9b000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x14f975d9b000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libresolv.so.2", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libresolv.so.2", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`C\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=68080, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 80008, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975d82000 mprotect(0x14f975d86000, 49152, PROT_NONE) = 0 mmap(0x14f975d86000, 36864, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x14f975d86000 mmap(0x14f975d8f000, 8192, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xd000) = 0x14f975d8f000 mmap(0x14f975d92000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0xf000) = 0x14f975d92000 mmap(0x14f975d94000, 6280, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f975d94000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libselinux.so.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libselinux.so.1", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\177\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=166320, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 177680, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975d56000 mmap(0x14f975d5d000, 102400, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x14f975d5d000 mmap(0x14f975d76000, 32768, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20000) = 0x14f975d76000 mmap(0x14f975d7e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x27000) = 0x14f975d7e000 mmap(0x14f975d80000, 5648, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f975d80000 close(3) = 0 openat(AT_FDCWD, "/usr/local/lib/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib64/libpcre2-8.so.0", O_RDONLY|O_CLOEXEC) = 3 read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\2002\0\0\0\0\0\0"..., 832) = 832 newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=625152, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x14f975d54000 mmap(NULL, 627304, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x14f975cba000 mprotect(0x14f975cbd000, 610304, PROT_NONE) = 0 mmap(0x14f975cbd000, 438272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x14f975cbd000 mmap(0x14f975d28000, 167936, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6e000) = 0x14f975d28000 mmap(0x14f975d52000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x97000) = 0x14f975d52000 close(3) = 0 mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x14f975cb8000 mmap(NULL, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x14f975cb4000 arch_prctl(ARCH_SET_FS, 0x14f975cb56c0) = 0 set_tid_address(0x14f975cb5990) = 1224331 set_robust_list(0x14f975cb59a0, 24) = 0 mprotect(0x14f976357000, 12288, PROT_READ) = 0 mprotect(0x14f975d52000, 4096, PROT_READ) = 0 mprotect(0x14f975d7e000, 4096, PROT_READ) = 0 mprotect(0x14f975d92000, 4096, PROT_READ) = 0 mprotect(0x14f975d9b000, 4096, PROT_READ) = 0 mprotect(0x14f976b16000, 4096, PROT_READ) = 0 mprotect(0x14f976801000, 180224, PROT_READ) = 0 mprotect(0x14f975dad000, 4096, PROT_READ) = 0 mprotect(0x14f975dc7000, 4096, PROT_READ) = 0 mprotect(0x14f976afc000, 4096, PROT_READ) = 0 mprotect(0x14f975fa0000, 45056, PROT_READ) = 0 mprotect(0x14f975fd8000, 4096, PROT_READ) = 0 mprotect(0x14f976008000, 16384, PROT_READ) = 0 mprotect(0x14f976011000, 4096, PROT_READ) = 0 mprotect(0x14f976028000, 8192, PROT_READ) = 0 mprotect(0x14f976103000, 61440, PROT_READ) = 0 mprotect(0x14f976166000, 8192, PROT_READ) = 0 mprotect(0x14f976394000, 4096, PROT_READ) = 0 mprotect(0x14f976999000, 36864, PROT_READ) = 0 mprotect(0x14f9769f0000, 12288, PROT_READ) = 0 mprotect(0x14f976a0b000, 4096, PROT_READ) = 0 mprotect(0x14f976a04000, 4096, PROT_READ) = 0 mprotect(0x14f976a15000, 4096, PROT_READ) = 0 mprotect(0x14f976a1e000, 4096, PROT_READ) = 0 mprotect(0x14f976b45000, 8192, PROT_READ) = 0 mprotect(0x14f976c94000, 4096, PROT_READ) = 0 mprotect(0x14f976b9e000, 4096, PROT_READ) = 0 mprotect(0x14f976b6e000, 4096, PROT_READ) = 0 mprotect(0x46b000, 4096, PROT_READ) = 0 mprotect(0x14f976cce000, 8192, PROT_READ) = 0 prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=RLIM64_INFINITY, rlim_max=RLIM64_INFINITY}) = 0 munmap(0x14f976b48000, 53395) = 0 statfs("/sys/fs/selinux", 0x7ffe3fa123c0) = -1 ENOENT (No such file or directory) statfs("/selinux", 0x7ffe3fa123c0) = -1 ENOENT (No such file or directory) brk(NULL) = 0x1070000 futex(0x14f976543b0c, FUTEX_WAKE_PRIVATE, 2147483647) = 0 brk(0x1870000) = 0x1870000 brk(0x1970000) = 0x1970000 openat(AT_FDCWD, "/proc/filesystems", O_RDONLY|O_CLOEXEC) = 3 newfstatat(3, "", {st_mode=S_IFREG|0444, st_size=0, ...}, AT_EMPTY_PATH) = 0 read(3, "nodev\tsysfs\nnodev\ttmpfs\nnodev\tbd"..., 1024) = 393 read(3, "", 1024) = 0 close(3) = 0 access("/etc/selinux/config", F_OK) = 0 openat(AT_FDCWD, "/proc/sys/crypto/fips_enabled", O_RDONLY) = 3 read(3, "0\n", 2) = 2 close(3) = 0 access("/etc/system-fips", F_OK) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC) = 3 read(3, "0-63\n", 1024) = 5 close(3) = 0 futex(0x7ffe3fa124b4, FUTEX_WAKE, 1) = 0 futex(0x7ffe3fa124b4, FUTEX_WAKE_PRIVATE, 1) = 0 membarrier(MEMBARRIER_CMD_QUERY, 0) = 0x1ff (MEMBARRIER_CMD_GLOBAL|MEMBARRIER_CMD_GLOBAL_EXPEDITED|MEMBARRIER_CMD_REGISTER_GLOBAL_EXPEDITED|MEMBARRIER_CMD_PRIVATE_EXPEDITED|MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED|MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE|MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE|MEMBARRIER_CMD_PRIVATE_EXPEDITED_RSEQ|MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_RSEQ) membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0) = 0 uname({sysname="Linux", nodename="gd16-145-gfs-012022", ...}) = 0 futex(0x14f976c99490, FUTEX_WAKE_PRIVATE, 2147483647) = 0 getrandom("\xd3\xed\x6f\x7d\xe2\xa0\x5f\xb4\x0a\xcd\x88\x9a\x8c\xdd\x04\xdc", 16, GRND_NONBLOCK) = 16 getpid() = 1224331 getuid() = 0 getpid() = 1224331 getppid() = 1224328 gettid() = 1224331 epoll_create(16384) = 3 prlimit64(0, RLIMIT_CORE, {rlim_cur=RLIM64_INFINITY, rlim_max=RLIM64_INFINITY}, NULL) = 0 access("/var/lib/glusterd/secure-access", F_OK) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 4 newfstatat(4, "", {st_mode=S_IFREG|0644, st_size=19037712, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 19037712, PROT_READ, MAP_PRIVATE, 4, 0) = 0x14f974a8c000 close(4) = 0 socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0) = 4 connect(4, {sa_family=AF_UNIX, sun_path="/dev/log"}, 110) = 0 newfstatat(AT_FDCWD, "/etc/glusterfs/logger.conf", 0x7ffe3fa12210, 0) = -1 ENOENT (No such file or directory) mkdir("/var", 0755) = -1 EEXIST (File exists) mkdir("/var/log", 0755) = -1 EEXIST (File exists) mkdir("/var/log/glusterfs", 0755) = -1 EEXIST (File exists) newfstatat(AT_FDCWD, "/var/log/glusterfs/", {st_mode=S_IFDIR|0755, st_size=166, ...}, 0) = 0 openat(AT_FDCWD, "/var/log/glusterfs/cli.log", O_WRONLY|O_CREAT|O_APPEND, 0600) = 5 fcntl(5, F_GETFL) = 0x8401 (flags O_WRONLY|O_APPEND|O_LARGEFILE) openat(AT_FDCWD, "/etc/localtime", O_RDONLY|O_CLOEXEC) = 6 newfstatat(6, "", {st_mode=S_IFREG|0644, st_size=561, ...}, AT_EMPTY_PATH) = 0 newfstatat(6, "", {st_mode=S_IFREG|0644, st_size=561, ...}, AT_EMPTY_PATH) = 0 read(6, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = 561 lseek(6, -342, SEEK_CUR) = 219 read(6, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 4096) = 342 close(6) = 0 newfstatat(5, "", {st_mode=S_IFREG|0600, st_size=14860, ...}, AT_EMPTY_PATH) = 0 write(5, "[2024-03-22 08:52:49.049418 +000"..., 103) = 103 openat(AT_FDCWD, "/usr/local/lib/glusterfs/11.1/rpc-transport/socket.so", O_RDONLY|O_CLOEXEC) = 6 read(6, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0000H\0\0\0\0\0\0"..., 832) = 832 newfstatat(6, "", {st_mode=S_IFREG|0755, st_size=142512, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 132760, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 6, 0) = 0x14f974a6b000 mmap(0x14f974a6f000, 49152, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 6, 0x4000) = 0x14f974a6f000 mmap(0x14f974a7b000, 20480, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 6, 0x10000) = 0x14f974a7b000 mmap(0x14f974a80000, 49152, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 6, 0x14000) = 0x14f974a80000 close(6) = 0 openat(AT_FDCWD, "/usr/local/lib/libssl.so.1.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/local/lib/libssl.so.1.1", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 6 newfstatat(6, "", {st_mode=S_IFREG|0644, st_size=53395, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 53395, PROT_READ, MAP_PRIVATE, 6, 0) = 0x14f976b48000 close(6) = 0 openat(AT_FDCWD, "/usr/lib64/libssl.so.1.1", O_RDONLY|O_CLOEXEC) = 6 read(6, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\374\1\0\0\0\0\0"..., 832) = 832 newfstatat(6, "", {st_mode=S_IFREG|0755, st_size=628816, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 631224, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 6, 0) = 0x14f9749d0000 mmap(0x14f9749ee000, 335872, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 6, 0x1e000) = 0x14f9749ee000 mmap(0x14f974a40000, 114688, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 6, 0x70000) = 0x14f974a40000 mmap(0x14f974a5c000, 57344, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 6, 0x8b000) = 0x14f974a5c000 mmap(0x14f974a6a000, 440, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x14f974a6a000 close(6) = 0 mprotect(0x14f974a5c000, 40960, PROT_READ) = 0 mprotect(0x14f974a80000, 4096, PROT_READ) = 0 munmap(0x14f976b48000, 53395) = 0 openat(AT_FDCWD, "/usr/lib64/gconv/gconv-modules.cache", O_RDONLY) = 6 newfstatat(6, "", {st_mode=S_IFREG|0644, st_size=26988, ...}, AT_EMPTY_PATH) = 0 mmap(NULL, 26988, PROT_READ, MAP_SHARED, 6, 0) = 0x14f976b4f000 close(6) = 0 futex(0x14f97635c56c, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f9768337b8, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f9768337ac, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f9768337a4, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f9768338a0, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f976833790, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f976833788, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f97683071c, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f9768331a4, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f97683313c, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f976833130, FUTEX_WAKE_PRIVATE, 2147483647) = 0 openat(AT_FDCWD, "/usr/share/locale/locale.alias", O_RDONLY|O_CLOEXEC) = 6 newfstatat(6, "", {st_mode=S_IFREG|0644, st_size=2998, ...}, AT_EMPTY_PATH) = 0 read(6, "# Locale name alias data base.\n#"..., 4096) = 2998 read(6, "", 4096) = 0 close(6) = 0 openat(AT_FDCWD, "/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory) openat(AT_FDCWD, "/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory) futex(0x14f97683379c, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f976833758, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f976833750, FUTEX_WAKE_PRIVATE, 2147483647) = 0 openat(AT_FDCWD, "/etc/pki/tls/openssl.cnf", O_RDONLY) = 6 newfstatat(6, "", {st_mode=S_IFREG|0644, st_size=10909, ...}, AT_EMPTY_PATH) = 0 read(6, "#\n# OpenSSL example configuratio"..., 4096) = 4096 read(6, "# WARNING: ancient versions of N"..., 4096) = 4096 read(6, "es of the usage of nsCertType. I"..., 4096) = 2717 read(6, "", 4096) = 0 close(6) = 0 futex(0x14f976833780, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f974a6a0c8, FUTEX_WAKE_PRIVATE, 2147483647) = 0 sysinfo({uptime=625449, loads=[38912, 37792, 35072], totalram=269477277696, freeram=70230937600, sharedram=4335357952, bufferram=3985408, totalswap=8589930496, freeswap=8589930496, procs=1368, totalhigh=0, freehigh=0, mem_unit=1}) = 0 futex(0x14f974a6a1b0, FUTEX_WAKE_PRIVATE, 2147483647) = 0 futex(0x14f974a6a1a4, FUTEX_WAKE_PRIVATE, 2147483647) = 0 socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0) = 6 fcntl(6, F_GETFL) = 0x2 (flags O_RDWR) fcntl(6, F_SETFL, O_RDWR|O_NONBLOCK) = 0 getsockopt(6, SOL_SOCKET, SO_ERROR, [0], [4]) = 0 connect(6, {sa_family=AF_UNIX, sun_path="/var/run/glusterd.socket"}, 110) = 0 epoll_ctl(3, EPOLL_CTL_ADD, 6, {events=EPOLLIN|EPOLLPRI|EPOLLOUT|EPOLLERR|EPOLLHUP|EPOLLONESHOT, data={u32=0, u64=4294967296}}) = 0 rt_sigprocmask(SIG_BLOCK, ~[ILL ABRT BUS FPE SEGV CONT SYS RTMIN RT_1], [], 8) = 0 rt_sigaction(SIGRT_1, {sa_handler=0x14f9761f1580, sa_mask=[], sa_flags=SA_RESTORER|SA_ONSTACK|SA_RESTART|SA_SIGINFO, sa_restorer=0x14f9761a9ed0}, NULL, 8) = 0 rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 mmap(NULL, 2101248, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x14f9747cf000 mprotect(0x14f9747d0000, 2097152, PROT_READ|PROT_WRITE) = 0 rt_sigprocmask(SIG_BLOCK, ~[], ~[ILL ABRT BUS FPE KILL SEGV CONT STOP SYS RTMIN RT_1], 8) = 0 clone3({flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, child_tid=0x14f9749cf910, parent_tid=0x14f9749cf910, exit_signal=0, stack=0x14f9747cf000, stack_size=0x1fef80, tls=0x14f9749cf640} => {parent_tid=[1224332]}, 88) = 1224332 rt_sigprocmask(SIG_SETMASK, ~[ILL ABRT BUS FPE KILL SEGV CONT STOP SYS RTMIN RT_1], NULL, 8) = 0 openat(AT_FDCWD, "/proc/self/task/1224332/comm", O_RDWR) = 7 write(7, "glfs_timer", 10) = 10 close(7) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 futex(0x19101b8, FUTEX_WAKE_PRIVATE, 1) = 1 futex(0x1910168, FUTEX_WAKE_PRIVATE, 1) = 1 access("/var/run/gluster/quotad/quotad.pid", F_OK) = -1 ENOENT (No such file or directory) pipe([7, 8]) = 0 fcntl(8, F_SETFD, FD_CLOEXEC) = 0 pipe([9, 10]) = 0 fcntl(9, F_GETFL) = 0 (flags O_RDONLY) prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 openat(AT_FDCWD, "/proc/self/fd", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 11 newfstatat(11, "", {st_mode=S_IFDIR|0500, st_size=0, ...}, AT_EMPTY_PATH) = 0 getdents64(11, 0x1922030 /* 14 entries */, 32768) = 336 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 prlimit64(0, RLIMIT_NOFILE, NULL, {rlim_cur=512*1024, rlim_max=512*1024}) = 0 getdents64(11, 0x1922030 /* 0 entries */, 32768) = 0 close(11) = 0 mmap(NULL, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x14f9747c6000 rt_sigprocmask(SIG_BLOCK, ~[], [], 8) = 0 clone3({flags=CLONE_VM|CLONE_VFORK, exit_signal=SIGCHLD, stack=0x14f9747c6000, stack_size=0x9000}, 88) = 1224333 munmap(0x14f9747c6000, 36864) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 write(8, "\0\0\0\0", 4) = 4 close(10) = 0 close(8) = 0 read(7, "\0\0\0\0", 4) = 4 close(7) = 0 newfstatat(9, "", {st_mode=S_IFIFO|0600, st_size=0, ...}, AT_EMPTY_PATH) = 0 read(9, "gsyncd.py 4.0.0\n", 4096) = 16 wait4(1224333, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 1224333 --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=1224333, si_uid=0, si_status=0, si_utime=5, si_stime=1} --- close(9) = 0 mmap(NULL, 2101248, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x14f9745ce000 mprotect(0x14f9745cf000, 2097152, PROT_READ|PROT_WRITE) = 0 rt_sigprocmask(SIG_BLOCK, ~[], [], 8) = 0 clone3({flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, child_tid=0x14f9747ce910, parent_tid=0x14f9747ce910, exit_signal=0, stack=0x14f9745ce000, stack_size=0x1fef80, tls=0x14f9747ce640} => {parent_tid=[1224334]}, 88) = 1224334 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 rt_sigprocmask(SIG_BLOCK, ~[ILL ABRT BUS FPE SEGV CONT SYS RTMIN RT_1], [], 8) = 0 mmap(NULL, 2101248, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x14f9743cd000 mprotect(0x14f9743ce000, 2097152, PROT_READ|PROT_WRITE) = 0 rt_sigprocmask(SIG_BLOCK, ~[], ~[ILL ABRT BUS FPE KILL SEGV CONT STOP SYS RTMIN RT_1], 8) = 0 clone3({flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, child_tid=0x14f9745cd910, parent_tid=0x14f9745cd910, exit_signal=0, stack=0x14f9743cd000, stack_size=0x1fef80, tls=0x14f9745cd640} => {parent_tid=[1224335]}, 88) = 1224335 rt_sigprocmask(SIG_SETMASK, ~[ILL ABRT BUS FPE KILL SEGV CONT STOP SYS RTMIN RT_1], NULL, 8) = 0 openat(AT_FDCWD, "/proc/self/task/1224335/comm", O_RDWR) = 7 write(7, "glfs_epoll000", 13) = 13 close(7) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 rt_sigprocmask(SIG_BLOCK, ~[ILL ABRT BUS FPE SEGV CONT SYS RTMIN RT_1], [], 8) = 0 mmap(NULL, 2101248, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x14f9741cc000 mprotect(0x14f9741cd000, 2097152, PROT_READ|PROT_WRITE) = 0 rt_sigprocmask(SIG_BLOCK, ~[], ~[ILL ABRT BUS FPE KILL SEGV CONT STOP SYS RTMIN RT_1], 8) = 0 clone3({flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, child_tid=0x14f9743cc910, parent_tid=0x14f9743cc910, exit_signal=0, stack=0x14f9741cc000, stack_size=0x1fef80, tls=0x14f9743cc640} => {parent_tid=[1224336]}, 88) = 1224336 rt_sigprocmask(SIG_SETMASK, ~[ILL ABRT BUS FPE KILL SEGV CONT STOP SYS RTMIN RT_1], NULL, 8) = 0 openat(AT_FDCWD, "/proc/self/task/1224336/comm", O_RDWR) = 7 write(7, "glfs_epoll001", 13) = 13 close(7) = 0 rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0 futex(0x1892068, FUTEX_WAKE_PRIVATE, 1) = 1 futex(0x14f9745cd910, FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, 1224335, NULL, FUTEX_BITSET_MATCH_ANY) = ? +++ killed by SIGSEGV +++ Segmentation fault ```

mohit84 commented 1 month ago

It seems you are getting crash for CLI process, can you please attach a coredump with gdb and share the command output "thread apply all bt full" to debug it more?

wonderful1984 commented 1 month ago

(No debugging symbols found in /usr/local/sbin/gluster)
(gdb) set pagination 0
(gdb) set logging on
Copying output to gdb.txt.
Copying debug output to gdb.txt.
(gdb) thread apply all bt
(gdb) run
Starting program: /usr/local/sbin/gluster 
Missing separate debuginfos, use: dnf debuginfo-install glibc-2.34-112.oe2203sp1.x86_64
Missing separate debuginfo for /lib64/libtcmalloc_minimal.so.4
Try: dnf --enablerepo='*debug*' install /usr/lib/debug/.build-id/e5/0805e57e7c53c2a57576a2643850ea6e9d5a4a.debug
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/usr/lib64/libthread_db.so.1".
[New Thread 0x155553253640 (LWP 294594)]
[Detaching after vfork from child process 294596]
[New Thread 0x155553052640 (LWP 294617)]
Welcome to gluster prompt, type 'help' to see the available commands.
[New Thread 0x155552e51640 (LWP 294618)]
[New Thread 0x155552c50640 (LWP 294619)]
gluster> volume status all clients

Thread 5 "glfs_epoll001" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x155552c50640 (LWP 294619)]
0x0000155555473744 in iobuf_get2 () from /usr/local/lib/libglusterfs.so.0
Missing separate debuginfos, use: dnf debuginfo-install e2fsprogs-1.46.4-17.oe2203sp1.x86_64 keyutils-libs-1.6.3-4.oe2203sp1.x86_64 krb5-libs-1.19.2-6.oe2203sp1.x86_64 libgcc-10.3.1-20.oe2203sp1.x86_64 libselinux-3.3-2.oe2203sp1.x86_64 libstdc++-10.3.1-20.oe2203sp1.x86_64 libtirpc-1.3.2-3.oe2203sp1.x86_64 libuuid-2.37.2-13.oe2203sp1.x86_64 libxml2-2.9.14-5.oe2203sp1.x86_64 ncurses-libs-6.3-5.oe2203sp1.x86_64 openssl-libs-1.1.1m-20.oe2203sp1.x86_64 pcre2-10.39-6.oe2203sp1.x86_64 readline-8.1-2.oe2203sp1.x86_64 userspace-rcu-0.13.0-5.oe2203sp1.x86_64 xz-libs-5.2.5-3.oe2203sp1.x86_64 zlib-1.2.11-22.oe2203sp1.x86_64
(gdb) bt
#0  0x0000155555473744 in iobuf_get2 () from /usr/local/lib/libglusterfs.so.0
#1  0x00001555532fb49c in socket_event_handler () from /usr/local/lib/glusterfs/11.1/rpc-transport/socket.so
#2  0x00001555554a836e in event_dispatch_epoll_worker.lto_priv () from /usr/local/lib/libglusterfs.so.0
#3  0x0000155554a7822a in ?? () from /usr/lib64/libc.so.6
#4  0x0000155554afacf0 in ?? () from /usr/lib64/libc.so.6
(gdb) bt
#0  0x0000155555473744 in iobuf_get2 () from /usr/local/lib/libglusterfs.so.0
#1  0x00001555532fb49c in socket_event_handler () from /usr/local/lib/glusterfs/11.1/rpc-transport/socket.so
#2  0x00001555554a836e in event_dispatch_epoll_worker.lto_priv () from /usr/local/lib/libglusterfs.so.0
#3  0x0000155554a7822a in ?? () from /usr/lib64/libc.so.6
#4  0x0000155554afacf0 in ?? () from /usr/lib64/libc.so.6
(gdb) f 0
#0  0x0000155555473744 in iobuf_get2 () from /usr/local/lib/libglusterfs.so.0

mohit84 commented 1 month ago

Please generate a coredump and attach a coredump to gdb and then run a command "thread apply all bt full" instead of running a cli process through gdb.

wonderful1984 commented 1 month ago

core-glfs_epoll001-318471-1711336118.zip

wonderful1984 commented 1 month ago

(gdb) set pagination 0
(gdb) thread apply all bt full

Thread 5 (Thread 0x14bdb8622640 (LWP 318474)):
#0  0x000014bdba044d92 in ?? () from /usr/lib64/libc.so.6
No symbol table info available.
#1  0x000014bdba047934 in pthread_cond_timedwait () from /usr/lib64/libc.so.6
No symbol table info available.
#2  0x000000000040cca4 in cli_cmd_submit ()
No symbol table info available.
#3  0x000000000044fb8a in cli_to_glusterd.constprop ()
No symbol table info available.
#4  0x000000000042afff in gf_cli_status_volume_all.lto_priv ()
No symbol table info available.
#5  0x0000000000416c94 in cli_cmd_volume_status_cbk ()
No symbol table info available.
#6  0x000000000040c017 in cli_cmd_process ()
No symbol table info available.
#7  0x000000000040c1dc in cli_batch ()
No symbol table info available.
#8  0x000014bdba04822a in ?? () from /usr/lib64/libc.so.6
No symbol table info available.
#9  0x000014bdba0cacf0 in ?? () from /usr/lib64/libc.so.6
No symbol table info available.

Thread 4 (Thread 0x14bdb8421640 (LWP 318475)):
#0  0x000014bdba0ca2be in epoll_wait () from /usr/lib64/libc.so.6
No symbol table info available.
#1  0x000014bdbaa7823c in event_dispatch_epoll_worker.lto_priv () from /usr/local/lib/libglusterfs.so.0
No symbol table info available.
#2  0x000014bdba04822a in ?? () from /usr/lib64/libc.so.6
No symbol table info available.
#3  0x000014bdba0cacf0 in ?? () from /usr/lib64/libc.so.6
No symbol table info available.

Thread 3 (Thread 0x14bdb8823640 (LWP 318472)):
#0  0x000014bdba044d92 in ?? () from /usr/lib64/libc.so.6
No symbol table info available.
#1  0x000014bdba047934 in pthread_cond_timedwait () from /usr/lib64/libc.so.6
No symbol table info available.
#2  0x000014bdbaa285a8 in gf_timer_proc () from /usr/local/lib/libglusterfs.so.0
No symbol table info available.
#3  0x000014bdba04822a in ?? () from /usr/lib64/libc.so.6
No symbol table info available.
#4  0x000014bdba0cacf0 in ?? () from /usr/lib64/libc.so.6
No symbol table info available.

Thread 2 (Thread 0x14bdb9b096c0 (LWP 318471)):
#0  0x000014bdba044d92 in ?? () from /usr/lib64/libc.so.6
No symbol table info available.
#1  0x000014bdba049b83 in ?? () from /usr/lib64/libc.so.6
No symbol table info available.
#2  0x000014bdbaa79f1f in event_dispatch_epoll.lto_priv () from /usr/local/lib/libglusterfs.so.0
No symbol table info available.
#3  0x0000000000409183 in main ()
No symbol table info available.

Thread 1 (Thread 0x14bdb8220640 (LWP 318476)):
#0  0x000014bdbaa43744 in iobuf_get2 () from /usr/local/lib/libglusterfs.so.0
No symbol table info available.
#1  0x000014bdb88cb49c in socket_event_handler () from /usr/local/lib/glusterfs/11.1/rpc-transport/socket.so
No symbol table info available.
#2  0x000014bdbaa7836e in event_dispatch_epoll_worker.lto_priv () from /usr/local/lib/libglusterfs.so.0
No symbol table info available.
#3  0x000014bdba04822a in ?? () from /usr/lib64/libc.so.6
No symbol table info available.
#4  0x000014bdba0cacf0 in ?? () from /usr/lib64/libc.so.6
No symbol table info available.

mohit84 commented 1 month ago

I think you need to install source after enable debug symbol, that;s why gdb is not able to fetch the symbol at specific address.

wonderful1984 commented 1 month ago

For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /usr/local/sbin/gluster...
[New LWP 1996244]
[New LWP 1996195]
[New LWP 1996201]
[New LWP 1996243]
[New LWP 1996242]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/usr/lib64/libthread_db.so.1".
Core was generated by `gluster vol status all clients'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  iobuf_get_from_stdalloc (page_size=3247444, iobuf_pool=0x0) at iobuf.c:423
423         list_for_each_entry(trav, &iobuf_pool->arenas[IOBUF_ARENA_MAX_INDEX], list)
[Current thread is 1 (Thread 0x14e328f87640 (LWP 1996244))]
Missing separate debuginfos, use: dnf debuginfo-install e2fsprogs-1.46.4-17.oe2203sp1.x86_64 keyutils-libs-1.6.3-4.oe2203sp1.x86_64 krb5-libs-1.19.2-6.oe2203sp1.x86_64 libgcc-10.3.1-20.oe2203sp1.x86_64 libselinux-3.3-2.oe2203sp1.x86_64 libstdc++-10.3.1-20.oe2203sp1.x86_64 libtirpc-1.3.2-3.oe2203sp1.x86_64 libuuid-2.37.2-13.oe2203sp1.x86_64 libxml2-2.9.14-5.oe2203sp1.x86_64 ncurses-libs-6.3-5.oe2203sp1.x86_64 openssl-libs-1.1.1m-20.oe2203sp1.x86_64 pcre2-10.39-6.oe2203sp1.x86_64 readline-8.1-2.oe2203sp1.x86_64 userspace-rcu-0.13.0-5.oe2203sp1.x86_64 xz-libs-5.2.5-3.oe2203sp1.x86_64 zlib-1.2.11-22.oe2203sp1.x86_64
(gdb) set pagination 0
(gdb) thread apply all bt full

Thread 5 (Thread 0x14e329389640 (LWP 1996242)):
#0  0x000014e32adaad92 in __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x14e3293878b0, op=393, expected=0, futex_word=0x46224c <cond+44>) at futex-internal.c:57
        sc_cancel_oldtype = 0
        __arg6 = <optimized out>
        __arg3 = <optimized out>
        _a5 = <optimized out>
        _a2 = <optimized out>
        sc_ret = <optimized out>
        __arg4 = <optimized out>
        __arg1 = <optimized out>
        _a6 = <optimized out>
        _a3 = <optimized out>
        resultvar = <optimized out>
        __arg5 = <optimized out>
        __arg2 = <optimized out>
        _a4 = <optimized out>
        _a1 = <optimized out>
#1  __futex_abstimed_wait_common (futex_word=futex_word@entry=0x46224c <cond+44>, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0x14e3293878b0, private=private@entry=0, cancel=cancel@entry=true) at futex-internal.c:87
        err = <optimized out>
        clockbit = 256
        op = 393
#2  0x000014e32adaadff in __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x46224c <cond+44>, expected=expected@entry=0, clockid=clockid@entry=0, abstime=abstime@entry=0x14e3293878b0, private=private@entry=0) at futex-internal.c:139
No locals.
#3  0x000014e32adad934 in __pthread_cond_wait_common (abstime=0x14e3293878b0, clockid=0, mutex=0x4621e0 <cond_mutex>, cond=0x462220 <cond>) at pthread_cond_wait.c:504
        spin = 0
        buffer = {__routine = 0x14e32adad3a0 <__condvar_cleanup_waiting>, __arg = 0x14e329387840, __canceltype = 4591584, __prev = 0x0}
        cbuffer = {wseq = 3, cond = 0x462220 <cond>, mutex = 0x4621e0 <cond_mutex>, private = 0}
        err = <optimized out>
        g = 1
        flags = <optimized out>
        g1_start = <optimized out>
        maxspin = 0
        signals = <optimized out>
        result = 0
        wseq = 3
        seq = 1
        private = 0
        maxspin = <optimized out>
        err = <optimized out>
        result = <optimized out>
        wseq = <optimized out>
        g = <optimized out>
        seq = <optimized out>
        flags = <optimized out>
        private = <optimized out>
        signals = <optimized out>
        done = <optimized out>
        g1_start = <optimized out>
        spin = <optimized out>
        buffer = <optimized out>
        cbuffer = <optimized out>
        s = <optimized out>
#4  ___pthread_cond_timedwait64 (cond=cond@entry=0x462220 <cond>, mutex=mutex@entry=0x4621e0 <cond_mutex>, abstime=abstime@entry=0x14e3293878b0) at pthread_cond_wait.c:644
        flags = <optimized out>
        clockid = 0
#5  0x000000000040aec2 in cli_cmd_await_response (time=time@entry=120) at cli-cmd.c:265
        ts = {tv_sec = 1711599554, tv_nsec = 858221356}
        ret = 0
#6  0x000000000040b462 in cli_cmd_submit (rpc=rpc@entry=0x0, req=req@entry=0x14e3293879a0, frame=frame@entry=0x17b2568, prog=prog@entry=0x460fe0 <cli_prog>, procnum=procnum@entry=27, this=this@entry=0x14e32b85e560 <global_xlator>, cbkfn=0x429a20 <gf_cli_status_cbk>, xdrproc=0x4084c0 <xdr_gf_cli_req@plt>) at cli-cmd.c:372
        ret = 0
        timeout = 120
        state = <optimized out>
        __FUNCTION__ = "cli_cmd_submit"
#7  0x000000000041c942 in cli_to_glusterd (req=req@entry=0x14e3293879a0, frame=frame@entry=0x17b2568, cbkfn=cbkfn@entry=0x429a20 <gf_cli_status_cbk>, dict=dict@entry=0x17a6428, procnum=procnum@entry=27, this=this@entry=0x14e32b85e560 <global_xlator>, prog=0x460fe0 <cli_prog>, xdrproc=<optimized out>) at cli-rpc-ops.c:10668
        ret = <optimized out>
        len = <optimized out>
        cmd = <optimized out>
        i = <optimized out>
        words = <optimized out>
        local = <optimized out>
        state = 0x7ffec163ba50
        __FUNCTION__ = "cli_to_glusterd"
#8  0x000000000041d633 in gf_cli_status_volume (frame=frame@entry=0x17b2568, this=this@entry=0x14e32b85e560 <global_xlator>, data=data@entry=0x17a6428) at cli-rpc-ops.c:7637
        req = {dict = {dict_len = 86, dict_val = 0x17a4388 ""}}
        ret = -1
        dict = 0x17a6428
        __FUNCTION__ = "gf_cli_status_volume"
#9  0x000000000041d868 in gf_cli_status_volume_all (frame=frame@entry=0x17b2568, this=0x14e32b85e560 <global_xlator>, data=data@entry=0x17a6328) at cli-rpc-ops.c:7728
        i = 0
        ret = <optimized out>
        vol_count = 1
        cmd = 258
        key = "vol0", '\000' <repeats 1019 times>
        volname = 0x17cc6a8 "backup-img"
        vol_dict = 0x17a64a8
        dict = 0x17a6428
        local = 0x17b6708
        state = 0x7ffec163ba50
        __PRETTY_FUNCTION__ = "gf_cli_status_volume_all"
        __FUNCTION__ = "gf_cli_status_volume_all"
        xml_end = <optimized out>
#10 0x0000000000410e73 in cli_cmd_volume_status_cbk (state=<optimized out>, word=<optimized out>, words=<optimized out>, wordcount=<optimized out>) at cli-cmd-volume.c:2315
        ret = <optimized out>
        proc = 0x4611e0 <gluster_cli_actors+448>
        frame = 0x17b2568
        dict = 0x17a6328
        cmd = 514
        local = 0x17b6708
        out = <optimized out>
        __FUNCTION__ = "cli_cmd_volume_status_cbk"
#11 0x000000000040b09e in cli_cmd_process (state=0x7ffec163ba50, argc=4, argv=0x7ffec163bcb0) at cli-cmd.c:131
        ret = <optimized out>
        word = 0x1778af0
        next = <optimized out>
        i = <optimized out>
        __FUNCTION__ = "cli_cmd_process"
#12 0x000000000040ab5c in cli_batch (d=<optimized out>) at input.c:29
        state = <optimized out>
        ret = 0
        __FUNCTION__ = "cli_batch"
#13 0x000014e32adae22a in start_thread (arg=<optimized out>) at pthread_create.c:443
        ret = <optimized out>
        pd = <optimized out>
        out = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140732142958640, 7385284747875049934, 22965881706048, 0, 22965909118816, 140732142958944, 5745360750633017806, 5745353891887440334}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#14 0x000014e32ae30cf0 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
No locals.

Thread 4 (Thread 0x14e329188640 (LWP 1996243)):
#0  0x000014e32ae302be in epoll_wait (epfd=3, events=events@entry=0x14e329186e84, maxevents=maxevents@entry=1, timeout=timeout@entry=-1) at ../sysdeps/unix/sysv/linux/epoll_wait.c:30
        sc_ret = -4
        sc_cancel_oldtype = 0
        sc_ret = <optimized out>
#1  0x000014e32b7d0e24 in event_dispatch_epoll_worker (data=0x17cc4a8) at event-epoll.c:715
        event = {events = 4, data = {ptr = 0x100000000, fd = 0, u32 = 0, u64 = 4294967296}}
        ret = <optimized out>
        ev_data = 0x17cc4a8
        event_pool = 0x1782028
        myindex = 1
        timetodie = 0
        gen = 0
        poller_death_notify = {next = 0x0, prev = 0x0}
        slot = 0x0
        tmp = 0x0
        __FUNCTION__ = "event_dispatch_epoll_worker"
        out = <optimized out>
#2  0x000014e32adae22a in start_thread (arg=<optimized out>) at pthread_create.c:443
        ret = <optimized out>
        pd = <optimized out>
        out = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140732142957984, 7385284747875049934, 22965879604800, 0, 22965909118816, 140732142958288, 5745361026047795662, 5745353891887440334}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#3  0x000014e32ae30cf0 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
No locals.

Thread 3 (Thread 0x14e32958a640 (LWP 1996201)):
#0  0x000014e32adaad92 in __futex_abstimed_wait_common64 (private=0, cancel=true, abstime=0x14e329588ea0, op=137, expected=0, futex_word=0x17b629c) at futex-internal.c:57
        sc_cancel_oldtype = 0
        __arg6 = <optimized out>
        __arg3 = <optimized out>
        _a5 = <optimized out>
        _a2 = <optimized out>
        sc_ret = <optimized out>
        __arg4 = <optimized out>
        __arg1 = <optimized out>
        _a6 = <optimized out>
        _a3 = <optimized out>
        resultvar = <optimized out>
        __arg5 = <optimized out>
        __arg2 = <optimized out>
        _a4 = <optimized out>
        _a1 = <optimized out>
#1  __futex_abstimed_wait_common (futex_word=futex_word@entry=0x17b629c, expected=expected@entry=0, clockid=clockid@entry=1, abstime=abstime@entry=0x14e329588ea0, private=private@entry=0, cancel=cancel@entry=true) at futex-internal.c:87
        err = <optimized out>
        clockbit = 0
        op = 137
#2  0x000014e32adaadff in __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x17b629c, expected=expected@entry=0, clockid=clockid@entry=1, abstime=abstime@entry=0x14e329588ea0, private=private@entry=0) at futex-internal.c:139
No locals.
#3  0x000014e32adad934 in __pthread_cond_wait_common (abstime=0x14e329588ea0, clockid=1, mutex=0x17b6248, cond=0x17b6270) at pthread_cond_wait.c:504
        spin = 0
        buffer = {__routine = 0x14e32adad3a0 <__condvar_cleanup_waiting>, __arg = 0x14e329588e30, __canceltype = 0, __prev = 0x0}
        cbuffer = {wseq = 7, cond = 0x17b6270, mutex = 0x17b6248, private = 0}
        err = <optimized out>
        g = 1
        flags = <optimized out>
        g1_start = <optimized out>
        maxspin = 0
        signals = <optimized out>
        result = 0
        wseq = 7
        seq = 3
        private = 0
        maxspin = <optimized out>
        err = <optimized out>
        result = <optimized out>
        wseq = <optimized out>
        g = <optimized out>
        seq = <optimized out>
        flags = <optimized out>
        private = <optimized out>
        signals = <optimized out>
        done = <optimized out>
        g1_start = <optimized out>
        spin = <optimized out>
        buffer = <optimized out>
        cbuffer = <optimized out>
        s = <optimized out>
#4  ___pthread_cond_timedwait64 (cond=cond@entry=0x17b6270, mutex=mutex@entry=0x17b6248, abstime=abstime@entry=0x14e329588ea0) at pthread_cond_wait.c:644
        flags = <optimized out>
        clockid = 1
#5  0x000014e32b788a38 in gf_timer_proc (data=0x17b6238) at timer.c:140
        now = {tv_sec = 1127334, tv_nsec = 67471480}
        reg = 0x17b6238
        event = 0x17ac178
        tmp = 0x0
        old_THIS = <optimized out>
#6  0x000014e32adae22a in start_thread (arg=<optimized out>) at pthread_create.c:443
        ret = <optimized out>
        pd = <optimized out>
        out = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140732142957904, 7385284747875049934, 22965883807296, 17, 22965909118816, 140732142958208, 5745361570434900430, 5745353891887440334}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#7  0x000014e32ae30cf0 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
No locals.

Thread 2 (Thread 0x14e32a86f6c0 (LWP 1996195)):
#0  0x000014e32adaad92 in __futex_abstimed_wait_common64 (private=128, cancel=true, abstime=0x0, op=265, expected=1996243, futex_word=0x14e329188910) at futex-internal.c:57
        sc_cancel_oldtype = 0
        __arg6 = <optimized out>
        __arg3 = <optimized out>
        _a5 = <optimized out>
        _a2 = <optimized out>
        sc_ret = <optimized out>
        __arg4 = <optimized out>
        __arg1 = <optimized out>
        _a6 = <optimized out>
        _a3 = <optimized out>
        resultvar = <optimized out>
        __arg5 = <optimized out>
        __arg2 = <optimized out>
        _a4 = <optimized out>
        _a1 = <optimized out>
#1  __futex_abstimed_wait_common (futex_word=futex_word@entry=0x14e329188910, expected=1996243, clockid=clockid@entry=0, abstime=abstime@entry=0x0, private=private@entry=128, cancel=cancel@entry=true) at futex-internal.c:87
        err = <optimized out>
        clockbit = 256
        op = 265
#2  0x000014e32adaadff in __GI___futex_abstimed_wait_cancelable64 (futex_word=futex_word@entry=0x14e329188910, expected=<optimized out>, clockid=clockid@entry=0, abstime=abstime@entry=0x0, private=private@entry=128) at futex-internal.c:139
No locals.
#3  0x000014e32adafb83 in __pthread_clockjoin_ex (threadid=22965879604800, thread_return=thread_return@entry=0x0, clockid=clockid@entry=0, abstime=abstime@entry=0x0, block=block@entry=true) at pthread_join_common.c:102
        ret = <optimized out>
        _buffer = {__routine = 0x14e32adafa60 <cleanup>, __arg = 0x14e329188c68, __canceltype = 0, __prev = 0x0}
        tid = <optimized out>
        pd = 0x14e329188640
        self = <optimized out>
        result = 0
        pd_result = <optimized out>
#4  0x000014e32adafa53 in ___pthread_join (threadid=<optimized out>, thread_return=thread_return@entry=0x0) at pthread_join.c:25
No locals.
#5  0x000014e32b7d06df in event_dispatch_epoll (event_pool=0x1782028) at event-epoll.c:809
        i = <optimized out>
        t_id = 22965877503552
        pollercount = 2
        ret = 0
        ev_data = <optimized out>
        __FUNCTION__ = "event_dispatch_epoll"
#6  0x0000000000409187 in main (argc=<optimized out>, argv=<optimized out>) at cli.c:811
        state = {argc = 4, argv = 0x7ffec163bcb0, debug = 0 '\000', ctx = 0x1772000, rpc = 0x17ca028, quotad_rpc = 0x0, tree = {state = 0x7ffec163ba50, root = {tree = 0x7ffec163ba80, word = 0x0, filler = 0x0, match = 0x0, cbkfn = 0x0, desc = 0x0, pattern = 0x0, nextwords_cnt = 11, nextwords = 0x17ac310}}, input = 22965881706048, prompt = 0x0, rl_enabled = 0, rl_async = 0, rl_processing = 0, matches = 0x0, matchesp = 0x0, remote_host = 0x0, remote_port = 0, mode = 0, await_connected = 120, default_conn_timeout = 120, log_file = 0x0, log_level = GF_LOG_NONE, glusterd_sock = 0x0, address_family = 0x0}
        ret = 0
        ctx = 0x1772000
        out = <optimized out>
        __FUNCTION__ = "main"

Thread 1 (Thread 0x14e328f87640 (LWP 1996244)):
#0  iobuf_get_from_stdalloc (page_size=3247444, iobuf_pool=0x0) at iobuf.c:423
        iobuf = 0x0
        iobuf_arena = 0x0
        trav = 0x0
#1  iobuf_get2 (iobuf_pool=0x0, page_size=3247444) at iobuf.c:493
        iobuf = 0x0
        rounded_size = <optimized out>
        index = <optimized out>
        __FUNCTION__ = "iobuf_get2"
        post_unlock = <optimized out>
#2  0x000014e32962fcf1 in socket_proto_state_machine (pollin=<synthetic pointer>, this=0x17ae5a8) at socket.c:2234
        ret = <optimized out>
        iobuf = <optimized out>
        frag = <optimized out>
        priv = <optimized out>
        iobref = 0x0
        vector = {{iov_base = 0x17ca560, iov_len = 256}, {iov_base = 0x0, iov_len = 0}}
        in = <optimized out>
        ret = <optimized out>
        priv = <optimized out>
        iobuf = <optimized out>
        iobref = <optimized out>
        vector = <optimized out>
        in = <optimized out>
        frag = <optimized out>
        __FUNCTION__ = "socket_proto_state_machine"
        out = <optimized out>
        count = <optimized out>
#3  socket_event_poll_in (notify_handled=true, this=0x17ae5a8) at socket.c:2384
        ret = -1
        pollin = 0x0
        priv = 0x1812028
        ctx = 0x1772000
        ret = <optimized out>
        pollin = <optimized out>
        priv = <optimized out>
        ctx = <optimized out>
#4  socket_event_handler (event_thread_died=0, poll_err=0, poll_out=<optimized out>, poll_in=<optimized out>, data=0x17ae5a8, gen=1, idx=0, fd=<optimized out>) at socket.c:2790
        this = <optimized out>
        ret = <optimized out>
        ctx = <optimized out>
        notify_handled = <optimized out>
        priv = 0x1812028
        socket_closed = <optimized out>
        this = <optimized out>
        priv = <optimized out>
        ret = <optimized out>
        ctx = <optimized out>
        socket_closed = <optimized out>
        notify_handled = <optimized out>
        __FUNCTION__ = "socket_event_handler"
        out = <optimized out>
        sock_type = <optimized out>
        sa = <optimized out>
#5  socket_event_handler (fd=fd@entry=6, idx=idx@entry=0, gen=gen@entry=1, data=data@entry=0x17ae5a8, poll_in=<optimized out>, poll_out=<optimized out>, poll_err=0, event_thread_died=0) at socket.c:2710
        this = 0x17ae5a8
        priv = 0x0
        ret = -1
        ctx = 0x0
        socket_closed = false
        notify_handled = false
        __FUNCTION__ = "socket_event_handler"
        out = <optimized out>
        sock_type = <optimized out>
        sa = <optimized out>
#6  0x000014e32b7d0f4d in event_dispatch_epoll_handler (event=0x14e328f85e84, event_pool=0x1782028) at event-epoll.c:614
        handler = <optimized out>
        gen = <optimized out>
        pre_unlock = <optimized out>
        slot = 0x17860e8
        data = <optimized out>
        ret = 0
        fd = 6
        ev_data = 0x14e328f85e88
        idx = 0
        handled_error_previously = false
        ev_data = <optimized out>
        slot = <optimized out>
        handler = <optimized out>
        data = <optimized out>
        idx = <optimized out>
        gen = <optimized out>
        ret = <optimized out>
        fd = <optimized out>
        handled_error_previously = <optimized out>
        __FUNCTION__ = "event_dispatch_epoll_handler"
        pre_unlock = <optimized out>
        out = <optimized out>
#7  event_dispatch_epoll_worker (data=0x17cc468) at event-epoll.c:725
        event = {events = 1, data = {ptr = 0x100000000, fd = 0, u32 = 0, u64 = 4294967296}}
        ret = <optimized out>
        ev_data = 0x17cc468
        event_pool = 0x1782028
        myindex = 2
        timetodie = 0
        gen = 0
        poller_death_notify = {next = 0x0, prev = 0x0}
        slot = 0x0
        tmp = 0x0
        __FUNCTION__ = "event_dispatch_epoll_worker"
        out = <optimized out>
#8  0x000014e32adae22a in start_thread (arg=<optimized out>) at pthread_create.c:443
        ret = <optimized out>
        pd = <optimized out>
        out = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {140732142957984, 7385284747875049934, 22965877503552, 0, 22965909118816, 140732142958288, 5745358002927690190, 5745353891887440334}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0, cleanup = 0x0, canceltype = 0}}}
        not_first_call = <optimized out>
#9  0x000014e32ae30cf0 in clone3 () at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:81
No locals.
(gdb)

mohit84 commented 1 month ago

As per backtrace it is showing cli process is getting crashed while trying to access a iobuf_pool list of entries, as we can see the function ( iobuf_get_from_stdalloc (page_size=3247444, iobuf_pool=0x0)) is calling with NULL iobuf_pool. It is a basic code path and no one would be able to run any cli command if iobuf_pool is NULL. The iobuf_pool is allocated by glusterfs_ctx_defaults_init the function call by main function as a part of cli.c. It is the starting function that allocates pool structure. Are you getting any error in cli.logs while running a command?

wonderful1984 commented 1 month ago

glusterfs.log 
[2024-03-28 05:02:16.592514 +0000] I [MSGID: 106061] [glusterd-utils.c:9576:glusterd_volume_status_copy_to_op_ctx_dict] 0-management: Dict get failed [{Key=count}] 
[2024-03-28 05:02:16.592667 +0000] I [MSGID: 106499] [glusterd-handler.c:4535:__glusterd_handle_status_volume] 0-management: Received status volume req for volume backup-img 

cli.log
[2024-03-28 05:02:16.515610 +0000] I [cli.c:788:main] 0-cli: Started running /usr/local/sbin/gluster with version 11.1
[2024-03-28 05:02:16.591492 +0000] I [MSGID: 101188] [event-epoll.c:643:event_dispatch_epoll_worker] 0-epoll: Started thread with index [{index=0}] 
[2024-03-28 05:02:16.591771 +0000] I [MSGID: 101188] [event-epoll.c:643:event_dispatch_epoll_worker] 0-epoll: Started thread with index [{index=1}]

gluster / glusterfs

Segmentation fault gluster volume status all clients #4323