search

stunnel / static-curl

static builds cURL with HTTP3
MIT License
231 stars 29 forks source link

SIGFPE on kernel 6.8 with 8.10.1 glibc version #93

Closed PAStheLoD closed 1 month ago

PAStheLoD commented 1 month ago

Hello,

First of all thanks for maintaining this!

The details of the crash

Tried with the following kernel versions:

6.8.0-45-generic #45~22.04.1-Ubuntu SMP PREEMPT_DYNAMIC Wed Sep 11 15:25:05 UTC 2 x86_64 x86_64 x86_64 GNU/Linux

6.11.0-8-generic #8-Ubuntu SMP PREEMPT_DYNAMIC Mon Sep 16 13:41:20 UTC 2024 x86_64 x86_64 x86_64 GNU/Linux

On Ubuntu 24.10 userspace.

It's a VM, and cpuinfo says it's a 

processor   : 0
vendor_id   : GenuineIntel
cpu family  : 6
model       : 85
model name  : Intel Xeon Processor (Skylake, IBRS)
stepping    : 4
microcode   : 0x1
cpu MHz     : 2099.998
cache size  : 16384 KB
physical id : 0
siblings    : 2
core id     : 0
cpu cores   : 2
apicid      : 0
initial apicid  : 0
fpu     : yes
fpu_exception   : yes
cpuid level : 13
wp      : yes
flags       : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology cpuid tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch cpuid_fault pti ssbd ibrs ibpb fsgsbase bmi1 hle avx2 smep bmi2 erms invpcid rtm avx512f avx512dq rdseed adx smap clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 arat pku ospke md_clear
bugs        : cpu_meltdown spectre_v1 spectre_v2 spec_store_bypass l1tf mds swapgs taa itlb_multihit mmio_stale_data retbleed gds bhi
bogomips    : 4199.99
clflush size    : 64
cache_alignment : 64
address sizes   : 40 bits physical, 48 bits virtual
power management:

execve("./glibc/curl", ["./glibc/curl"], 0x7ffff7346a38 /* 26 vars */) = 0
brk(NULL)                               = 0xf01000
brk(0xf01d80)                           = 0xf01d80
arch_prctl(ARCH_SET_FS, 0xf01400)       = 0
set_tid_address(0xf016d0)               = 410633
set_robust_list(0xf016e0, 24)           = 0
rseq(0xf01d20, 0x20, 0, 0x53053053)     = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
readlink("/proc/self/exe", "/opt/curl-http3/glibc/curl", 4096) = 26
getrandom("\x00\xa5\x79\x15\xf1\x6c\x43\xf7", 8, GRND_NONBLOCK) = 8
brk(NULL)                               = 0xf01d80
brk(0xf22d80)                           = 0xf22d80
brk(0xf23000)                           = 0xf23000
mprotect(0xbe9000, 450560, PROT_READ)   = 0
fcntl(0, F_GETFD)                       = 0
fcntl(1, F_GETFD)                       = 0
fcntl(2, F_GETFD)                       = 0
rt_sigaction(SIGPIPE, {sa_handler=SIG_IGN, sa_mask=[PIPE], sa_flags=SA_RESTORER|SA_RESTART, sa_restorer=0x90c800}, {sa_handler=SIG_DFL, sa_mask=[], sa_flags=0}, 8) = 0
openat(AT_FDCWD, "/data/curl-x86_64/ssl/openssl.cnf", O_RDONLY) = -1 ENOENT (No such file or directory)
sysinfo({uptime=69883, loads=[7392, 8256, 15488], totalram=4006031360, freeram=273661952, sharedram=15052800, bufferram=96718848, totalswap=0, freeswap=0, procs=471, totalhigh=0, freehigh=0, mem_unit=1}) = 0
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=3055776, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 3055776, PROT_READ, MAP_PRIVATE, 3, 0) = 0x79c97b600000
close(3)                                = 0
openat(AT_FDCWD, "/root/.curlrc", O_RDONLY) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/root/.config/curlrc", O_RDONLY) = -1 ENOENT (No such file or directory)
geteuid()                               = 0
socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_UNIX, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3)                                = 0
socket(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC|SOCK_NONBLOCK, 0) = 3
connect(3, {sa_family=AF_UNIX, sun_path="/var/run/nscd/socket"}, 110) = -1 ENOENT (No such file or directory)
close(3)                                = 0
newfstatat(AT_FDCWD, "/etc/nsswitch.conf", {st_mode=S_IFREG|0644, st_size=513, ...}, 0) = 0
newfstatat(AT_FDCWD, "/", {st_mode=S_IFDIR|0755, st_size=4096, ...}, 0) = 0
openat(AT_FDCWD, "/etc/nsswitch.conf", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=513, ...}, AT_EMPTY_PATH) = 0
read(3, "# /etc/nsswitch.conf\n#\n# Example"..., 4096) = 513
read(3, "", 4096)                       = 0
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=513, ...}, AT_EMPTY_PATH) = 0
close(3)                                = 0
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=29629, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 29629, PROT_READ, MAP_PRIVATE, 3, 0) = 0x79c97ba20000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libnss_compat.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\0\0\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0644, st_size=43816, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 46816, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x79c97ba14000
mmap(0x79c97ba16000, 28672, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x79c97ba16000
mmap(0x79c97ba1d000, 4096, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x9000) = 0x79c97ba1d000
mmap(0x79c97ba1e000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x9000) = 0x79c97ba1e000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200\245\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=2182752, ...}, AT_EMPTY_PATH) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
mmap(NULL, 2227672, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x79c97b200000
mmap(0x79c97b228000, 1662976, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x28000) = 0x79c97b228000
mmap(0x79c97b3be000, 323584, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1be000) = 0x79c97b3be000
mmap(0x79c97b40d000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x20c000) = 0x79c97b40d000
mmap(0x79c97b413000, 52696, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x79c97b413000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0@\t\2\0\0\0\0\0"..., 832) = 832
newfstatat(3, "", {st_mode=S_IFREG|0755, st_size=244816, ...}, AT_EMPTY_PATH) = 0
mmap(NULL, 242400, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x79c97b9d8000
mmap(0x79c97b9d9000, 184320, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1000) = 0x79c97b9d9000
mmap(0x79c97ba06000, 40960, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2e000) = 0x79c97ba06000
mmap(0x79c97ba10000, 16384, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x38000) = 0x79c97ba10000
close(3)                                = 0
mprotect(0x79c97ba10000, 8192, PROT_READ) = 0
mprotect(0x79c97b40d000, 16384, PROT_READ) = 0
mprotect(0x79c97ba1e000, 4096, PROT_READ) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
--- SIGFPE {si_signo=SIGFPE, si_code=FPE_INTDIV, si_addr=0x79c97b392141} ---
+++ killed by SIGFPE (core dumped) +++
Floating point exception (core dumped)```

Also the musl version runs flawlessly.
travislee89 commented 1 month ago

curl is a statically linked program, so it shouldn't load /etc/ld.so.cache.
I checked kernels 5.15, 6.8, and 6.11 on Ubuntu 22.04, and they never load /etc/ld.so.cache.

Could you show me the result of ldd ./glibc/curl?

PAStheLoD commented 1 month ago

Hello,

Yes of course.

/opt/curl-http3/glibc# ./curl
Floating point exception (core dumped)

/opt/curl-http3/glibc# ldd ./curl
    not a dynamic executable

/opt/curl-http3/glibc# file curl
curl: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), statically linked, BuildID[sha1]=c2de21ff7b244331dda32053f3b6a33c13e99ab1, for GNU/Linux 3.2.0, stripped

Many thanks for the fast response!

Interestingly trurl runs. (Well sort of, but probably this is a bug, as both versions crash. Maybe a double-free.)

/opt/curl-http3/glibc# ./trurl  'https://example.com/aa?a/b"'
https://example.com/aa?a%2fb%22

# ./trurl  'https://example.com/aa?a/b"=aaa'
https://example.com/aa?a%2fb%22=aaa
free(): invalid pointer
Aborted (core dumped)

/opt/curl-http3/musl# ./trurl  'https://example.com/aa?a/b"=aaa'
https://example.com/aa?a%2fb%22=aaa
Segmentation fault (core dumped)
travislee89 commented 1 month ago
  1. echo $LD_PRELOAD; echo $LD_LIBRARY_PATH
  2. Please show me the content of these files.
    • /etc/nsswitch.conf
    • /etc/ld.so.conf

And all files in directory /etc/ld.so.conf.d/

PAStheLoD commented 1 month ago

1.

# echo " ___ $LD_PRELOAD __ $LD_LIBRARY_PATH ___"
 ___  __  ___

# ( set -o posix ; set ) | grep LD_
#

# env | grep LD_
#

They should be unset (and they appear to be).

2.

# cat /etc/nsswitch.conf
# /etc/nsswitch.conf
#
# Example configuration of GNU Name Service Switch functionality.
# If you have the `glibc-doc-reference' and `info' packages installed, try:
# `info libc "Name Service Switch"' for information about this file.

passwd:         compat systemd
group:          compat systemd
shadow:         compat
gshadow:        files

hosts:          files dns
networks:       files

protocols:      db files
services:       db files
ethers:         db files
rpc:            db files

netgroup:       nis

(I tried removing systemd from nsswitch.conf, it seems to have made no difference.)

# cat /etc/ld.so.conf
include /etc/ld.so.conf.d/*.conf
# find /etc/ld.so.conf.d/ -type f
/etc/ld.so.conf.d/x86_64-linux-gnu.conf
/etc/ld.so.conf.d/libc.conf
# find /etc/ld.so.conf.d/ -type f -exec echo -e "\n\n---" {} "---"  \; -exec cat {} \;

--- /etc/ld.so.conf.d/x86_64-linux-gnu.conf ---
# Multiarch support
/usr/local/lib/x86_64-linux-gnu
/lib/x86_64-linux-gnu
/usr/lib/x86_64-linux-gnu

--- /etc/ld.so.conf.d/libc.conf ---
# libc default configuration
/usr/local/lib
travislee89 commented 1 month ago

It seems like everything is fine, but I have no idea why it loads libnss_compat.so.2, libc.so.6, and ld-linux-x86-64.so.2.
There is also a similar issue, #62.
If the cause cannot be found, you can use the musl version.

PAStheLoD commented 1 month ago

If I empty out nsswitch.conf it starts working :o

Also if the first value is "files" for the keys, it works.

So for example it works with this:


passwd: files systemd
group: files systemd

gshadow: files
shadow: files

hosts:          files dns
networks:       files

protocols:      db files
services:       db files
ethers:         db files
rpc:            db files

netgroup:       nis

So probably it's something in glibc + compat libs ... probably it wants to parse UID ... because it seems to happen only when "passwd" contains systemd or compat on first place, and the rest doesn't seem to matter (at least when I tested with "group".)

Thanks again!

travislee89 commented 1 month ago

Thanks @PAStheLoD , I tested on Ubuntu and Debian, if /etc/nsswitch.conf file is configured with passwd: compat, glibc will attempt to load libnss_compat.so, libnss_nis.so, libpthread.so, etc. These libraries may not be compatible with the statically linked glibc, and the program might crash.
Currently, there is no good solution for this issue, it is recommended to use the musl version in this case.