search

prometheus / node_exporter

Exporter for machine metrics
https://prometheus.io/
Apache License 2.0
11k stars 2.34k forks source link

Go panic: fatal error: checkdead: runnable g #2730

Closed Evesy closed 6 months ago

Evesy commented 1 year ago

Related https://github.com/prometheus/node_exporter/issues/1881

Host operating system: output of uname -a

Linux gke-delivery-platform-normal-20220831-b14ed261-plov 5.15.89+ #1 SMP Sat Mar 18 09:27:02 UTC 2023 x86_64 Intel(R) Xeon(R) CPU @ 2.60GHz GenuineIntel GNU/Linux

node_exporter version: output of node_exporter --version

1.6.0

node_exporter command line flags

        - --path.sysfs=/host/sys
        - --web.listen-address=0.0.0.0:9100
        - --collector.disable-defaults
        - --collector.conntrack
        - --collector.cpu
        - --collector.meminfo
        - --collector.filesystem
        - --collector.diskstats
        - --collector.netdev
        - --collector.timex
        - --collector.vmstat
        - --collector.vmstat.fields=^(oom_kill|pgpg|pswp|pg.*fault|nr_mapped).*
        - --collector.netstat
        - --collector.netstat.fields=^(TcpExt_TCPSynRetrans|Tcp_RetransSegs)$

node_exporter log output

Are you running node_exporter in Docker?

Kubernetes pod

What did you do that produced an error?

Node exporter attempted to startup on a new node

What did you expect to see?

What did you see instead?

ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.6.0, branch=HEAD, revision=ff7f9d69b645cb691dd3e84dc3afc88f5c006962)"
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.20.4, platform=linux/amd64, user=root@f9c3ed0cfbd3, date=20230527-12:03:54, tags=netgo osusergo static_build)"
ts=2023-06-30T07:31:51.768Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$
ts=2023-06-30T07:31:51.768Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data
ts=2023-06-30T07:31:51.768Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/)
ts=2023-06-30T07:31:51.768Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:110 level=info msg="Enabled collectors"
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=conntrack
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=cpu
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=diskstats
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=filesystem
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=meminfo
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=netdev
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=netstat
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=timex
ts=2023-06-30T07:31:51.768Z caller=node_exporter.go:117 level=info collector=vmstat
ts=2023-06-30T07:31:51.769Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100
ts=2023-06-30T07:31:51.769Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100
runtime: checkdead: find g 62359 in status 1
fatal error: checkdead: runnable g

runtime stack:
runtime.throw({0xbdbcc7?, 0x0?})
    /usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0xc000111d58 sp=0xc000111d28 pc=0x438d5d
runtime.checkdead.func1(0xc0002fb6c0)
    /usr/local/go/src/runtime/proc.go:5216 +0xe5 fp=0xc000111d90 sp=0xc000111d58 pc=0x446a85
runtime.forEachG(0xc000111df0)
    /usr/local/go/src/runtime/proc.go:591 +0x5c fp=0xc000111dc0 sp=0xc000111d90 pc=0x43c4fc
runtime.checkdead()
    /usr/local/go/src/runtime/proc.go:5203 +0xf7 fp=0xc000111e10 sp=0xc000111dc0 pc=0x446777
runtime.mput(0x0?)
    /usr/local/go/src/runtime/proc.go:5696 +0x33 fp=0xc000111e20 sp=0xc000111e10 pc=0x447cf3
runtime.stopm()
    /usr/local/go/src/runtime/proc.go:2332 +0x67 fp=0xc000111e50 sp=0xc000111e20 pc=0x43f927
runtime.findRunnable()
    /usr/local/go/src/runtime/proc.go:3007 +0xabc fp=0xc000111f58 sp=0xc000111e50 pc=0x44119c
runtime.schedule()
    /usr/local/go/src/runtime/proc.go:3360 +0xb1 fp=0xc000111f90 sp=0xc000111f58 pc=0x441fd1
runtime.park_m(0xc0002fb6c0?)
    /usr/local/go/src/runtime/proc.go:3511 +0x12d fp=0xc000111fc0 sp=0xc000111f90 pc=0x4424ed
runtime.mcall()
    /usr/local/go/src/runtime/asm_amd64.s:452 +0x43 fp=0xc000111fd0 sp=0xc000111fc0 pc=0x46cb03

goroutine 1 [semacquire, 119 minutes]:
runtime.gopark(0xce8f00?, 0xc000040030?, 0x0?, 0x80?, 0x6?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010f950 sp=0xc00010f930 pc=0x43ba76
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0xc000145890, 0x40?, 0x1, 0x0, 0xe0?)
    /usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc00010f9b8 sp=0xc00010f950 pc=0x44cdaf
sync.runtime_Semacquire(0xc0000061a0?)
    /usr/local/go/src/runtime/sema.go:62 +0x27 fp=0xc00010f9f0 sp=0xc00010f9b8 pc=0x46ac07
sync.(*WaitGroup).Wait(0x7f2bec846501?)
    /usr/local/go/src/sync/waitgroup.go:116 +0x4b fp=0xc00010fa18 sp=0xc00010f9f0 pc=0x48a76b
golang.org/x/sync/errgroup.(*Group).Wait(0xc000145880)
    /go/pkg/mod/golang.org/x/sync@v0.2.0/errgroup/errgroup.go:53 +0x27 fp=0xc00010fa30 sp=0xc00010fa18 pc=0x92c867
github.com/prometheus/exporter-toolkit/web.ServeMultiple({0xc0001f3a00, 0x1, 0x7ffe4b72d247?}, 0xc0001f1770, 0xc0001ea3c0, {0xce3360?, 0xc000043a00})
    /go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.10.0/web/tls_config.go:235 +0x151 fp=0xc00010fa80 sp=0xc00010fa30 pc=0x92f831
github.com/prometheus/exporter-toolkit/web.ListenAndServe(0x11381a0?, 0xc0001ea3c0, {0xce3360, 0xc000043a00})
    /go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.10.0/web/tls_config.go:268 +0x488 fp=0xc00010fb58 sp=0xc00010fa80 pc=0x92fd88
main.main()
    /app/node_exporter.go:210 +0x164e fp=0xc00010ff80 sp=0xc00010fb58 pc=0xa645ee
runtime.main()
    /usr/local/go/src/runtime/proc.go:250 +0x207 fp=0xc00010ffe0 sp=0xc00010ff80 pc=0x43b647
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010ffe8 sp=0xc00010ffe0 pc=0x46ec01

goroutine 2 [force gc (idle), 119 minutes]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000064fb0 sp=0xc000064f90 pc=0x43ba76
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:387
runtime.forcegchelper()
    /usr/local/go/src/runtime/proc.go:305 +0xb0 fp=0xc000064fe0 sp=0xc000064fb0 pc=0x43b8b0
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000064fe8 sp=0xc000064fe0 pc=0x46ec01
created by runtime.init.6
    /usr/local/go/src/runtime/proc.go:293 +0x25

goroutine 3 [GC sweep wait]:
runtime.gopark(0x1?, 0x0?, 0x0?, 0x0?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000065780 sp=0xc000065760 pc=0x43ba76
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:387
runtime.bgsweep(0x0?)
    /usr/local/go/src/runtime/mgcsweep.go:319 +0xde fp=0xc0000657c8 sp=0xc000065780 pc=0x425ede
runtime.gcenable.func1()
    /usr/local/go/src/runtime/mgc.go:178 +0x26 fp=0xc0000657e0 sp=0xc0000657c8 pc=0x41b146
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000657e8 sp=0xc0000657e0 pc=0x46ec01
created by runtime.gcenable
    /usr/local/go/src/runtime/mgc.go:178 +0x6b

goroutine 4 [runnable]:
runtime.gopark(0x67143f4e771?, 0x2fc273?, 0x0?, 0x0?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000065f70 sp=0xc000065f50 pc=0x43ba76
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:387
runtime.(*scavengerState).park(0x11384a0)
    /usr/local/go/src/runtime/mgcscavenge.go:400 +0x53 fp=0xc000065fa0 sp=0xc000065f70 pc=0x423db3
runtime.bgscavenge(0x0?)
    /usr/local/go/src/runtime/mgcscavenge.go:633 +0x65 fp=0xc000065fc8 sp=0xc000065fa0 pc=0x4243a5
runtime.gcenable.func2()
    /usr/local/go/src/runtime/mgc.go:179 +0x26 fp=0xc000065fe0 sp=0xc000065fc8 pc=0x41b0e6
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000065fe8 sp=0xc000065fe0 pc=0x46ec01
created by runtime.gcenable
    /usr/local/go/src/runtime/mgc.go:179 +0xaa

goroutine 5 [finalizer wait, 119 minutes]:
runtime.gopark(0x43bdf2?, 0x7f2c13be5a48?, 0x0?, 0x0?, 0xc000064770?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000064628 sp=0xc000064608 pc=0x43ba76
runtime.runfinq()
    /usr/local/go/src/runtime/mfinal.go:193 +0x107 fp=0xc0000647e0 sp=0xc000064628 pc=0x41a187
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0000647e8 sp=0xc0000647e0 pc=0x46ec01
created by runtime.createfing
    /usr/local/go/src/runtime/mfinal.go:163 +0x45

goroutine 36 [IO wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00009aa78 sp=0xc00009aa58 pc=0x43ba76
runtime.netpollblock(0xc00009ab08?, 0x40482f?, 0x0?)
    /usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc00009aab0 sp=0xc00009aa78 pc=0x4343d7
internal/poll.runtime_pollWait(0x7f2bec8159a0, 0x72)
    /usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc00009aad0 sp=0xc00009aab0 pc=0x469349
internal/poll.(*pollDesc).wait(0xc0001b3b80?, 0x5?, 0x0)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc00009aaf8 sp=0xc00009aad0 pc=0x4d9bf2
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0001b3b80)
    /usr/local/go/src/internal/poll/fd_unix.go:614 +0x2bd fp=0xc00009aba0 sp=0xc00009aaf8 pc=0x4df4fd
net.(*netFD).accept(0xc0001b3b80)
    /usr/local/go/src/net/fd_unix.go:172 +0x35 fp=0xc00009ac58 sp=0xc00009aba0 pc=0x5c12b5
net.(*TCPListener).accept(0xc00011ecd8)
    /usr/local/go/src/net/tcpsock_posix.go:148 +0x25 fp=0xc00009ac80 sp=0xc00009ac58 pc=0x5d85a5
net.(*TCPListener).Accept(0xc00011ecd8)
    /usr/local/go/src/net/tcpsock.go:297 +0x3d fp=0xc00009acb0 sp=0xc00009ac80 pc=0x5d769d
net/http.(*onceCloseListener).Accept(0xc0000ee1b0?)
    <autogenerated>:1 +0x2a fp=0xc00009acc8 sp=0xc00009acb0 pc=0x6fd76a
net/http.(*Server).Serve(0xc0001f1770, {0xce8800, 0xc00011ecd8})
    /usr/local/go/src/net/http/server.go:3059 +0x385 fp=0xc00009adf8 sp=0xc00009acc8 pc=0x6d7ba5
github.com/prometheus/exporter-toolkit/web.Serve({0xce8800, 0xc00011ecd8}, 0xc0001f1770, 0xc0001ea3c0, {0xce3360?, 0xc000043a00})
    /go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.10.0/web/tls_config.go:278 +0x35f fp=0xc00009af38 sp=0xc00009adf8 pc=0x9301bf
github.com/prometheus/exporter-toolkit/web.ServeMultiple.func1()
    /go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.10.0/web/tls_config.go:232 +0x31 fp=0xc00009af78 sp=0xc00009af38 pc=0x92f8d1
golang.org/x/sync/errgroup.(*Group).Go.func1()
    /go/pkg/mod/golang.org/x/sync@v0.2.0/errgroup/errgroup.go:75 +0x64 fp=0xc00009afe0 sp=0xc00009af78 pc=0x92ca04
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00009afe8 sp=0xc00009afe0 pc=0x46ec01
created by golang.org/x/sync/errgroup.(*Group).Go
    /go/pkg/mod/golang.org/x/sync@v0.2.0/errgroup/errgroup.go:72 +0xa5

goroutine 41 [IO wait]:
runtime.gopark(0x8c?, 0xb?, 0x0?, 0x0?, 0x7?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000075900 sp=0xc0000758e0 pc=0x43ba76
runtime.netpollblock(0x4be225?, 0x40482f?, 0x0?)
    /usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc000075938 sp=0xc000075900 pc=0x4343d7
internal/poll.runtime_pollWait(0x7f2bec8158b0, 0x72)
    /usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc000075958 sp=0xc000075938 pc=0x469349
internal/poll.(*pollDesc).wait(0xc00012e380?, 0xc000203000?, 0x0)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc000075980 sp=0xc000075958 pc=0x4d9bf2
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00012e380, {0xc000203000, 0x1000, 0x1000})
    /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc000075a18 sp=0xc000075980 pc=0x4dafd9
net.(*netFD).Read(0xc00012e380, {0xc000203000?, 0x4da0e6?, 0x0?})
    /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc000075a60 sp=0xc000075a18 pc=0x5bf129
net.(*conn).Read(0xc000012b98, {0xc000203000?, 0x0?, 0xc000236878?})
    /usr/local/go/src/net/net.go:183 +0x45 fp=0xc000075aa8 sp=0xc000075a60 pc=0x5cf545
net.(*TCPConn).Read(0xc000236870?, {0xc000203000?, 0x0?, 0xc000075ac8?})
    <autogenerated>:1 +0x29 fp=0xc000075ad8 sp=0xc000075aa8 pc=0x5e1ea9
net/http.(*connReader).Read(0xc000236870, {0xc000203000, 0x1000, 0x1000})
    /usr/local/go/src/net/http/server.go:782 +0x171 fp=0xc000075b28 sp=0xc000075ad8 pc=0x6cd1b1
bufio.(*Reader).fill(0xc000089ec0)
    /usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc000075b60 sp=0xc000075b28 pc=0x65a0bf
bufio.(*Reader).Peek(0xc000089ec0, 0x4)
    /usr/local/go/src/bufio/bufio.go:144 +0x5d fp=0xc000075b80 sp=0xc000075b60 pc=0x65a21d
net/http.(*conn).serve(0xc0000ee750, {0xce8f70, 0xc0002363f0})
    /usr/local/go/src/net/http/server.go:2030 +0x77c fp=0xc000075fb8 sp=0xc000075b80 pc=0x6d317c
net/http.(*Server).Serve.func3()
    /usr/local/go/src/net/http/server.go:3089 +0x2e fp=0xc000075fe0 sp=0xc000075fb8 pc=0x6d7f8e
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000075fe8 sp=0xc000075fe0 pc=0x46ec01
created by net/http.(*Server).Serve
    /usr/local/go/src/net/http/server.go:3089 +0x5ed

goroutine 158 [GC worker (idle)]:
runtime.gopark(0x68d347731d6?, 0x0?, 0x80?, 0x5c?, 0x0?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc0002cf750 sp=0xc0002cf730 pc=0x43ba76
runtime.gcBgMarkWorker()
    /usr/local/go/src/runtime/mgc.go:1275 +0xf1 fp=0xc0002cf7e0 sp=0xc0002cf750 pc=0x41ceb1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc0002cf7e8 sp=0xc0002cf7e0 pc=0x46ec01
created by runtime.gcBgMarkStartWorkers
    /usr/local/go/src/runtime/mgc.go:1199 +0x25

goroutine 62359 [runnable]:
runtime.gopark(0x4d?, 0xb?, 0x0?, 0x0?, 0x8?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc000098478 sp=0xc000098458 pc=0x43ba76
runtime.netpollblock(0x4be225?, 0x40482f?, 0x0?)
    /usr/local/go/src/runtime/netpoll.go:527 +0xf7 fp=0xc0000984b0 sp=0xc000098478 pc=0x4343d7
internal/poll.runtime_pollWait(0x7f2bec8154f0, 0x72)
    /usr/local/go/src/runtime/netpoll.go:306 +0x89 fp=0xc0000984d0 sp=0xc0000984b0 pc=0x469349
internal/poll.(*pollDesc).wait(0xc0004ea300?, 0xc000341000?, 0x0)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0x32 fp=0xc0000984f8 sp=0xc0000984d0 pc=0x4d9bf2
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc0004ea300, {0xc000341000, 0x1000, 0x1000})
    /usr/local/go/src/internal/poll/fd_unix.go:167 +0x299 fp=0xc000098590 sp=0xc0000984f8 pc=0x4dafd9
net.(*netFD).Read(0xc0004ea300, {0xc000341000?, 0x0?, 0xc000098618?})
    /usr/local/go/src/net/fd_posix.go:55 +0x29 fp=0xc0000985d8 sp=0xc000098590 pc=0x5bf129
net.(*conn).Read(0xc000120020, {0xc000341000?, 0xc000098678?, 0xc000236938?})
    /usr/local/go/src/net/net.go:183 +0x45 fp=0xc000098620 sp=0xc0000985d8 pc=0x5cf545
net.(*TCPConn).Read(0xc000236930?, {0xc000341000?, 0xc0002ec410?, 0x0?})
    <autogenerated>:1 +0x29 fp=0xc000098650 sp=0xc000098620 pc=0x5e1ea9
net/http.(*connReader).Read(0xc000236930, {0xc000341000, 0x1000, 0x1000})
    /usr/local/go/src/net/http/server.go:782 +0x171 fp=0xc0000986a0 sp=0xc000098650 pc=0x6cd1b1
bufio.(*Reader).fill(0xc000335e00)
    /usr/local/go/src/bufio/bufio.go:106 +0xff fp=0xc0000986d8 sp=0xc0000986a0 pc=0x65a0bf
bufio.(*Reader).ReadSlice(0xc000335e00, 0x5?)
    /usr/local/go/src/bufio/bufio.go:372 +0x2f fp=0xc000098728 sp=0xc0000986d8 pc=0x65acaf
bufio.(*Reader).ReadLine(0xc000335e00)
    /usr/local/go/src/bufio/bufio.go:401 +0x27 fp=0xc000098778 sp=0xc000098728 pc=0x65af07
net/textproto.(*Reader).readLineSlice(0xc000236510)
    /usr/local/go/src/net/textproto/reader.go:56 +0x99 fp=0xc000098860 sp=0xc000098778 pc=0x65f419
net/textproto.(*Reader).ReadLine(...)
    /usr/local/go/src/net/textproto/reader.go:39
net/http.readRequest(0xc000100000?)
    /usr/local/go/src/net/http/request.go:1042 +0xba fp=0xc0000989f8 sp=0xc000098860 pc=0x6c855a
net/http.(*conn).readRequest(0xc0000ee1b0, {0xce8ec8, 0xc00045b130})
    /usr/local/go/src/net/http/server.go:990 +0x24a fp=0xc000098b80 sp=0xc0000989f8 pc=0x6ce72a
net/http.(*conn).serve(0xc0000ee1b0, {0xce8f70, 0xc0002363f0})
    /usr/local/go/src/net/http/server.go:1920 +0x34b fp=0xc000098fb8 sp=0xc000098b80 pc=0x6d2d4b
net/http.(*Server).Serve.func3()
    /usr/local/go/src/net/http/server.go:3089 +0x2e fp=0xc000098fe0 sp=0xc000098fb8 pc=0x6d7f8e
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc000098fe8 sp=0xc000098fe0 pc=0x46ec01
created by net/http.(*Server).Serve
    /usr/local/go/src/net/http/server.go:3089 +0x5ed
discordianfish commented 1 year ago

Looks like a go issue.. I found https://github.com/golang/go/issues/40368 but that is suppose to be fixed for a while..

discordianfish commented 1 year ago

Maybe related to #2705

SuperQ commented 1 year ago

Please test the latest release: https://github.com/prometheus/node_exporter/releases/tag/v1.6.1

rexagod commented 6 months ago 
...
goroutine 1 [semacquire, 119 minutes]:
runtime.gopark(0xce8f00?, 0xc000040030?, 0x0?, 0x80?, 0x6?)
    /usr/local/go/src/runtime/proc.go:381 +0xd6 fp=0xc00010f950 sp=0xc00010f930 pc=0x43ba76
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:387
runtime.semacquire1(0xc000145890, 0x40?, 0x1, 0x0, 0xe0?)
    /usr/local/go/src/runtime/sema.go:160 +0x20f fp=0xc00010f9b8 sp=0xc00010f950 pc=0x44cdaf
sync.runtime_Semacquire(0xc0000061a0?)
    /usr/local/go/src/runtime/sema.go:62 +0x27 fp=0xc00010f9f0 sp=0xc00010f9b8 pc=0x46ac07
sync.(*WaitGroup).Wait(0x7f2bec846501?)
    /usr/local/go/src/sync/waitgroup.go:116 +0x4b fp=0xc00010fa18 sp=0xc00010f9f0 pc=0x48a76b
golang.org/x/sync/errgroup.(*Group).Wait(0xc000145880)
    /go/pkg/mod/golang.org/x/sync@v0.2.0/errgroup/errgroup.go:53 +0x27 fp=0xc00010fa30 sp=0xc00010fa18 pc=0x92c867
github.com/prometheus/exporter-toolkit/web.ServeMultiple({0xc0001f3a00, 0x1, 0x7ffe4b72d247?}, 0xc0001f1770, 0xc0001ea3c0, {0xce3360?, 0xc000043a00})
    /go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.10.0/web/tls_config.go:235 +0x151 fp=0xc00010fa80 sp=0xc00010fa30 pc=0x92f831
github.com/prometheus/exporter-toolkit/web.ListenAndServe(0x11381a0?, 0xc0001ea3c0, {0xce3360, 0xc000043a00})
    /go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.10.0/web/tls_config.go:268 +0x488 fp=0xc00010fb58 sp=0xc00010fa80 pc=0x92fd88
main.main()
    /app/node_exporter.go:210 +0x164e fp=0xc00010ff80 sp=0xc00010fb58 pc=0xa645ee
runtime.main()
    /usr/local/go/src/runtime/proc.go:250 +0x207 fp=0xc00010ffe0 sp=0xc00010ff80 pc=0x43b647
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1598 +0x1 fp=0xc00010ffe8 sp=0xc00010ffe0 pc=0x46ec01
...

This seems to stem from https://github.com/prometheus/exporter-toolkit/blob/v0.10.0/web/tls_config.go#L273. I believe this can be closed as its not specific to node_exporter (but to exporter-toolkit), given this bug still exists (cc @Evesy).

SuperQ commented 6 months ago

@rexagod I think you're reading the stacktrace backwards. The error starts within /usr/local/go/src/runtime/proc.go. The exporter-toolkit is the end of the trace, basically the victim.

SuperQ commented 6 months ago

Unrelated, I do think this issue can be closed, we've had no more reports since 1.6.1.