search

QubitProducts / exporter_exporter

A reverse proxy designed for Prometheus exporters
Apache License 2.0
344 stars 55 forks source link

curl: (52) Empty reply from server #87

Closed mritou closed 1 year ago

mritou commented 2 years ago

Hello, Is there a limit to scrape metrics with exporter_exporter ?

I have an exporter with 5102 metrics lines, but when i request it :

curl: (52) Empty reply from server

Config :

modules: gitlab: method: http http: address: 10.0.0.152 port: 8080

All other modules works, but this one always failed

Any idea ?

Regards

tcolgate commented 2 years ago

I don't think there is a size limit, though it is possible. You can try setting verify: false under http

mritou commented 2 years ago

Hello, Thank a lot it's working ! Regards

tcolgate commented 2 years ago

Did you check the exporter_exporter logs for any errors? I'd be interested to know why it can't parse. It's unlikely to be size. I wouldn't be super surprised if it's not logging anything.

mritou commented 2 years ago

Hello, here is the log :

runtime: mlock of signal stack failed: 12 runtime: increase the mlock limit (ulimit -l) or runtime: update your kernel to 5.3.15+, 5.4.2+, or 5.5+

fatal error: mlock failed

runtime stack: runtime.throw(0x9412a7, 0xc) /usr/local/go/src/runtime/panic.go:1112 +0x72 runtime.mlockGsignal(0xc000581980) /usr/local/go/src/runtime/os_linux_x86.go:72 +0x107 runtime.mpreinit(0xc000600700) /usr/local/go/src/runtime/os_linux.go:341 +0x78 runtime.mcommoninit(0xc000600700) /usr/local/go/src/runtime/proc.go:630 +0x108 runtime.allocm(0xc00004e000, 0x963ee8, 0x0) /usr/local/go/src/runtime/proc.go:1390 +0x14e runtime.newm(0x963ee8, 0xc00004e000)

    /usr/local/go/src/runtime/proc.go:1704 +0x39

runtime.startm(0x0, 0xc00030ac01) /usr/local/go/src/runtime/proc.go:1869 +0x12a runtime.wakep(...) /usr/local/go/src/runtime/proc.go:1953 runtime.resetspinning() /usr/local/go/src/runtime/proc.go:2415 +0x93 runtime.schedule() /usr/local/go/src/runtime/proc.go:2527 +0x2de runtime.park_m(0xc000182a80) /usr/local/go/src/runtime/proc.go:2690 +0x9d runtime.mcall(0x800000) /usr/local/go/src/runtime/asm_amd64.s:318 +0x5b

goroutine 1 [semacquire]: sync.runtime_Semacquire(0xc000259a20) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(WaitGroup).Wait(0xc000259a18) /usr/local/go/src/sync/waitgroup.go:130 +0x64 golang.org/x/sync/errgroup.(Group).Wait(0xc000259a10, 0xc000150f80, 0x9f7ae0) /home/tristan/go/pkg/mod/golang.org/x/sync@v0.0.0-20190227155943-e225da77a7e6/errgroup/errgroup.go:40 +0x31 main.main() /home/tristan/src/exporter_exporter/main.go:390 +0x420

goroutine 18 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0f18, 0x72, 0x0) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc00023c518, 0x72, 0x0, 0x0, 0x93dc8d) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0xc00023c500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4 net.(netFD).accept(0xc00023c500, 0x0, 0x0, 0xc15992c0dfb23310) /usr/local/go/src/net/fd_unix.go:238 +0x42 net.(TCPListener).accept(0xc00000fc20, 0x63872f18, 0xc000139d48, 0x4cd136) /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 net.(TCPListener).Accept(0xc00000fc20, 0xc000139d98, 0x18, 0xc000001980, 0x6c81dc) /usr/local/go/src/net/tcpsock.go:261 +0x64 net/http.(Server).Serve(0xc0002a8000, 0x9f6520, 0xc00000fc20, 0x0, 0x0) /usr/local/go/src/net/http/server.go:2901 +0x25d main.runListener(0x9f7ae0, 0xc000150f40, 0x93ba72, 0x4, 0x9f6520, 0xc00000fc20, 0x9edd60, 0xc00028ac80, 0xc000128798, 0x40729f) /home/tristan/src/exporter_exporter/main.go:300 +0xf1 main.main.func2(0x0, 0xc000152d80) /home/tristan/src/exporter_exporter/main.go:380 +0x6c golang.org/x/sync/errgroup.(Group).Go.func1(0xc000259a10, 0xc000150f80) /home/tristan/go/pkg/mod/golang.org/x/sync@v0.0.0-20190227155943-e225da77a7e6/errgroup/errgroup.go:57 +0x59 created by golang.org/x/sync/errgroup.(*Group).Go /home/tristan/go/pkg/mod/golang.org/x/sync@v0.0.0-20190227155943-e225da77a7e6/errgroup/errgroup.go:54 +0x66

goroutine 19 [chan receive]: main.runListener.func1(0x9f7ae0, 0xc000150f40, 0xc0002a8000) /home/tristan/src/exporter_exporter/main.go:296 +0x48 created by main.runListener /home/tristan/src/exporter_exporter/main.go:295 +0xc9

goroutine 34 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0e38, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc000190018, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000190000, 0xc0001ac000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc000190000, 0xc0001ac000, 0x1000, 0x1000, 0x3, 0x0, 0x556f72) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc000194000, 0xc0001ac000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).Read(0xc0001920c0, 0xc0001ac000, 0x1000, 0x1000, 0x2, 0x16, 0x0) /usr/local/go/src/net/http/server.go:786 +0xf4 bufio.(Reader).fill(0xc0001aa000) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc0001aa000, 0xc00023790a, 0x43ed3f, 0xc0002379c0, 0x40d9c6, 0xc000454b00, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc0001aa000, 0xc0002379c8, 0xc00012a700, 0x7f69a4c30e98, 0x0, 0x2, 0xc0001ad000) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc000192120, 0xc000454b00, 0x0, 0x43790c, 0xc000237a28, 0x0) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc0001aa000, 0x0, 0xc000454b00, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1015 +0xa4 net/http.(conn).readRequest(0xc000196000, 0x9f7ae0, 0xc00019e000, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:966 +0x191 net/http.(conn).serve(0xc000196000, 0x9f7ae0, 0xc00019e000) /usr/local/go/src/net/http/server.go:1822 +0x6d4 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 37 [select]: net/http.(persistConn).writeLoop(0xc00017b8c0) /usr/local/go/src/net/http/transport.go:2277 +0x11c created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1648 +0xc7b

goroutine 50 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0d58, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc000302018, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000302000, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc000302000, 0xc000326000, 0x1000, 0x1000, 0x4, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc000306000, 0xc000326000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).Read(0xc000304090, 0xc000326000, 0x1000, 0x1000, 0x44f92f, 0x12, 0x203000) /usr/local/go/src/net/http/server.go:786 +0xf4 bufio.(Reader).fill(0xc000324000) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc000324000, 0xa, 0x28, 0xc0005919c0, 0x40d9c6, 0xc0001be200, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc000324000, 0xc0005919c8, 0xc000180000, 0x7f69a4c31560, 0x0, 0x40e238, 0x30) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc0004b4810, 0xc0001be200, 0x0, 0x43790c, 0xc000591a28, 0x0) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(*Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc000324000, 0x0, 0xc0001be200, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1015 +0xa4

net/http.(conn).readRequest(0xc000308000, 0x9f7ae0, 0xc000320000, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:966 +0x191 net/http.(conn).serve(0xc000308000, 0x9f7ae0, 0xc000320000) /usr/local/go/src/net/http/server.go:1822 +0x6d4 created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 21 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0b98, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc00023c898, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc00023c880, 0xc000426000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc00023c880, 0xc000426000, 0x1000, 0x1000, 0x203000, 0x203000, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0003d4af0, 0xc000426000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).Read(0xc000402810, 0xc000426000, 0x1000, 0x1000, 0x7f697dc2efff, 0xc00, 0x7f697da5a800) /usr/local/go/src/net/http/server.go:786 +0xf4 bufio.(Reader).fill(0xc000152fc0) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc000152fc0, 0xc00034390a, 0x0, 0xc0003439c0, 0x40d9c6, 0xc0001be800, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc000152fc0, 0xc0003439c8, 0xc00012a700, 0x7f69a4c30108, 0x0, 0x917880, 0xc00070fe30) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc00070fe30, 0xc0001be800, 0x0, 0xc0001524e0, 0xc000343a28, 0x0) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc000152fc0, 0x0, 0xc0001be800, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1015 +0xa4 net/http.(conn).readRequest(0xc000175ae0, 0x9f7ae0, 0xc0001511c0, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:966 +0x191 net/http.(conn).serve(0xc000175ae0, 0x9f7ae0, 0xc0001511c0) /usr/local/go/src/net/http/server.go:1822 +0x6d4 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 56 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0c78, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc00023c718, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc00023c700, 0xc00037c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc00023c700, 0xc00037c000, 0x1000, 0x1000, 0x43790c, 0xc00013ab70, 0x461180) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc000306040, 0xc00037c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(persistConn).Read(0xc00034a000, 0xc00037c000, 0x1000, 0x1000, 0xc00015e360, 0xc00013ac70, 0x406855) /usr/local/go/src/net/http/transport.go:1825 +0x75 bufio.(Reader).fill(0xc0003246c0) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).Peek(0xc0003246c0, 0x1, 0x0, 0x0, 0x1, 0xc000332400, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(persistConn).readLoop(0xc00034a000) /usr/local/go/src/net/http/transport.go:1978 +0x1a8 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1647 +0xc56

goroutine 57 [select]: net/http.(persistConn).writeLoop(0xc00034a000) /usr/local/go/src/net/http/transport.go:2277 +0x11c created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1648 +0xc7b

goroutine 149 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f09d8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc00023d398, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc00023d380, 0xc0001df000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc00023d380, 0xc0001df000, 0x1000, 0x1000, 0x4, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0004bca78, 0xc0001df000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).Read(0xc0001e00f0, 0xc0001df000, 0x1000, 0x1000, 0x44f92f, 0x18, 0x203000) /usr/local/go/src/net/http/server.go:786 +0xf4 bufio.(Reader).fill(0xc0004d0060) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc0004d0060, 0xa, 0x28, 0xc0002359c0, 0x40d9c6, 0xc0001bed00, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc0004d0060, 0xc0002359c8, 0xc000180380, 0x7f69a4c30108, 0x0, 0x40e238, 0x30) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc0002c5e00, 0xc0001bed00, 0x0, 0x43790c, 0xc000235a28, 0x0) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc0004d0060, 0x0, 0xc0001bed00, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1015 +0xa4 net/http.(conn).readRequest(0xc000174320, 0x9f7ae0, 0xc0004c0c40, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:966 +0x191 net/http.(conn).serve(0xc000174320, 0x9f7ae0, 0xc0004c0c40) /usr/local/go/src/net/http/server.go:1822 +0x6d4 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 36 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0ab8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc00023ca98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc00023ca80, 0xc0001f2000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc00023ca80, 0xc0001f2000, 0x1000, 0x1000, 0x43790c, 0xc00013bb70, 0x461180) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc000194010, 0xc0001f2000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(persistConn).Read(0xc00017b8c0, 0xc0001f2000, 0x1000, 0x1000, 0xc00015e720, 0xc00013bc70, 0x406855) /usr/local/go/src/net/http/transport.go:1825 +0x75 bufio.(Reader).fill(0xc0001aa120) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).Peek(0xc0001aa120, 0x1, 0x0, 0x0, 0x1, 0xc00015e300, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(persistConn).readLoop(0xc00017b8c0) /usr/local/go/src/net/http/transport.go:1978 +0x1a8 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1647 +0xc56

goroutine 97 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f08f8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc000190f18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000190f00, 0xc0002c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc000190f00, 0xc0002c0000, 0x1000, 0x1000, 0x4, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc000010d08, 0xc0002c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).Read(0xc0002b0f90, 0xc0002c0000, 0x1000, 0x1000, 0x44f92f, 0x12, 0xc000360600) /usr/local/go/src/net/http/server.go:786 +0xf4 bufio.(Reader).fill(0xc000152360) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc000152360, 0xc00033f90a, 0x43ed3f, 0xc00033f9c0, 0x40d9c6, 0xc0001bf000, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc000152360, 0xc00033f9c8, 0xc000180380, 0x7f69a4c30108, 0x0, 0x0, 0xc0002c1000) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc0001e0120, 0xc0001bf000, 0x0, 0x43790c, 0xc00033fa28, 0x0) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc000152360, 0x0, 0xc0001bf000, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1015 +0xa4 net/http.(conn).readRequest(0xc0001960a0, 0x9f7ae0, 0xc000461600, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:966 +0x191 net/http.(conn).serve(0xc0001960a0, 0x9f7ae0, 0xc000461600) /usr/local/go/src/net/http/server.go:1822 +0x6d4 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 153 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0818, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc00023d618, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc00023d600, 0xc000446000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc00023d600, 0xc000446000, 0x1000, 0x1000, 0x203000, 0x203000, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0004bcf10, 0xc000446000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).Read(0xc000431bf0, 0xc000446000, 0x1000, 0x1000, 0x7f697dc2efff, 0xc00, 0x7f697da46400) /usr/local/go/src/net/http/server.go:786 +0xf4 bufio.(Reader).fill(0xc0004d02a0) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc0004d02a0, 0xc00023390a, 0x43ed3f, 0xc0002339c0, 0x40d9c6, 0xc0001bec00, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc0004d02a0, 0xc0002339c8, 0xc000180000, 0x7f69a4c30108, 0x0, 0x917880, 0xc0003ac240) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc0003ac240, 0xc0001bec00, 0x0, 0xc000152420, 0xc000233a28, 0x0) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc0004d02a0, 0x0, 0xc0001bec00, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1015 +0xa4 net/http.(conn).readRequest(0xc000174c80, 0x9f7ae0, 0xc0004c0e80, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:966 +0x191 net/http.(conn).serve(0xc000174c80, 0x9f7ae0, 0xc0004c0e80) /usr/local/go/src/net/http/server.go:1822 +0x6d4 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 122 [select]: net/http.(persistConn).roundTrip(0xc0001b4000, 0xc0004ba0f0, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/transport.go:2498 +0x756 net/http.(Transport).roundTrip(0xc000216f00, 0xc000454400, 0x891b60, 0xc000214201, 0xc0002142c0) /usr/local/go/src/net/http/transport.go:565 +0xad9 net/http.(Transport).RoundTrip(0xc000216f00, 0xc000454400, 0x942743, 0xf, 0xc0002143a8) /usr/local/go/src/net/http/roundtrip.go:17 +0x35 net/http/httputil.(ReverseProxy).ServeHTTP(0xc00015c910, 0x9f64e0, 0xc00061c000, 0xc000285c00) /usr/local/go/src/net/http/httputil/reverseproxy.go:257 +0x2c2 main.moduleConfig.ServeHTTP(0xc000280238, 0x4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /home/tristan/src/exporter_exporter/main.go:500 +0x288 main.(config).doProxy(0xc000150cc0, 0x9f64e0, 0xc00061c000, 0xc000285c00) /home/tristan/src/exporter_exporter/main.go:443 +0x199 net/http.HandlerFunc.ServeHTTP(0xc00028ab10, 0x9f64e0, 0xc00061c000, 0xc000285c00) /usr/local/go/src/net/http/server.go:2012 +0x44 net/http.(ServeMux).ServeHTTP(0xd45880, 0x9f64e0, 0xc00061c000, 0xc000285c00) /usr/local/go/src/net/http/server.go:2387 +0x1a5 main.AccessLogMiddleware.ServeHTTP(0x9edf60, 0xd45880, 0x9f67e0, 0xc0002a80e0, 0xc000285c00) /home/tristan/src/exporter_exporter/main.go:420 +0x10c net/http.serverHandler.ServeHTTP(0xc0002a8000, 0x9f67e0, 0xc0002a80e0, 0xc000285c00) /usr/local/go/src/net/http/server.go:2807 +0xa3 net/http.(conn).serve(0xc000196140, 0x9f7ae0, 0xc00019e100) /usr/local/go/src/net/http/server.go:1895 +0x86c created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 161 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f03b8, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc000302298, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000302280, 0xc000170000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc000302280, 0xc000170000, 0x1000, 0x1000, 0x4, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc000718250, 0xc000170000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).Read(0xc00070fe00, 0xc000170000, 0x1000, 0x1000, 0x44f92f, 0x12, 0x203000) /usr/local/go/src/net/http/server.go:786 +0xf4 bufio.(Reader).fill(0xc000324300) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).ReadSlice(0xc000324300, 0xa, 0x28, 0xc0005939c0, 0x40d9c6, 0xc000454a00, 0x100) /usr/local/go/src/bufio/bufio.go:359 +0x3d bufio.(Reader).ReadLine(0xc000324300, 0xc0005939c8, 0xc000180000, 0x7f69a4c30e98, 0x0, 0x40e238, 0x30) /usr/local/go/src/bufio/bufio.go:388 +0x34 net/textproto.(Reader).readLineSlice(0xc0002d5800, 0xc000454a00, 0x0, 0x43790c, 0xc000593a28, 0x0) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc000324300, 0x0, 0xc000454a00, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1015 +0xa4 net/http.(conn).readRequest(0xc000196320, 0x9f7ae0, 0xc0004600c0, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:966 +0x191 net/http.(conn).serve(0xc000196320, 0x9f7ae0, 0xc0004600c0) /usr/local/go/src/net/http/server.go:1822 +0x6d4 created by net/http.(Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 169 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0578, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc000191a98, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000191a80, 0xc00039c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc000191a80, 0xc00039c000, 0x1000, 0x1000, 0x43790c, 0xc00038fb70, 0x461180) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0004bd410, 0xc00039c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(persistConn).Read(0xc0001b4000, 0xc00039c000, 0x1000, 0x1000, 0xc00015e180, 0xc00038fc70, 0x406855) /usr/local/go/src/net/http/transport.go:1825 +0x75 bufio.(Reader).fill(0xc0001525a0) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).Peek(0xc0001525a0, 0x1, 0x0, 0x0, 0x1, 0xc0001c6d00, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(persistConn).readLoop(0xc0001b4000) /usr/local/go/src/net/http/transport.go:1978 +0x1a8 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1647 +0xc56

goroutine 159 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0658, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc000191318, 0x72, 0x1000, 0x1000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000191300, 0xc000686000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc000191300, 0xc000686000, 0x1000, 0x1000, 0x43790c, 0xc000315b70, 0x461180) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0004bcf30, 0xc000686000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(persistConn).Read(0xc0001b4120, 0xc000686000, 0x1000, 0x1000, 0xc0001c65a0, 0xc000315c70, 0x406855) /usr/local/go/src/net/http/transport.go:1825 +0x75 bufio.(Reader).fill(0xc0004d0720) /usr/local/go/src/bufio/bufio.go:100 +0x103 bufio.(Reader).Peek(0xc0004d0720, 0x1, 0x0, 0x0, 0x1, 0xc0001c6a00, 0x0) /usr/local/go/src/bufio/bufio.go:138 +0x4f net/http.(persistConn).readLoop(0xc0001b4120) /usr/local/go/src/net/http/transport.go:1978 +0x1a8 created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1647 +0xc56

goroutine 160 [select]: net/http.(persistConn).writeLoop(0xc0001b4120) /usr/local/go/src/net/http/transport.go:2277 +0x11c created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1648 +0xc7b

goroutine 128 [running]: goroutine running on other thread; stack unavailable created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 170 [select]: net/http.(persistConn).writeLoop(0xc0001b4000) /usr/local/go/src/net/http/transport.go:2277 +0x11c created by net/http.(Transport).dialConn /usr/local/go/src/net/http/transport.go:1648 +0xc7b

goroutine 211 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0738, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc000302098, 0x72, 0x0, 0x1, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000302080, 0xc0003ac311, 0x1, 0x1, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc000302080, 0xc0003ac311, 0x1, 0x1, 0x0, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0003d4010, 0xc0003ac311, 0x1, 0x1, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).backgroundRead(0xc0003ac300) /usr/local/go/src/net/http/server.go:678 +0x58 created by net/http.(*connReader).startBackgroundRead /usr/local/go/src/net/http/server.go:674 +0xd0

goroutine 184 [IO wait]: internal/poll.runtime_pollWait(0x7f697c0f0498, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:203 +0x55 internal/poll.(pollDesc).wait(0xc000302398, 0x72, 0x0, 0x1, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc000302380, 0xc0003ac761, 0x1, 0x1, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc000302380, 0xc0003ac761, 0x1, 0x1, 0x0, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc0003d4030, 0xc0003ac761, 0x1, 0x1, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:184 +0x8e net/http.(connReader).backgroundRead(0xc0003ac750) /usr/local/go/src/net/http/server.go:678 +0x58 created by net/http.(*connReader).startBackgroundRead /usr/local/go/src/net/http/server.go:674 +0xd0

tcolgate commented 2 years ago

That's an extremely odd error for parsing metrics. Googling for that error seems to highlight a combination of a kernel bug and some stuff within go 1.14. Is this an exporter_exporter binary you have built yourself? If so, try updating to a newer Go version and/or update the kernel version has per the log maybe?

mritou commented 2 years ago

Hi, Yes that's what we have on mind, once we check on google. I use a binary i download here, not one built by myself. Yes maybe that could be a good idea to try an upgrade ? Thanks for your help Regards