search

prometheus / client_golang

Prometheus instrumentation library for Go applications
https://pkg.go.dev/github.com/prometheus/client_golang
Apache License 2.0
5.4k stars 1.18k forks source link

multiple goroutines panic #1172

Open huyujie opened 1 year ago

huyujie commented 1 year ago 
import (
proclientApi "github.com/prometheus/client_golang/api"
)

var ProClientMap = make(map[string]proclientApi.Client, 0)

func PromQueryRange(){
    ProClient := proclientV1.NewAPI(ProClientMap[dataSource])
    value, _, err := ProClient.QueryRange(ctx, sql, r)
    if err != nil {
         logger.Errorf("[proQuery]error,[%v]", err)
         return PromMonitorData, err
    }
}

when concurrent PromQueryRange,ProClient.QueryRange panic,why?

[2022-11-21 02:02:51]

[2022-11-21 02:02:51 time.Sleep(0x3b9aca00)] [2022-11-21 02:02:51 /usr/local/go/src/runtime/time.go:194 +0x12e] [2022-11-21 02:02:51 github.com/valyala/fasthttp.(TCPDialer).tcpAddrsClean(0x1493740)] [2022-11-21 02:02:51 /build/vendor/github.com/valyala/fasthttp/tcpdialer.go:372 +0x33] [2022-11-21 02:02:51 created by github.com/valyala/fasthttp.(TCPDialer).dial.func1] [2022-11-21 02:02:51 /build/vendor/github.com/valyala/fasthttp/tcpdialer.go:282 +0xaa] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 internal/poll.runtime_pollWait(0x7efc942d7e88, 0x72)] [2022-11-21 02:02:51 /usr/local/go/src/runtime/netpoll.go:302 +0x89] [2022-11-21 02:02:51 internal/poll.(pollDesc).wait(0xc001611a80?, 0xc000b95000?, 0x0)] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32] [2022-11-21 02:02:51 internal/poll.(pollDesc).waitRead(...)] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_poll_runtime.go:88] [2022-11-21 02:02:51 internal/poll.(FD).Read(0xc001611a80, {0xc000b95000, 0x1000, 0x1000})] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a] [2022-11-21 02:02:51 net.(netFD).Read(0xc001611a80, {0xc000b95000?, 0x405ea9?, 0x4?})] [2022-11-21 02:02:51 /usr/local/go/src/net/fd_posix.go:55 +0x29] [2022-11-21 02:02:51 net.(conn).Read(0xc000222ae0, {0xc000b95000?, 0xc0003d16b0?, 0x1?})] [2022-11-21 02:02:51 /usr/local/go/src/net/net.go:183 +0x45] [2022-11-21 02:02:51 net/http.(persistConn).Read(0xc00164ac60, {0xc000b95000?, 0xc0012d4540?, 0xc000a5fd30?})] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1929 +0x4e] [2022-11-21 02:02:51 bufio.(Reader).fill(0xc0004c8ae0)] [2022-11-21 02:02:51 /usr/local/go/src/bufio/bufio.go:106 +0x103] [2022-11-21 02:02:51 bufio.(Reader).Peek(0xc0004c8ae0, 0x1)] [2022-11-21 02:02:51 /usr/local/go/src/bufio/bufio.go:144 +0x5d] [2022-11-21 02:02:51 net/http.(persistConn).readLoop(0xc00164ac60)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:2093 +0x1ac] [2022-11-21 02:02:51 created by net/http.(Transport).dialConn] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1750 +0x173e] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 net/http.(persistConn).writeLoop(0xc0017eed80)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:2392 +0xf5] [2022-11-21 02:02:51 created by net/http.(Transport).dialConn] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1751 +0x1791] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 internal/poll.runtime_pollWait(0x7efc942d7ca8, 0x72)] [2022-11-21 02:02:51 /usr/local/go/src/runtime/netpoll.go:302 +0x89] [2022-11-21 02:02:51 internal/poll.(pollDesc).wait(0xc000601d80?, 0xc000a30000?, 0x0)] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32] [2022-11-21 02:02:51 internal/poll.(pollDesc).waitRead(...)] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_poll_runtime.go:88] [2022-11-21 02:02:51 internal/poll.(FD).Read(0xc000601d80, {0xc000a30000, 0x1000, 0x1000})] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a] [2022-11-21 02:02:51 net.(netFD).Read(0xc000601d80, {0xc000a30000?, 0x405ea9?, 0x4?})] [2022-11-21 02:02:51 /usr/local/go/src/net/fd_posix.go:55 +0x29] [2022-11-21 02:02:51 net.(conn).Read(0xc0003d0640, {0xc000a30000?, 0x0?, 0x0?})] [2022-11-21 02:02:51 /usr/local/go/src/net/net.go:183 +0x45] [2022-11-21 02:02:51 net/http.(persistConn).Read(0xc001185b00, {0xc000a30000?, 0xc0005c7620?, 0xc000a5cd30?})] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1929 +0x4e] [2022-11-21 02:02:51 bufio.(Reader).fill(0xc000cd6f00)] [2022-11-21 02:02:51 /usr/local/go/src/bufio/bufio.go:106 +0x103] [2022-11-21 02:02:51 bufio.(Reader).Peek(0xc000cd6f00, 0x1)] [2022-11-21 02:02:51 /usr/local/go/src/bufio/bufio.go:144 +0x5d] [2022-11-21 02:02:51 net/http.(persistConn).readLoop(0xc001185b00)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:2093 +0x1ac] [2022-11-21 02:02:51 created by net/http.(Transport).dialConn] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1750 +0x173e] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 internal/poll.runtime_pollWait(0x7efc942d7d98, 0x72)] [2022-11-21 02:02:51 /usr/local/go/src/runtime/netpoll.go:302 +0x89] [2022-11-21 02:02:51 internal/poll.(pollDesc).wait(0xc000410880?, 0xc000e42000?, 0x0)] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_poll_runtime.go:83 +0x32] [2022-11-21 02:02:51 internal/poll.(pollDesc).waitRead(...)] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_poll_runtime.go:88] [2022-11-21 02:02:51 internal/poll.(FD).Read(0xc000410880, {0xc000e42000, 0x1000, 0x1000})] [2022-11-21 02:02:51 /usr/local/go/src/internal/poll/fd_unix.go:167 +0x25a] [2022-11-21 02:02:51 net.(netFD).Read(0xc000410880, {0xc000e42000?, 0x405ea9?, 0x4?})] [2022-11-21 02:02:51 /usr/local/go/src/net/fd_posix.go:55 +0x29] [2022-11-21 02:02:51 net.(conn).Read(0xc000306888, {0xc000e42000?, 0x0?, 0x0?})] [2022-11-21 02:02:51 /usr/local/go/src/net/net.go:183 +0x45] [2022-11-21 02:02:51 net/http.(persistConn).Read(0xc000d14c60, {0xc000e42000?, 0xc000043b60?, 0xc000a59d30?})] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1929 +0x4e] [2022-11-21 02:02:51 bufio.(Reader).fill(0xc000cd7a40)] [2022-11-21 02:02:51 /usr/local/go/src/bufio/bufio.go:106 +0x103] [2022-11-21 02:02:51 bufio.(Reader).Peek(0xc000cd7a40, 0x1)] [2022-11-21 02:02:51 /usr/local/go/src/bufio/bufio.go:144 +0x5d] [2022-11-21 02:02:51 net/http.(persistConn).readLoop(0xc000d14c60)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:2093 +0x1ac] [2022-11-21 02:02:51 created by net/http.(Transport).dialConn] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1750 +0x173e] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 net/http.(persistConn).writeLoop(0xc000d14c60)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:2392 +0xf5] [2022-11-21 02:02:51 created by net/http.(Transport).dialConn] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1751 +0x1791] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 net/http.(persistConn).writeLoop(0xc001253c20)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:2392 +0xf5] [2022-11-21 02:02:51 created by net/http.(Transport).dialConn] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1751 +0x1791] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 github.com/go-sql-driver/mysql.(mysqlConn).startWatcher.func1()] [2022-11-21 02:02:51 /build/vendor/github.com/go-sql-driver/mysql/connection.go:614 +0xaa] [2022-11-21 02:02:51 created by github.com/go-sql-driver/mysql.(mysqlConn).startWatcher] [2022-11-21 02:02:51 /build/vendor/github.com/go-sql-driver/mysql/connection.go:611 +0xfe] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 net/http.(persistConn).writeLoop(0xc00164ac60)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:2392 +0xf5] [2022-11-21 02:02:51 created by net/http.(Transport).dialConn] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:1751 +0x1791] [2022-11-21 02:02:51]

[2022-11-21 02:02:51 net/http.(persistConn).roundTrip(0xc000bbf0e0, 0xc000d27240)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:2620 +0x974] [2022-11-21 02:02:51 net/http.(Transport).roundTrip(0x1495240, 0xc000dc9200)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/transport.go:594 +0x7c9] [2022-11-21 02:02:51 net/http.(Transport).RoundTrip(0x40d325?, 0xeeb400?)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/roundtrip.go:17 +0x19] [2022-11-21 02:02:51 net/http.send(0xc000dc9200, {0xeeb400, 0x1495240}, {0xd71040?, 0x7efc9477df01?, 0x0?})] [2022-11-21 02:02:51 /usr/local/go/src/net/http/client.go:252 +0x5d8] [2022-11-21 02:02:51 net/http.(Client).send(0xc0000444c8, 0xc000dc9200, {0x7efc9477dfff?, 0x7efc945f0c00?, 0x0?})] [2022-11-21 02:02:51 /usr/local/go/src/net/http/client.go:176 +0x9b] [2022-11-21 02:02:51 net/http.(Client).do(0xc0000444c8, 0xc000dc9200)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/client.go:725 +0x8f5] [2022-11-21 02:02:51 net/http.(Client).Do(...)] [2022-11-21 02:02:51 /usr/local/go/src/net/http/client.go:593] [2022-11-21 02:02:51 github.com/prometheus/client_golang/api.(httpClient).Do(0xc0000444c0, {0xef08d0, 0xc00003a068}, 0xc000dc9100)] [2022-11-21 02:02:51 /build/vendor/github.com/prometheus/client_golang/api/client.go:100 +0x285] [2022-11-21 02:02:51 github.com/prometheus/client_golang/api/prometheus/v1.(apiClientImpl).Do(0xd1d0c0?, {0xef08d0?, 0xc00003a068?}, 0xc?)] [2022-11-21 02:02:51 /build/vendor/github.com/prometheus/client_golang/api/prometheus/v1/api.go:1088 +0x43] [2022-11-21 02:02:51 github.com/prometheus/client_golang/api/prometheus/v1.(apiClientImpl).DoGetFallback(0xd2b1c0?, {0xef08d0, 0xc00003a068}, 0xc00103a630, 0xc0003a019b?)] [2022-11-21 02:02:51 /build/vendor/github.com/prometheus/client_golang/api/prometheus/v1/api.go:1142 +0x1d4] [2022-11-21 02:02:51 github.com/prometheus/client_golang/api/prometheus/v1.(httpAPI).QueryRange(0xc000c49b78, {0xef08d0, 0xc00003a068}, {0xc00076b880, 0x66}, {{0x0, 0xedb0cd297, 0x14ab420}, {0x0, 0xedb0cd34b, ...}, ...})] [2022-11-21 02:02:51 /build/vendor/github.com/prometheus/client_golang/api/prometheus/v1/api.go:848 +0x3e2] [2022-11-21 02:02:51 ***/pkg/util/prometheus.PromQueryRange({0xc00076b880, 0x66}, {0xc001237dfe?, 0xa?}, {0xc001237e30, 0xc})]

huyujie commented 1 year ago

@beorn7

dswarbrick commented 1 year ago

Did you read the docs?

NewAPI returns a new API for the client.

It is safe to use the returned API from multiple goroutines.

cf. https://pkg.go.dev/github.com/prometheus/client_golang@v1.14.0/api/prometheus/v1#NewAPI