search

grpc-ecosystem / grpc-opentracing

OpenTracing is a set of consistent, expressive, vendor-neutral APIs for distributed tracing and context propagation
BSD 3-Clause "New" or "Revised" License
472 stars 98 forks source link

fatal error: concurrent map read and map write #12

Closed tomwilkie closed 7 years ago

tomwilkie commented 7 years ago 
fatal error: concurrent map read and map write
fatal error: concurrent map writes

goroutine 119 [running]:
runtime.throw(0xa4df85, 0x21)
    /usr/local/go/src/runtime/panic.go:566 +0x95 fp=0xc42003a608 sp=0xc42003a5e8
runtime.mapaccess1_faststr(0x9c25e0, 0xc42046cb70, 0xa41473, 0xc, 0x10)
    /usr/local/go/src/runtime/hashmap_fast.go:201 +0x4f3 fp=0xc42003a668 sp=0xc42003a608
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.metadataReaderWriter.Set(0xc42046cb70, 0xa41473, 0xc, 0xc42062ca10, 0x10)
    /go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/shared.go:29 +0x88 fp=0xc42003a6f8 sp=0xc42003a668
github.com/weaveworks/cortex/vendor/github.com/openzipkin/zipkin-go-opentracing.(*textMapPropagator).Inject(0xc420266020, 0xf05500, 0xc420054e40, 0x9eaae0, 0xc42046cb70, 0x96f420, 0xc42062ca00)
    /go/src/github.com/weaveworks/cortex/vendor/github.com/openzipkin/zipkin-go-opentracing/propagation_ot.go:48 +0x170 fp=0xc42003a858 sp=0xc42003a6f8
github.com/weaveworks/cortex/vendor/github.com/openzipkin/zipkin-go-opentracing.(*tracerImpl).Inject(0xc4202842a0, 0xf05500, 0xc420054e40, 0x96f420, 0xc42062ca00, 0x9eaae0, 0xc42046cb70, 0xc4201c8540, 0x17)
    /go/src/github.com/weaveworks/cortex/vendor/github.com/openzipkin/zipkin-go-opentracing/tracer.go:394 +0x9e fp=0xc42003a8a8 sp=0xc42003a858
github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x7f3998e484e8, 0xc42062fad0, 0xa46908, 0x15, 0x9cece0, 0xc420293460, 0x9bf0c0, 0xf47750, 0xc420474120, 0xc42054ee40, ...)
    /go/src/github.com/weaveworks/cortex/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:55 +0x356 fp=0xc42003a9f0 sp=0xc42003a8a8
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryClient.func1.1.1(0x7f3998e484e8, 0xc42062fad0, 0xa46908, 0x15, 0x9cece0, 0xc420293460, 0x9bf0c0, 0xf47750, 0xc420474120, 0x0, ...)
    /go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:58 +0xd4 fp=0xc42003aa78 sp=0xc42003a9f0
github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware.ChainUnaryClient.func1(0x7f3998e484e8, 0xc42062fad0, 0xa46908, 0x15, 0x9cece0, 0xc420293460, 0x9bf0c0, 0xf47750, 0xc420474120, 0xa86048, ...)
    /go/src/github.com/weaveworks/cortex/vendor/github.com/mwitkow/go-grpc-middleware/chain.go:65 +0x141 fp=0xc42003ab08 sp=0xc42003aa78
github.com/weaveworks/cortex/vendor/google.golang.org/grpc.Invoke(0x7f3998e484e8, 0xc42062fad0, 0xa46908, 0x15, 0x9cece0, 0xc420293460, 0x9bf0c0, 0xf47750, 0xc420474120, 0x0, ...)
    /go/src/github.com/weaveworks/cortex/vendor/google.golang.org/grpc/call.go:141 +0xdd fp=0xc42003ab90 sp=0xc42003ab08
github.com/weaveworks/cortex.(*ingesterClient).Push(0xc42011c9c8, 0x7f3998e484e8, 0xc42062fad0, 0xc420293460, 0x0, 0x0, 0x0, 0x6010107, 0x1, 0xffffffffffffffff)
    /go/src/github.com/weaveworks/cortex/cortex.pb.go:351 +0xd2 fp=0xc42003ac18 sp=0xc42003ab90
github.com/weaveworks/cortex/distributor.(*ingesterClient).Push(0xc420293400, 0x7f3998e484e8, 0xc42062fad0, 0xc420293460, 0x0, 0x0, 0x0, 0x514152, 0xc42003acd8, 0x5391a2)
    <autogenerated>:8 +0x92 fp=0xc42003ac78 sp=0xc42003ac18
github.com/weaveworks/cortex/distributor.(*Distributor).sendSamples.func1(0x7f3998e484e8, 0xc42062fad0, 0xf23dc0, 0xc4200170a0)
    /go/src/github.com/weaveworks/cortex/distributor/distributor.go:333 +0xa5 fp=0xc42003ace8 sp=0xc42003ac78
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogramStatus(0x7f3998e484e8, 0xc42046d0b0, 0xa47bec, 0x17, 0xc42011c190, 0xa85e18, 0xc42003aec8, 0xc42011c9c8, 0xc420474120)
    /go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:45 +0x14d fp=0xc42003adc8 sp=0xc42003ace8
github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument.TimeRequestHistogram(0x7f3998e484e8, 0xc42046d0b0, 0xa47bec, 0x17, 0xc42011c190, 0xc42003aec8, 0x0, 0x0)
    /go/src/github.com/weaveworks/cortex/vendor/github.com/weaveworks/common/instrument/instrument.go:27 +0x70 fp=0xc42003ae20 sp=0xc42003adc8
github.com/weaveworks/cortex/distributor.(*Distributor).sendSamples(0xc420206000, 0x7f3998e484e8, 0xc42046d0b0, 0xc420245260, 0xc420287c00, 0x27, 0x40, 0x0, 0x0)
    /go/src/github.com/weaveworks/cortex/distributor/distributor.go:335 +0x1d5 fp=0xc42003af08 sp=0xc42003ae20
github.com/weaveworks/cortex/distributor.(*Distributor).Push.func1(0xc420270360, 0xc420206000, 0x7f3998e484e8, 0xc42046d0b0, 0xc420245260, 0xc420287c00, 0x27, 0x40)
    /go/src/github.com/weaveworks/cortex/distributor/distributor.go:304 +0x7a fp=0xc42003af70 sp=0xc42003af08
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:2086 +0x1 fp=0xc42003af78 sp=0xc42003af70
created by github.com/weaveworks/cortex/distributor.(*Distributor).Push
    /go/src/github.com/weaveworks/cortex/distributor/distributor.go:305 +0x7e7

Problem AFAICT lies at client.go:49 - I don't think its safe to pick the gRPC metadata out the context and start modifying it, as at that point you don't know if the context is being used / shared elsewhere. The whole context pattern is for copy-on-write. Will file a PR shortly.