search

grafana / loki

Like Prometheus, but for logs.
https://grafana.com/loki
GNU Affero General Public License v3.0
23.96k stars 3.46k forks source link

Ingester crash with fatal error: concurrent map read and map write #2982

Closed scoof closed 3 years ago

scoof commented 3 years ago

Describe the bug Ingester crash with fatal error: concurrent map read and map write

To Reproduce Steps to reproduce the behavior: I am not sure what I did to provoke this error. This may or may not be related to me setting max_chunk_age: 24h and chunk_idle_period: 4h - this crash definitely happened during a query.

Expected behavior No crashes

Environment:

Screenshots, Promtail config, or terminal output

fatal error: concurrent map read and map write

goroutine 319913 [running]:
runtime.throw(0x21e09f1, 0x21)
    /usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0xc017314cd0 sp=0xc017314ca0 pc=0x434ae2
runtime.mapaccess2_fast64(0x1e2e9c0, 0xc00062ef30, 0x7de0052e1fea2702, 0x7de0052e1fea2702, 0x0)
    /usr/local/go/src/runtime/map_fast64.go:61 +0x1a5 fp=0xc017314cf8 sp=0xc017314cd0 pc=0x411b25
github.com/grafana/loki/pkg/logql/log.(*noopPipeline).ForStream(0xc000628268, 0xc002070180, 0xc, 0xc, 0xed74ded69, 0x0)
    /src/loki/pkg/logql/log/pipeline.go:45 +0x6f fp=0xc017314d58 sp=0xc017314cf8 pc=0x1512e3f
github.com/grafana/loki/pkg/ingester.(*instance).Query.func1(0xc0009a0780, 0xc000615a10, 0xc3179ef54bc48060)
    /src/loki/pkg/ingester/instance.go:214 +0xa2 fp=0xc017314e18 sp=0xc017314d58 pc=0x1ae2852
github.com/grafana/loki/pkg/ingester.(*instance).forMatchingStreams(0xc000514fd0, 0xc000902510, 0x1, 0x1, 0xc017314f58, 0x0, 0x0)
    /src/loki/pkg/ingester/instance.go:374 +0x225 fp=0xc017314ec8 sp=0xc017314e18 pc=0x1ad8d15
github.com/grafana/loki/pkg/ingester.(*instance).Query(0xc000514fd0, 0x2741300, 0xc0018d65d0, 0xc0005c6770, 0xc00569b770, 0xc00697f588, 0x0, 0x26f20e0, 0xc00007c060)
    /src/loki/pkg/ingester/instance.go:210 +0x1f2 fp=0xc017314fa8 sp=0xc017314ec8 pc=0x1ad7df2
github.com/grafana/loki/pkg/ingester.(*Ingester).Query(0xc00058e000, 0xc0005c6770, 0x275bd60, 0xc00569b780, 0x0, 0x0)
    /src/loki/pkg/ingester/ingester.go:297 +0x1d4 fp=0xc017315608 sp=0xc017314fa8 pc=0x1ad39d4
github.com/grafana/loki/pkg/logproto._Querier_Query_Handler(0x215ff20, 0xc00058e000, 0x2758880, 0xc006078700, 0x2741300, 0xc0018d6450)
    /src/loki/pkg/logproto/logproto.pb.go:2863 +0x109 fp=0xc017315660 sp=0xc017315608 pc=0x14c81b9
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x215ff20, 0xc00058e000, 0x2756de0, 0xc0060786e0, 0xc0060785a0, 0x22fd0d8, 0xc008b95d40, 0xc000f0f760)
    /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe fp=0xc0173156d0 sp=0xc017315660 pc=0xaeca2e
github.com/grafana/loki/pkg/loki.glob..func4(0x215ff20, 0xc00058e000, 0x2756de0, 0xc0060786e0, 0xc0060785a0, 0x22fd0d8, 0xc000f0f7f0, 0x40c736)
    /src/loki/pkg/loki/loki.go:198 +0x78 fp=0xc017315720 sp=0xc0173156d0 pc=0x1bdce38
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x2756de0, 0xc0060786e0, 0x20, 0x20)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f fp=0xc017315770 sp=0xc017315720 pc=0xb0496f
github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1(0x215ff20, 0xc00058e000, 0x2756de0, 0xc0060786e0, 0xc0060785a0, 0xc0060785c0, 0x0, 0x0)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:44 +0xae fp=0xc0173157e0 sp=0xc017315770 pc=0x1ae838e
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x2756de0, 0xc0060786e0, 0x2741300, 0xc0018d6420)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f fp=0xc017315830 sp=0xc0173157e0 pc=0xb0496f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0001efd40, 0xc0060785a0, 0xc0060785e0, 0x0, 0x0)
    /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x34a fp=0xc0173159b0 sp=0xc017315830 pc=0xb08e4a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0001efd40, 0x1, 0xc006078560)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f fp=0xc017315a00 sp=0xc0173159b0 pc=0xb0496f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0001efd40, 0xc0060785a0, 0xc006078600, 0x23aae646, 0x9ffe7367c2bce9)
    /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89 fp=0xc017315a70 sp=0xc017315a00 pc=0xaefd39
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0001efd40, 0x203001, 0x86789a)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f fp=0xc017315ac0 sp=0xc017315a70 pc=0xb0496f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2767b40, 0xc000674880, 0x40c700, 0x215ff20, 0xc00058e000, 0x27572c0, 0xc0001efd40, 0xc0060785a0, 0xc006078620, 0xc0060785a0, ...)
    /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98 fp=0xc017315b60 sp=0xc017315ac0 pc=0xaed3c8
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor-fm(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0001efd40, 0xc0060785a0, 0xc006078620, 0xc006078640, 0xc0060785a0)
    /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:47 +0x84 fp=0xc017315bc8 sp=0xc017315b60 pc=0xb5aea4
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0001efd40, 0xc013bafc68, 0x40cfa8)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f fp=0xc017315c18 sp=0xc017315bc8 pc=0xb0496f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0001efd40, 0xc0060785a0, 0x22fd0d8, 0x2741300, 0xc0018d63c0)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf fp=0xc017315c78 sp=0xc017315c18 pc=0xb04b2f
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00047a680, 0x2765540, 0xc0010b4180, 0xc0101f1a00, 0xc000996810, 0x391bf00, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/server.go:1336 +0x511 fp=0xc017315e40 sp=0xc017315c78 pc=0x9e7f11
google.golang.org/grpc.(*Server).handleStream(0xc00047a680, 0x2765540, 0xc0010b4180, 0xc0101f1a00, 0x0)
    /src/loki/vendor/google.golang.org/grpc/server.go:1409 +0xc66 fp=0xc017315f68 sp=0xc017315e40 pc=0x9e9dc6
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc005efa0f0, 0xc00047a680, 0x2765540, 0xc0010b4180, 0xc0101f1a00)
    /src/loki/vendor/google.golang.org/grpc/server.go:746 +0xa1 fp=0xc017315fb8 sp=0xc017315f68 pc=0x9f7fb1
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc017315fc0 sp=0xc017315fb8 pc=0x467811
created by google.golang.org/grpc.(*Server).serveStreams.func1
    /src/loki/vendor/google.golang.org/grpc/server.go:744 +0xa1

goroutine 1 [select, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*Manager).AwaitStopped(0xc000906de0, 0x2741280, 0xc00005a480, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/manager.go:145 +0xbb
github.com/grafana/loki/pkg/loki.(*Loki).Run(0xc000622000, 0xc0005dfcc0, 0x4)
    /src/loki/pkg/loki/loki.go:264 +0x69e
main.main()
    /src/loki/cmd/loki/main.go:130 +0x566

goroutine 38 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc00028c400)
    /src/loki/vendor/go.opencensus.io/stats/view/worker.go:276 +0x100
created by go.opencensus.io/stats/view.init.0
    /src/loki/vendor/go.opencensus.io/stats/view/worker.go:34 +0x68

goroutine 139 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc000117980)
    /src/loki/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
    /src/loki/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80

goroutine 168 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc0004491a0)
    /src/loki/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
    /src/loki/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80

goroutine 171 [chan receive]:
go.etcd.io/etcd/pkg/logutil.(*MergeLogger).outputLoop(0xc000449340)
    /src/loki/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:173 +0x3a5
created by go.etcd.io/etcd/pkg/logutil.NewMergeLogger
    /src/loki/vendor/go.etcd.io/etcd/pkg/logutil/merge_logger.go:91 +0x80

goroutine 297 [select, 2 minutes]:
github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollControllerWithTicker(0xc000595450, 0xc0003053b0)
    /src/loki/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:144 +0xd0
github.com/uber/jaeger-client-go.(*RemotelyControlledSampler).pollController(0xc000595450)
    /src/loki/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:139 +0x6e
created by github.com/uber/jaeger-client-go.NewRemotelyControlledSampler
    /src/loki/vendor/github.com/uber/jaeger-client-go/sampler_remote.go:86 +0x119

goroutine 232 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/alertmanager.init.0.func1()
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager.go:87 +0x55
created by github.com/cortexproject/cortex/pkg/alertmanager.init.0
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/alertmanager/alertmanager.go:83 +0x35

goroutine 299 [select]:
github.com/uber/jaeger-client-go/utils.(*reconnectingUDPConn).reconnectLoop(0xc0004f7880, 0x6fc23ac00)
    /src/loki/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:70 +0xed
created by github.com/uber/jaeger-client-go/utils.newReconnectingUDPConn
    /src/loki/vendor/github.com/uber/jaeger-client-go/utils/reconnecting_udp_conn.go:60 +0x10f

goroutine 300 [select]:
github.com/uber/jaeger-client-go.(*remoteReporter).processQueue(0xc00064faa0)
    /src/loki/vendor/github.com/uber/jaeger-client-go/reporter.go:296 +0x120
created by github.com/uber/jaeger-client-go.NewRemoteReporter
    /src/loki/vendor/github.com/uber/jaeger-client-go/reporter.go:237 +0x1a0

goroutine 284 [select, 2 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*memcachedClient).updateLoop(0xc000595ee0, 0xdf8475800)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached_client.go:205 +0x113
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcachedClient
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached_client.go:145 +0x5d6

goroutine 286 [chan receive, 212 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 287 [chan receive, 205 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 288 [chan receive, 201 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 337 [chan receive, 197 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 338 [chan receive, 194 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 339 [chan receive, 191 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 340 [chan receive, 184 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 341 [chan receive, 180 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 342 [chan receive, 176 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 343 [chan receive, 166 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 344 [chan receive, 217 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 345 [chan receive, 158 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 346 [chan receive, 152 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 347 [chan receive, 148 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 348 [chan receive, 142 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 349 [chan receive, 134 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 350 [chan receive, 132 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 351 [chan receive, 130 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 352 [chan receive, 126 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 353 [chan receive, 120 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 354 [chan receive, 114 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 355 [chan receive, 110 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 356 [chan receive, 104 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 357 [chan receive, 100 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 358 [chan receive, 96 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 359 [chan receive, 88 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 360 [chan receive, 86 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 361 [chan receive, 84 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 362 [chan receive, 80 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 363 [chan receive, 74 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 364 [chan receive, 68 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 365 [chan receive, 64 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 366 [chan receive, 60 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 367 [chan receive, 54 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 368 [chan receive, 50 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 369 [chan receive, 46 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 370 [chan receive, 44 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 371 [chan receive, 40 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 372 [chan receive, 34 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 373 [chan receive, 28 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 374 [chan receive, 24 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 375 [chan receive, 18 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 376 [chan receive, 14 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 377 [chan receive, 8 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 378 [chan receive, 6 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 379 [chan receive, 4 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 380 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 381 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 382 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 383 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 384 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 385 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 386 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 387 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 388 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 389 [chan receive, 162 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 390 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 391 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 392 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 393 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 394 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 395 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 396 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 397 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 398 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 399 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 400 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 401 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 402 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 403 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 404 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 405 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 406 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 407 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 408 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 409 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 410 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 411 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 412 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 413 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 414 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 415 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 416 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 417 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 418 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 419 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 420 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 421 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 422 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 423 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 424 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 425 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 426 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 427 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 428 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 429 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 430 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 431 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 432 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 433 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached.func1(0xc00047c930)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:90 +0x99
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewMemcached
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/memcached.go:89 +0x351

goroutine 440 [select, 24 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 441 [select, 18 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 442 [select, 14 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 443 [select, 8 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 444 [select, 28 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 445 [select, 4 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 446 [select, 6 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 447 [select, 44 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 448 [select, 40 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 449 [select, 34 minutes]:
github.com/cortexproject/cortex/pkg/chunk/cache.(*backgroundCache).writeBackLoop(0xc0006c2060)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:115 +0x1b9
created by github.com/cortexproject/cortex/pkg/chunk/cache.NewBackground
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/cache/background.go:67 +0x460

goroutine 451 [select, 6 minutes]:
github.com/cortexproject/cortex/pkg/chunk/local.(*BoltIndexClient).loop(0xc000305810)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/local/boltdb_index_client.go:81 +0x132
created by github.com/cortexproject/cortex/pkg/chunk/local.NewBoltDBIndexClient
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/local/boltdb_index_client.go:70 +0x103

goroutine 453 [select, 2 minutes]:
github.com/grafana/loki/pkg/storage/stores/shipper/uploads.(*TableManager).loop(0xc0004ab2c0)
    /src/loki/pkg/storage/stores/shipper/uploads/table_manager.go:77 +0x1c1
created by github.com/grafana/loki/pkg/storage/stores/shipper/uploads.NewTableManager
    /src/loki/pkg/storage/stores/shipper/uploads/table_manager.go:63 +0x2b2

goroutine 454 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 455 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 456 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 457 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 458 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 459 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 460 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 461 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 462 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 463 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 464 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 465 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 466 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 467 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 468 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 469 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/chunk.(*Fetcher).worker(0xc000307ec0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:128 +0x20f
created by github.com/cortexproject/cortex/pkg/chunk.NewChunkFetcher
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/chunk/chunk_store_utils.go:112 +0xf6

goroutine 520 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000906c00, 0x2748100, 0xc000996330)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:320 +0x11c

goroutine 521 [select, 224 minutes]:
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KV).running(0xc000946000, 0x2741240, 0xc00094e180, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/ring/kv/memberlist/memberlist_client.go:419 +0x127
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0004f4090)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 484 [syscall, 224 minutes]:
os/signal.signal_recv(0x0)
    /usr/local/go/src/runtime/sigqueue.go:147 +0x9c
os/signal.loop()
    /usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.Notify.func1
    /usr/local/go/src/os/signal/signal.go:127 +0x44

goroutine 525 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000906cc0, 0x2748100, 0xc0009966f0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:320 +0x11c

goroutine 526 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000906e40, 0x2748140, 0xc00098f760)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:320 +0x11c

goroutine 527 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000906ea0, 0x2748140, 0xc00098f7a0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:320 +0x11c

goroutine 528 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000906f00, 0x2748140, 0xc00098f7e0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:320 +0x11c

goroutine 529 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener.func1(0xc000906f60, 0x2748140, 0xc00098f840)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:321 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AddListener
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:320 +0x11c

goroutine 530 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener.func1(0xc000907020, 0x27295c0, 0xc00098f900)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/manager.go:244 +0x66
created by github.com/cortexproject/cortex/pkg/util/services.(*Manager).AddListener
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/manager.go:243 +0xfb

goroutine 531 [select, 224 minutes]:
github.com/weaveworks/common/signals.(*Handler).Loop(0xc000996d50)
    /src/loki/vendor/github.com/weaveworks/common/signals/signals.go:48 +0x1d7
github.com/grafana/loki/pkg/loki.(*Loki).Run.func4(0xc000996d50, 0xc000906de0)
    /src/loki/pkg/loki/loki.go:253 +0x2b
created by github.com/grafana/loki/pkg/loki.(*Loki).Run
    /src/loki/pkg/loki/loki.go:252 +0x4a9

goroutine 532 [select, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0004f41b0, 0x2741240, 0xc00094e380, 0x4, 0xc00090e5a0, 0x1, 0xc000efbe88)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:271 +0xcc
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc0004f41b0, 0x2741240, 0xc00094e380, 0xc000906e40, 0x22fc4f0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:267 +0x51
github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc00094e280, 0x2741240, 0xc00094e380, 0x2300da8, 0xc0004f4258)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/module_service.go:70 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0004f4240)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 533 [select, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc0006527e0, 0x2741240, 0xc00094e3c0, 0x4, 0xc000151980, 0x1, 0xc00080ee88)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:271 +0xcc
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc0006527e0, 0x2741240, 0xc00094e3c0, 0xc000906ea0, 0x22fc4f0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:267 +0x51
github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc0005dfd40, 0x2741240, 0xc00094e3c0, 0x2300da8, 0xc000652888)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/module_service.go:70 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000652870)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 534 [select, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc000652900, 0x2741240, 0xc00094e400, 0x4, 0xc000151c80, 0xc000906f00, 0x22fc4f0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:271 +0xcc
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc000652900, 0x2741240, 0xc00094e400, 0x0, 0xc000652990)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:267 +0x51
github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc0005dff00, 0x2741240, 0xc00094e400, 0x2300da8, 0xc0006529a8)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/module_service.go:70 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000652990)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 535 [select, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).awaitState(0xc000254cf0, 0x2741240, 0xc00094e440, 0x4, 0xc0004ca660, 0xc000906f60, 0x22fc4f0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:271 +0xcc
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).AwaitTerminated(0xc000254cf0, 0x2741240, 0xc00094e440, 0x0, 0xc000254d80)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:267 +0x51
github.com/cortexproject/cortex/pkg/util.(*moduleService).run(0xc000307f00, 0x2741240, 0xc00094e440, 0x2300da8, 0xc000254d98)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/module_service.go:70 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000254d80)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 536 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/util/services.NewIdleService.func1(0x2741240, 0xc00094e500, 0x2, 0xc000f05f68)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/services.go:12 +0x48
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000254cf0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 547 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34a88, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0006b8018, 0x72, 0x0, 0x0, 0x21a3329)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0006b8000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4
net.(*netFD).accept(0xc0006b8000, 0x37e11d600, 0x0, 0x0)
    /usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0009d8140, 0xc000890f28, 0xc000890f30, 0x10)
    /usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).AcceptTCP(0xc0009d8140, 0x22fc458, 0xc000913200, 0xc00011f2c0)
    /usr/local/go/src/net/tcpsock.go:248 +0x65
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*TCPTransport).tcpListen(0xc000913200, 0xc0009d8140)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/ring/kv/memberlist/tcp_transport.go:173 +0x86
created by github.com/cortexproject/cortex/pkg/ring/kv/memberlist.NewTCPTransport
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/ring/kv/memberlist/tcp_transport.go:149 +0x220

goroutine 561 [select, 224 minutes]:
github.com/cortexproject/cortex/pkg/ring/kv/memberlist.(*KVInitService).running(0xc00064fbc0, 0x2741240, 0xc000814080, 0x2300da8, 0xc0006527f8)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/ring/kv/memberlist/kv_init_service.go:52 +0xd6
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0006527e0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 336 [select, 224 minutes]:
github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1(0x2741240, 0xc00028f7c0, 0x2, 0xc000f04f68)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/cortex/server_service.go:28 +0xfb
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc000652900)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 577 [chan receive, 224 minutes]:
github.com/weaveworks/common/server.(*Server).Run(0xc00061d340, 0xc000151f80, 0x0)
    /src/loki/vendor/github.com/weaveworks/common/server/server.go:402 +0x15b
github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1.1(0xc0001706c0, 0xc00061d340)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/cortex/server_service.go:25 +0x57
created by github.com/cortexproject/cortex/pkg/cortex.NewServerService.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/cortex/server_service.go:23 +0x5e

goroutine 578 [chan receive, 224 minutes]:
github.com/cortexproject/cortex/pkg/cortex.ignoreSignalHandler.Loop(0xc000151aa0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/cortex/server_service.go:65 +0x34
github.com/weaveworks/common/server.(*Server).Run.func1(0xc00061d340, 0xc000280e40)
    /src/loki/vendor/github.com/weaveworks/common/server/server.go:362 +0x3a
created by github.com/weaveworks/common/server.(*Server).Run
    /src/loki/vendor/github.com/weaveworks/common/server/server.go:361 +0x6b

goroutine 579 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34d28, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0006dfe18, 0x72, 0x0, 0x0, 0x21a3329)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0006dfe00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4
net.(*netFD).accept(0xc0006dfe00, 0x5e1727552a9fe549, 0x0, 0x5e1727552a9fe549)
    /usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0003005e0, 0x5fbbf793, 0xc000ef8e08, 0x4cee96)
    /usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0003005e0, 0xc000ef8e58, 0x18, 0xc000b18780, 0x80dbdc)
    /usr/local/go/src/net/tcpsock.go:261 +0x64
net/http.(*Server).Serve(0xc00031c000, 0x2731dc0, 0xc0003005e0, 0x0, 0x0)
    /usr/local/go/src/net/http/server.go:2901 +0x25d
github.com/weaveworks/common/server.(*Server).Run.func2(0xc00061d340, 0xc000280e40)
    /src/loki/vendor/github.com/weaveworks/common/server/server.go:372 +0x126
created by github.com/weaveworks/common/server.(*Server).Run
    /src/loki/vendor/github.com/weaveworks/common/server/server.go:369 +0x97

goroutine 580 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34c48, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0006dff18, 0x72, 0x0, 0x0, 0x21a3329)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc0006dff00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1d4
net.(*netFD).accept(0xc0006dff00, 0x37e11d600, 0x0, 0x0)
    /usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc000300600, 0xc000ef7e50, 0xc000ef7e58, 0x18)
    /usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc000300600, 0x22fee60, 0xc00047a680, 0x27631a0, 0xc018376428)
    /usr/local/go/src/net/tcpsock.go:261 +0x64
google.golang.org/grpc.(*Server).Serve(0xc00047a680, 0x2731dc0, 0xc000300600, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/server.go:621 +0x210
github.com/weaveworks/common/server.(*Server).Run.func3(0xc00061d340, 0xc000280e40)
    /src/loki/vendor/github.com/weaveworks/common/server/server.go:391 +0x4e
created by github.com/weaveworks/common/server.(*Server).Run
    /src/loki/vendor/github.com/weaveworks/common/server/server.go:390 +0x13b

goroutine 581 [select, 224 minutes]:
github.com/grafana/loki/pkg/ingester.(*Ingester).running(0xc00058e000, 0x2741240, 0xc00028fa00, 0x2300da8, 0xc0004f41c8)
    /src/loki/pkg/ingester/ingester.go:202 +0xee
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0004f41b0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 582 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00028fc10, 0xc000000000)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00028fc00)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090cd20, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x0)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 583 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00028fc50, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00028fc40)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090cd70, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x1)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 584 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00028fc90, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00028fc80)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090cdc0, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x2)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 585 [sync.Cond.Wait, 2 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00028fdd0, 0xb5)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00028fdc0)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090ce10, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x3)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 586 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00028fe10, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00028fe00)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090ce60, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x4)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 587 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00028fe50, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00028fe40)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090ceb0, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x5)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 588 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00028fed0, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00028fec0)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090cf00, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x6)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 589 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00028ff10, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00028ff00)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090cf50, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x7)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 590 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00094e790, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00094e780)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090cfa0, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x8)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 591 [sync.Cond.Wait, 6 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00094e7d0, 0x4)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00094e7c0)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090cff0, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0x9)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 592 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00094e810, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00094e800)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090d040, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0xa)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 593 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00094e850, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00094e840)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090d090, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0xb)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 594 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00094e890, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00094e880)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090d0e0, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0xc)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 595 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00094e8d0, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00094e8c0)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090d130, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0xd)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 596 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00094e910, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00094e900)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090d180, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0xe)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 597 [sync.Cond.Wait, 224 minutes]:
runtime.goparkunlock(...)
    /usr/local/go/src/runtime/proc.go:310
sync.runtime_notifyListWait(0xc00094e950, 0x0)
    /usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00094e940)
    /usr/local/go/src/sync/cond.go:56 +0x9d
github.com/cortexproject/cortex/pkg/util.(*PriorityQueue).Dequeue(0xc00090d1d0, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/priority_queue.go:115 +0x76
github.com/grafana/loki/pkg/ingester.(*Ingester).flushLoop(0xc00058e000, 0xf)
    /src/loki/pkg/ingester/flush.go:189 +0x99
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:180 +0x68

goroutine 598 [select]:
github.com/cortexproject/cortex/pkg/ring.(*Lifecycler).loop(0xc00096a000, 0x2741240, 0xc00094e980, 0x0, 0x0)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/ring/lifecycler.go:388 +0x246
github.com/cortexproject/cortex/pkg/util/services.(*BasicService).main(0xc0004f4120)
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:167 +0x3f7
created by github.com/cortexproject/cortex/pkg/util/services.(*BasicService).StartAsync.func1
    /src/loki/vendor/github.com/cortexproject/cortex/pkg/util/services/basic_service.go:96 +0xa8

goroutine 599 [select]:
github.com/grafana/loki/pkg/ingester.(*Ingester).loop(0xc00058e000)
    /src/loki/pkg/ingester/ingester.go:245 +0x13a
created by github.com/grafana/loki/pkg/ingester.(*Ingester).starting
    /src/loki/pkg/ingester/ingester.go:196 +0x1cd

goroutine 600 [select]:
github.com/hashicorp/memberlist.(*Memberlist).streamListen(0xc00098a000)
    /src/loki/vendor/github.com/hashicorp/memberlist/net.go:211 +0x10b
created by github.com/hashicorp/memberlist.newMemberlist
    /src/loki/vendor/github.com/hashicorp/memberlist/memberlist.go:216 +0x4cc

goroutine 601 [select]:
github.com/hashicorp/memberlist.(*Memberlist).packetListen(0xc00098a000)
    /src/loki/vendor/github.com/hashicorp/memberlist/net.go:314 +0x139
created by github.com/hashicorp/memberlist.newMemberlist
    /src/loki/vendor/github.com/hashicorp/memberlist/memberlist.go:217 +0x4f1

goroutine 602 [select]:
github.com/hashicorp/memberlist.(*Memberlist).packetHandler(0xc00098a000)
    /src/loki/vendor/github.com/hashicorp/memberlist/net.go:433 +0x2f6
created by github.com/hashicorp/memberlist.newMemberlist
    /src/loki/vendor/github.com/hashicorp/memberlist/memberlist.go:218 +0x516

goroutine 603 [select]:
github.com/hashicorp/memberlist.(*Memberlist).triggerFunc(0xc00098a000, 0x3b9aca00, 0xc000281f80, 0xc00090ec60, 0xc0002878c0)
    /src/loki/vendor/github.com/hashicorp/memberlist/state.go:155 +0x17b
created by github.com/hashicorp/memberlist.(*Memberlist).schedule
    /src/loki/vendor/github.com/hashicorp/memberlist/state.go:121 +0x3a2

goroutine 604 [select]:
github.com/hashicorp/memberlist.(*Memberlist).pushPullTrigger(0xc00098a000, 0xc00090ec60)
    /src/loki/vendor/github.com/hashicorp/memberlist/state.go:182 +0x1c2
created by github.com/hashicorp/memberlist.(*Memberlist).schedule
    /src/loki/vendor/github.com/hashicorp/memberlist/state.go:127 +0x2f7

goroutine 605 [select]:
github.com/hashicorp/memberlist.(*Memberlist).triggerFunc(0xc00098a000, 0xbebc200, 0xc000907980, 0xc00090ec60, 0xc0002878d0)
    /src/loki/vendor/github.com/hashicorp/memberlist/state.go:155 +0x17b
created by github.com/hashicorp/memberlist.(*Memberlist).schedule
    /src/loki/vendor/github.com/hashicorp/memberlist/state.go:133 +0x1e2

goroutine 17777 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002c65a40, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001caf020, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc001d5fb00)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 690 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac349a8, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00028dd98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00028dd80, 0xc000dce000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00028dd80, 0xc000dce000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000f0c8, 0xc000dce000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc00012ff80, 0xc00025a658, 0x9, 0x9, 0xc000580000, 0x1007efd019017d0, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc00012ff80, 0xc00025a658, 0x9, 0x9, 0x9, 0x5fbbf795, 0xc0021dbdc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00025a658, 0x9, 0x9, 0x26f1c80, 0xc00012ff80, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00025a620, 0xc003c45740, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000823200, 0xc000a0cab0, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc000823200)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc000823200)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 689 [select, 104 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000823200)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 640 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000885810, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00058b320, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000823200)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 787 [select, 104 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0006e8480)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 788 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac348c8, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00094c898, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00094c880, 0xc000538000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00094c880, 0xc000538000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0009020b8, 0xc000538000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc00016c4e0, 0xc00025a118, 0x9, 0x9, 0xc00007a380, 0x1007efd01901e98, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc00016c4e0, 0xc00025a118, 0x9, 0x9, 0x9, 0x5fbbf795, 0xc00080bdc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00025a118, 0x9, 0x9, 0x26f1c80, 0xc00016c4e0, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00025a0e0, 0xc004734b40, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0006e8480, 0xc0003e3800, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc0006e8480)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc0006e8480)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 319272 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda672a68, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0058c8618, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0058c8600, 0xc0043cc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0058c8600, 0xc0043cc000, 0x8000, 0x8000, 0x203000, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc018376428, 0xc0043cc000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc01478c900, 0xc0008e5458, 0x9, 0x9, 0xc000100700, 0x1007efd019039b8, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc01478c900, 0xc0008e5458, 0x9, 0x9, 0x9, 0x5fbbf78d, 0xc013baedc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0008e5458, 0x9, 0x9, 0x26f1c80, 0xc01478c900, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008e5420, 0xc005835b20, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0010b5800, 0xc00bcce600, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc0010b5800)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc0010b5800)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 786 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000304b90, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000906840, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0006e8480)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 2039 [select, 104 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000b5b080)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 170179 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda672ec8, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0104d1d18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0104d1d00, 0xc00c0ed000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0104d1d00, 0xc00c0ed000, 0x1000, 0x1000, 0x4ce9da, 0x9ffe78a8e201d5, 0x0)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000288430, 0xc00c0ed000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
net/http.(*connReader).Read(0xc00b8a6b10, 0xc00c0ed000, 0x1000, 0x1000, 0xbfe71be18a40d27d, 0xc5f8f4125d4, 0x393c400)
    /usr/local/go/src/net/http/server.go:786 +0xf4
bufio.(*Reader).fill(0xc00430ad80)
    /usr/local/go/src/bufio/bufio.go:100 +0x103
bufio.(*Reader).Peek(0xc00430ad80, 0x4, 0xc5f8f4125d4, 0x393c400, 0x0, 0x0, 0x393c400)
    /usr/local/go/src/bufio/bufio.go:138 +0x4f
net/http.(*conn).serve(0xc00932fe00, 0x2741240, 0xc003371900)
    /usr/local/go/src/net/http/server.go:1920 +0x9d7
created by net/http.(*Server).Serve
    /usr/local/go/src/net/http/server.go:2933 +0x35c

goroutine 319270 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0101f33b0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00a35d7a0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0010b5800)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 17778 [select, 94 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001d5fb00)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 2038 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001302050, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000812360, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000b5b080)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 18048 [select, 92 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0021bd500)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 1852 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00164bd60, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00058a360, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002071380)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 1853 [select, 104 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002071380)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 1796 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00164bbd0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0013cc660, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002193080)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 1854 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34548, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0008e2898, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0008e2880, 0xc001a00000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0008e2880, 0xc001a00000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0002884b0, 0xc001a00000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc00162e660, 0xc0003f8498, 0x9, 0x9, 0xc000100700, 0x1007efd01901e98, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc00162e660, 0xc0003f8498, 0x9, 0x9, 0x9, 0x5fbbf78e, 0xc001d57dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0003f8498, 0x9, 0x9, 0x26f1c80, 0xc00162e660, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003f8460, 0xc0093a8be0, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002071380, 0xc001d1a030, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc002071380)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc002071380)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 1797 [select, 104 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002193080)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 1798 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34708, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc000125218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000125200, 0xc0014a6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc000125200, 0xc0014a6000, 0x8000, 0x8000, 0x203000, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000902228, 0xc0014a6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0013cc5a0, 0xc001218118, 0x9, 0x9, 0xc00007a700, 0x1007efd019039b8, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc0013cc5a0, 0xc001218118, 0x9, 0x9, 0x9, 0x5fbbf78e, 0xc0021dddc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc001218118, 0x9, 0x9, 0x26f1c80, 0xc0013cc5a0, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0012180e0, 0xc004cbc860, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002193080, 0xc00143fbc0, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc002193080)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc002193080)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 1801 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001522460, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0013cd140, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002193380)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 1802 [select, 104 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002193380)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 1803 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34b68, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc000991a18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000991a00, 0xc0014f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc000991a00, 0xc0014f6000, 0x8000, 0x8000, 0x203000, 0x7efcdac70b30, 0x41)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000902390, 0xc0014f6000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0013cd0e0, 0xc0012181f8, 0x9, 0x9, 0xc0005e0000, 0x1007efd01901108, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc0013cd0e0, 0xc0012181f8, 0x9, 0x9, 0x9, 0x5fbbf793, 0xc001d58dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0012181f8, 0x9, 0x9, 0x26f1c80, 0xc0013cd0e0, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0012181c0, 0xc003634460, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002193380, 0xc0016c73b0, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc002193380)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc002193380)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 17779 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34388, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc006275798, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc006275780, 0xc003c62000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc006275780, 0xc003c62000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000902e58, 0xc003c62000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc000812a80, 0xc001218738, 0x9, 0x9, 0xc00007a380, 0x1007efd019017d0, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc000812a80, 0xc001218738, 0x9, 0x9, 0x9, 0x5fbbf795, 0xc000769dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc001218738, 0x9, 0x9, 0x26f1c80, 0xc000812a80, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc001218700, 0xc003c459a0, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001d5fb00, 0xc002d10f90, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc001d5fb00)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc001d5fb00)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 2040 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34628, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00071e018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00071e000, 0xc000778000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00071e000, 0xc000778000, 0x8000, 0x8000, 0x203002, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000902000, 0xc000778000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc001480000, 0xc001218038, 0x9, 0x9, 0xc000100700, 0x1007efd019039b8, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc001480000, 0xc001218038, 0x9, 0x9, 0x9, 0x5fbbf78d, 0xc00129fdc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc001218038, 0x9, 0x9, 0x26f1c80, 0xc001480000, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc001218000, 0xc004a87380, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000b5b080, 0xc0015e0c60, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc000b5b080)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc000b5b080)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 181515 [select, 96 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0021bc480)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 20311 [select, 92 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00282a000)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 20578 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc002c64d20, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000709b60, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc001d5e180)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 45872 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00f268f50, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00a35c840, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00a0cbb00)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 17666 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001522aa0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00012ff20, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc004893380)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 18047 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00269fdb0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0061ae060, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0021bd500)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 19817 [select, 92 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0035f9c80)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 46033 [select, 74 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00a0cbb00)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 17772 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34468, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0039cfe98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0039cfe80, 0xc003d52000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0039cfe80, 0xc003d52000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000fad8, 0xc003d52000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc001ebf800, 0xc0008e4f18, 0x9, 0x9, 0xc000100700, 0x1007efd01901e98, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc001ebf800, 0xc0008e4f18, 0x9, 0x9, 0x9, 0x5fbbf78f, 0xc0021d6dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0008e4f18, 0x9, 0x9, 0x26f1c80, 0xc001ebf800, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008e4ee0, 0xc004ac2040, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001cf0d80, 0xc002d7d3e0, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc001cf0d80)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc001cf0d80)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 17807 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0039200a0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc002d69260, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0010b4180)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 62821 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00cbf94f0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00bc14c60, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00a4b5500)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 21547 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00164af50, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00430a120, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0015a0180)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 18081 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac341c8, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0058c8798, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0058c8780, 0xc0059ec000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0058c8780, 0xc0059ec000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00030ea60, 0xc0059ec000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0061ae000, 0xc00025a9d8, 0x9, 0x9, 0xc000100700, 0x1007efd01901e98, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc0061ae000, 0xc00025a9d8, 0x9, 0x9, 0x9, 0x5fbbf78e, 0xc006314dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00025a9d8, 0x9, 0x9, 0x26f1c80, 0xc0061ae000, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00025a9a0, 0xc0093a8b60, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0021bd500, 0xc00601dc20, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc0021bd500)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc0021bd500)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 46034 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda672fa8, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0036d4598, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0036d4580, 0xc00dd14000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0036d4580, 0xc00dd14000, 0x8000, 0x8000, 0x203002, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00030fef0, 0xc00dd14000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0098261e0, 0xc001219a78, 0x9, 0x9, 0xc000100700, 0x1007efd01901108, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc0098261e0, 0xc001219a78, 0x9, 0x9, 0x9, 0x5fbbf78a, 0xc000811dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc001219a78, 0x9, 0x9, 0x26f1c80, 0xc0098261e0, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc001219a40, 0xc0057f76c0, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00a0cbb00, 0xc00f253110, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc00a0cbb00)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc00a0cbb00)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 20579 [select, 92 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001d5e180)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 22332 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc005b81ef0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001913aa0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0008a5800)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 19816 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0029b3180, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc005bb4fc0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0035f9c80)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 17808 [select, 92 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0010b4180)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 22333 [select, 90 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0008a5800)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 181514 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc004dba3c0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00498e1e0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc0021bc480)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 17667 [select, 94 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc004893380)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 17771 [select, 94 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001cf0d80)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 19818 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac34008, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc0006b9018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0006b9000, 0xc0066fe000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0006b9000, 0xc0066fe000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00011e958, 0xc0066fe000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc000709da0, 0xc00025a3b8, 0x9, 0x9, 0xc00007a380, 0x1007efd01901e98, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc000709da0, 0xc00025a3b8, 0x9, 0x9, 0x9, 0x5fbbf791, 0xc00129cdc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00025a3b8, 0x9, 0x9, 0x26f1c80, 0xc000709da0, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00025a380, 0xc0029b0040, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0035f9c80, 0xc00125f800, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc0035f9c80)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc0035f9c80)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 62823 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda672d08, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00b817918, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00b817900, 0xc00bcf4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00b817900, 0xc00bcf4000, 0x8000, 0x8000, 0x203000, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00030f148, 0xc00bcf4000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc00bc14c00, 0xc00b4121f8, 0x9, 0x9, 0xc00007a380, 0x1007efd019039b8, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc00bc14c00, 0xc00b4121f8, 0x9, 0x9, 0x9, 0x5fbbf78e, 0xc00d8aedc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00b4121f8, 0x9, 0x9, 0x26f1c80, 0xc00bc14c00, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00b4121c0, 0xc00a6d1a40, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00a4b5500, 0xc00d3a9d70, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc00a4b5500)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc00a4b5500)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 181516 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda672c28, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc004ec2418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc004ec2400, 0xc012e22000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc004ec2400, 0xc012e22000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00011e050, 0xc012e22000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc00498e180, 0xc0046961f8, 0x9, 0x9, 0xc00007a380, 0x1007efd01902560, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc00498e180, 0xc0046961f8, 0x9, 0x9, 0x9, 0x5fbbf792, 0xc00d8b5dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0046961f8, 0x9, 0x9, 0x26f1c80, 0xc00498e180, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0046961c0, 0xc0033a7660, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0021bc480, 0xc005966090, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc0021bc480)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc0021bc480)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 20312 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda673328, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc004af2d18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc004af2d00, 0xc0016ce000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc004af2d00, 0xc0016ce000, 0x8000, 0x8000, 0x203000, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00011e310, 0xc0016ce000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc003613140, 0xc0012182d8, 0x9, 0x9, 0xc00007a700, 0x1007efd019039b8, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc003613140, 0xc0012182d8, 0x9, 0x9, 0x9, 0x5fbbf78a, 0xc001d5ddc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0012182d8, 0x9, 0x9, 0x26f1c80, 0xc003613140, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0012182a0, 0xc00a23d8a0, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00282a000, 0xc003ec2c90, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc00282a000)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc00282a000)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 20580 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac340e8, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc004540298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc004540280, 0xc006412000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc004540280, 0xc006412000, 0x8000, 0x8000, 0x203000, 0x0, 0x601042e2cc0)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000288538, 0xc006412000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc000709200, 0xc0008e50d8, 0x9, 0x9, 0xc00007a700, 0x1007efd019039b8, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc000709200, 0xc0008e50d8, 0x9, 0x9, 0x9, 0x5fbbf78a, 0xc0042e2dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0008e50d8, 0x9, 0x9, 0x26f1c80, 0xc000709200, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008e50a0, 0xc00a23d820, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001d5e180, 0xc0012c0150, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc001d5e180)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc001d5e180)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 21548 [select, 90 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0015a0180)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 20310 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc003a40c80, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0036131a0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00282a000)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 18097 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda673248, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc007dfc018, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc007dfc000, 0xc001128000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc007dfc000, 0xc001128000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc004284000, 0xc001128000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0056a8000, 0xc00565a038, 0x9, 0x9, 0xc000580000, 0x1007efd019017d0, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc0056a8000, 0xc00565a038, 0x9, 0x9, 0x9, 0x5fbbf795, 0xc0024fcdc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00565a038, 0x9, 0x9, 0x26f1c80, 0xc0056a8000, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00565a000, 0xc003c45960, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0010b4180, 0xc0061522d0, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc0010b4180)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc0010b4180)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 17668 [IO wait]:
internal/poll.runtime_pollWait(0x7efcdac342a8, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc004ec2c98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc004ec2c80, 0xc002bd8000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc004ec2c80, 0xc002bd8000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000f9d0, 0xc002bd8000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc00012fec0, 0xc0008e4038, 0x9, 0x9, 0xc0005e0000, 0x1007efd019017d0, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc00012fec0, 0xc0008e4038, 0x9, 0x9, 0x9, 0x5fbbf78e, 0xc006313dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0008e4038, 0x9, 0x9, 0x26f1c80, 0xc00012fec0, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0008e4000, 0xc004cbcce0, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc004893380, 0xc001fe44e0, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc004893380)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc004893380)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 17770 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0033001e0, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc001ebf8c0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc001cf0d80)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 21549 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda673408, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc000ad9b98, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000ad9b80, 0xc007192000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc000ad9b80, 0xc007192000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00000f960, 0xc007192000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc00430a060, 0xc0012189d8, 0x9, 0x9, 0xc0005e0000, 0x1007efd01901e98, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc00430a060, 0xc0012189d8, 0x9, 0x9, 0x9, 0x5fbbf793, 0xc0042e3dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc0012189d8, 0x9, 0x9, 0x26f1c80, 0xc00430a060, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0012189a0, 0xc0029b0160, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0015a0180, 0xc0028b6c60, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc0015a0180)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc0015a0180)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 22334 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda673088, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc004ec3918, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc004ec3900, 0xc001dda000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc004ec3900, 0xc001dda000, 0x8000, 0x8000, 0x203000, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00030e578, 0xc001dda000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc001913a40, 0xc00563c738, 0x9, 0x9, 0xc000100700, 0x1007efd019039b8, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc001913a40, 0xc00563c738, 0x9, 0x9, 0x9, 0x5fbbf78e, 0xc0042e9dc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00563c738, 0x9, 0x9, 0x26f1c80, 0xc001913a40, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00563c700, 0xc00a6d19c0, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc0008a5800, 0xc00530a660, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc0008a5800)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc0008a5800)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 319839 [runnable]:
github.com/grafana/loki/pkg/ingester.(*stream).Iterator(0xc00cbad9a0, 0x2741300, 0xc003753ec0, 0x0, 0xed74dee78, 0x0, 0x0, 0xed74dee96, 0x0, 0x1, ...)
    /src/loki/pkg/ingester/stream.go:261 +0x4d7
github.com/grafana/loki/pkg/ingester.(*instance).Query.func1(0xc00cbad9a0, 0xc000615a10, 0x95b552cfae02c2de)
    /src/loki/pkg/ingester/instance.go:214 +0x126
github.com/grafana/loki/pkg/ingester.(*instance).forMatchingStreams(0xc000514fd0, 0xc000628ef0, 0x1, 0x1, 0xc0071b2f58, 0x0, 0x0)
    /src/loki/pkg/ingester/instance.go:374 +0x225
github.com/grafana/loki/pkg/ingester.(*instance).Query(0xc000514fd0, 0x2741300, 0xc003753ec0, 0xc0006609a0, 0xc009275bc0, 0xc012c76780, 0x0, 0x26f20e0, 0xc000580000)
    /src/loki/pkg/ingester/instance.go:210 +0x1f2
github.com/grafana/loki/pkg/ingester.(*Ingester).Query(0xc00058e000, 0xc0006609a0, 0x275bd60, 0xc009275bd0, 0x0, 0x0)
    /src/loki/pkg/ingester/ingester.go:297 +0x1d4
github.com/grafana/loki/pkg/logproto._Querier_Query_Handler(0x215ff20, 0xc00058e000, 0x2758880, 0xc000841760, 0x2741300, 0xc003753d10)
    /src/loki/pkg/logproto/logproto.pb.go:2863 +0x109
github.com/weaveworks/common/middleware.StreamServerUserHeaderInterceptor(0x215ff20, 0xc00058e000, 0x2756de0, 0xc000841740, 0xc000841600, 0x22fd0d8, 0xc0110f46c0, 0xc00c55f760)
    /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_auth.go:48 +0xfe
github.com/grafana/loki/pkg/loki.glob..func4(0x215ff20, 0xc00058e000, 0x2756de0, 0xc000841740, 0xc000841600, 0x22fd0d8, 0xc00c55f7f0, 0x40c736)
    /src/loki/pkg/loki/loki.go:198 +0x78
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x2756de0, 0xc000841740, 0x20, 0x20)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware/recovery.StreamServerInterceptor.func1(0x215ff20, 0xc00058e000, 0x2756de0, 0xc000841740, 0xc000841600, 0xc000841620, 0x0, 0x0)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/recovery/interceptors.go:44 +0xae
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x2756de0, 0xc000841740, 0x2741300, 0xc003753ce0)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/opentracing-contrib/go-grpc.OpenTracingStreamServerInterceptor.func1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0009fa600, 0xc000841600, 0xc000841640, 0x0, 0x0)
    /src/loki/vendor/github.com/opentracing-contrib/go-grpc/server.go:114 +0x34a
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0009fa600, 0x1, 0xc0008415c0)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.StreamServerInstrumentInterceptor.func1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0009fa600, 0xc000841600, 0xc000841660, 0x24298b92, 0x9ffe7368416234)
    /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_instrumentation.go:42 +0x89
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0009fa600, 0x203000, 0x86789a)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/weaveworks/common/middleware.GRPCServerLog.StreamServerInterceptor(0x2767b40, 0xc000674880, 0x40c700, 0x215ff20, 0xc00058e000, 0x27572c0, 0xc0009fa600, 0xc000841600, 0xc000841680, 0xc000841600, ...)
    /src/loki/vendor/github.com/weaveworks/common/middleware/grpc_logging.go:49 +0x98
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0009fa600, 0xc013bb3c68, 0x40cfa8)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x215ff20, 0xc00058e000, 0x27572c0, 0xc0009fa600, 0xc000841600, 0x22fd0d8, 0x2741300, 0xc003753c80)
    /src/loki/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xcf
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00047a680, 0x2765540, 0xc001d5fb00, 0xc00fd42c00, 0xc000996810, 0x391bf00, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/server.go:1336 +0x511
google.golang.org/grpc.(*Server).handleStream(0xc00047a680, 0x2765540, 0xc001d5fb00, 0xc00fd42c00, 0x0)
    /src/loki/vendor/google.golang.org/grpc/server.go:1409 +0xc66
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc002649880, 0xc00047a680, 0x2765540, 0xc001d5fb00, 0xc00fd42c00)
    /src/loki/vendor/google.golang.org/grpc/server.go:746 +0xa1
created by google.golang.org/grpc.(*Server).serveStreams.func1
    /src/loki/vendor/google.golang.org/grpc/server.go:744 +0xa1

goroutine 42990 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001bfbf40, 0x1, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:395 +0x122
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000813d40, 0x0, 0x0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:513 +0x1cd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc001802900)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:291 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:288 +0x10c1

goroutine 42991 [select, 76 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001802900)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 42992 [IO wait]:
internal/poll.runtime_pollWait(0x7efcda673168, 0x72, 0xffffffffffffffff)
    /usr/local/go/src/runtime/netpoll.go:203 +0x55
internal/poll.(*pollDesc).wait(0xc00a689498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00a689480, 0xc005442000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00a689480, 0xc005442000, 0x8000, 0x8000, 0x203001, 0x0, 0x60100000008)
    /usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000288a20, 0xc005442000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
    /usr/local/go/src/net/net.go:184 +0x8e
bufio.(*Reader).Read(0xc0023f1f20, 0xc004696038, 0x9, 0x9, 0xc000580000, 0x1007efd01901108, 0x0)
    /usr/local/go/src/bufio/bufio.go:226 +0x24f
io.ReadAtLeast(0x26f1c80, 0xc0023f1f20, 0xc004696038, 0x9, 0x9, 0x9, 0x5fbbf790, 0xc001d5cdc8, 0x4cee96)
    /usr/local/go/src/io/io.go:310 +0x87
io.ReadFull(...)
    /usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc004696038, 0x9, 0x9, 0x26f1c80, 0xc0023f1f20, 0x0, 0x0, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:237 +0x87
golang.org/x/net/http2.(*Framer).ReadFrame(0xc004696000, 0xc0037128a0, 0x393c400, 0x0, 0x0)
    /src/loki/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001802900, 0xc001bd0060, 0x22feea0)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:453 +0x98
google.golang.org/grpc.(*Server).serveStreams(0xc00047a680, 0x2765540, 0xc001802900)
    /src/loki/vendor/google.golang.org/grpc/server.go:742 +0xdb
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00047a680, 0x2765540, 0xc001802900)
    /src/loki/vendor/google.golang.org/grpc/server.go:703 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
    /src/loki/vendor/google.golang.org/grpc/server.go:702 +0x4fd

goroutine 62822 [select, 62 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00a4b5500)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6

goroutine 319271 [select]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc0010b5800)
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:966 +0x285
created by google.golang.org/grpc/internal/transport.newHTTP2Server
    /src/loki/vendor/google.golang.org/grpc/internal/transport/http2_server.go:297 +0x10e6
scoof commented 3 years ago

Fixed in master (I tested master-14a5e22)