search

grafana / mimir

Grafana Mimir provides horizontally scalable, highly available, multi-tenant, long-term storage for Prometheus.
https://grafana.com/oss/mimir/
GNU Affero General Public License v3.0
4.02k stars 509 forks source link

Flaky `TestMultitenantAlertmanager_FirewallShouldBlockHTTPBasedReceiversWhenEnabled` #6725

Open dimitarvdimitrov opened 9 months ago

dimitarvdimitrov commented 9 months ago

The test timed out waiting for alertmanager to start. This is the goroutine dump

Details

``` panic: test timed out after 30m0s running tests: TestMultitenantAlertmanager_FirewallShouldBlockHTTPBasedReceiversWhenEnabled (29m58s) TestMultitenantAlertmanager_FirewallShouldBlockHTTPBasedReceiversWhenEnabled/receiver=webhook_firewall_enabled=false (29m51s) goroutine 4056 [running]: testing.(*M).startAlarm.func1() /usr/local/go/src/testing/testing.go:2259 +0x1fc created by time.goFunc /usr/local/go/src/time/sleep.go:176 +0x45 goroutine 1 [chan receive, 29 minutes]: testing.(*T).Run(0xc0002add40, {0x218b1c0, 0x4c}, 0x228d9d0) /usr/local/go/src/testing/testing.go:1649 +0x856 testing.runTests.func1(0x0?) /usr/local/go/src/testing/testing.go:2054 +0x85 testing.tRunner(0xc0002add40, 0xc0008b3b08) /usr/local/go/src/testing/testing.go:1595 +0x239 testing.runTests(0xc0005bfb80?, {0x403c8a0, 0x2d, 0x2d}, {0x1c?, 0x4ad0f9?, 0x404dcc0?}) /usr/local/go/src/testing/testing.go:2052 +0x897 testing.(*M).Run(0xc0005bfb80) /usr/local/go/src/testing/testing.go:1925 +0xb58 main.main() _testmain.go:135 +0x2be goroutine 7 [select]: github.com/golang/glog.(*fileSink).flushDaemon(0x404dd58) /__w/mimir/mimir/vendor/github.com/golang/glog/glog_file.go:351 +0x15c created by github.com/golang/glog.init.1 in goroutine 1 /__w/mimir/mimir/vendor/github.com/golang/glog/glog_file.go:166 +0x225 goroutine 8 [select, 30 minutes]: github.com/dgraph-io/ristretto.(*defaultPolicy).processItems(0xc000041340) /__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/policy.go:102 +0x105 created by github.com/dgraph-io/ristretto.newDefaultPolicy in goroutine 1 /__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/policy.go:86 +0x2bd goroutine 9 [select]: github.com/dgraph-io/ristretto.(*Cache).processItems(0xc000099980) /__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/cache.go:452 +0x285 created by github.com/dgraph-io/ristretto.NewCache in goroutine 1 /__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/cache.go:207 +0xe86 goroutine 10 [select]: go.opencensus.io/stats/view.(*worker).start(0xc000099b80) /__w/mimir/mimir/vendor/go.opencensus.io/stats/view/worker.go:292 +0x199 created by go.opencensus.io/stats/view.init.0 in goroutine 1 /__w/mimir/mimir/vendor/go.opencensus.io/stats/view/worker.go:34 +0xfd goroutine 39 [chan receive, 30 minutes]: github.com/grafana/mimir/pkg/alertmanager.init.0.func1() /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:138 +0x54 created by github.com/grafana/mimir/pkg/alertmanager.init.0 in goroutine 1 /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:133 +0x25 goroutine 225 [chan receive, 29 minutes]: testing.(*T).Parallel(0xc00077aea0) /usr/local/go/src/testing/testing.go:1403 +0x570 github.com/grafana/mimir/pkg/alertmanager.TestIsHealthyForAlertmanagerOperations(0x0?) /__w/mimir/mimir/pkg/alertmanager/alertmanager_ring_test.go:17 +0x3f testing.tRunner(0xc00077aea0, 0x228d9b8) /usr/local/go/src/testing/testing.go:1595 +0x239 created by testing.(*T).Run in goroutine 1 /usr/local/go/src/testing/testing.go:1648 +0x82b goroutine 964 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000f7b710, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000f7b700) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000aa83c0, {0x2120734, 0xc}, 0xc000abc1a0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000be67e0, {0x3415dd8?, 0xc000aa8500}, {0x2120734, 0xc}, 0xc0009229[60](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:61)) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc00027bb80, {0x3415dd8, 0xc000aa8500}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0001335e0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 962 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1026 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000e195d0, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000e195c0) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000e00c30, {0x2120734, 0xc}, 0xc000e20820) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000c08cf0, {0x3415dd8?, 0xc000e00d70}, {0x2120734, 0xc}, 0xc000be3990) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000e162c0, {0x3415dd8, 0xc000e00d70}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000e0c820) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 1008 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 968 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000aba350, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000aba340) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000aa8780, {0x2120734, 0xc}, 0xc000abcc30) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000be6870, {0x3415dd8?, 0xc000aa88c0}, {0x2120734, 0xc}, 0xc000923150) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000eee000, {0x3415dd8, 0xc000aa88c0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000133900) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 966 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1006 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000c7da10, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000c7da00) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000e00500, {0x2120734, 0xc}, 0xc000c21930) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000c08990, {0x3415dd8?, 0xc000e00640}, {0x2120734, 0xc}, 0xc000be2f50) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000e16000, {0x3415dd8, 0xc000e00640}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000e0c280) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 1004 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 972 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000abbe50, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000abbe40) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000aa8e10, {0x2120734, 0xc}, 0xc000abd860) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000be6ea0, {0x3415dd8?, 0xc000aa8f50}, {0x2120734, 0xc}, 0xc0009239d0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000eee2c0, {0x3415dd8, 0xc000aa8f50}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000133ea0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 970 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1434 [chan receive, 29 minutes]: testing.tRunner.func1() /usr/local/go/src/testing/testing.go:15[61](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:62) +0x8cc testing.tRunner(0xc0009a89c0, 0x228d9d0) /usr/local/go/src/testing/testing.go:1601 +0x26c created by testing.(*T).Run in goroutine 1 /usr/local/go/src/testing/testing.go:1648 +0x82b goroutine 943 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000b96cd0, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000b96cc0) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000927d60, {0x2120734, 0xc}, 0xc000b94dd0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bd9ef0, {0x3415dd8?, 0xc000927ea0}, {0x2120734, 0xc}, 0xc000a83330) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000215b80, {0x3415dd8, 0xc000927ea0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000785220) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 941 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 921 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000ed1190, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000ed1180) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc0009e0e60, {0x2120734, 0xc}, 0xc000a984e0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000b9d3b0, {0x3415dd8?, 0xc0009e0fa0}, {0x2120734, 0xc}, 0xc000acf5a0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000f958c0, {0x3415dd8, 0xc0009e0fa0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000132c80) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 919 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1002 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000c07e50, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000c07e40) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000ba5db0, {0x2120734, 0xc}, 0xc000c20ea0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000c086c0, {0x3415dd8?, 0xc000ba5ef0}, {0x2120734, 0xc}, 0xc000be24a0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc00056db80, {0x3415dd8, 0xc000ba5ef0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0000b3a40) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 1000 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 900 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000ade190, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000ade180) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000768c30, {0x2120734, 0xc}, 0xc000ea6000) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bca000, {0x3415dd8?, 0xc000768d70}, {0x2120734, 0xc}, 0xc000767330) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc001158000, {0x3415dd8, 0xc000768d70}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0005bef00) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 898 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 993 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000cf4950, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000cf4940) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000d00690, {0x2120734, 0xc}, 0xc000d232b0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000c2f710, {0x3415dd8?, 0xc000d007d0}, {0x2120734, 0xc}, 0xc000bc4390) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc001159340, {0x3415dd8, 0xc000d007d0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc00028b040) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 991 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 975 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000e70590, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000e70580) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000aa9180, {0x2120734, 0xc}, 0xc000e760d0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000be7050, {0x3415dd8?, 0xc000aa92c0}, {0x2120734, 0xc}, 0xc000eac110) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000eee580, {0x3415dd8, 0xc000aa92c0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000e741e0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 973 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 959 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000bbe650, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000bbe640) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000ba4dc0, {0x2120734, 0xc}, 0xc000bb7380) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bdd7a0, {0x3415dd8?, 0xc000ba4f00}, {0x2120734, 0xc}, 0xc0008d9000) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc00056d080, {0x3415dd8, 0xc000ba4f00}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0000b23c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 957 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 981 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000c84f10, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000c84f00) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc00105d090, {0x2120734, 0xc}, 0xc000b68dd0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bcb9e0, {0x3415dd8?, 0xc00105d1d0}, {0x2120734, 0xc}, 0xc000b4ecd0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc001158dc0, {0x3415dd8, 0xc00105d1d0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc00026b180) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 979 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 961 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000bbedd0, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000bbedc0) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000ba51d0, {0x2120734, 0xc}, 0xc000bb7d40) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bdd950, {0x3415dd8?, 0xc000ba5310}, {0x2120734, 0xc}, 0xc0008d9770) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc00056d8c0, {0x3415dd8, 0xc000ba5310}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0000b2a00) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 965 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 985 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000ca8ad0, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000ca8ac0) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc00105d810, {0x2120734, 0xc}, 0xc000b69c70) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bcbd40, {0x3415dd8?, 0xc00105d950}, {0x2120734, 0xc}, 0xc000b4f6d0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc001159080, {0x3415dd8, 0xc00105d950}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc00028a500) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 983 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 935 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000a7b050, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000a7b040) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000926f50, {0x2120734, 0xc}, 0xc000ae7ad0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bd9170, {0x3415dd8?, 0xc0009270e0}, {0x2120734, 0xc}, 0xc000a827b0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000214b00, {0x3415dd8, 0xc0009270e0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc00051ba40) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 933 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 951 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000bb0950, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000bb0940) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000ba4550, {0x2120734, 0xc}, 0xc000bb61a0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bdc900, {0x3415dd8?, 0xc000ba4690}, {0x2120734, 0xc}, 0xc000a83f30) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc00056cb00, {0x3415dd8, 0xc000ba4690}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000785d60) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 949 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 909 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc00107b310, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc00107b300) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc00105c780, {0x2120734, 0xc}, 0xc000ea7ad0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000bcb050, {0x3415dd8?, 0xc00105c8c0}, {0x2120734, 0xc}, 0xc000b4e200) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc001158b00, {0x3415dd8, 0xc00105c8c0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc00026a0a0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 907 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 929 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000f7af90, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000f7af80) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc0009e1f40, {0x2120734, 0xc}, 0xc000a996c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000be66c0, {0x3415dd8?, 0xc000aa80a0}, {0x2120734, 0xc}, 0xc000922210) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000f95b80, {0x3415dd8, 0xc000aa80a0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0001332c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 927 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1042 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000e70cd0, 0x0) /usr/local/go/src/runtime/sema.go:527 +0x159 sync.(*Cond).Wait(0xc000e70cc0) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000aa94f0, {0x2120734, 0xc}, 0xc000e76b60) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x16f6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000be7170, {0x3415dd8?, 0xc000aa9630}, {0x2120734, 0xc}, 0xc000eac7a0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x592 github.com/grafana/dskit/ring.(*Ring).loop(0xc000eee840, {0x3415dd8, 0xc000aa9630}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:287 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000e74500) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x34d created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 976 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 3881 [sync.Mutex.Lock, 29 minutes]: sync.runtime_SemacquireMutex(0x0?, 0xac?, 0x4ad0f9?) /usr/local/go/src/runtime/sema.go:77 +0x25 sync.(*Mutex).lockSlow(0xc0019f[62](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:63)d0) /usr/local/go/src/sync/mutex.go:171 +0x213 sync.(*Mutex).Lock(0xc0019f62d0) /usr/local/go/src/sync/mutex.go:90 +0x55 sync.(*RWMutex).Lock(0xc0019f62d0) /usr/local/go/src/sync/rwmutex.go:147 +0x45 github.com/prometheus/alertmanager/nflog.(*Log).GC(0xc0019f62a0) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:426 +0x119 github.com/prometheus/alertmanager/nflog.(*Log).Maintenance.func1() /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:310 +0x45 github.com/prometheus/alertmanager/nflog.(*Log).Maintenance.func2(0xc000afdf18) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:335 +0x3cc github.com/prometheus/alertmanager/nflog.(*Log).Maintenance(0xc0019f62a0, 0xd18c2e2800, {0xc000125b80, 0x9a}, 0xc0013034a0, 0x0) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:361 +0x713 github.com/grafana/mimir/pkg/alertmanager.New.func1() /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:209 +0xa7 created by github.com/grafana/mimir/pkg/alertmanager.New in goroutine 3850 /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:208 +0xcc6 goroutine 1558 [select, 29 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0x0?, {0x3415b38, 0x4080380}, 0x4, 0xc0017532c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0xc9 github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc0001bb7c0, {0x3415b38, 0x4080380}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x57 github.com/grafana/dskit/services.StopAndAwaitTerminated({0x3415b38, 0x4080380}, {0x341a7d0, 0xc001890a00}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/services.go:122 +0x5c github.com/grafana/mimir/pkg/alertmanager.setupSingleMultitenantAlertmanager.func2() /__w/mimir/mimir/pkg/alertmanager/multitenant_test.go:122 +0x49 testing.(*common).Cleanup.func1() /usr/local/go/src/testing/testing.go:1169 +0x183 testing.(*common).runCleanup(0xc000183860, 0x0) /usr/local/go/src/testing/testing.go:1347 +0x1c3 testing.tRunner.func2() /usr/local/go/src/testing/testing.go:1589 +0x51 testing.tRunner(0xc000183860, 0xc0011b3[63](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:64)8) /usr/local/go/src/testing/testing.go:1601 +0x26c created by testing.(*T).Run in goroutine 1434 /usr/local/go/src/testing/testing.go:1[64](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:65)8 +0x82b goroutine 3643 [chan send, 29 minutes]: github.com/grafana/mimir/pkg/alertmanager.(*state).broadcast(...) /__w/mimir/mimir/pkg/alertmanager/state_replication.go:312 github.com/grafana/mimir/pkg/alertmanager.(*stateChannel).Broadcast(0xc0001bf620, {0xc00097f500, 0x46, 0x80}) /__w/mimir/mimir/pkg/alertmanager/state_replication.go:325 +0x1a5 github.com/prometheus/alertmanager/nflog.(*Log).Log(0xc0019f62a0, 0xc00093[66](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:67)90, {0xc001a7f18b, 0x5}, {0xc001a7f218, 0x1, 0x1}, {0x4080380, 0x0, 0x0}, ...) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:413 +0x74b github.com/prometheus/alertmanager/notify.SetNotifiesStage.Exec({{0x340aed0?, 0xc0019f62a0?}, 0xc000936690?}, {0x3415da0, 0xc000b075c0}, {0xc001f79da0?, 0x441014?}, {0xc0008ca6c0, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:876 +0x253 github.com/prometheus/alertmanager/notify.MultiStage.Exec({0xc001fb6980, 0x4, 0x3415da0?}, {0x3415da0, 0xc000b07530}, {0x3403a80, 0xc00093[67](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:68)30}, {0xc0008ca6c0, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:455 +0x169 github.com/prometheus/alertmanager/notify.FanoutStage.Exec.func1({0x3403c00, 0xc000a03908}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:477 +0xc7 created by github.com/prometheus/alertmanager/notify.FanoutStage.Exec in goroutine 3642 /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:476 +0xfa goroutine 3642 [semacquire, 29 minutes]: sync.runtime_Semacquire(0xc001a7f208?) /usr/local/go/src/runtime/sema.go:62 +0x25 sync.(*WaitGroup).Wait(0xc001a7f200) /usr/local/go/src/sync/waitgroup.go:116 +0xa5 github.com/prometheus/alertmanager/notify.FanoutStage.Exec({0xc0020bef30, 0x1, 0xc000783c70?}, {0x3415da0?, 0xc000b07530}, {0x3403a80?, 0xc000936730}, {0xc0008ca6c0, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:483 +0x397 github.com/prometheus/alertmanager/notify.MultiStage.Exec({0xc001fb5200, 0x6, 0x44d905?}, {0x3415da0, 0xc000b07530}, {0x3403a80, 0xc000936730}, {0xc0008ca6b0, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:455 +0x169 github.com/prometheus/alertmanager/notify.RoutingStage.Exec(0xa2be36?, {0x3415da0, 0xc000b07530}, {0x3403a80, 0xc000936730}, {0xc0008ca6b0, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:441 +0x212 github.com/prometheus/alertmanager/dispatch.(*Dispatcher).processAlert.func1({0x3415da0, 0xc000b07530}, {0xc0008ca6b0, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:343 +0x106 github.com/prometheus/alertmanager/dispatch.(*aggrGroup).run.func1({0xc0008ca6b0, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:457 +0x6c github.com/prometheus/alertmanager/dispatch.(*aggrGroup).flush(0xc001a86ab0, 0xc000af7f60) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:517 +0x[68](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:69)c github.com/prometheus/alertmanager/dispatch.(*aggrGroup).run(0xc001a86ab0, 0xc0012a2d60) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:456 +0x714 created by github.com/prometheus/alertmanager/dispatch.(*Dispatcher).processAlert in goroutine 3925 /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:342 +0x72c goroutine 3827 [select]: github.com/grafana/dskit/kv/consul.(*mockKV).loop(0xc000dc55e0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:87 +0x191 created by github.com/grafana/dskit/kv/consul.NewInMemoryClientWithConfig in goroutine 1558 /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:58 +0x29d goroutine 3850 [semacquire, 29 minutes]: sync.runtime_Semacquire(0xc000b94d80?) /usr/local/go/src/runtime/sema.go:62 +0x25 sync.(*WaitGroup).Wait(0xc000b94d78) /usr/local/go/src/sync/waitgroup.go:116 +0xa5 github.com/grafana/mimir/pkg/alertmanager.(*Alertmanager).StopAndWait(0xc000b94d00) /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:448 +0x625 github.com/grafana/mimir/pkg/alertmanager.(*MultitenantAlertmanager).stopping(0xc001890a00, {0x34026e0?, 0x401fa40?}) /__w/mimir/mimir/pkg/alertmanager/multitenant.go:5[69](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:70) +0xca github.com/grafana/dskit/services.(*BasicService).main(0xc0001bb7c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:207 +0x4b9 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 1558 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:[119](https://github.com/grafana/mimir/actions/runs/6973715887/job/18978192226?pr=6710#step:8:120) +0x1dd ```

dimitarvdimitrov commented 3 months ago

another instance

Details

``` panic: test timed out after 30m0s running tests: TestMultitenantAlertmanager_FirewallShouldBlockHTTPBasedReceiversWhenEnabled/receiver=pagerduty_firewall_enabled=false (29m51s) goroutine 4283 [running]: testing.(*M).startAlarm.func1() /usr/local/go/src/testing/testing.go:2366 +0x265 created by time.goFunc /usr/local/go/src/time/sleep.go:177 +0x45 goroutine 1 [chan receive, 29 minutes]: testing.(*T).Run(0xc0000d6000, {0x244216d, 0x4c}, 0x2554940) /usr/local/go/src/testing/testing.go:1750 +0x851 testing.runTests.func1(0xc0000d6000) /usr/local/go/src/testing/testing.go:2161 +0x86 testing.tRunner(0xc0000d6000, 0xc000c27b10) /usr/local/go/src/testing/testing.go:1689 +0x21f testing.runTests(0xc000406438, {0x4bcf6e0, 0x36, 0x36}, {0xc000c27bb8?, 0xc000c27c00?, 0x4be3360?}) /usr/local/go/src/testing/testing.go:2159 +0x8bf testing.(*M).Run(0xc0009968c0) /usr/local/go/src/testing/testing.go:2027 +0xf18 main.main() _testmain.go:153 +0x2be goroutine 35 [select]: github.com/golang/glog.(*fileSink).flushDaemon(0x4be33f8) /__w/mimir/mimir/vendor/github.com/golang/glog/glog_file.go:351 +0x17c created by github.com/golang/glog.init.1 in goroutine 1 /__w/mimir/mimir/vendor/github.com/golang/glog/glog_file.go:166 +0x225 goroutine 36 [select, 30 minutes]: github.com/dgraph-io/ristretto.(*defaultPolicy).processItems(0xc000469a00) /__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/policy.go:102 +0x106 created by github.com/dgraph-io/ristretto.newDefaultPolicy in goroutine 1 /__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/policy.go:86 +0x2bd goroutine 37 [select]: github.com/dgraph-io/ristretto.(*Cache).processItems(0xc000461280) /__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/cache.go:452 +0x2c5 created by github.com/dgraph-io/ristretto.NewCache in goroutine 1 /__w/mimir/mimir/vendor/github.com/dgraph-io/ristretto/cache.go:207 +0xea6 goroutine 38 [select]: go.opencensus.io/stats/view.(*worker).start(0xc000461480) /__w/mimir/mimir/vendor/go.opencensus.io/stats/view/worker.go:292 +0x1a5 created by go.opencensus.io/stats/view.init.0 in goroutine 1 /__w/mimir/mimir/vendor/go.opencensus.io/stats/view/worker.go:34 +0xfd goroutine 360 [chan receive, 29 minutes]: testing.(*T).Parallel(0xc0000d61a0) /usr/local/go/src/testing/testing.go:1483 +0x525 github.com/grafana/mimir/pkg/alertmanager.TestIsHealthyForAlertmanagerOperations(0xc0000d61a0) /__w/mimir/mimir/pkg/alertmanager/alertmanager_ring_test.go:17 +0x3f testing.tRunner(0xc0000d61a0, 0x2554918) /usr/local/go/src/testing/testing.go:1689 +0x21f created by testing.(*T).Run in goroutine 1 /usr/local/go/src/testing/testing.go:1742 +0x826 goroutine 46 [chan receive, 30 minutes]: github.com/grafana/mimir/pkg/alertmanager.init.0.func1() /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:139 +0x54 created by github.com/grafana/mimir/pkg/alertmanager.init.0 in goroutine 1 /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:134 +0x25 goroutine 1342 [chan receive, 29 minutes]: testing.tRunner.func1() /usr/local/go/src/testing/testing.go:1650 +0x9b2 testing.tRunner(0xc000c75040, 0x2554940) /usr/local/go/src/testing/testing.go:1695 +0x25e created by testing.(*T).Run in goroutine 1 /usr/local/go/src/testing/testing.go:1742 +0x826 goroutine 845 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc00148afd0, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc00148afc0) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000ceae10, {0x23cff63, 0xc}, 0xc000e609c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000f47170, {0x3ec3338, 0xc000ceaf50}, {0x23cff63, 0xc}, 0xc0012d74f0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000ce8908, {0x3ec3338, 0xc000ceaf50}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000cba640) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 843 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 983 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc00157d290, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc00157d280) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000527720, {0x23cff63, 0xc}, 0xc000e84b60) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000afb830, {0x3ec3338, 0xc000527860}, {0x23cff63, 0xc}, 0xc000c19f60) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc0011c4908, {0x3ec3338, 0xc000527860}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc001401680) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 981 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 976 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000dc8a50, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000dc8a40) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000dac9b0, {0x23cff63, 0xc}, 0xc000e44750) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000814630, {0x3ec3338, 0xc000dacaf0}, {0x23cff63, 0xc}, 0xc000491540) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000bf0908, {0x3ec3338, 0xc000dacaf0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000997720) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 974 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 995 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000dc9210, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000dc9200) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000dacd70, {0x23cff63, 0xc}, 0xc000e452b0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc0008147e0, {0x3ec3338, 0xc000daceb0}, {0x23cff63, 0xc}, 0xc000534400) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000bf0c08, {0x3ec3338, 0xc000daceb0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0009979a0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 977 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1018 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000ed2510, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000ed2500) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000e91770, {0x23cff63, 0xc}, 0xc000ed09c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc0008f0120, {0x3ec3338, 0xc000e918b0}, {0x23cff63, 0xc}, 0xc000852670) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc0011c5208, {0x3ec3338, 0xc000e918b0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc00075e460) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 1016 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 998 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000dc9950, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000dc9940) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000dad180, {0x23cff63, 0xc}, 0xc000e45c70) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000814870, {0x3ec3338, 0xc000dad2c0}, {0x23cff63, 0xc}, 0xc000534ba0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000bf0f08, {0x3ec3338, 0xc000dad2c0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000997c20) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 996 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 435 [chan receive, 29 minutes]: github.com/oklog/run.(*Group).Run(0xc000eadf10) /__w/mimir/mimir/vendor/github.com/oklog/run/group.go:43 +0x251 github.com/prometheus/alertmanager/inhibit.(*Inhibitor).Run(0xc0007b1490) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/inhibit/inhibit.go:108 +0x505 created by github.com/grafana/mimir/pkg/alertmanager.(*Alertmanager).ApplyConfig in goroutine 370 /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:407 +0x1af5 goroutine 841 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000e2ca10, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000e2ca00) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000cea550, {0x23cff63, 0xc}, 0xc0014768f0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000f46c60, {0x3ec3338, 0xc000cea690}, {0x23cff63, 0xc}, 0xc0012d6d80) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000ce8608, {0x3ec3338, 0xc000cea690}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0004fb180) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 839 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 424 [select, 29 minutes]: github.com/prometheus/alertmanager/inhibit.(*Inhibitor).run(0xc0007b1490, {0x3ec3338, 0xc000ca2fa0}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/inhibit/inhibit.go:65 +0x1d9 github.com/prometheus/alertmanager/inhibit.(*Inhibitor).Run.func1() /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/inhibit/inhibit.go:102 +0x47 github.com/oklog/run.(*Group).Run.func1({0xc000f1e720?, 0xc000c30400?}) /__w/mimir/mimir/vendor/github.com/oklog/run/group.go:38 +0x3a created by github.com/oklog/run.(*Group).Run in goroutine 435 /__w/mimir/mimir/vendor/github.com/oklog/run/group.go:37 +0xa5 goroutine 968 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc0017a8b50, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc0017a8b40) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000dac0a0, {0x23cff63, 0xc}, 0xc000da6f70) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc00077d710, {0x3ec3338, 0xc000dac1e0}, {0x23cff63, 0xc}, 0xc000b9ba50) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000bf0608, {0x3ec3338, 0xc000dac1e0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000997220) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 966 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 838 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000e2c050, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000e2c040) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000405590, {0x23cff63, 0xc}, 0xc001209520) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000f46120, {0x3ec3338, 0xc000405770}, {0x23cff63, 0xc}, 0xc0012d6520) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000ce8008, {0x3ec3338, 0xc000405770}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0004fa3c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 836 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 941 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc0012e0350, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc0012e0340) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000484be0, {0x23cff63, 0xc}, 0xc000cae820) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc00077c360, {0x3ec3338, 0xc000484d20}, {0x23cff63, 0xc}, 0xc001573bb0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000bf0008, {0x3ec3338, 0xc000484d20}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000996aa0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 939 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 922 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000ee26d0, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000ee26c0) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc0005269b0, {0x23cff63, 0xc}, 0xc0015768f0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000b53ef0, {0x3ec3338, 0xc000526af0}, {0x23cff63, 0xc}, 0xc000c19450) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc0011c4308, {0x3ec3338, 0xc000526af0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc001400e60) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 920 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1021 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000ed2d50, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000ed2d40) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000e91bd0, {0x23cff63, 0xc}, 0xc000ed1520) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc0008f0480, {0x3ec3338, 0xc000e91d10}, {0x23cff63, 0xc}, 0xc000853100) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc0011c5508, {0x3ec3338, 0xc000e91d10}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc00075e960) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 1019 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 965 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc0017a8210, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc0017a8200) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000485950, {0x23cff63, 0xc}, 0xc000da64e0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc00077d320, {0x3ec3338, 0xc000485b30}, {0x23cff63, 0xc}, 0xc000b9b2e0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000bf0308, {0x3ec3338, 0xc000485b30}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000996fa0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 963 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 955 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000c68010, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000c68000) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc00108de50, {0x23cff63, 0xc}, 0xc00132ca90) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000d26090, {0x3ec3338, 0xc000d0a050}, {0x23cff63, 0xc}, 0xc000fa04d0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000161b08, {0x3ec3338, 0xc000d0a050}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc000969400) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 953 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1013 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000ee52d0, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000ee52c0) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000e90d20, {0x23cff63, 0xc}, 0xc000e45380) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc00098abd0, {0x3ec3338, 0xc000e90e60}, {0x23cff63, 0xc}, 0xc0000521e0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc0011c4f08, {0x3ec3338, 0xc000e90e60}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc0004640a0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 1011 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 991 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000e8cd50, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000e8cd40) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000e90000, {0x23cff63, 0xc}, 0xc000e85a00) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc00098a360, {0x3ec3338, 0xc000e90140}, {0x23cff63, 0xc}, 0xc000411030) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc0011c4c08, {0x3ec3338, 0xc000e90140}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc001401b80) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 989 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1027 [sync.Cond.Wait, 29 minutes]: sync.runtime_notifyListWait(0xc000c69e50, 0x0) /usr/local/go/src/runtime/sema.go:569 +0x159 sync.(*Cond).Wait(0xc000c69e40) /usr/local/go/src/sync/cond.go:70 +0x75 github.com/grafana/dskit/kv/consul.(*mockKV).Get(0xc000d0af50, {0x23cff63, 0xc}, 0xc000d400d0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:178 +0x17d6 github.com/grafana/dskit/kv/consul.(*Client).WatchKey(0xc000d27290, {0x3ec3338, 0xc000d0b090}, {0x23cff63, 0xc}, 0xc000fa1250) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/client.go:239 +0x6b2 github.com/grafana/dskit/ring.(*Ring).loop(0xc000d32008, {0x3ec3338, 0xc000d0b090}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/ring/ring.go:308 +0x131 github.com/grafana/dskit/services.(*BasicService).main(0xc00085a780) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:190 +0x3b8 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 961 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 1344 [select, 29 minutes]: github.com/grafana/dskit/services.(*BasicService).awaitState(0xc00075f0e0, {0x3ec3098, 0x4c481c0}, 0x4, 0xc0010409c0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:294 +0xca github.com/grafana/dskit/services.(*BasicService).AwaitTerminated(0xc00075f0e0, {0x3ec3098, 0x4c481c0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:290 +0x57 github.com/grafana/dskit/services.StopAndAwaitTerminated({0x3ec3098, 0x4c481c0}, {0x3ec9470, 0xc00227dce0}) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/services.go:122 +0x5c github.com/grafana/mimir/pkg/alertmanager.setupSingleMultitenantAlertmanager.func2() /__w/mimir/mimir/pkg/alertmanager/multitenant_test.go:123 +0x49 testing.(*common).Cleanup.func1() /usr/local/go/src/testing/testing.go:1175 +0x17a testing.(*common).runCleanup(0xc000c756c0, 0x0) /usr/local/go/src/testing/testing.go:1353 +0x2[62](https://github.com/grafana/mimir/actions/runs/9302586918/job/25603103012?pr=8219#step:8:63) testing.tRunner.func2() /usr/local/go/src/testing/testing.go:1683 +0x51 testing.tRunner(0xc000c756c0, 0xc0014ad188) /usr/local/go/src/testing/testing.go:1695 +0x25e created by testing.(*T).Run in goroutine 1342 /usr/local/go/src/testing/testing.go:1742 +0x826 goroutine 3939 [semacquire, 29 minutes]: sync.runtime_Semacquire(0xc00253b4d0?) /usr/local/go/src/runtime/sema.go:62 +0x25 sync.(*WaitGroup).Wait(0xc00253b4c8) /usr/local/go/src/sync/waitgroup.go:116 +0xa5 github.com/grafana/mimir/pkg/alertmanager.(*Alertmanager).StopAndWait(0xc00253b450) /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:441 +0x645 github.com/grafana/mimir/pkg/alertmanager.(*MultitenantAlertmanager).stopping(0xc00227dce0, {0x3eaf020?, 0x4ba3550?}) /__w/mimir/mimir/pkg/alertmanager/multitenant.go:581 +0x10f github.com/grafana/dskit/services.(*BasicService).main(0xc00075f0e0) /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:207 +0x559 created by github.com/grafana/dskit/services.(*BasicService).StartAsync.func1 in goroutine 1344 /__w/mimir/mimir/vendor/github.com/grafana/dskit/services/basic_service.go:119 +0x1dd goroutine 4005 [semacquire, 29 minutes]: sync.runtime_Semacquire(0xc001765a28?) /usr/local/go/src/runtime/sema.go:62 +0x25 sync.(*WaitGroup).Wait(0xc001765a20) /usr/local/go/src/sync/waitgroup.go:116 +0xa5 github.com/prometheus/alertmanager/notify.FanoutStage.Exec({0xc0022e2070, 0x1, 0xc00287aec0?}, {0x3ec3300, 0xc0023d84e0}, {0x3eb0420, 0xc00158c8c0}, {0xc001493158, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:491 +0x390 github.com/prometheus/alertmanager/notify.MultiStage.Exec({0xc001cd6d80, 0x6, 0x44f885?}, {0x3ec3300, 0xc0023d84e0}, {0x3eb0420, 0xc00158c8c0}, {0xc001493148, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:4[63](https://github.com/grafana/mimir/actions/runs/9302586918/job/25603103012?pr=8219#step:8:64) +0x169 github.com/prometheus/alertmanager/notify.RoutingStage.Exec(0xc001cdd860, {0x3ec3300, 0xc0023d84e0}, {0x3eb0420, 0xc00158c8c0}, {0xc001493148, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:449 +0x1b8 github.com/prometheus/alertmanager/dispatch.(*Dispatcher).processAlert.func1({0x3ec3300, 0xc0023d84e0}, {0xc001493148, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:344 +0x106 github.com/prometheus/alertmanager/dispatch.(*aggrGroup).run.func1({0xc001493148, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:458 +0x6c github.com/prometheus/alertmanager/dispatch.(*aggrGroup).flush(0xc002253200, 0xc0007edeb8) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:518 +0x6b5 github.com/prometheus/alertmanager/dispatch.(*aggrGroup).run(0xc002253200, 0xc00251fcb0) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:457 +0x7a5 created by github.com/prometheus/alertmanager/dispatch.(*Dispatcher).processAlert in goroutine 3872 /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/dispatch/dispatch.go:343 +0x76c goroutine 4006 [chan send, 29 minutes]: github.com/grafana/mimir/pkg/alertmanager.(*state).broadcast(...) /__w/mimir/mimir/pkg/alertmanager/state_replication.go:312 github.com/grafana/mimir/pkg/alertmanager.(*stateChannel).Broadcast(0xc000f90[64](https://github.com/grafana/mimir/actions/runs/9302586918/job/25603103012?pr=8219#step:8:65)8, {0xc0000b3d00, 0x4a, 0x80}) /__w/mimir/mimir/pkg/alertmanager/state_replication.go:325 +0x1a5 github.com/prometheus/alertmanager/nflog.(*Log).Log(0xc00287c360, 0xc00158c820, {0xc0017[65](https://github.com/grafana/mimir/actions/runs/9302586918/job/25603103012?pr=8219#step:8:66)98b, 0x5}, {0xc001765a38, 0x1, 0x1}, {0x4c481c0, 0x0, 0x0}, ...) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:417 +0x76e github.com/prometheus/alertmanager/notify.SetNotifiesStage.Exec({{0x3eb7c10?, 0xc00287c360?}, 0xc00158c820?}, {0x3ec3300, 0xc0023d8540}, {0xc00250dda0?, 0x4426b4?}, {0xc001493158, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:922 +0x253 github.com/prometheus/alertmanager/notify.MultiStage.Exec({0xc00245e300, 0x4, 0xc001c0d270?}, {0x3ec3300, 0xc0023d84e0}, {0x3eb0420, 0xc00158c8c0}, {0xc001493158, 0x1, 0x1}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:463 +0x169 github.com/prometheus/alertmanager/notify.FanoutStage.Exec.func1({0x3eb05e0, 0xc0016c8060}) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:485 +0xc7 created by github.com/prometheus/alertmanager/notify.FanoutStage.Exec in goroutine 4005 /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/notify/notify.go:484 +0xf0 goroutine 3916 [select]: github.com/grafana/dskit/kv/consul.(*mockKV).loop(0xc0018b3310) /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:87 +0x1af created by github.com/grafana/dskit/kv/consul.NewInMemoryClientWithConfig in goroutine 1344 /__w/mimir/mimir/vendor/github.com/grafana/dskit/kv/consul/mock.go:58 +0x28e goroutine 3963 [sync.Mutex.Lock, 29 minutes]: sync.runtime_SemacquireMutex(0x4b4d19?, 0x80?, 0x4b4d19?) /usr/local/go/src/runtime/sema.go:77 +0x25 sync.(*Mutex).lockSlow(0xc00287c390) /usr/local/go/src/sync/mutex.go:171 +0x213 sync.(*Mutex).Lock(0xc00287c390) /usr/local/go/src/sync/mutex.go:90 +0x55 sync.(*RWMutex).Lock(0xc00287c390) /usr/local/go/src/sync/rwmutex.go:146 +0x45 github.com/prometheus/alertmanager/nflog.(*Log).GC(0xc00287c360) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:430 +0x1[66](https://github.com/grafana/mimir/actions/runs/9302586918/job/25603103012?pr=8219#step:8:67) github.com/prometheus/alertmanager/nflog.(*Log).Maintenance.func1() /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:314 +0x45 github.com/prometheus/alertmanager/nflog.(*Log).Maintenance.func2(0xc0015e7ef0) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:339 +0x3e2 github.com/prometheus/alertmanager/nflog.(*Log).Maintenance(0xc00287c360, 0xd18c2e2800, {0xc0000d0780, 0x9d}, 0xc0010419e0, 0x0) /__w/mimir/mimir/vendor/github.com/prometheus/alertmanager/nflog/nflog.go:365 +0x7a7 github.com/grafana/mimir/pkg/alertmanager.New.func1() /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:210 +0x8f created by github.com/grafana/mimir/pkg/alertmanager.New in goroutine 3939 /__w/mimir/mimir/pkg/alertmanager/alertmanager.go:209 +0xd7a ```