ingester panic

[shichanglin5]

ts=2024-06-13T03:06:57.641429905Z caller=compact.go:709 level=info user=default msg="write block" mint=1718244000000 maxt=1718251200000 ulid=01J07S03EM40YXG89G6MMRQCQ2 duration=276.632077ms
ts=2024-06-13T03:06:57.641710321Z caller=db.go:1279 level=info user=default msg="out-of-order compaction completed" duration=276.911749ms ulids=[01J07S03EM40YXG89G6MMRQCQ2]
unexpected fault address 0x7f9d2c6b0e01
fatal error: fault
ts=2024-06-13T03:06:57.759060723Z caller=db.go:1616 level=info user=default msg="Deleting obsolete block" block=01HXQYMN2MRP6CTCEZRVM27FBR
[signal SIGSEGV: segmentation violation code=0x1 addr=0x7f9d2c6b0e01 pc=0x471510]

goroutine 33439826870 [running]:
runtime.throw({0x24d583f?, 0xc291263618?})
    /usr/local/go/src/runtime/panic.go:1047 +0x5d fp=0xc2912635a8 sp=0xc291263578 pc=0x43905d
runtime.sigpanic()
    /usr/local/go/src/runtime/signal_unix.go:855 +0x28a fp=0xc291263608 sp=0xc2912635a8 pc=0x45032a
runtime.memmove()
    /usr/local/go/src/runtime/memmove_amd64.s:178 +0x130 fp=0xc291263610 sp=0xc291263608 pc=0x471510
github.com/grafana/mimir/pkg/ingester/client.(*LabelValuesResponse).MarshalToSizedBuffer(0xc35b428678, {0xc7c2bf0000, 0x550c9, 0x550c9})
    /__w/mimir/mimir/pkg/ingester/client/ingester.pb.go:5132 +0xe9 fp=0xc291263650 sp=0xc291263610 pc=0x1b33949
github.com/grafana/mimir/pkg/ingester/client.(*LabelValuesResponse).Marshal(0xc35b428678)
    /__w/mimir/mimir/pkg/ingester/client/ingester.pb.go:5112 +0x6e fp=0xc291263690 sp=0xc291263650 pc=0x1b3368e
google.golang.org/protobuf/internal/impl.legacyMarshal({{}, {0x2bac280, 0xc35b0f76f0}, {0x0, 0x0, 0x0}, 0x0})
    /__w/mimir/mimir/vendor/google.golang.org/protobuf/internal/impl/legacy_message.go:402 +0xa2 fp=0xc291263710 sp=0xc291263690 pc=0x65ee82
google.golang.org/protobuf/proto.MarshalOptions.marshal({{}, 0xe8?, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x2bac280, 0xc35b0f76f0})
    /__w/mimir/mimir/vendor/google.golang.org/protobuf/proto/encode.go:166 +0x27b fp=0xc2912637b0 sp=0xc291263710 pc=0x5ee41b
google.golang.org/protobuf/proto.MarshalOptions.MarshalAppend({{}, 0x40?, 0x58?, 0x40?}, {0x0, 0x0, 0x0}, {0x2b792a0?, 0xc35b0f76f0?})
    /__w/mimir/mimir/vendor/google.golang.org/protobuf/proto/encode.go:125 +0x79 fp=0xc2912637f8 sp=0xc2912637b0 pc=0x5ee059
github.com/golang/protobuf/proto.marshalAppend({0x0, 0x0, 0x0}, {0x7fcdb11bf298?, 0xc35b428678?}, 0x60?)
    /__w/mimir/mimir/vendor/github.com/golang/protobuf/proto/wire.go:40 +0xa5 fp=0xc291263878 sp=0xc2912637f8 pc=0x68a6e5
github.com/golang/protobuf/proto.Marshal(...)

[shichanglin5]

exit code: 2

[pstibrany]

I believe this is now fixed in https://github.com/grafana/mimir/pull/8003 (issue https://github.com/grafana/mimir/issues/7897).

[aknuds1]

Ah nice @pstibrany - I didn't recognize that as the source of the bug :man_facepalming:

[aknuds1]

Closing as probable duplicate of #7897. Please let us know if it turns out to not be the case after all.