golang / protobuf

Go support for Google's protocol buffers
BSD 3-Clause "New" or "Revised" License
9.8k stars 1.58k forks source link

appendMap Panic #1648

Closed winey-dev closed 1 month ago

winey-dev commented 2 months ago

Go Version

protoc Version

Log

reflect.mapiterinit(0x4ac5a5?, 0x1b12b80?, 0x7b0e?)
        /home/smlee/.dev-package/go/src/runtime/map.go:1387 +0x13
reflect.(*MapIter).Next(0x1c07380?)
        /home/smlee/.dev-package/go/src/reflect/value.go:2000 +0x57
google.golang.org/protobuf/internal/impl.appendMap({0xc002bc6000, 0x6cb0, 0x7b0e}, {0x1c07380?, 0xc0015a4ba0?, 0x6cb0?}, 0xc0015fa3c0, 0xc00049de28, {0x74?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_map.go:274 +0x1d3
google.golang.org/protobuf/internal/impl.encoderFuncsForMap.func2({0xc002bc6000, 0x6cb0, 0x7b0e}, {0xc0015a51e0?}, 0xc00049de28, {0x50?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_map.go:56 +0x7e
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshalAppendPointer(0xc000426010, {0xc002bc6000?, 0x5d42c8?, 0x2d8?}, {0xc000795400?}, {0x80?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:139 +0x13f
google.golang.org/protobuf/internal/impl.appendMessageSliceInfo({0xc002bc6000?, 0xc0017ab578?, 0x5e9345?}, {0xc002bc6000?}, 0xc00154e450, {0x0?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_field.go:485 +0xf1
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshalAppendPointer(0xc000426158, {0xc002bc6000?, 0x5bc0aa?, 0x612?}, {0xc002bc6000?}, {0x2?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:139 +0x13f
google.golang.org/protobuf/internal/impl.appendMessageSliceInfo({0xc002bc6000?, 0xc0017ab638?, 0x41b128?}, {0xc005e30b00?}, 0xc00055a948, {0x0?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_field.go:485 +0xf1
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshalAppendPointer(0xc0004262a0, {0xc002bc6000?, 0xc0004262a0?, 0x412c?}, {0x7f7ed79378d8?}, {0x5?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:139 +0x13f
google.golang.org/protobuf/internal/impl.appendMessageSliceInfo({0xc002bc6000?, 0xc0017ab728?, 0x4113be?}, {0x7f7ed79378d8?}, 0xc001698090, {0x28?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_field.go:485 +0xf1
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshalAppendPointer(0xc0004263e8, {0xc002bc6000?, 0x8000?, 0x7f7f1ef92b38?}, {0x8000?}, {0x8?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:139 +0x13f
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshal(0x0?, {{}, {0x22901b8, 0xc000778e40}, {0xc002bc6000, 0x0, 0x7b0e}, 0x2})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:107 +0x85
google.golang.org/protobuf/proto.MarshalOptions.marshal({{}, 0xc0?, 0xb8?, 0x7a?}, {0x0, 0x0, 0x0}, {0x22901b8, 0xc000778e40})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/proto/encode.go:163 +0x24e
google.golang.org/protobuf/proto.MarshalOptions.MarshalAppend({{}, 0x40?, 0xc4?, 0xd3?}, {0x0, 0x0, 0x0}, {0x2264de0?, 0xc000778e40?})
        /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/proto/encode.go:122 +0x73
github.com/golang/protobuf/proto.marshalAppend({0x0, 0x0, 0x0}, {0x7f7ed8456a78?, 0xc000778e40?}, 0x0)
        /home/smlee/go/pkg/mod/github.com/golang/protobuf@v1.5.2/proto/wire.go:40 +0x97
github.com/golang/protobuf/proto.Marshal(...)
        /home/smlee/go/pkg/mod/github.com/golang/protobuf@v1.5.2/proto/wire.go:23
google.golang.org/grpc/encoding/proto.codec.Marshal({}, {0x1d3c440?, 0xc000778e40?})
        /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/encoding/proto/proto.go:45 +0x5a
google.golang.org/grpc.encode({0x7f7ed849d260?, 0x323eb40?}, {0x1d3c440?, 0xc000778e40?})
        /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/rpc_util.go:594 +0x3e
google.golang.org/grpc.(*Server).sendResponse(0xc000c0ce00, {0x228abc0, 0xc000b05a00}, 0xc000c9efc0, {0x1d3c440, 0xc000778e40}, {0x0, 0x0}, 0xc00392cfb8, {0x0, ...})
        /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:1062 +0x167
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c0ce00, {0x228abc0, 0xc000b05a00}, 0xc000c9efc0, 0xc001089dd0, 0x31b1190, 0x0)
        /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:1318 +0xd75
google.golang.org/grpc.(*Server).handleStream(0xc000c0ce00, {0x228abc0, 0xc000b05a00}, 0xc000c9efc0, 0x0)
        /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:1619 +0x9da
google.golang.org/grpc.(*Server).serveStreams.func1.2()
        /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:921 +0x8d
created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1385081
        /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:919 +0x226

For services that operate on the background, Store/Load is performed through SyncMap, etc

When the gRPC Call comes, the corresponding sync.Map will go around the Range and check it.

I'm getting Panic from AppendMap intermittently when I'm doing that... Is it my code error?

puellanivis commented 2 months ago

What is the panic message that you’re receiving? Like the panic: foo bar baz line.

winey-dev commented 2 months ago

@puellanivis . Sorry .

fatal error: concurrent map iteration and map write

goroutine 1385082 [running]:
reflect.mapiterinit(0x4ac5a5?, 0x1b12b80?, 0x7b0e?)
    /home/smlee/.dev-package/go/src/runtime/map.go:1387 +0x13
reflect.(*MapIter).Next(0x1c07380?)
    /home/smlee/.dev-package/go/src/reflect/value.go:2000 +0x57
google.golang.org/protobuf/internal/impl.appendMap({0xc002bc6000, 0x6cb0, 0x7b0e}, {0x1c07380?, 0xc0015a4ba0?, 0x6cb0?}, 0xc0015fa3c0, 0xc00049de28, {0x74?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_map.go:274 +0x1d3
google.golang.org/protobuf/internal/impl.encoderFuncsForMap.func2({0xc002bc6000, 0x6cb0, 0x7b0e}, {0xc0015a51e0?}, 0xc00049de28, {0x50?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_map.go:56 +0x7e
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshalAppendPointer(0xc000426010, {0xc002bc6000?, 0x5d42c8?, 0x2d8?}, {0xc000795400?}, {0x80?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:139 +0x13f
google.golang.org/protobuf/internal/impl.appendMessageSliceInfo({0xc002bc6000?, 0xc0017ab578?, 0x5e9345?}, {0xc002bc6000?}, 0xc00154e450, {0x0?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_field.go:485 +0xf1
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshalAppendPointer(0xc000426158, {0xc002bc6000?, 0x5bc0aa?, 0x612?}, {0xc002bc6000?}, {0x2?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:139 +0x13f
google.golang.org/protobuf/internal/impl.appendMessageSliceInfo({0xc002bc6000?, 0xc0017ab638?, 0x41b128?}, {0xc005e30b00?}, 0xc00055a948, {0x0?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_field.go:485 +0xf1
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshalAppendPointer(0xc0004262a0, {0xc002bc6000?, 0xc0004262a0?, 0x412c?}, {0x7f7ed79378d8?}, {0x5?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:139 +0x13f
google.golang.org/protobuf/internal/impl.appendMessageSliceInfo({0xc002bc6000?, 0xc0017ab728?, 0x4113be?}, {0x7f7ed79378d8?}, 0xc001698090, {0x28?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/codec_field.go:485 +0xf1
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshalAppendPointer(0xc0004263e8, {0xc002bc6000?, 0x8000?, 0x7f7f1ef92b38?}, {0x8000?}, {0x8?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:139 +0x13f
google.golang.org/protobuf/internal/impl.(*MessageInfo).marshal(0x0?, {{}, {0x22901b8, 0xc000778e40}, {0xc002bc6000, 0x0, 0x7b0e}, 0x2})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/internal/impl/encode.go:107 +0x85
google.golang.org/protobuf/proto.MarshalOptions.marshal({{}, 0xc0?, 0xb8?, 0x7a?}, {0x0, 0x0, 0x0}, {0x22901b8, 0xc000778e40})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/proto/encode.go:163 +0x24e
google.golang.org/protobuf/proto.MarshalOptions.MarshalAppend({{}, 0x40?, 0xc4?, 0xd3?}, {0x0, 0x0, 0x0}, {0x2264de0?, 0xc000778e40?})
    /home/smlee/go/pkg/mod/google.golang.org/protobuf@v1.28.0/proto/encode.go:122 +0x73
github.com/golang/protobuf/proto.marshalAppend({0x0, 0x0, 0x0}, {0x7f7ed8456a78?, 0xc000778e40?}, 0x0)
    /home/smlee/go/pkg/mod/github.com/golang/protobuf@v1.5.2/proto/wire.go:40 +0x97
github.com/golang/protobuf/proto.Marshal(...)
    /home/smlee/go/pkg/mod/github.com/golang/protobuf@v1.5.2/proto/wire.go:23
google.golang.org/grpc/encoding/proto.codec.Marshal({}, {0x1d3c440?, 0xc000778e40?})
    /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/encoding/proto/proto.go:45 +0x5a
google.golang.org/grpc.encode({0x7f7ed849d260?, 0x323eb40?}, {0x1d3c440?, 0xc000778e40?})
    /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/rpc_util.go:594 +0x3e
google.golang.org/grpc.(*Server).sendResponse(0xc000c0ce00, {0x228abc0, 0xc000b05a00}, 0xc000c9efc0, {0x1d3c440, 0xc000778e40}, {0x0, 0x0}, 0xc00392cfb8, {0x0, ...})
    /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:1062 +0x167
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000c0ce00, {0x228abc0, 0xc000b05a00}, 0xc000c9efc0, 0xc001089dd0, 0x31b1190, 0x0)
    /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:1318 +0xd75
google.golang.org/grpc.(*Server).handleStream(0xc000c0ce00, {0x228abc0, 0xc000b05a00}, 0xc000c9efc0, 0x0)
    /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:1619 +0x9da
google.golang.org/grpc.(*Server).serveStreams.func1.2()
    /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:921 +0x8d
created by google.golang.org/grpc.(*Server).serveStreams.func1 in goroutine 1385081
    /home/smlee/go/pkg/mod/google.golang.org/grpc@v1.45.0/server.go:919 +0x226
puellanivis commented 2 months ago

I don’t know what your problem is, but I know it is going to be somewhere in your own code.

I’m taking a stab here, because I don’t know what your code is, but I think there is a map somewhere in the message itself, which is not concurrent safe like sync.Map is.

Another common read/write race condition can happen when you’re attempting to marshal a message at the same time as accessing it. You need to ensure you are only doing the one or the other at any given time.

stapelberg commented 1 month ago

As @puellanivis explained, almost certainly the issue is within your code.

You can use Go’s race detector to locate the issue: https://go.dev/blog/race-detector

I’ll close this issue as it’s not a Go Protobuf issue.