Open dhiaayachi opened 2 weeks ago
Temporal server should not panic and exits
temporal | {"level":"info","ts":"2023-04-12T19:43:54.839Z","msg":"Failing the workflow task.","shard-id":1,"address":"172.30.2.7:7234","component":"history-engine","value":"UnhandledCommand","wf-id":"any_command_combination1681328632774897000","wf-run-id":"88da57bb-b604-4107-8e5d-0499c052d85f","wf-namespace-id":"5d797742-e8dc-4166-bca0-7a7a6b72ed1e","logging-call-at":"workflowTaskHandlerCallbacks.go:510"} temporal | {"level":"info","ts":"2023-04-12T19:43:54.847Z","msg":"history client encountered error","service":"frontend","error":"UnhandledCommand","service-error-type":"serviceerror.InvalidArgument","logging-call-at":"metric_client.go:90"} temporal | panic: runtime error: slice bounds out of range [-24:] [recovered] temporal | panic: runtime error: slice bounds out of range [-24:] temporal | temporal | goroutine 34529409 [running]: temporal | go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End.func1() temporal | /go/pkg/mod/go.opentelemetry.io/otel/sdk@v1.10.0/trace/span.go:393 +0x2c temporal | go.opentelemetry.io/otel/sdk/trace.(*recordingSpan).End(0x4000893b00, {0x0, 0x0, 0x40044288e0?}) temporal | /go/pkg/mod/go.opentelemetry.io/otel/sdk@v1.10.0/trace/span.go:432 +0x6e4 temporal | panic({0x1ceab40, 0x400489cf18}) temporal | /usr/local/go/src/runtime/panic.go:884 +0x20c temporal | go.temporal.io/api/common/v1.(*WorkflowExecution).MarshalToSizedBuffer(0x4004672a80, {0x4002364280, 0xc, 0x400236429d?}) temporal | /go/pkg/mod/go.temporal.io/api@v1.13.1-0.20221110200459-6a3cb21a3415/common/v1/message.pb.go:1407 +0x290 temporal | go.temporal.io/api/workflow/v1.(*WorkflowExecutionInfo).MarshalToSizedBuffer(0x400433aea0, {0x4002364280, 0x0?, 0x123}) temporal | /go/pkg/mod/go.temporal.io/api@v1.13.1-0.20221110200459-6a3cb21a3415/workflow/v1/message.pb.go:1725 +0xa14 temporal | go.temporal.io/server/api/historyservice/v1.(*DescribeWorkflowExecutionResponse).MarshalToSizedBuffer(0x4003835540, {0x4002364280, 0x123, 0x123}) temporal | /home/builder/temporal/api/historyservice/v1/request_response.pb.go:12548 +0x134 temporal | go.temporal.io/server/api/historyservice/v1.(*DescribeWorkflowExecutionResponse).Marshal(0x4001403158?) temporal | /home/builder/temporal/api/historyservice/v1/request_response.pb.go:12489 +0x50 temporal | google.golang.org/protobuf/internal/impl.legacyMarshal({{}, {0x2426668, 0x40049e28b0}, {0x0, 0x0, 0x0}, 0x0}) temporal | /go/pkg/mod/google.golang.org/protobuf@v1.28.1/internal/impl/legacy_message.go:402 +0x84 temporal | google.golang.org/protobuf/proto.MarshalOptions.size({{}, 0x1?, 0xb2?, 0xde?}, {0x2426668, 0x40049e28b0}) temporal | /go/pkg/mod/google.golang.org/protobuf@v1.28.1/proto/size.go:43 +0x8c temporal | google.golang.org/protobuf/proto.MarshalOptions.Size({{}, 0x80?, 0x1b?, 0xdc?}, {0x23fc3c0?, 0x40049e28b0?}) temporal | /go/pkg/mod/google.golang.org/protobuf@v1.28.1/proto/size.go:26 +0x54 temporal | google.golang.org/protobuf/proto.Size(...) temporal | /go/pkg/mod/google.golang.org/protobuf@v1.28.1/proto/size.go:16 temporal | github.com/golang/protobuf/proto.Size({0xffff8dde1fc8?, 0x4003835540?}) temporal | /go/pkg/mod/github.com/golang/protobuf@v1.5.2/proto/wire.go:18 +0x48 temporal | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.messageType.Event({{0x1e762ec, 0xc}, {0x4, 0x0, {0x1e64581, 0x4}, {0x0, 0x0}}}, {0x2412a50?, 0x40040ce390?}, ...) temporal | /go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.36.1/interceptor.go:50 +0xe4 temporal | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1({0x2412a50, 0x40040ce210}, {0x1d94a80, 0x40019b19e0}, 0x4004fae120, 0x4002c67d40) temporal | /go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc@v0.36.1/interceptor.go:360 +0x920 temporal | google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x2412a50?, 0x40040ce210?}, {0x1d94a80?, 0x40019b19e0?}) temporal | /go/pkg/mod/google.golang.org/grpc@v1.50.1/server.go:1165 +0x8c temporal | go.temporal.io/server/common/rpc.ServiceErrorInterceptor({0x2412a50?, 0x40040ce210?}, {0x1d94a80?, 0x40019b19e0?}, 0xffffb5469501?, 0x18?) temporal | /home/builder/temporal/common/rpc/grpc.go:137 +0x34 temporal | google.golang.org/grpc.chainUnaryInterceptors.func1.1({0x2412a50?, 0x40040ce210?}, {0x1d94a80?, 0x40019b19e0?}) temporal | /go/pkg/mod/google.golang.org/grpc@v1.50.1/server.go:1165 +0x8c temporal | google.golang.org/grpc.chainUnaryInterceptors.func1({0x2412a50, 0x40040ce210}, {0x1d94a80, 0x40019b19e0}, 0x4004fae120, 0x40019b1a10) temporal | /go/pkg/mod/google.golang.org/grpc@v1.50.1/server.go:1167 +0x128 temporal | go.temporal.io/server/api/historyservice/v1._HistoryService_DescribeWorkflowExecution_Handler({0x1e26fc0?, 0x400054a9a0}, {0x2412a50, 0x40040ce210}, 0x400428ac40, 0x400039b960) temporal | /home/builder/temporal/api/historyservice/v1/service.pb.go:1462 +0x134 temporal | google.golang.org/grpc.(*Server).processUnaryRPC(0x40007bc960, {0x241e700, 0x400173e9c0}, 0x400037e480, 0x4000b73050, 0x361ab88, 0x0) temporal | /go/pkg/mod/google.golang.org/grpc@v1.50.1/server.go:1340 +0xb68 temporal | google.golang.org/grpc.(*Server).handleStream(0x40007bc960, {0x241e700, 0x400173e9c0}, 0x400037e480, 0x0) temporal | /go/pkg/mod/google.golang.org/grpc@v1.50.1/server.go:1713 +0x840 temporal | google.golang.org/grpc.(*Server).serveStreams.func1.2() temporal | /go/pkg/mod/google.golang.org/grpc@v1.50.1/server.go:965 +0x84 temporal | created by google.golang.org/grpc.(*Server).serveStreams.func1 temporal | /go/pkg/mod/google.golang.org/grpc@v1.50.1/server.go:963 +0x294
I ran the docker container for a few days and just saw it.
Expected Behavior
Temporal server should not panic and exits
Actual Behavior
Steps to Reproduce the Problem
I ran the docker container for a few days and just saw it.
Specifications