search

cadence-workflow / cadence-go-client

Framework for authoring workflows and activities running on top of the Cadence orchestration engine.
https://cadenceworkflow.io
MIT License
349 stars 132 forks source link

Concurrent map writes of search attributes due to potential racing condition #1198

Open longquanzheng opened 2 years ago

longquanzheng commented 2 years ago

Describe the bug Upseart search attributes could got error of concurrent map writes

To Reproduce Is the issue reproducible?

Steps to reproduce the behavior: Uncomment the test for Cadence Run the integ test from iWF

Expected behavior The integ test should succeed.

Screenshots If applicable, add screenshots to help explain your problem.

Additional context


runtime.throw({0x103003d8f?, 0x1400084d7e8?})
    /usr/local/go-dists/1.18.3/go/src/runtime/panic.go:992 +0x50 fp=0x14000c4ae90 sp=0x14000c4ae60 pc=0x1021b2500
runtime.mapassign_faststr(0x102fe0d6b?, 0x103fadc00?, {0x103001b1f, 0x14})
    /usr/local/go-dists/1.18.3/go/src/runtime/map_faststr.go:212 +0x3bc fp=0x14000c4af00 sp=0x14000c4ae90 pc=0x10219016c
go.uber.org/cadence/internal.mergeSearchAttributes(...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_event_handlers.go:354
go.uber.org/cadence/internal.(*workflowEnvironmentImpl).updateWorkflowInfoWithSearchAttributes(...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_event_handlers.go:339
go.uber.org/cadence/internal.(*workflowEnvironmentImpl).UpsertSearchAttributes(0x1400091c900, 0x102a26c8c?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_event_handlers.go:334 +0x258 fp=0x14000c4aff0 sp=0x14000c4af00 pc=0x1029e4f38
go.uber.org/cadence/internal.(*workflowEnvironmentInterceptor).UpsertSearchAttributes(0x14000917980, {0x1032f8e20?, 0x103562ea0?}, 0x10344b7a0?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/workflow.go:1347 +0x94 fp=0x14000c4b030 sp=0x14000c4aff0 pc=0x102a26d84
go.uber.org/cadence/internal.UpsertSearchAttributes({0x10357a788, 0x140005e7f40}, 0x140005e7f40?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/workflow.go:1340 +0x78 fp=0x14000c4b070 sp=0x14000c4b030 pc=0x102a26ca8
go.uber.org/cadence/workflow.UpsertSearchAttributes(...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/workflow/workflow.go:452
github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).UpsertSearchAttributes(0x10335ccc0?, {0x1035656b8?, 0x140005f1570?}, 0x14?)
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/cadence/workflowProvider.go:28 +0x54 fp=0x14000c4b0c0 sp=0x14000c4b070 pc=0x102a37cb4
github.com/indeedeng/iwf/service/interpreter.(*stateExecutingManager).updateSearchAttribute(0x1400092e0c0)
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/stateExecutingManager.go:60 +0x19c fp=0x14000c4b1c0 sp=0x14000c4b0c0 pc=0x10269d64c
github.com/indeedeng/iwf/service/interpreter.(*stateExecutingManager).completeStates(0x1400092e0c0, {{0x14000297918?, 0x140004f6750?}, 0x140005c5e90?, 0x140005c5ea0?})
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/stateExecutingManager.go:46 +0xb0 fp=0x14000c4b200 sp=0x14000c4b1c0 pc=0x10269d450
github.com/indeedeng/iwf/service/interpreter.InterpreterImpl.func3.1()
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/workflowImpl.go:83 +0x40 fp=0x14000c4b240 sp=0x14000c4b200 pc=0x10269ea40
runtime.deferCallSave(0x14000c4b2e0, 0x14000c4be68?)
    /usr/local/go-dists/1.18.3/go/src/runtime/panic.go:750 +0x8c fp=0x14000c4b250 sp=0x14000c4b240 pc=0x1021b1d3c
runtime.runOpenDeferFrame(0x0?, 0x1400073a0a0)
    /usr/local/go-dists/1.18.3/go/src/runtime/panic.go:723 +0x1b8 fp=0x14000c4b2a0 sp=0x14000c4b250 pc=0x1021b1b68
runtime.Goexit()
    /usr/local/go-dists/1.18.3/go/src/runtime/panic.go:482 +0x188 fp=0x14000c4b320 sp=0x14000c4b2a0 pc=0x1021b11f8
go.uber.org/cadence/internal.(*coroutineState).exit.func1({0x14000c4b368?, 0x102a0fbe0?}, 0x14000746d90?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:850 +0x20 fp=0x14000c4b330 sp=0x14000c4b320 pc=0x102a11040
go.uber.org/cadence/internal.(*coroutineState).initialYield(0x140005f7a80, 0x3, {0x1400027e9e0, 0x19})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:797 +0xbc fp=0x14000c4b370 sp=0x14000c4b330 pc=0x102a10c0c
go.uber.org/cadence/internal.(*coroutineState).yield(...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:806
go.uber.org/cadence/internal.(*channelImpl).Receive(0x1400072dc20, {0x10357a980, 0x1400092e270}, {0x0, 0x0})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:621 +0x214 fp=0x14000c4b400 sp=0x14000c4b370 pc=0x102a0fbf4
go.uber.org/cadence/internal.(*decodeFutureImpl).Get(0x14000738408, {0x10357a980, 0x1400092e270}, {0x1032af200?, 0x14000011578})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:1282 +0x4c fp=0x14000c4b4a0 sp=0x14000c4b400 pc=0x102a148cc
github.com/indeedeng/iwf/service/interpreter/cadence.(*futureImpl).Get(0x140005f1960, {0x1035656b8?, 0x140005f1740?}, {0x1032af200, 0x14000011578})
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/cadence/workflowProvider.go:130 +0x7c fp=0x14000c4b4e0 sp=0x14000c4b4a0 pc=0x102a386ec
github.com/indeedeng/iwf/service/interpreter.executeState({0x1035656b8?, 0x140005f1700?}, {0x1035813e0, 0x104035540}, {{0x14000297918?, 0x140004e2700?}, 0x140005c5e90?, 0x140005c5ea0?}, {{0x1400032b020, 0x15}, ...}, ...)
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/workflowImpl.go:210 +0x4b0 fp=0x14000c4bce0 sp=0x14000c4b4e0 pc=0x10269f770
github.com/indeedeng/iwf/service/interpreter.InterpreterImpl.func3({0x1035656b8, 0x140005f1700})
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/workflowImpl.go:90 +0x24c fp=0x14000c4bee0 sp=0x14000c4bce0 pc=0x10269e59c
github.com/indeedeng/iwf/service/interpreter/cadence.workflowProvider.GoNamed.func1({0x10357a980?, 0x1400092e1e0})
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/cadence/workflowProvider.go:80 +0x8c fp=0x14000c4bf20 sp=0x14000c4bee0 pc=0x102a3839c
go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0x1400077ae20?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:884 +0xc4 fp=0x14000c4bfb0 sp=0x14000c4bf20 pc=0x102a117b4
go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func2()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:885 +0x34 fp=0x14000c4bfd0 sp=0x14000c4bfb0 pc=0x102a116b4
runtime.goexit()
    /usr/local/go-dists/1.18.3/go/src/runtime/asm_arm64.s:1263 +0x4 fp=0x14000c4bfd0 sp=0x14000c4bfd0 pc=0x1021e6074
created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:875 +0x314
zedongh commented 1 year ago

Yeah, i meet this problem recently ...

fatal error: concurrent map read and map write
  /go/src/gitlab.momenta.works/hdmap-workflow/cadence-system-worker/internal/workflowimpl/workflow_interpreter.go:429 +0x25d
  gitlab.momenta.works/hdmap-workflow/cadence-system-worker/internal/workflowimpl.(*ExecutionVisitor).visitActivityInvocation.func1()
  go.uber.org/cadence/internal.(*workflowEnvironmentImpl).UpsertSearchAttributes(0xc0165d2120, 0xc00d6db8f0?)
      /go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/workflow.go:1347 +0x90
  go.uber.org/cadence/internal.UpsertSearchAttributes({0x1190758, 0xc000bc61b0}, 0xfef115?)
      /go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_decision_state_machine.go:951 +0x89
  go.uber.org/cadence/internal.(*decisionsHelper).addDecision(0xc007b885d0, {0x1199208, 0xc0144f9b20})

It seems like using go routine execute upsert attributes would cause this problem. I've to use explicit lock to avoid this concurrent problem

longquanzheng commented 1 year ago

more repro:

/usr/bin/make -f /Users/qlong/indeed/iwf/iwf-server/Makefile -C /Users/qlong/indeed/iwf/iwf-server integTests
go test -v ./integ
=== RUN   TestAnyCommandCloseWorkflowTemporal
[GIN-debug] [WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached.

[GIN-debug] [WARNING] Running in "debug" mode. Switch to "release" mode in production.
 - using env:   export GIN_MODE=release
 - using code:  gin.SetMode(gin.ReleaseMode)

[GIN-debug] POST   /api/v1/workflowState/start --> github.com/indeedeng/iwf/integ/workflow/common.WorkflowHandler.ApiV1WorkflowStateStart-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflowState/decide --> github.com/indeedeng/iwf/integ/workflow/common.WorkflowHandler.ApiV1WorkflowStateDecide-fm (3 handlers)
2022/12/29 09:22:27 INFO  No logger configured for temporal client. Created default one.
[GIN-debug] [WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached.

[GIN-debug] [WARNING] Running in "debug" mode. Switch to "release" mode in production.
 - using env:   export GIN_MODE=release
 - using code:  gin.SetMode(gin.ReleaseMode)

[GIN-debug] GET    /                         --> github.com/indeedeng/iwf/service/api.(*handler).index-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/start    --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowStart-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/signal   --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowSignal-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/dataobjects/get --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGetDataObjects-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/searchattributes/get --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGetSearchAttributes-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/get      --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGet-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/getWithWait --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGetWithWait-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/search   --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowSearch-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/reset    --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowReset-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/stop     --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowStop-fm (3 handlers)
2022/12/29 09:22:27 INFO  Started Worker Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@
2022/12/29 09:22:27 received API request {any_command_close1672334547 any_command_close 10 http://localhost:9714 S1 <nil> <nil> <nil>}
2022/12/29 09:22:27 Started workflow WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8
[GIN] 2022/12/29 - 09:22:27 | 200 |    8.913833ms |             ::1 | POST     "/api/v1/workflow/start"
2022/12/29 09:22:27 received API request {any_command_close1672334547 <nil> test-signal-name2 0x140003aa410}
[GIN] 2022/12/29 - 09:22:27 | 200 |    7.184583ms |             ::1 | POST     "/api/v1/workflow/signal"
2022/12/29 09:22:27 received API request {any_command_close1672334547 <nil> <nil> <nil>}
2022/12/29 09:22:27 DEBUG ExecuteActivity Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@ WorkflowType Interpreter WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8 Attempt 1 ActivityID 11 ActivityType StateStart
2022/12/29 09:22:27 INFO  StateStartActivity Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@ ActivityID 11 ActivityType StateStart Attempt 1 WorkflowType Interpreter WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8 input {http://localhost:9714 {{any_command_close1672334547 3c9b39a7-4779-4ddd-839e-bf0477debca8 1672334547 S1-1} any_command_close S1 0x1400042c920 [] []}}
2022/12/29 09:22:27 received state start request,  {{any_command_close1672334547 3c9b39a7-4779-4ddd-839e-bf0477debca8 1672334547 S1-1} any_command_close S1 0x140003ab1d0 [] []}
[GIN] 2022/12/29 - 09:22:27 | 200 |         155µs |             ::1 | POST     "/api/v1/workflowState/start"
2022/12/29 09:22:27 DEBUG ExecuteActivity Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@ WorkflowType Interpreter WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8 Attempt 1 ActivityID 17 ActivityType StateDecide
2022/12/29 09:22:27 INFO  StateDecideActivity Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@ ActivityID 17 ActivityType StateDecide Attempt 1 WorkflowType Interpreter WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8 input {http://localhost:9714 {{any_command_close1672334547 3c9b39a7-4779-4ddd-839e-bf0477debca8 1672334547 S1-1} any_command_close S1 0x140008d07a0 [] [] [] 0x1400041db80}}
2022/12/29 09:22:27 received state decide request,  {{any_command_close1672334547 3c9b39a7-4779-4ddd-839e-bf0477debca8 1672334547 S1-1} any_command_close S1 0x1400042d3d0 [] [] [] 0x140004f53b0}
[GIN] 2022/12/29 - 09:22:27 | 200 |      90.708µs |             ::1 | POST     "/api/v1/workflowState/decide"
2022/12/29 09:22:27 DEBUG ExecuteActivity Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@ WorkflowType Interpreter WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8 Attempt 1 ActivityID 25 ActivityType StateStart
2022/12/29 09:22:27 INFO  StateStartActivity Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@ ActivityID 25 ActivityType StateStart Attempt 1 WorkflowType Interpreter WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8 input {http://localhost:9714 {{any_command_close1672334547 3c9b39a7-4779-4ddd-839e-bf0477debca8 1672334547 S2-1} any_command_close S2 <nil> [] []}}
2022/12/29 09:22:27 received state start request,  {{any_command_close1672334547 3c9b39a7-4779-4ddd-839e-bf0477debca8 1672334547 S2-1} any_command_close S2 <nil> [] []}
[GIN] 2022/12/29 - 09:22:27 | 200 |      43.208µs |             ::1 | POST     "/api/v1/workflowState/start"
2022/12/29 09:22:27 DEBUG ExecuteActivity Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@ WorkflowType Interpreter WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8 Attempt 1 ActivityID 31 ActivityType StateDecide
2022/12/29 09:22:27 INFO  StateDecideActivity Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@ ActivityID 31 ActivityType StateDecide Attempt 1 WorkflowType Interpreter WorkflowID any_command_close1672334547 RunID 3c9b39a7-4779-4ddd-839e-bf0477debca8 input {http://localhost:9714 {{any_command_close1672334547 3c9b39a7-4779-4ddd-839e-bf0477debca8 1672334547 S2-1} any_command_close S2 <nil> [] [] [] 0x1400041df40}}
2022/12/29 09:22:27 received state decide request,  {{any_command_close1672334547 3c9b39a7-4779-4ddd-839e-bf0477debca8 1672334547 S2-1} any_command_close S2 <nil> [] [] [] 0x14000136f50}
[GIN] 2022/12/29 - 09:22:27 | 200 |      69.208µs |             ::1 | POST     "/api/v1/workflowState/decide"
[GIN] 2022/12/29 - 09:22:27 | 200 |  103.351375ms |             ::1 | POST     "/api/v1/workflow/getWithWait"
2022/12/29 09:22:27 INFO  Stopped Worker Namespace default TaskQueue Interpreter_DEFAULT WorkerID 6745@IT-USA-VX8131@
--- PASS: TestAnyCommandCloseWorkflowTemporal (0.21s)
=== RUN   TestAnyCommandCloseWorkflowCadence
[GIN-debug] [WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached.

[GIN-debug] [WARNING] Running in "debug" mode. Switch to "release" mode in production.
 - using env:   export GIN_MODE=release
 - using code:  gin.SetMode(gin.ReleaseMode)

[GIN-debug] POST   /api/v1/workflowState/start --> github.com/indeedeng/iwf/integ/workflow/common.WorkflowHandler.ApiV1WorkflowStateStart-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflowState/decide --> github.com/indeedeng/iwf/integ/workflow/common.WorkflowHandler.ApiV1WorkflowStateDecide-fm (3 handlers)
[GIN-debug] [WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached.

[GIN-debug] [WARNING] Running in "debug" mode. Switch to "release" mode in production.
 - using env:   export GIN_MODE=release
 - using code:  gin.SetMode(gin.ReleaseMode)

[GIN-debug] GET    /                         --> github.com/indeedeng/iwf/service/api.(*handler).index-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/start    --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowStart-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/signal   --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowSignal-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/dataobjects/get --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGetDataObjects-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/searchattributes/get --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGetSearchAttributes-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/get      --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGet-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/getWithWait --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGetWithWait-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/search   --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowSearch-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/reset    --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowReset-fm (3 handlers)
[GIN-debug] POST   /api/v1/workflow/stop     --> github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowStop-fm (3 handlers)
{"level":"info","ts":"2022-12-29T09:22:27.363-0800","caller":"internal/internal_worker.go:223","msg":"No logger configured for cadence worker. Created default one."}
{"level":"info","ts":"2022-12-29T09:22:27.363-0800","caller":"internal/internal_worker.go:227","msg":"No metrics scope configured for cadence worker. Use NoopScope as default."}
{"level":"info","ts":"2022-12-29T09:22:27.434-0800","caller":"internal/internal_worker.go:834","msg":"Started Workflow Worker","Domain":"default","TaskList":"Interpreter_DEFAULT","WorkerID":"6745@IT-USA-VX8131@Interpreter_DEFAULT@2f8d29d1-e39a-433e-a7a2-f00411651f29"}
{"level":"info","ts":"2022-12-29T09:22:27.448-0800","caller":"internal/internal_worker.go:859","msg":"Started Activity Worker","Domain":"default","TaskList":"Interpreter_DEFAULT","WorkerID":"6745@IT-USA-VX8131@Interpreter_DEFAULT@2f8d29d1-e39a-433e-a7a2-f00411651f29"}
2022/12/29 09:22:27 received API request {any_command_close1672334547 any_command_close 10 http://localhost:9714 S1 <nil> <nil> <nil>}
2022/12/29 09:22:27 Started workflow WorkflowID any_command_close1672334547 RunID eaf1acf2-9109-4d32-9e28-db1a9c40737b
[GIN] 2022/12/29 - 09:22:27 | 200 |   21.851791ms |             ::1 | POST     "/api/v1/workflow/start"
2022/12/29 09:22:27 received API request {any_command_close1672334547 <nil> test-signal-name2 0x140007d4f90}
[GIN] 2022/12/29 - 09:22:27 | 200 |   20.215042ms |             ::1 | POST     "/api/v1/workflow/signal"
2022/12/29 09:22:27 received API request {any_command_close1672334547 <nil> <nil> <nil>}
fatal error: concurrent map writes

goroutine 268 [running]:
runtime.throw({0x1050c9ae5?, 0x14000b8a768?})
    /usr/local/go-dists/1.18.3/go/src/runtime/panic.go:992 +0x50 fp=0x1400049ec90 sp=0x1400049ec60 pc=0x10424cbb0
runtime.mapassign_faststr(0x10542fc00?, 0x14000ce06b0?, {0x1050c7518, 0x14})
    /usr/local/go-dists/1.18.3/go/src/runtime/map_faststr.go:212 +0x3bc fp=0x1400049ed00 sp=0x1400049ec90 pc=0x10422829c
go.uber.org/cadence/internal.mergeSearchAttributes(...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_event_handlers.go:354
go.uber.org/cadence/internal.(*workflowEnvironmentImpl).updateWorkflowInfoWithSearchAttributes(...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_event_handlers.go:339
go.uber.org/cadence/internal.(*workflowEnvironmentImpl).UpsertSearchAttributes(0x1400087d560, 0x1400049ee58?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_event_handlers.go:334 +0x258 fp=0x1400049edf0 sp=0x1400049ed00 pc=0x104d72998
go.uber.org/cadence/internal.(*workflowEnvironmentImpl).GetVersion(0x1400087d560, {0x1050b0fc9, 0x6}, 0x1055f75a0?, 0x1)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_event_handlers.go:623 +0x148 fp=0x1400049efa0 sp=0x1400049edf0 pc=0x104d74a38
go.uber.org/cadence/internal.(*workflowEnvironmentInterceptor).GetVersion(0x1400049f058?, {0x1053c9b00?, 0x10563eea8?}, {0x1050b0fc9?, 0x1400049f038?}, 0x104e86684?, 0x105490940?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/workflow.go:1614 +0x40 fp=0x1400049efe0 sp=0x1400049efa0 pc=0x104db58a0
go.uber.org/cadence/internal.GetVersion({0x105656c48, 0x14000e17ae0}, {0x1050b0fc9, 0x6}, 0x78?, 0x105554a80?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/workflow.go:1610 +0x90 fp=0x1400049f040 sp=0x1400049efe0 pc=0x104db5800
go.uber.org/cadence/workflow.GetVersion(...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/workflow/workflow.go:344
github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GetVersion(0x1400049f0d8?, {0x1056416c8?, 0x14000e21a90?}, {0x1050b0fc9, 0x6}, 0x0?, 0x1400049f108?)
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/cadence/workflowProvider.go:161 +0x70 fp=0x1400049f090 sp=0x1400049f040 pc=0x104e866a0
github.com/indeedeng/iwf/service/interpreter.(*globalVersioner).isAfterVersionOfUsingGlobalVersioning(...)
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/globalVersionProvider.go:21
github.com/indeedeng/iwf/service/interpreter.InterpreterImpl({0x1056416c8?, 0x14000e21a90}, {0x10565dbe8, 0x106133940}, {{0x140003bceb8, 0x11}, {0x140003bced0, 0x15}, {0x14000ce0688, 0x2}, ...})
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/workflowImpl.go:12 +0xc8 fp=0x1400049f3a0 sp=0x1400049f090 pc=0x104e80638
github.com/indeedeng/iwf/service/interpreter/cadence.Interpreter({0x105656c48?, 0x14000e17ae0}, {{0x140003bceb8, 0x11}, {0x140003bced0, 0x15}, {0x14000ce0688, 0x2}, {0x0, 0x0}, ...})
    /Users/qlong/indeed/iwf/iwf-server/service/interpreter/cadence/workflow.go:10 +0xb0 fp=0x1400049f450 sp=0x1400049f3a0 pc=0x104e85790
runtime.call256(0x140001ebcb0, 0x10563b0b8, 0x14000cd9100, 0x78, 0x78, 0x88, 0x1400049fa80)
    /usr/local/go-dists/1.18.3/go/src/runtime/asm_arm64.s:515 +0x84 fp=0x1400049f560 sp=0x1400049f450 pc=0x10427ea34
runtime.reflectcall(0x105554a80?, 0x14000cd9000?, 0x4?, 0x50c5041?, 0x1?, 0x12?, 0x105554a80?)
    <autogenerated>:1 +0x34 fp=0x1400049f5a0 sp=0x1400049f560 pc=0x104282394
reflect.Value.call({0x1054210e0?, 0x10563b0b8?, 0x1400049fc88?}, {0x1050a7a2d, 0x4}, {0x14000bc9680, 0x2, 0x104d686dc?})
    /usr/local/go-dists/1.18.3/go/src/reflect/value.go:556 +0x5e4 fp=0x1400049fc10 sp=0x1400049f5a0 pc=0x1042e15c4
reflect.Value.Call({0x1054210e0?, 0x10563b0b8?, 0x1400049fcd8?}, {0x14000bc9680, 0x2, 0x2})
    /usr/local/go-dists/1.18.3/go/src/reflect/value.go:339 +0x98 fp=0x1400049fc90 sp=0x1400049fc10 pc=0x1042e0dd8
go.uber.org/cadence/internal.(*workflowEnvironmentInterceptor).ExecuteWorkflow(0x14000bc92c0?, {0x105656c48?, 0x14000e17ae0?}, {0x14000e21a50?, 0x14000d2e120?}, {0x14000e21a70?, 0x1?, 0x14000003c28?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/workflow.go:681 +0x200 fp=0x1400049fdd0 sp=0x1400049fc90 pc=0x104db0330
go.uber.org/cadence/internal.(*workflowExecutor).Execute(0x14000ca89e0, {0x105656c48, 0x14000e17ae0}, {0x14000d2e120, 0x85, 0x90})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker.go:700 +0x3b8 fp=0x1400049feb0 sp=0x1400049fdd0 pc=0x104d96168
go.uber.org/cadence/internal.(*syncWorkflowDefinition).Execute.func1({0x105656e40, 0x14000bc9590})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:467 +0xe4 fp=0x1400049ff20 sp=0x1400049feb0 pc=0x104d9ce04
go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0x14000cc1060?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:884 +0xc4 fp=0x1400049ffb0 sp=0x1400049ff20 pc=0x104d9f214
go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func2()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:885 +0x34 fp=0x1400049ffd0 sp=0x1400049ffb0 pc=0x104d9f114
runtime.goexit()
    /usr/local/go-dists/1.18.3/go/src/runtime/asm_arm64.s:1263 +0x4 fp=0x1400049ffd0 sp=0x1400049ffd0 pc=0x104280884
created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:875 +0x314

goroutine 1 [chan receive]:
testing.(*T).Run(0x14000582820, {0x1050e1465?, 0x9b7f7eca3f5f?}, 0x10563b028)
    /usr/local/go-dists/1.18.3/go/src/testing/testing.go:1487 +0x314
testing.runTests.func1(0x0?)
    /usr/local/go-dists/1.18.3/go/src/testing/testing.go:1839 +0x74
testing.tRunner(0x14000582820, 0x140004fbcb8)
    /usr/local/go-dists/1.18.3/go/src/testing/testing.go:1439 +0x110
testing.runTests(0x14000147f40?, {0x1060ed340, 0x10, 0x10}, {0xfa00000000000000?, 0x104222190?, 0x106102da0?})
    /usr/local/go-dists/1.18.3/go/src/testing/testing.go:1837 +0x3f0
testing.(*M).Run(0x14000147f40)
    /usr/local/go-dists/1.18.3/go/src/testing/testing.go:1719 +0x500
main.main()
    _testmain.go:77 +0x1d0

goroutine 59 [select]:
net/http.(*persistConn).roundTrip(0x140007dd680, 0x1400095b340)
    /usr/local/go-dists/1.18.3/go/src/net/http/transport.go:2620 +0x810
net/http.(*Transport).roundTrip(0x1060ece20, 0x14000257c00)
    /usr/local/go-dists/1.18.3/go/src/net/http/transport.go:594 +0x6bc
net/http.(*Transport).RoundTrip(0x14000c82b00?, 0x105644c48?)
    /usr/local/go-dists/1.18.3/go/src/net/http/roundtrip.go:17 +0x20
net/http.send(0x14000257c00, {0x105644c48, 0x1060ece20}, {0x1044e4d10?, 0x8?, 0x0?})
    /usr/local/go-dists/1.18.3/go/src/net/http/client.go:252 +0x4fc
net/http.(*Client).send(0x1061023e0, 0x14000257c00, {0x140005b1428?, 0x104221d20?, 0x0?})
    /usr/local/go-dists/1.18.3/go/src/net/http/client.go:176 +0x9c
net/http.(*Client).do(0x1061023e0, 0x14000257c00)
    /usr/local/go-dists/1.18.3/go/src/net/http/client.go:725 +0x6ec
net/http.(*Client).Do(...)
    /usr/local/go-dists/1.18.3/go/src/net/http/client.go:593
github.com/indeedeng/iwf/gen/iwfidl.(*APIClient).callAPI(0x1400016b140, 0x105655208?)
    /Users/qlong/indeed/iwf/iwf-server/gen/iwfidl/client.go:171 +0x9c
github.com/indeedeng/iwf/gen/iwfidl.(*DefaultApiService).ApiV1WorkflowGetWithWaitPostExecute(0x1400016b148, {{0x105655208, 0x14000128008}, {0x10565eca8, 0x1400016b148}, 0x14000961170})
    /Users/qlong/indeed/iwf/iwf-server/gen/iwfidl/api_default.go:459 +0x314
github.com/indeedeng/iwf/gen/iwfidl.ApiApiV1WorkflowGetWithWaitPostRequest.Execute(...)
    /Users/qlong/indeed/iwf/iwf-server/gen/iwfidl/api_default.go:398
github.com/indeedeng/iwf/integ.doTestAnyCommandCloseWorkflow(0x1400040bba0, {0x1050b30c0, 0x7})
    /Users/qlong/indeed/iwf/iwf-server/integ/any_command_close_test.go:79 +0x6d0
github.com/indeedeng/iwf/integ.TestAnyCommandCloseWorkflowCadence(0x1400040bba0)
    /Users/qlong/indeed/iwf/iwf-server/integ/any_command_close_test.go:29 +0x68
testing.tRunner(0x1400040bba0, 0x10563b028)
    /usr/local/go-dists/1.18.3/go/src/testing/testing.go:1439 +0x110
created by testing.(*T).Run
    /usr/local/go-dists/1.18.3/go/src/testing/testing.go:1486 +0x300

goroutine 94 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0x14000b84900)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:324 +0x70
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:339
google.golang.org/grpc.(*csAttempt).recvMsg(0x1400050cd80, {0x10537e980?, 0x14000b88258}, 0x104915610?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:1011 +0x7c
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x140005f9768?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:865 +0x2c
google.golang.org/grpc.(*clientStream).withRetry(0x14000b846c0, 0x140005f9798, 0x140005f9768)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:724 +0x1c8
google.golang.org/grpc.(*clientStream).RecvMsg(0x14000b846c0, {0x10537e980?, 0x14000b88258?})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:864 +0xbc
google.golang.org/grpc.invoke({0x105655278?, 0x140001eb230?}, {0x14000044f40?, 0x140007d54b0?}, {0x1053a8bc0, 0x14000b882a0}, {0x10537e980, 0x14000b88258}, 0x0?, {0x140007e51c0?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:73 +0xb8
google.golang.org/grpc.(*ClientConn).Invoke(0x14000866000?, {0x105655278?, 0x140001eb230?}, {0x14000044f40?, 0x3c?}, {0x1053a8bc0?, 0x14000b882a0?}, {0x10537e980?, 0x14000b88258?}, {0x140007d54b0?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:37 +0x1a8
go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0x14000850ea0, {0x105655240, 0x140007d9560}, 0x1400050ccc0, 0x1050b5b05?, 0x14000124c00, {0x1?, 0x10426c2c4?, 0x106102da0?})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x368
go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0x14000850ea0, {0x105655240, 0x140007d9560}, 0x1400050ccc0)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0xc0
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240?, 0x140007d9560?}, 0x1050a82fd?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
go.uber.org/yarpc/internal/observability.(*Middleware).Call(0x14000844a80, {0x105655240, 0x140007d9560}, 0x1400050ccc0, {0x105658fb0, 0x140001eb110})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0xf8
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x140007d9560}, 0x104759b78?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x105580d00?, {0x105655240, 0x140007d9560}, 0x1400050ccc0, {0x105658fb0?, 0x140001eb0e0})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xbc
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x1, 0x1}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x140007d9560}, 0x140001eb0b0?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x105658eb0?, 0x14000851020?}, {0x12d78cb18?, 0x14000851020?}}, {0x105655240, 0x140007d9560}, 0x140003a8600?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x90
go.uber.org/yarpc/encoding/protobuf.(*client).Call(0x140008514a0, {0x105655240?, 0x140007d9560?}, {0x1050d348a?, 0x1010140005faee8?}, {0x10564d008?, 0x140002caa10?}, 0x10563b460, {0x140007d95c0?, 0x7?, ...})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0xb8
github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkflowAPIYARPCCaller).GetWorkflowExecutionHistory(0x140007e6c80?, {0x105655240?, 0x140007d9560?}, 0x10426c94c?, {0x140007d95c0?, 0x104222190?, 0x140001eb020?})
    /Users/qlong/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220505064515-7bb6b0808383/go/proto/api/v1/service_workflow.pb.yarpc.go:505 +0x64
go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.GetWorkflowExecutionHistory({{0x1056581f0, 0x1400086a260}, {0x10565d0c0, 0x1400086a280}, {0x10565d038, 0x1400086a2a0}, {0x10565ae50, 0x1400086a2c0}}, {0x105655240, 0x140007d9560}, ...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/compatibility/adapter.go:86 +0x88
go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).GetWorkflowExecutionHistory(0x140008516e0, {0x105655240, 0x140007d9560}, 0x140005fb0e8?, {0x140007d95c0, 0x4, 0x6})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/common/metrics/service_wrapper.go:163 +0x80
go.uber.org/cadence/internal.(*workflowClient).GetWorkflowHistory.func1.1()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow_client.go:524 +0xcc
go.uber.org/cadence/internal/common/backoff.Retry({0x1056551d0, 0x140005d0380}, 0x140005fb2c8, {0x105643488, 0x140001948d0}, 0x10563bfc0)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/common/backoff/retry.go:98 +0x8c
go.uber.org/cadence/internal.(*workflowClient).GetWorkflowHistory.func1({0x0, 0x0, 0x0})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow_client.go:515 +0x334
go.uber.org/cadence/internal.(*historyEventIteratorImpl).HasNext(0x140007d9500)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow_client.go:1071 +0x8c
go.uber.org/cadence/internal.(*workflowRunImpl).Get(0x140007d9440, {0x1056551d0, 0x140005d0380}, {0x105382dc0?, 0x14000b881f8})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow_client.go:1124 +0x74
github.com/indeedeng/iwf/service/api/cadence.(*cadenceClient).GetWorkflowResult(0x0?, {0x1056551d0, 0x140005d0380}, {0x105382dc0, 0x14000b881f8}, {0x140003420a0?, 0x0?}, {0x0?, 0x0?})
    /Users/qlong/indeed/iwf/iwf-server/service/api/cadence/client.go:177 +0x70
github.com/indeedeng/iwf/service/api.(*serviceImpl).doApiV1WorkflowGetPost(0x140004711c0, {0x1056551d0, 0x140005d0380}, {{0x140003420a0, 0x1b}, 0x0, 0x0, 0x0}, 0x1)
    /Users/qlong/indeed/iwf/iwf-server/service/api/service.go:138 +0x160
github.com/indeedeng/iwf/service/api.(*serviceImpl).ApiV1WorkflowGetWithWaitPost(0x140005fb600?, {0x1056551d0?, 0x140005d0380?}, {{0x140003420a0, 0x1b}, 0x0, 0x0, 0x0})
    /Users/qlong/indeed/iwf/iwf-server/service/api/service.go:126 +0x44
github.com/indeedeng/iwf/service/api.(*handler).doApiV1WorkflowGetPost(0x1400086a340, 0x140005dc200, 0x1)
    /Users/qlong/indeed/iwf/iwf-server/service/api/handler.go:158 +0x1f8
github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowGetWithWait(0x140005fb6a8?, 0x1042294d0?)
    /Users/qlong/indeed/iwf/iwf-server/service/api/handler.go:144 +0x24
github.com/gin-gonic/gin.(*Context).Next(...)
    /Users/qlong/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/context.go:173
github.com/gin-gonic/gin.CustomRecoveryWithWriter.func1(0x140005dc200)
    /Users/qlong/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/recovery.go:101 +0x80
github.com/gin-gonic/gin.(*Context).Next(...)
    /Users/qlong/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/context.go:173
github.com/gin-gonic/gin.LoggerWithConfig.func1(0x140005dc200)
    /Users/qlong/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/logger.go:240 +0xb0
github.com/gin-gonic/gin.(*Context).Next(...)
    /Users/qlong/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/context.go:173
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0x1400086e340, 0x140005dc200)
    /Users/qlong/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/gin.go:616 +0x568
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0x1400086e340, {0x105654118?, 0x140005d80e0}, 0x140005dc100)
    /Users/qlong/go/pkg/mod/github.com/gin-gonic/gin@v1.8.1/gin.go:572 +0x1d8
net/http.serverHandler.ServeHTTP({0x105649508?}, {0x105654118, 0x140005d80e0}, 0x140005dc100)
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:2916 +0x3fc
net/http.(*conn).serve(0x140003fe820, {0x105655278, 0x14000a58c60})
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:1966 +0x56c
created by net/http.(*Server).Serve
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:3071 +0x450

goroutine 147 [IO wait]:
internal/poll.runtime_pollWait(0x12d53d800, 0x72)
    /usr/local/go-dists/1.18.3/go/src/runtime/netpoll.go:302 +0xa4
internal/poll.(*pollDesc).wait(0x1400095e280?, 0x14000960851?, 0x0)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:83 +0x2c
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0x1400095e280, {0x14000960851, 0x1, 0x1})
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_unix.go:167 +0x1e4
net.(*netFD).Read(0x1400095e280, {0x14000960851?, 0x14000860fa8?, 0x104d99d2c?})
    /usr/local/go-dists/1.18.3/go/src/net/fd_posix.go:55 +0x2c
net.(*conn).Read(0x1400018aa38, {0x14000960851?, 0x100000001?, 0x104d99ef0?})
    /usr/local/go-dists/1.18.3/go/src/net/net.go:183 +0x38
net/http.(*connReader).backgroundRead(0x14000960840)
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:672 +0x48
created by net/http.(*connReader).startBackgroundRead
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:668 +0xd0

goroutine 299 [select]:
go.uber.org/cadence/internal.(*locallyDispatchedActivityTunnel).getTask(0x140004b1728)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:193 +0x60
go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).pollLocallyDispatchedActivity(0x14000874080, {0x1043160f8?, 0x140008440d8?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1122 +0x2c
go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0x14000874080, {0x105655240?, 0x14000a62f60?}, 0x14000878418?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1019 +0x3c
go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetricsFunc.func1({0x105655240?, 0x14000a62f60?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1013 +0x3c
go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:256 +0x50
created by go.uber.org/cadence/internal.(*basePoller).doPoll
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:255 +0x18c

goroutine 104 [select]:
go.uber.org/cadence/internal.(*basePoller).doPoll(0x14000874080, {0xd0?}, 0x14000799278)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:261 +0x1c4
go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask(0x14000874080)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1113 +0x8c
go.uber.org/cadence/internal.(*baseWorker).pollTask(0x14000852540)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:263 +0x90
go.uber.org/cadence/internal.(*baseWorker).runPoller(0x14000852540)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:227 +0x8c
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:190 +0x70

goroutine 337 [select]:
go.uber.org/cadence/internal.(*locallyDispatchedActivityTunnel).getTask(0x140004b1728)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:193 +0x60
go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).pollLocallyDispatchedActivity(0x14000874080, {0x1043160f8?, 0x14000844018?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1122 +0x2c
go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0x14000874080, {0x105655240?, 0x1400060e360?}, 0x14000878238?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1019 +0x3c
go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetricsFunc.func1({0x105655240?, 0x1400060e360?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1013 +0x3c
go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:256 +0x50
created by go.uber.org/cadence/internal.(*basePoller).doPoll
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:255 +0x18c

goroutine 119 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0x14000a49c20)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:324 +0x70
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:339
google.golang.org/grpc.(*csAttempt).recvMsg(0x140009772c0, {0x10537e980?, 0x140007999e0}, 0x104915610?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:1011 +0x7c
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x140004a8488?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:865 +0x2c
google.golang.org/grpc.(*clientStream).withRetry(0x14000a499e0, 0x140004a84b8, 0x140004a8488)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:724 +0x1c8
google.golang.org/grpc.(*clientStream).RecvMsg(0x14000a499e0, {0x10537e980?, 0x140007999e0?})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:864 +0xbc
google.golang.org/grpc.invoke({0x105655278?, 0x14000a58ff0?}, {0x1400047cc40?, 0x140008d1a40?}, {0x1053a8bc0, 0x14000799a28}, {0x10537e980, 0x140007999e0}, 0x0?, {0x14000a5a540?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:73 +0xb8
google.golang.org/grpc.(*ClientConn).Invoke(0x14000866000?, {0x105655278?, 0x14000a58ff0?}, {0x1400047cc40?, 0x32?}, {0x1053a8bc0?, 0x14000799a28?}, {0x10537e980?, 0x140007999e0?}, {0x140008d1a40?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:37 +0x1a8
go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0x14000850ea0, {0x105655240, 0x14000899a40}, 0x14000977200, 0x1050b5b05?, 0x140001cc968, {0x1?, 0x18?, 0x106102da0?})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x368
go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0x14000850ea0, {0x105655240, 0x14000899a40}, 0x14000977200)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0xc0
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240?, 0x14000899a40?}, 0x1050a82fd?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
go.uber.org/yarpc/internal/observability.(*Middleware).Call(0x14000844a80, {0x105655240, 0x14000899a40}, 0x14000977200, {0x105658fb0, 0x14000a58ed0})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0xf8
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x14000899a40}, 0x104759b78?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x105580d00?, {0x105655240, 0x14000899a40}, 0x14000977200, {0x105658fb0?, 0x14000a58ea0})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xbc
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x1, 0x1}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x14000899a40}, 0x14000a58e70?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x105658eb0?, 0x14000851020?}, {0x12d78cb18?, 0x14000851020?}}, {0x105655240, 0x14000899a40}, 0x12d552708?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x90
go.uber.org/yarpc/encoding/protobuf.(*client).Call(0x14000851500, {0x105655240?, 0x14000899a40?}, {0x1050c5f4c?, 0x14000903c08?}, {0x10564dde8?, 0x14000a54d20?}, 0x10563b3d0, {0x14000899b00?, 0x14000a58d80?, ...})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0xb8
github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForActivityTask(0x1400009b900?, {0x105655240?, 0x14000899a40?}, 0x10426c94c?, {0x14000899b00?, 0x104222190?, 0x14000a58d50?})
    /Users/qlong/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220505064515-7bb6b0808383/go/proto/api/v1/service_worker.pb.yarpc.go:397 +0x64
go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForActivityTask({{0x1056581f0, 0x1400086a260}, {0x10565d0c0, 0x1400086a280}, {0x10565d038, 0x1400086a2a0}, {0x10565ae50, 0x1400086a2c0}}, {0x105655240, 0x14000899a40}, ...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/compatibility/adapter.go:121 +0x88
go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForActivityTask(0x14000851b60, {0x105655240, 0x14000899a40}, 0x4?, {0x14000899b00, 0x4, 0x6})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/common/metrics/service_wrapper.go:212 +0x80
go.uber.org/cadence/internal.(*activityTaskPoller).poll(0x1400080de80, {0x105655240, 0x14000899a40})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:991 +0x348
go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0x1400080de80, {0x105655240?, 0x14000899a40?}, 0x14000055e00?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1019 +0x3c
go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetricsFunc.func1({0x105655240?, 0x14000899a40?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1013 +0x3c
go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:256 +0x50
created by go.uber.org/cadence/internal.(*basePoller).doPoll
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:255 +0x18c

goroutine 102 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0x140007e6370, 0x1)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/controlbuf.go:408 +0x15c
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0x140007d8840)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/controlbuf.go:535 +0x78
google.golang.org/grpc/internal/transport.newHTTP2Client.func3()
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/http2_client.go:419 +0x68
created by google.golang.org/grpc/internal/transport.newHTTP2Client
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/http2_client.go:417 +0x16e0

goroutine 105 [select]:
go.uber.org/cadence/internal.(*basePoller).doPoll(0x14000874080, {0xe0?}, 0x14000bfdde8)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:261 +0x1c4
go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask(0x14000874080)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1113 +0x8c
go.uber.org/cadence/internal.(*baseWorker).pollTask(0x14000852540)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:263 +0x90
go.uber.org/cadence/internal.(*baseWorker).runPoller(0x14000852540)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:227 +0x8c
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:190 +0x70

goroutine 96 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0x140009639e0)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:324 +0x70
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:339
google.golang.org/grpc.(*csAttempt).recvMsg(0x1400079f500, {0x10537e980?, 0x140005a9680}, 0x104915610?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:1011 +0x7c
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x140009164e8?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:865 +0x2c
google.golang.org/grpc.(*clientStream).withRetry(0x140009637a0, 0x14000916518, 0x140009164e8)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:724 +0x1c8
google.golang.org/grpc.(*clientStream).RecvMsg(0x140009637a0, {0x10537e980?, 0x140005a9680?})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:864 +0xbc
google.golang.org/grpc.invoke({0x105655278?, 0x14000961b90?}, {0x14000c00440?, 0x14000c80710?}, {0x1053a8bc0, 0x140005a96c8}, {0x10537e980, 0x140005a9680}, 0x0?, {0x1400095b780?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:73 +0xb8
google.golang.org/grpc.(*ClientConn).Invoke(0x14000866000?, {0x105655278?, 0x14000961b90?}, {0x14000c00440?, 0x32?}, {0x1053a8bc0?, 0x140005a96c8?}, {0x10537e980?, 0x140005a9680?}, {0x14000c80710?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:37 +0x1a8
go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0x14000850ea0, {0x105655240, 0x1400095ccc0}, 0x1400079f440, 0x0?, 0x1400018aaf0, {0x14000864348?, 0x18?, 0x106102da0?})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x368
go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0x14000850ea0, {0x105655240, 0x1400095ccc0}, 0x1400079f440)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0xc0
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240?, 0x1400095ccc0?}, 0x1050a82fd?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
go.uber.org/yarpc/internal/observability.(*Middleware).Call(0x14000844a80, {0x105655240, 0x1400095ccc0}, 0x1400079f440, {0x105658fb0, 0x14000961aa0})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0xf8
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x1400095ccc0}, 0x104759b78?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x105580d00?, {0x105655240, 0x1400095ccc0}, 0x1400079f440, {0x105658fb0?, 0x14000961a70})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xbc
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x1, 0x1}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x1400095ccc0}, 0x14000961a40?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x105658eb0?, 0x14000851020?}, {0x12d78cb18?, 0x14000851020?}}, {0x105655240, 0x1400095ccc0}, 0x0?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x90
go.uber.org/yarpc/encoding/protobuf.(*client).Call(0x14000851500, {0x105655240?, 0x1400095ccc0?}, {0x1050c5f5f?, 0x14000682d58?}, {0x10564de48?, 0x1400095cde0?}, 0x10563b3d8, {0x1400095cd80?, 0x13?, ...})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0xb8
github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForDecisionTask(0x1400018fce0?, {0x105655240?, 0x1400095ccc0?}, 0x10426c94c?, {0x1400095cd80?, 0x104222190?, 0x140009619b0?})
    /Users/qlong/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220505064515-7bb6b0808383/go/proto/api/v1/service_worker.pb.yarpc.go:361 +0x64
go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForDecisionTask({{0x1056581f0, 0x1400086a260}, {0x10565d0c0, 0x1400086a280}, {0x10565d038, 0x1400086a2a0}, {0x10565ae50, 0x1400086a2c0}}, {0x105655240, 0x1400095ccc0}, ...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/compatibility/adapter.go:126 +0x88
go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForDecisionTask(0x14000851b60, {0x105655240, 0x1400095ccc0}, 0x0?, {0x1400095cd80, 0x4, 0x6})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/common/metrics/service_wrapper.go:219 +0x80
go.uber.org/cadence/internal.(*workflowTaskPoller).poll(0x140008209c0, {0x105655240, 0x1400095ccc0})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:767 +0x20c
go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:256 +0x50
created by go.uber.org/cadence/internal.(*basePoller).doPoll
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:255 +0x18c

goroutine 101 [IO wait]:
internal/poll.runtime_pollWait(0x12d53d710, 0x72)
    /usr/local/go-dists/1.18.3/go/src/runtime/netpoll.go:302 +0xa4
internal/poll.(*pollDesc).wait(0x14000a56580?, 0x140007e8000?, 0x0)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:83 +0x2c
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0x14000a56580, {0x140007e8000, 0x8000, 0x8000})
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_unix.go:167 +0x1e4
net.(*netFD).Read(0x14000a56580, {0x140007e8000?, 0x1400008ebf8?, 0x1046b74c8?})
    /usr/local/go-dists/1.18.3/go/src/net/fd_posix.go:55 +0x2c
net.(*conn).Read(0x14000124780, {0x140007e8000?, 0x1400008ec58?, 0x10426c2c4?})
    /usr/local/go-dists/1.18.3/go/src/net/net.go:183 +0x38
bufio.(*Reader).Read(0x140007d87e0, {0x140007ba200, 0x9, 0x14000bc8900?})
    /usr/local/go-dists/1.18.3/go/src/bufio/bufio.go:236 +0x1e4
io.ReadAtLeast({0x105641168, 0x140007d87e0}, {0x140007ba200, 0x9, 0x9}, 0x9)
    /usr/local/go-dists/1.18.3/go/src/io/io.go:331 +0xa4
io.ReadFull(...)
    /usr/local/go-dists/1.18.3/go/src/io/io.go:350
golang.org/x/net/http2.readFrameHeader({0x140007ba200?, 0x9?, 0x1400008ed58?}, {0x105641168?, 0x140007d87e0?})
    /Users/qlong/go/pkg/mod/golang.org/x/net@v0.1.0/http2/frame.go:237 +0x5c
golang.org/x/net/http2.(*Framer).ReadFrame(0x140007ba1c0)
    /Users/qlong/go/pkg/mod/golang.org/x/net@v0.1.0/http2/frame.go:498 +0x84
google.golang.org/grpc/internal/transport.(*http2Client).reader(0x14000947860)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/http2_client.go:1516 +0x2e8
created by google.golang.org/grpc/internal/transport.newHTTP2Client
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/http2_client.go:369 +0x1204

goroutine 163 [chan receive]:
go.uber.org/cadence/internal.(*coroutineState).call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:839
go.uber.org/cadence/internal.(*dispatcherImpl).ExecuteUntilAllBlocked(0x140007e4f00)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:923 +0x1fc
go.uber.org/cadence/internal.executeDispatcher({0x105656c48, 0x14000e17ae0}, {0x105656cf0, 0x140007e4f00})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:540 +0x7c
go.uber.org/cadence/internal.(*syncWorkflowDefinition).OnDecisionTaskStarted(0x1400087d560?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_workflow.go:514 +0x34
go.uber.org/cadence/internal.(*workflowExecutionEventHandlerImpl).ProcessEvent(0x14000e0ae70, 0x14000d32180, 0xd8?, 0x1)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_event_handlers.go:820 +0x7a0
go.uber.org/cadence/internal.(*workflowExecutionContextImpl).ProcessWorkflowTask(0x1400050ce40, 0x14000bc8ea0)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_handlers.go:895 +0x15ec
go.uber.org/cadence/internal.(*workflowTaskHandlerImpl).ProcessWorkflowTask(0x14000816b00, 0x14000bc8ea0, 0x14000bc8ed0)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_handlers.go:787 +0x320
go.uber.org/cadence/internal.(*workflowTaskPoller).processWorkflowTask(0x140008209c0, 0x140009618c0)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:341 +0x260
go.uber.org/cadence/internal.(*workflowTaskPoller).ProcessTask(0x140008209c0, {0x10538a0c0?, 0x140009618c0?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:314 +0x74
go.uber.org/cadence/internal.(*baseWorker).processTask(0x14000852620, {0x105389b80?, 0x14000c805a0?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:324 +0xe0
created by go.uber.org/cadence/internal.(*baseWorker).runTaskDispatcher
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:253 +0x98

goroutine 112 [IO wait]:
internal/poll.runtime_pollWait(0x12d53d260, 0x72)
    /usr/local/go-dists/1.18.3/go/src/runtime/netpoll.go:302 +0xa4
internal/poll.(*pollDesc).wait(0x140001ec080?, 0x140007ab000?, 0x0)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:83 +0x2c
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Read(0x140001ec080, {0x140007ab000, 0x1000, 0x1000})
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_unix.go:167 +0x1e4
net.(*netFD).Read(0x140001ec080, {0x140007ab000?, 0x14000596c58?, 0x10421a668?})
    /usr/local/go-dists/1.18.3/go/src/net/fd_posix.go:55 +0x2c
net.(*conn).Read(0x14000124bb8, {0x140007ab000?, 0x0?, 0x0?})
    /usr/local/go-dists/1.18.3/go/src/net/net.go:183 +0x38
net/http.(*persistConn).Read(0x140007dd680, {0x140007ab000?, 0x10421a710?, 0x14000956960?})
    /usr/local/go-dists/1.18.3/go/src/net/http/transport.go:1929 +0x54
bufio.(*Reader).fill(0x140007d8fc0)
    /usr/local/go-dists/1.18.3/go/src/bufio/bufio.go:106 +0x100
bufio.(*Reader).Peek(0x140007d8fc0, 0x1)
    /usr/local/go-dists/1.18.3/go/src/bufio/bufio.go:144 +0x64
net/http.(*persistConn).readLoop(0x140007dd680)
    /usr/local/go-dists/1.18.3/go/src/net/http/transport.go:2093 +0x148
created by net/http.(*Transport).dialConn
    /usr/local/go-dists/1.18.3/go/src/net/http/transport.go:1750 +0x11f4

goroutine 113 [select]:
net/http.(*persistConn).writeLoop(0x140007dd680)
    /usr/local/go-dists/1.18.3/go/src/net/http/transport.go:2392 +0xa4
created by net/http.(*Transport).dialConn
    /usr/local/go-dists/1.18.3/go/src/net/http/transport.go:1751 +0x1240

goroutine 92 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0x14000962120)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:324 +0x70
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:339
google.golang.org/grpc.(*csAttempt).recvMsg(0x1400079f080, {0x10537e980?, 0x140005a9020}, 0x104915610?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:1011 +0x7c
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x1400076e4e8?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:865 +0x2c
google.golang.org/grpc.(*clientStream).withRetry(0x1400090be60, 0x1400076e518, 0x1400076e4e8)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:724 +0x1c8
google.golang.org/grpc.(*clientStream).RecvMsg(0x1400090be60, {0x10537e980?, 0x140005a9020?})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:864 +0xbc
google.golang.org/grpc.invoke({0x105655278?, 0x14000960210?}, {0x14000122740?, 0x1400042db50?}, {0x1053a8bc0, 0x140005a9068}, {0x10537e980, 0x140005a9020}, 0x0?, {0x1400095ab40?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:73 +0xb8
google.golang.org/grpc.(*ClientConn).Invoke(0x14000866000?, {0x105655278?, 0x14000960210?}, {0x14000122740?, 0x32?}, {0x1053a8bc0?, 0x140005a9068?}, {0x10537e980?, 0x140005a9020?}, {0x1400042db50?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:37 +0x1a8
go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0x14000850ea0, {0x105655240, 0x1400095c3c0}, 0x1400079efc0, 0x1050b5b05?, 0x1400018aa08, {0x1?, 0x18?, 0x106102da0?})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x368
go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0x14000850ea0, {0x105655240, 0x1400095c3c0}, 0x1400079efc0)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0xc0
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240?, 0x1400095c3c0?}, 0x1050a82fd?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
go.uber.org/yarpc/internal/observability.(*Middleware).Call(0x14000844a80, {0x105655240, 0x1400095c3c0}, 0x1400079efc0, {0x105658fb0, 0x140009600f0})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0xf8
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x1400095c3c0}, 0x104759b78?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x105580d00?, {0x105655240, 0x1400095c3c0}, 0x1400079efc0, {0x105658fb0?, 0x140009600c0})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xbc
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x1, 0x1}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x1400095c3c0}, 0x14000960090?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x105658eb0?, 0x14000851020?}, {0x12d78cb18?, 0x14000851020?}}, {0x105655240, 0x1400095c3c0}, 0x1?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x90
go.uber.org/yarpc/encoding/protobuf.(*client).Call(0x14000851500, {0x105655240?, 0x1400095c3c0?}, {0x1050c5f5f?, 0x1400008bc68?}, {0x10564de48?, 0x1400095c4e0?}, 0x10563b3d8, {0x1400095c480?, 0x32?, ...})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0xb8
github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForDecisionTask(0x1400018f860?, {0x105655240?, 0x1400095c3c0?}, 0x10426c94c?, {0x1400095c480?, 0x104222190?, 0x14000919fb0?})
    /Users/qlong/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220505064515-7bb6b0808383/go/proto/api/v1/service_worker.pb.yarpc.go:361 +0x64
go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForDecisionTask({{0x1056581f0, 0x1400086a260}, {0x10565d0c0, 0x1400086a280}, {0x10565d038, 0x1400086a2a0}, {0x10565ae50, 0x1400086a2c0}}, {0x105655240, 0x1400095c3c0}, ...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/compatibility/adapter.go:126 +0x88
go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForDecisionTask(0x14000851b60, {0x105655240, 0x1400095c3c0}, 0x1042b5fbc?, {0x1400095c480, 0x4, 0x6})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/common/metrics/service_wrapper.go:219 +0x80
go.uber.org/cadence/internal.(*workflowTaskPoller).poll(0x140008209c0, {0x105655240, 0x1400095c3c0})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:767 +0x20c
go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:256 +0x50
created by go.uber.org/cadence/internal.(*basePoller).doPoll
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:255 +0x18c

goroutine 106 [select]:
go.uber.org/cadence/internal.(*baseWorker).runTaskDispatcher(0x14000852540)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:241 +0xdc
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:194 +0x130

goroutine 60 [IO wait]:
internal/poll.runtime_pollWait(0x12d53d350, 0x72)
    /usr/local/go-dists/1.18.3/go/src/runtime/netpoll.go:302 +0xa4
internal/poll.(*pollDesc).wait(0x140000b3d80?, 0x105512080?, 0x0)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:83 +0x2c
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Accept(0x140000b3d80)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_unix.go:614 +0x1d0
net.(*netFD).accept(0x140000b3d80)
    /usr/local/go-dists/1.18.3/go/src/net/fd_unix.go:172 +0x28
net.(*TCPListener).accept(0x140001c9ae8)
    /usr/local/go-dists/1.18.3/go/src/net/tcpsock_posix.go:139 +0x2c
net.(*TCPListener).Accept(0x140001c9ae8)
    /usr/local/go-dists/1.18.3/go/src/net/tcpsock.go:288 +0x30
net/http.(*Server).Serve(0x140008521c0, {0x105653f08, 0x140001c9ae8})
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:3039 +0x318
net/http.(*Server).ListenAndServe(0x140008521c0)
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:2968 +0x88
github.com/indeedeng/iwf/integ.startWorkflowWorker.func1()
    /Users/qlong/indeed/iwf/iwf-server/integ/util.go:42 +0x24
created by github.com/indeedeng/iwf/integ.startWorkflowWorker
    /Users/qlong/indeed/iwf/iwf-server/integ/util.go:41 +0x20c

goroutine 132 [IO wait]:
internal/poll.runtime_pollWait(0x12d53d530, 0x72)
    /usr/local/go-dists/1.18.3/go/src/runtime/netpoll.go:302 +0xa4
internal/poll.(*pollDesc).wait(0x14000a56600?, 0x4?, 0x0)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:83 +0x2c
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_poll_runtime.go:88
internal/poll.(*FD).Accept(0x14000a56600)
    /usr/local/go-dists/1.18.3/go/src/internal/poll/fd_unix.go:614 +0x1d0
net.(*netFD).accept(0x14000a56600)
    /usr/local/go-dists/1.18.3/go/src/net/fd_unix.go:172 +0x28
net.(*TCPListener).accept(0x14000799950)
    /usr/local/go-dists/1.18.3/go/src/net/tcpsock_posix.go:139 +0x2c
net.(*TCPListener).Accept(0x14000799950)
    /usr/local/go-dists/1.18.3/go/src/net/tcpsock.go:288 +0x30
net/http.(*Server).Serve(0x14000852380, {0x105653f08, 0x14000799950})
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:3039 +0x318
net/http.(*Server).ListenAndServe(0x14000852380)
    /usr/local/go-dists/1.18.3/go/src/net/http/server.go:2968 +0x88
github.com/indeedeng/iwf/integ.startIwfService.func3()
    /Users/qlong/indeed/iwf/iwf-server/integ/util.go:84 +0x24
created by github.com/indeedeng/iwf/integ.startIwfService
    /Users/qlong/indeed/iwf/iwf-server/integ/util.go:83 +0x4ec

goroutine 65 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0x140003cd600)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/balancer_conn_wrappers.go:112 +0x68
created by google.golang.org/grpc.newCCBalancerWrapper
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/balancer_conn_wrappers.go:73 +0x230

goroutine 103 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0x140007dd440)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:324 +0x70
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/internal/transport/transport.go:339
google.golang.org/grpc.(*csAttempt).recvMsg(0x1400050c9c0, {0x10537e980?, 0x140001c9bc0}, 0x104915610?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:1011 +0x7c
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x14000a52488?)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:865 +0x2c
google.golang.org/grpc.(*clientStream).withRetry(0x140007dd200, 0x14000a524b8, 0x14000a52488)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:724 +0x1c8
google.golang.org/grpc.(*clientStream).RecvMsg(0x140007dd200, {0x10537e980?, 0x140001c9bc0?})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/stream.go:864 +0xbc
google.golang.org/grpc.invoke({0x105655278?, 0x140007df6b0?}, {0x14000044b40?, 0x140007d4b00?}, {0x1053a8bc0, 0x140001c9c08}, {0x10537e980, 0x140001c9bc0}, 0x0?, {0x140007e45c0?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:73 +0xb8
google.golang.org/grpc.(*ClientConn).Invoke(0x14000866000?, {0x105655278?, 0x140007df6b0?}, {0x14000044b40?, 0x32?}, {0x1053a8bc0?, 0x140001c9c08?}, {0x10537e980?, 0x140001c9bc0?}, {0x140007d4b00?, ...})
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/call.go:37 +0x1a8
go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0x14000850ea0, {0x105655240, 0x140007d8960}, 0x1400050c900, 0x0?, 0x14000124988, {0x14000865868?, 0x18?, 0x106102da0?})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x368
go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0x14000850ea0, {0x105655240, 0x140007d8960}, 0x1400050c900)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0xc0
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240?, 0x140007d8960?}, 0x1050a82fd?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
go.uber.org/yarpc/internal/observability.(*Middleware).Call(0x14000844a80, {0x105655240, 0x140007d8960}, 0x1400050c900, {0x105658fb0, 0x140007df5c0})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0xf8
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x0, 0x0}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x140007d8960}, 0x104759b78?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x105580d00?, {0x105655240, 0x140007d8960}, 0x1400050c900, {0x105658fb0?, 0x140007df590})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xbc
go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0x14000470e50, 0x1, 0x1}, {0x105658530, 0x14000850ea0}}, {0x105655240, 0x140007d8960}, 0x140007df560?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0x100
go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x105658eb0?, 0x14000851020?}, {0x12d78cb18?, 0x14000851020?}}, {0x105655240, 0x140007d8960}, 0x14000907a18?)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x90
go.uber.org/yarpc/encoding/protobuf.(*client).Call(0x14000851500, {0x105655240?, 0x140007d8960?}, {0x1050c5f4c?, 0x14000682d58?}, {0x10564dde8?, 0x140007e6500?}, 0x10563b3d0, {0x140007d8a20?, 0x140007df470?, ...})
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0xb8
github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForActivityTask(0x1400016afa0?, {0x105655240?, 0x140007d8960?}, 0x10426c94c?, {0x140007d8a20?, 0x104222190?, 0x140007df440?})
    /Users/qlong/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220505064515-7bb6b0808383/go/proto/api/v1/service_worker.pb.yarpc.go:397 +0x64
go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForActivityTask({{0x1056581f0, 0x1400086a260}, {0x10565d0c0, 0x1400086a280}, {0x10565d038, 0x1400086a2a0}, {0x10565ae50, 0x1400086a2c0}}, {0x105655240, 0x140007d8960}, ...)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/compatibility/adapter.go:121 +0x88
go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForActivityTask(0x14000851b60, {0x105655240, 0x140007d8960}, 0x0?, {0x140007d8a20, 0x4, 0x6})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/common/metrics/service_wrapper.go:212 +0x80
go.uber.org/cadence/internal.(*activityTaskPoller).poll(0x1400080de80, {0x105655240, 0x140007d8960})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:991 +0x348
go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0x1400080de80, {0x105655240?, 0x140007d8960?}, 0x14000055e00?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1019 +0x3c
go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetricsFunc.func1({0x105655240?, 0x140007d8960?})
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1013 +0x3c
go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:256 +0x50
created by go.uber.org/cadence/internal.(*basePoller).doPoll
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:255 +0x18c

goroutine 131 [select]:
google.golang.org/grpc.(*ClientConn).WaitForStateChange(0x14000866000, {0x1056551d0, 0x140003cd700}, 0x2)
    /Users/qlong/go/pkg/mod/google.golang.org/grpc@v1.50.1/clientconn.go:515 +0x98
go.uber.org/yarpc/transport/grpc.(*grpcPeer).monitorConnectionStatus(0x140003cd740)
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/peer.go:82 +0x84
created by go.uber.org/yarpc/transport/grpc.(*Transport).newPeer
    /Users/qlong/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/peer.go:69 +0x52c

goroutine 133 [select]:
go.uber.org/cadence/internal.(*localActivityTunnel).getTask(0x0?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:167 +0x5c
go.uber.org/cadence/internal.(*localActivityTaskPoller).PollTask(0x14000471340?)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:540 +0x24
go.uber.org/cadence/internal.(*baseWorker).pollTask(0x14000852700)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:263 +0x90
go.uber.org/cadence/internal.(*baseWorker).runPoller(0x14000852700)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:227 +0x8c
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:190 +0x70

goroutine 134 [select]:
go.uber.org/cadence/internal.(*baseWorker).runTaskDispatcher(0x14000852700)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:241 +0xdc
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:194 +0x130

goroutine 135 [select]:
go.uber.org/cadence/internal.(*basePoller).doPoll(0x140008209c0, {0xa0?}, 0x1400042d9b0)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:261 +0x1c4
go.uber.org/cadence/internal.(*workflowTaskPoller).PollTask(0x140008209c0)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:298 +0x70
go.uber.org/cadence/internal.(*baseWorker).pollTask(0x14000852620)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:263 +0x90
go.uber.org/cadence/internal.(*baseWorker).runPoller(0x14000852620)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:227 +0x8c
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:190 +0x70

goroutine 136 [select]:
go.uber.org/cadence/internal.(*basePoller).doPoll(0x140008209c0, {0xa0?}, 0x14000c805b0)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:261 +0x1c4
go.uber.org/cadence/internal.(*workflowTaskPoller).PollTask(0x140008209c0)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:298 +0x70
go.uber.org/cadence/internal.(*baseWorker).pollTask(0x14000852620)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:263 +0x90
go.uber.org/cadence/internal.(*baseWorker).runPoller(0x14000852620)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:227 +0x8c
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:190 +0x70

goroutine 137 [select]:
go.uber.org/cadence/internal.(*baseWorker).runTaskDispatcher(0x14000852620)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:241 +0xdc
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:194 +0x130

goroutine 138 [select]:
go.uber.org/cadence/internal.(*basePoller).doPoll(0x1400080de80, {0xf0?}, 0x14000799968)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:261 +0x1c4
go.uber.org/cadence/internal.(*activityTaskPoller).PollTask(0x1400080de80)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1042 +0x88
go.uber.org/cadence/internal.(*baseWorker).pollTask(0x14000852460)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:263 +0x90
go.uber.org/cadence/internal.(*baseWorker).runPoller(0x14000852460)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:227 +0x8c
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:190 +0x70

goroutine 139 [select]:
go.uber.org/cadence/internal.(*basePoller).doPoll(0x1400080de80, {0xb0?}, 0x140001c9b78)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:261 +0x1c4
go.uber.org/cadence/internal.(*activityTaskPoller).PollTask(0x1400080de80)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_task_pollers.go:1042 +0x88
go.uber.org/cadence/internal.(*baseWorker).pollTask(0x14000852460)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:263 +0x90
go.uber.org/cadence/internal.(*baseWorker).runPoller(0x14000852460)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:227 +0x8c
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:190 +0x70

goroutine 140 [select]:
go.uber.org/cadence/internal.(*baseWorker).runTaskDispatcher(0x14000852460)
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:241 +0xdc
created by go.uber.org/cadence/internal.(*baseWorker).Start
    /Users/qlong/go/pkg/mod/go.uber.org/cadence@v0.19.0/internal/internal_worker_base.go:194 +0x130
FAIL    github.com/indeedeng/iwf/integ  1.891s
FAIL
make: *** [integTests] Error 1

Process finished with exit code 2
longquanzheng commented 1 year ago

I got more clues about how this happens:

longquanzheng commented 1 year ago

Workaround seems working after removing the defer function: https://github.com/uber-go/cadence-client/issues/1198

longquanzheng commented 6 months ago

More repro again:

2024-05-20T19:36:41.8763137Z fatal error: concurrent map writes
2024-05-20T19:36:41.8794825Z 
2024-05-20T19:36:41.8795349Z goroutine 15556 [running]:
2024-05-20T19:36:41.8798377Z github.com/indeedeng/iwf/service/interpreter.(*ContinueAsNewer).allThreadsDrained(0xc001156240, {0x1a9bb20, 0xc000ff3ac0})
2024-05-20T19:36:41.8800225Z    /home/runner/work/iwf/iwf/service/interpreter/continueAsNewer.go:200 +0x779
2024-05-20T19:36:41.8802201Z github.com/indeedeng/iwf/service/interpreter.(*ContinueAsNewer).DrainThreads.func1()
2024-05-20T19:36:41.8803536Z    /home/runner/work/iwf/iwf/service/interpreter/continueAsNewer.go:160 +0x29
2024-05-20T19:36:41.8805022Z go.uber.org/cadence/internal.Await({0x1ab2890, 0xc000d7aba0}, 0xc0005b3aa0)
2024-05-20T19:36:41.8806708Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:597 +0xbb
2024-05-20T19:36:41.8807918Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.8809130Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.8810576Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x0?, {0x1a9bb20?, 0xc000ff3ac0?}, 0x0?)
2024-05-20T19:36:41.8812134Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.8813561Z github.com/indeedeng/iwf/service/interpreter.(*ContinueAsNewer).DrainThreads(0xc001156240, {0x1a9bb20?, 0xc000ff3ac0})
2024-05-20T19:36:41.8815570Z    /home/runner/work/iwf/iwf/service/interpreter/continueAsNewer.go:159 +0xc4
2024-05-20T19:36:41.8817629Z github.com/indeedeng/iwf/service/interpreter.InterpreterImpl({0x1a9bb20?, 0xc000ff3ac0}, {0x1ac6300, 0xc000ff3ad0}, {{0xc001132950, 0xb}, {0xc001424768, 0x15}, 0xc000ff3970, {0x0, ...}, ...})
2024-05-20T19:36:41.8819580Z    /home/runner/work/iwf/iwf/service/interpreter/workflowImpl.go:258 +0x1f68
2024-05-20T19:36:41.8821469Z github.com/indeedeng/iwf/service/interpreter/cadence.Interpreter({0x1ab2890?, 0xc000d7aba0}, {{0xc001132950, 0xb}, {0xc001424768, 0x15}, 0xc000ff3970, {0x0, 0x0, 0x0}, ...})
2024-05-20T19:36:41.8823356Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflow.go:10 +0x17d
2024-05-20T19:36:41.8824661Z reflect.Value.call({0x15f8fa0?, 0x19028a0?, 0x1834e00?}, {0x1838b1a, 0x4}, {0xc000a74e70, 0x2, 0x18?})
2024-05-20T19:36:41.8825998Z    /opt/hostedtoolcache/go/1.21.10/x64/src/reflect/value.go:596 +0xce7
2024-05-20T19:36:41.8827135Z reflect.Value.Call({0x15f8fa0?, 0x19028a0?, 0x108f808?}, {0xc000a74e70?, 0x1834e00?, 0x1a99670?})
2024-05-20T19:36:41.8828274Z    /opt/hostedtoolcache/go/1.21.10/x64/src/reflect/value.go:380 +0xb9
2024-05-20T19:36:41.8830097Z go.uber.org/cadence/internal.(*workflowEnvironmentInterceptor).ExecuteWorkflow(0xc00101cd50, {0x1ab2890?, 0xc000d7aba0?}, {0x15f8fa0?, 0x15d3d00?}, {0xc000ff3aa0?, 0x1, 0xc000a2be60?})
2024-05-20T19:36:41.8832484Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:688 +0x44a
2024-05-20T19:36:41.8834409Z go.uber.org/cadence/internal.(*workflowExecutor).Execute(0xc0004d0d00, {0x1ab2890, 0xc000d7aba0}, {0xc0009d4580, 0x2a2, 0x2c0})
2024-05-20T19:36:41.8836412Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker.go:667 +0x392
2024-05-20T19:36:41.8838033Z go.uber.org/cadence/internal.(*syncWorkflowDefinition).Execute.func1({0x1ab0868, 0xc00101d020})
2024-05-20T19:36:41.8839821Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:467 +0xcc
2024-05-20T19:36:41.8841376Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc001817630?)
2024-05-20T19:36:41.8843101Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.8844718Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 15555
2024-05-20T19:36:41.8846509Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.8847551Z 
2024-05-20T19:36:41.8847717Z goroutine 1 [chan receive]:
2024-05-20T19:36:41.8848359Z testing.(*T).Run(0xc0004aa820, {0x1885022?, 0x5873bc?}, 0x1901b70)
2024-05-20T19:36:41.8849310Z    /opt/hostedtoolcache/go/1.21.10/x64/src/testing/testing.go:1649 +0x3c8
2024-05-20T19:36:41.8850258Z testing.runTests.func1(0xc00058a800?)
2024-05-20T19:36:41.8851038Z    /opt/hostedtoolcache/go/1.21.10/x64/src/testing/testing.go:2054 +0x3e
2024-05-20T19:36:41.8852010Z testing.tRunner(0xc0004aa820, 0xc000907950)
2024-05-20T19:36:41.8852810Z    /opt/hostedtoolcache/go/1.21.10/x64/src/testing/testing.go:1595 +0xff
2024-05-20T19:36:41.8853978Z testing.runTests(0xc0005d3cc0?, {0x26d0e20, 0x55, 0x55}, {0xc0006c40c0?, 0xc0000f8408?, 0x26e0980?})
2024-05-20T19:36:41.8855152Z    /opt/hostedtoolcache/go/1.21.10/x64/src/testing/testing.go:2052 +0x445
2024-05-20T19:36:41.8856079Z testing.(*M).Run(0xc0005d3cc0)
2024-05-20T19:36:41.8856800Z    /opt/hostedtoolcache/go/1.21.10/x64/src/testing/testing.go:1925 +0x636
2024-05-20T19:36:41.8857672Z github.com/indeedeng/iwf/integ.TestMain(0x1838020?)
2024-05-20T19:36:41.8858433Z    /home/runner/work/iwf/iwf/integ/main_test.go:110 +0x152
2024-05-20T19:36:41.8859079Z main.main()
2024-05-20T19:36:41.8859435Z    _testmain.go:251 +0x1e6
2024-05-20T19:36:41.8859754Z 
2024-05-20T19:36:41.8859914Z goroutine 15237 [select]:
2024-05-20T19:36:41.8860762Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000fce580, {0xe0?, 0x53?}, 0xc000f71cb0)
2024-05-20T19:36:41.8862839Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.8864543Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask(0xc000fce580)
2024-05-20T19:36:41.8866398Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1142 +0xb5
2024-05-20T19:36:41.8867904Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc000)
2024-05-20T19:36:41.8869607Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.8871938Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc000)
2024-05-20T19:36:41.8873577Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.8875126Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.8876923Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.8878011Z 
2024-05-20T19:36:41.8878191Z goroutine 15573 [chan receive]:
2024-05-20T19:36:41.8879136Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc000796640, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.8881194Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.8882532Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.8883970Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.8885381Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc000a75500}, 0xc0007968c0)
2024-05-20T19:36:41.8886924Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.8888049Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.8889390Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.8891396Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x10?, {0x1a9bb20?, 0xc000ff3d60?}, 0x7fd73c2b05c0?)
2024-05-20T19:36:41.8893208Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.8894729Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func6({0x1a9bb20?, 0xc000ff3d60})
2024-05-20T19:36:41.8896408Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:169 +0x124
2024-05-20T19:36:41.8898016Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc000a75500})
2024-05-20T19:36:41.8899831Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.8901254Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc0015ac870?)
2024-05-20T19:36:41.8903064Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.8905153Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 15556
2024-05-20T19:36:41.8907131Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.8908206Z 
2024-05-20T19:36:41.8908396Z goroutine 11474 [chan receive]:
2024-05-20T19:36:41.8909747Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc00060d940, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.8911636Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.8913038Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.8914820Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.8916286Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc000ae5b60}, 0xc00060dac0)
2024-05-20T19:36:41.8917895Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.8918936Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.8920278Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.8922119Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x10?, {0x1a9bb20?, 0xc001190d00?}, 0x7fd73c137158?)
2024-05-20T19:36:41.8923626Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.8925101Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func6({0x1a9bb20?, 0xc001190d00})
2024-05-20T19:36:41.8926163Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:169 +0x124
2024-05-20T19:36:41.8927455Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc000ae5b60})
2024-05-20T19:36:41.8929138Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.8930494Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc0017af380?)
2024-05-20T19:36:41.8932411Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.8933541Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 11468
2024-05-20T19:36:41.8934910Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.8936020Z 
2024-05-20T19:36:41.8936201Z goroutine 15531 [IO wait]:
2024-05-20T19:36:41.8936759Z internal/poll.runtime_pollWait(0x7fd78727e310, 0x72)
2024-05-20T19:36:41.8937635Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/netpoll.go:343 +0x85
2024-05-20T19:36:41.8938591Z internal/poll.(*pollDesc).wait(0xc001001580?, 0xc001158f11?, 0x0)
2024-05-20T19:36:41.8939627Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-05-20T19:36:41.8940528Z internal/poll.(*pollDesc).waitRead(...)
2024-05-20T19:36:41.8941366Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:89
2024-05-20T19:36:41.8942360Z internal/poll.(*FD).Read(0xc001001580, {0xc001158f11, 0x1, 0x1})
2024-05-20T19:36:41.8943310Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_unix.go:164 +0x27a
2024-05-20T19:36:41.8944361Z net.(*netFD).Read(0xc001001580, {0xc001158f11?, 0xc000942f40?, 0x470930?})
2024-05-20T19:36:41.8945318Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/fd_posix.go:55 +0x25
2024-05-20T19:36:41.8946411Z net.(*conn).Read(0xc001584388, {0xc001158f11?, 0x37?, 0xc000942ee8?})
2024-05-20T19:36:41.8947304Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/net.go:179 +0x45
2024-05-20T19:36:41.8948098Z net/http.(*connReader).backgroundRead(0xc001158f00)
2024-05-20T19:36:41.8948954Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/server.go:683 +0x37
2024-05-20T19:36:41.8949992Z created by net/http.(*connReader).startBackgroundRead in goroutine 15288
2024-05-20T19:36:41.8951019Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/server.go:679 +0xba
2024-05-20T19:36:41.8951637Z 
2024-05-20T19:36:41.8951794Z goroutine 15217 [select]:
2024-05-20T19:36:41.8952646Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000fce580, {0xe0?, 0x53?}, 0xc00139eb40)
2024-05-20T19:36:41.8954490Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.8956152Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask(0xc000fce580)
2024-05-20T19:36:41.8958221Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1142 +0xb5
2024-05-20T19:36:41.8959679Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc000)
2024-05-20T19:36:41.8961293Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.8962717Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc000)
2024-05-20T19:36:41.8964322Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.8965841Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.8967520Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.8968551Z 
2024-05-20T19:36:41.8968740Z goroutine 12 [IO wait, 7 minutes]:
2024-05-20T19:36:41.8969409Z internal/poll.runtime_pollWait(0x7fd78727edb8, 0x72)
2024-05-20T19:36:41.8970299Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/netpoll.go:343 +0x85
2024-05-20T19:36:41.8971245Z internal/poll.(*pollDesc).wait(0xc0006ba000?, 0xc0006c6000?, 0x0)
2024-05-20T19:36:41.8972292Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-05-20T19:36:41.8973209Z internal/poll.(*pollDesc).waitRead(...)
2024-05-20T19:36:41.8974046Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:89
2024-05-20T19:36:41.8975444Z internal/poll.(*FD).Read(0xc0006ba000, {0xc0006c6000, 0x8000, 0x8000})
2024-05-20T19:36:41.8976491Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_unix.go:164 +0x27a
2024-05-20T19:36:41.8977505Z net.(*netFD).Read(0xc0006ba000, {0xc0006c6000?, 0x1040100000000?, 0x0?})
2024-05-20T19:36:41.8978430Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/fd_posix.go:55 +0x25
2024-05-20T19:36:41.8979374Z net.(*conn).Read(0xc0004cc010, {0xc0006c6000?, 0xc0006b1d68?, 0xbd3aff?})
2024-05-20T19:36:41.8980276Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/net.go:179 +0x45
2024-05-20T19:36:41.8981116Z bufio.(*Reader).Read(0xc0006c4000, {0xc0006de040, 0x9, 0x30?})
2024-05-20T19:36:41.8981992Z    /opt/hostedtoolcache/go/1.21.10/x64/src/bufio/bufio.go:244 +0x197
2024-05-20T19:36:41.8982960Z io.ReadAtLeast({0x1a9ae40, 0xc0006c4000}, {0xc0006de040, 0x9, 0x9}, 0x9)
2024-05-20T19:36:41.8983863Z    /opt/hostedtoolcache/go/1.21.10/x64/src/io/io.go:335 +0x90
2024-05-20T19:36:41.8984532Z io.ReadFull(...)
2024-05-20T19:36:41.8984927Z    /opt/hostedtoolcache/go/1.21.10/x64/src/io/io.go:354
2024-05-20T19:36:41.8985708Z golang.org/x/net/http2.readFrameHeader({0xc0006de040, 0x9, 0xc0006b4450?}, {0x1a9ae40?, 0xc0006c4000?})
2024-05-20T19:36:41.8986611Z    /home/runner/go/pkg/mod/golang.org/x/net@v0.21.0/http2/frame.go:237 +0x65
2024-05-20T19:36:41.8987316Z golang.org/x/net/http2.(*Framer).ReadFrame(0xc0006de000)
2024-05-20T19:36:41.8988169Z    /home/runner/go/pkg/mod/golang.org/x/net@v0.21.0/http2/frame.go:498 +0x85
2024-05-20T19:36:41.8989101Z google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00054ed80, 0xc000794000?)
2024-05-20T19:36:41.8990656Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x245
2024-05-20T19:36:41.8992129Z created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 25
2024-05-20T19:36:41.8993587Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x1e9e
2024-05-20T19:36:41.8994518Z 
2024-05-20T19:36:41.8994699Z goroutine 23 [select, 7 minutes]:
2024-05-20T19:36:41.8995762Z google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000440300, {0x1ab04e8, 0xc0004d6140})
2024-05-20T19:36:41.8997422Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x112
2024-05-20T19:36:41.8998970Z created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20
2024-05-20T19:36:41.9000541Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x129
2024-05-20T19:36:41.9001727Z 
2024-05-20T19:36:41.9001934Z goroutine 24 [select, 7 minutes]:
2024-05-20T19:36:41.9002980Z google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000440360, {0x1ab04e8, 0xc0004d6190})
2024-05-20T19:36:41.9004967Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x112
2024-05-20T19:36:41.9006548Z created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 20
2024-05-20T19:36:41.9008138Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x129
2024-05-20T19:36:41.9009117Z 
2024-05-20T19:36:41.9009299Z goroutine 26 [select, 7 minutes]:
2024-05-20T19:36:41.9010307Z google.golang.org/grpc.(*ClientConn).WaitForStateChange(0xc000018000, {0x1ab04e8, 0xc0004d6370}, 0x2)
2024-05-20T19:36:41.9011676Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/clientconn.go:684 +0xb1
2024-05-20T19:36:41.9012943Z go.uber.org/yarpc/transport/grpc.(*grpcPeer).monitorConnectionStatus(0xc0003d8300)
2024-05-20T19:36:41.9014219Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/peer.go:82 +0x85
2024-05-20T19:36:41.9015637Z created by go.uber.org/yarpc/transport/grpc.(*Transport).newPeer in goroutine 20
2024-05-20T19:36:41.9016918Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/peer.go:69 +0x578
2024-05-20T19:36:41.9017834Z 
2024-05-20T19:36:41.9018048Z goroutine 6551 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9019079Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc0000f0480, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9020979Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9022355Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9023875Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9025341Z go.uber.org/cadence/internal.Await({0x1ab2890, 0xc000a56480}, 0xc0000f07c0)
2024-05-20T19:36:41.9026938Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9028125Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9029519Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9031269Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0xc000a91110?, {0x1a9bb20?, 0xc000a91100?}, 0xc001784f40?)
2024-05-20T19:36:41.9033076Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9035216Z github.com/indeedeng/iwf/service/interpreter.InterpreterImpl({0x1a9bb20?, 0xc000a91100}, {0x1ac6300, 0xc000a91110}, {{0xc0002e5648, 0x6}, {0xc0018a52d8, 0x15}, 0xc000a910d0, {0x0, ...}, ...})
2024-05-20T19:36:41.9037315Z    /home/runner/work/iwf/iwf/service/interpreter/workflowImpl.go:248 +0x1efb
2024-05-20T19:36:41.9038717Z github.com/indeedeng/iwf/service/interpreter/cadence.Interpreter({0x1ab2890?, 0xc000a56480}, {{0xc0002e5648, 0x6}, {0xc0018a52d8, 0x15}, 0xc000a910d0, {0x0, 0x0, 0x0}, ...})
2024-05-20T19:36:41.9040144Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflow.go:10 +0x17d
2024-05-20T19:36:41.9041216Z reflect.Value.call({0x15f8fa0?, 0x19028a0?, 0x1834e00?}, {0x1838b1a, 0x4}, {0xc001176ae0, 0x2, 0x18?})
2024-05-20T19:36:41.9042432Z    /opt/hostedtoolcache/go/1.21.10/x64/src/reflect/value.go:596 +0xce7
2024-05-20T19:36:41.9043618Z reflect.Value.Call({0x15f8fa0?, 0x19028a0?, 0x108f808?}, {0xc001176ae0?, 0x1834e00?, 0x1a99670?})
2024-05-20T19:36:41.9044788Z    /opt/hostedtoolcache/go/1.21.10/x64/src/reflect/value.go:380 +0xb9
2024-05-20T19:36:41.9046619Z go.uber.org/cadence/internal.(*workflowEnvironmentInterceptor).ExecuteWorkflow(0xc0011766f0, {0x1ab2890?, 0xc000a56480?}, {0x15f8fa0?, 0x15d3d00?}, {0xc000a910e0?, 0x1, 0xc000bbae60?})
2024-05-20T19:36:41.9049294Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:688 +0x44a
2024-05-20T19:36:41.9051105Z go.uber.org/cadence/internal.(*workflowExecutor).Execute(0xc000441900, {0x1ab2890, 0xc000a56480}, {0xc0007384d0, 0xa4, 0xb0})
2024-05-20T19:36:41.9053130Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker.go:667 +0x392
2024-05-20T19:36:41.9054347Z go.uber.org/cadence/internal.(*syncWorkflowDefinition).Execute.func1({0x1ab0868, 0xc0011769f0})
2024-05-20T19:36:41.9055917Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:467 +0xcc
2024-05-20T19:36:41.9057221Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000c08c50?)
2024-05-20T19:36:41.9058620Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9059913Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 6550
2024-05-20T19:36:41.9061374Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9062077Z 
2024-05-20T19:36:41.9062248Z goroutine 15471 [select]:
2024-05-20T19:36:41.9062641Z net/http.(*persistConn).writeLoop(0xc000cadc20)
2024-05-20T19:36:41.9063605Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:2421 +0xe5
2024-05-20T19:36:41.9064302Z created by net/http.(*Transport).dialConn in goroutine 15516
2024-05-20T19:36:41.9065364Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:1777 +0x16f1
2024-05-20T19:36:41.9065950Z 
2024-05-20T19:36:41.9066133Z goroutine 15134 [select]:
2024-05-20T19:36:41.9066535Z net/http.(*persistConn).roundTrip(0xc00161cb40, 0xc00069f980)
2024-05-20T19:36:41.9067363Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:2652 +0x979
2024-05-20T19:36:41.9068166Z net/http.(*Transport).roundTrip(0x26c9ae0, 0xc0011c5400)
2024-05-20T19:36:41.9068832Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:604 +0x79a
2024-05-20T19:36:41.9069622Z net/http.(*Transport).RoundTrip(0x4279bc?, 0x1a9b880?)
2024-05-20T19:36:41.9070372Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/roundtrip.go:17 +0x13
2024-05-20T19:36:41.9071132Z net/http.send(0xc0011c5400, {0x1a9b880, 0x26c9ae0}, {0x412a01?, 0xc0011c5400?, 0x0?})
2024-05-20T19:36:41.9072084Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/client.go:260 +0x606
2024-05-20T19:36:41.9072895Z net/http.(*Client).send(0x26dfd40, 0xc0011c5400, {0xf8?, 0x1814560?, 0x0?})
2024-05-20T19:36:41.9073594Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/client.go:181 +0x98
2024-05-20T19:36:41.9074363Z net/http.(*Client).do(0x26dfd40, 0xc0011c5400)
2024-05-20T19:36:41.9075387Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/client.go:724 +0x912
2024-05-20T19:36:41.9076192Z net/http.(*Client).Do(...)
2024-05-20T19:36:41.9076853Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/client.go:590
2024-05-20T19:36:41.9077967Z github.com/indeedeng/iwf/gen/iwfidl.(*APIClient).callAPI(0xc000be05e0, 0xc0006e8de8?)
2024-05-20T19:36:41.9079069Z    /home/runner/work/iwf/iwf/gen/iwfidl/client.go:248 +0x107
2024-05-20T19:36:41.9080456Z github.com/indeedeng/iwf/gen/iwfidl.(*DefaultApiService).ApiV1WorkflowGetWithWaitPostExecute(0xc000be05e8, {{0x1ab0328, 0x2712020}, {0x1aca7c0, 0xc000be05e8}, 0xc0006b5560})
2024-05-20T19:36:41.9081815Z    /home/runner/work/iwf/iwf/gen/iwfidl/api_default.go:660 +0x45f
2024-05-20T19:36:41.9082747Z github.com/indeedeng/iwf/gen/iwfidl.ApiApiV1WorkflowGetWithWaitPostRequest.Execute(...)
2024-05-20T19:36:41.9083677Z    /home/runner/work/iwf/iwf/gen/iwfidl/api_default.go:598
2024-05-20T19:36:41.9084643Z github.com/indeedeng/iwf/integ.doTestPersistenceWorkflow(0xc000ba3380?, {0x184257c, 0x7}, 0x0, 0x0?, 0xc0013b68a0)
2024-05-20T19:36:41.9085754Z    /home/runner/work/iwf/iwf/integ/persistence_test.go:334 +0x3466
2024-05-20T19:36:41.9086897Z github.com/indeedeng/iwf/integ.TestPersistenceWorkflowCadenceContinueAsNew(0xc000ba3380)
2024-05-20T19:36:41.9087958Z    /home/runner/work/iwf/iwf/integ/persistence_test.go:94 +0x5b
2024-05-20T19:36:41.9088433Z testing.tRunner(0xc000ba3380, 0x1901b70)
2024-05-20T19:36:41.9089120Z    /opt/hostedtoolcache/go/1.21.10/x64/src/testing/testing.go:1595 +0xff
2024-05-20T19:36:41.9089771Z created by testing.(*T).Run in goroutine 1
2024-05-20T19:36:41.9090613Z    /opt/hostedtoolcache/go/1.21.10/x64/src/testing/testing.go:1648 +0x3ad
2024-05-20T19:36:41.9091271Z 
2024-05-20T19:36:41.9091479Z goroutine 6554 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9092524Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc0000f0600, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9094403Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9095791Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9096969Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9098015Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc001177020}, 0xc001177590)
2024-05-20T19:36:41.9099279Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9100082Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9101354Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9102965Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0xc00082fea0?, {0x1a9bb20?, 0xc000a91350?}, 0x186e6a3?)
2024-05-20T19:36:41.9104555Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9106308Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func3({0x1a9bb20, 0xc000a91350})
2024-05-20T19:36:41.9107710Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:99 +0x172
2024-05-20T19:36:41.9109269Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc001177020})
2024-05-20T19:36:41.9110914Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9112283Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000c09df0?)
2024-05-20T19:36:41.9114065Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9115714Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 6551
2024-05-20T19:36:41.9117569Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9118780Z 
2024-05-20T19:36:41.9118971Z goroutine 13 [select, 7 minutes]:
2024-05-20T19:36:41.9119833Z google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00007ec80, 0x1)
2024-05-20T19:36:41.9121272Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:418 +0x113
2024-05-20T19:36:41.9122621Z google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000452070)
2024-05-20T19:36:41.9124437Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/controlbuf.go:552 +0x86
2024-05-20T19:36:41.9125330Z google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
2024-05-20T19:36:41.9126202Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:451 +0x85
2024-05-20T19:36:41.9127290Z created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 25
2024-05-20T19:36:41.9128246Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:449 +0x2450
2024-05-20T19:36:41.9128902Z 
2024-05-20T19:36:41.9128998Z goroutine 15435 [select]:
2024-05-20T19:36:41.9129456Z google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0008e0240)
2024-05-20T19:36:41.9130455Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x7c
2024-05-20T19:36:41.9131205Z google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
2024-05-20T19:36:41.9131907Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346
2024-05-20T19:36:41.9132706Z google.golang.org/grpc.(*csAttempt).recvMsg(0xc001441520, {0x155e160?, 0xc000e744e0}, 0xc0009224f8?)
2024-05-20T19:36:41.9133438Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0xc9
2024-05-20T19:36:41.9134060Z google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x2712020?)
2024-05-20T19:36:41.9134655Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x1f
2024-05-20T19:36:41.9135635Z google.golang.org/grpc.(*clientStream).withRetry(0xc0008e0000, 0xc000922528, 0xc0009224f8)
2024-05-20T19:36:41.9136344Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x13a
2024-05-20T19:36:41.9137030Z google.golang.org/grpc.(*clientStream).RecvMsg(0xc0008e0000, {0x155e160?, 0xc000e744e0?})
2024-05-20T19:36:41.9137709Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x12e
2024-05-20T19:36:41.9138671Z google.golang.org/grpc.invoke({0x1ab04b0?, 0xc000e93140?}, {0xc000d7cd00?, 0x412a05?}, {0x155f420, 0xc000e74528}, {0x155e160, 0xc000e744e0}, 0x412d85?, {0xc000a03780, ...})
2024-05-20T19:36:41.9139714Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0xcb
2024-05-20T19:36:41.9140696Z google.golang.org/grpc.(*ClientConn).Invoke(0xc000814800, {0x1ab04b0, 0xc000e93140}, {0xc000d7cd00, 0x32}, {0x155f420, 0xc000e74528}, {0x155e160, 0xc000e744e0}, {0xc0013d4a90, ...})
2024-05-20T19:36:41.9141669Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:42 +0x2cb
2024-05-20T19:36:41.9142693Z go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0xc0013b6ab0, {0x1ab0558, 0xc00043af50}, 0xc0009a4540, 0x1838f92?, 0xc001776a40, {0xc18b07726b75ad41?, 0xc001018ea8?, 0x26e0980?})
2024-05-20T19:36:41.9143777Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x4e5
2024-05-20T19:36:41.9144581Z go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0xc0013b6ab0, {0x1ab0558, 0xc00043af50}, 0xc0009a4540)
2024-05-20T19:36:41.9145371Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0x126
2024-05-20T19:36:41.9146433Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558?, 0xc00043af50?}, 0x184f9bf?)
2024-05-20T19:36:41.9147527Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
2024-05-20T19:36:41.9148529Z go.uber.org/yarpc/internal/observability.(*Middleware).Call(0xc00183bc80, {0x1ab0558, 0xc00043af50}, 0xc0009a4540, {0x1ab4e00, 0xc000e93050})
2024-05-20T19:36:41.9149619Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0x162
2024-05-20T19:36:41.9150718Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc00043af50}, 0xc0013b6ab0?)
2024-05-20T19:36:41.9151801Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9153372Z go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x17706e0?, {0x1ab0558, 0xc00043af50}, 0xc0009a4540, {0x1ab4e00?, 0xc000e93020})
2024-05-20T19:36:41.9155835Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xa8
2024-05-20T19:36:41.9162861Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x1, 0x1}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc00043af50}, 0xc000814c00?)
2024-05-20T19:36:41.9165578Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9170553Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
2024-05-20T19:36:41.9172357Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
2024-05-20T19:36:41.9177522Z go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
2024-05-20T19:36:41.9178793Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
2024-05-20T19:36:41.9180659Z go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x1ab4c80?, 0xc0013b6c30?}, {0x7fd73c34b008?, 0xc0013b6c30?}}, {0x1ab0558, 0xc00043af50}, 0x412a05?)
2024-05-20T19:36:41.9182712Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x92
2024-05-20T19:36:41.9184099Z go.uber.org/yarpc/encoding/protobuf.(*client).Call(0xc0013b7260, {0x1ab0558?, 0xc00043af50?}, {0x18556fb?, 0x7?}, {0x1aa9be0?, 0xc0008739a0?}, 0x1903898, {0xc0018a61e0, 0x4, ...})
2024-05-20T19:36:41.9185180Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0x10c
2024-05-20T19:36:41.9186562Z github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForActivityTask(0xc001562960?, {0x1ab0558?, 0xc00043af50?}, 0xc000c37cc0?, {0xc0018a61e0?, 0xc000c37c90?, 0x184f117?})
2024-05-20T19:36:41.9188072Z    /home/runner/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220713235846-fda89e95df1e/go/proto/api/v1/service_worker.pb.yarpc.go:397 +0x66
2024-05-20T19:36:41.9189794Z go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForActivityTask({{0x1ab3c80, 0xc0010956f0}, {0x1abd1d8, 0xc001095710}, {0x1abd260, 0xc001095730}, {0x1ab7ea0, 0xc001095750}}, {0x1ab0558, 0xc00043af50}, ...)
2024-05-20T19:36:41.9191337Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/compatibility/adapter.go:121 +0x95
2024-05-20T19:36:41.9192626Z go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForActivityTask(0xc001586fc0, {0x1ab0558, 0xc00043af50}, 0xc000c6fd40?, {0xc0018a61e0, 0x4, 0x6})
2024-05-20T19:36:41.9194052Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/metrics/service_wrapper.go:213 +0x9d
2024-05-20T19:36:41.9195027Z go.uber.org/cadence/internal.(*activityTaskPoller).poll(0xc000fce500, {0x1ab0558, 0xc00043af50})
2024-05-20T19:36:41.9196046Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1005 +0x33e
2024-05-20T19:36:41.9197300Z go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0xc000fce500, {0x1ab0558?, 0xc00043af50?}, 0xc000c1a1d8?)
2024-05-20T19:36:41.9198457Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1048 +0x42
2024-05-20T19:36:41.9199559Z go.uber.org/cadence/internal.(*activityTaskPoller).PollTask.(*activityTaskPoller).pollWithMetricsFunc.func1({0x1ab0558?, 0xc00043af50?})
2024-05-20T19:36:41.9200885Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1042 +0x2e
2024-05-20T19:36:41.9201834Z go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
2024-05-20T19:36:41.9202704Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:256 +0x4a
2024-05-20T19:36:41.9203558Z created by go.uber.org/cadence/internal.(*basePoller).doPoll in goroutine 15226
2024-05-20T19:36:41.9204525Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:255 +0x1b7
2024-05-20T19:36:41.9205121Z 
2024-05-20T19:36:41.9205220Z goroutine 15226 [select]:
2024-05-20T19:36:41.9205715Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000fce500, {0xe0?, 0x53?}, 0xc000e744b0)
2024-05-20T19:36:41.9206705Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9207546Z go.uber.org/cadence/internal.(*activityTaskPoller).PollTask(0xc000fce500)
2024-05-20T19:36:41.9208665Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1071 +0xb5
2024-05-20T19:36:41.9209708Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fc9b00)
2024-05-20T19:36:41.9210629Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9211448Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fc9b00)
2024-05-20T19:36:41.9212337Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9213181Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9214124Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9214702Z 
2024-05-20T19:36:41.9214804Z goroutine 15238 [select]:
2024-05-20T19:36:41.9215486Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000fce580, {0xe0?, 0x53?}, 0xc000e743d8)
2024-05-20T19:36:41.9216502Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9217427Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask(0xc000fce580)
2024-05-20T19:36:41.9218572Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1142 +0xb5
2024-05-20T19:36:41.9219374Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc000)
2024-05-20T19:36:41.9220263Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9221067Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc000)
2024-05-20T19:36:41.9221946Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9222795Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9223734Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9224307Z 
2024-05-20T19:36:41.9224407Z goroutine 15188 [select]:
2024-05-20T19:36:41.9224975Z google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000691300, {0x1ab04e8, 0xc001182370})
2024-05-20T19:36:41.9225902Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:83 +0x112
2024-05-20T19:36:41.9226782Z created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 15137
2024-05-20T19:36:41.9227650Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/grpcsync/callback_serializer.go:55 +0x129
2024-05-20T19:36:41.9228276Z 
2024-05-20T19:36:41.9228372Z goroutine 15470 [IO wait]:
2024-05-20T19:36:41.9228707Z internal/poll.runtime_pollWait(0x7fd78727e7e8, 0x72)
2024-05-20T19:36:41.9229217Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/netpoll.go:343 +0x85
2024-05-20T19:36:41.9229762Z internal/poll.(*pollDesc).wait(0xc0010acd00?, 0xc000b1a000?, 0x0)
2024-05-20T19:36:41.9230368Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-05-20T19:36:41.9230887Z internal/poll.(*pollDesc).waitRead(...)
2024-05-20T19:36:41.9231370Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:89
2024-05-20T19:36:41.9231964Z internal/poll.(*FD).Read(0xc0010acd00, {0xc000b1a000, 0x1000, 0x1000})
2024-05-20T19:36:41.9232556Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_unix.go:164 +0x27a
2024-05-20T19:36:41.9233160Z net.(*netFD).Read(0xc0010acd00, {0xc000b1a000?, 0xc0010acd00?, 0x6fc23ac00?})
2024-05-20T19:36:41.9233718Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/fd_posix.go:55 +0x25
2024-05-20T19:36:41.9234223Z net.(*conn).Read(0xc001584848, {0xc000b1a000?, 0x0?, 0x1ab6100?})
2024-05-20T19:36:41.9234840Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/net.go:179 +0x45
2024-05-20T19:36:41.9235396Z net/http.(*persistConn).Read(0xc000cadc20, {0xc000b1a000?, 0x40a8c5?, 0x60?})
2024-05-20T19:36:41.9235998Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:1954 +0x4a
2024-05-20T19:36:41.9236464Z bufio.(*Reader).fill(0xc000d7a120)
2024-05-20T19:36:41.9236863Z    /opt/hostedtoolcache/go/1.21.10/x64/src/bufio/bufio.go:113 +0x103
2024-05-20T19:36:41.9237727Z bufio.(*Reader).Peek(0xc000d7a120, 0x1)
2024-05-20T19:36:41.9238163Z    /opt/hostedtoolcache/go/1.21.10/x64/src/bufio/bufio.go:151 +0x53
2024-05-20T19:36:41.9238622Z net/http.(*persistConn).readLoop(0xc000cadc20)
2024-05-20T19:36:41.9239120Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:2118 +0x1b9
2024-05-20T19:36:41.9239681Z created by net/http.(*Transport).dialConn in goroutine 15516
2024-05-20T19:36:41.9240237Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:1776 +0x169f
2024-05-20T19:36:41.9240612Z 
2024-05-20T19:36:41.9240708Z goroutine 14773 [sleep]:
2024-05-20T19:36:41.9240970Z time.Sleep(0x46dcc5f0)
2024-05-20T19:36:41.9241334Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/time.go:195 +0x125
2024-05-20T19:36:41.9242036Z go.uber.org/cadence/internal/common/backoff.(*ConcurrentRetrier).throttleInternal(0xc0011733c0)
2024-05-20T19:36:41.9243099Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/backoff/retry.go:65 +0x8b
2024-05-20T19:36:41.9244068Z go.uber.org/cadence/internal/common/backoff.(*ConcurrentRetrier).Throttle(...)
2024-05-20T19:36:41.9244990Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/backoff/retry.go:51
2024-05-20T19:36:41.9245757Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc00161de60)
2024-05-20T19:36:41.9246651Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:291 +0x21c
2024-05-20T19:36:41.9247447Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc00161de60)
2024-05-20T19:36:41.9248331Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9249173Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 14758
2024-05-20T19:36:41.9250106Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9250689Z 
2024-05-20T19:36:41.9250782Z goroutine 15465 [select]:
2024-05-20T19:36:41.9251231Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTunnel).getTask(...)
2024-05-20T19:36:41.9252156Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:193
2024-05-20T19:36:41.9253332Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).pollLocallyDispatchedActivity(0xc000fce580, {0xc000505120?, 0x26b40f0?})
2024-05-20T19:36:41.9254558Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1151 +0x8b
2024-05-20T19:36:41.9255809Z go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0xc000fce580, {0x1ab0558?, 0xc000407e30?}, 0x3?)
2024-05-20T19:36:41.9256921Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1048 +0x42
2024-05-20T19:36:41.9258100Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask.(*activityTaskPoller).pollWithMetricsFunc.func1({0x1ab0558?, 0xc000407e30?})
2024-05-20T19:36:41.9259369Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1042 +0x2e
2024-05-20T19:36:41.9260139Z go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
2024-05-20T19:36:41.9261000Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:256 +0x4a
2024-05-20T19:36:41.9262019Z created by go.uber.org/cadence/internal.(*basePoller).doPoll in goroutine 15214
2024-05-20T19:36:41.9262987Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:255 +0x1b7
2024-05-20T19:36:41.9263578Z 
2024-05-20T19:36:41.9263679Z goroutine 15227 [select]:
2024-05-20T19:36:41.9264104Z go.uber.org/cadence/internal.(*baseWorker).runTaskDispatcher(0xc000fc9b00)
2024-05-20T19:36:41.9265036Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:256 +0x11d
2024-05-20T19:36:41.9265885Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9266821Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:209 +0x138
2024-05-20T19:36:41.9267403Z 
2024-05-20T19:36:41.9267496Z goroutine 15236 [select]:
2024-05-20T19:36:41.9267989Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000fce580, {0xe0?, 0x53?}, 0xc000be8ac8)
2024-05-20T19:36:41.9268983Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9269942Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask(0xc000fce580)
2024-05-20T19:36:41.9270971Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1142 +0xb5
2024-05-20T19:36:41.9271857Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc000)
2024-05-20T19:36:41.9272749Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9273545Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc000)
2024-05-20T19:36:41.9274436Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9275283Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9276215Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9276785Z 
2024-05-20T19:36:41.9276877Z goroutine 15276 [select]:
2024-05-20T19:36:41.9277193Z net/http.(*persistConn).writeLoop(0xc000f27d40)
2024-05-20T19:36:41.9277698Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:2421 +0xe5
2024-05-20T19:36:41.9278257Z created by net/http.(*Transport).dialConn in goroutine 15257
2024-05-20T19:36:41.9278813Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:1777 +0x16f1
2024-05-20T19:36:41.9279183Z 
2024-05-20T19:36:41.9279287Z goroutine 11814 [semacquire]:
2024-05-20T19:36:41.9279582Z sync.runtime_Semacquire(0xc000944fd0?)
2024-05-20T19:36:41.9280092Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/sema.go:62 +0x25
2024-05-20T19:36:41.9280518Z sync.(*WaitGroup).Wait(0x10c5be5?)
2024-05-20T19:36:41.9280942Z    /opt/hostedtoolcache/go/1.21.10/x64/src/sync/waitgroup.go:116 +0x48
2024-05-20T19:36:41.9281506Z go.uber.org/cadence/internal/common/util.AwaitWaitGroup.func1()
2024-05-20T19:36:41.9282569Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/util/util.go:52 +0x25
2024-05-20T19:36:41.9283443Z created by go.uber.org/cadence/internal/common/util.AwaitWaitGroup in goroutine 11484
2024-05-20T19:36:41.9284417Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/util/util.go:51 +0x7d
2024-05-20T19:36:41.9284973Z 
2024-05-20T19:36:41.9285095Z goroutine 6401 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9285664Z go.temporal.io/sdk/internal.(*coroutineState).initialYield(0xc000890cd0, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9286438Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:945 +0x5f
2024-05-20T19:36:41.9287065Z go.temporal.io/sdk/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9287789Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:955
2024-05-20T19:36:41.9288654Z go.temporal.io/sdk/internal.(*workflowEnvironmentInterceptor).Await(0x1ab08a0?, {0x1ab08a0, 0xc000a8d920}, 0xc000a8df50)
2024-05-20T19:36:41.9289490Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/workflow.go:364 +0xa5
2024-05-20T19:36:41.9290140Z go.temporal.io/sdk/internal.Await({0x1ab08a0, 0xc000a8d920}, 0xc000a8d920?)
2024-05-20T19:36:41.9290788Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/workflow.go:349 +0x58
2024-05-20T19:36:41.9291298Z go.temporal.io/sdk/workflow.Await(...)
2024-05-20T19:36:41.9291840Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/workflow/deterministic_wrappers.go:84
2024-05-20T19:36:41.9292782Z github.com/indeedeng/iwf/service/interpreter/temporal.(*workflowProvider).Await(0x10?, {0x1a9bb20?, 0xc0004cf450?}, 0x1865c58?)
2024-05-20T19:36:41.9293737Z    /home/runner/work/iwf/iwf/service/interpreter/temporal/workflowProvider.go:203 +0x7e
2024-05-20T19:36:41.9294591Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func2({0x1a9bb20, 0xc0004cf450})
2024-05-20T19:36:41.9295642Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:75 +0x1c4
2024-05-20T19:36:41.9296608Z github.com/indeedeng/iwf/service/interpreter/temporal.(*workflowProvider).GoNamed.func1({0x1ab08a0?, 0xc000a8d920})
2024-05-20T19:36:41.9297814Z    /home/runner/work/iwf/iwf/service/interpreter/temporal/workflowProvider.go:180 +0xfd
2024-05-20T19:36:41.9298557Z go.temporal.io/sdk/internal.(*dispatcherImpl).NewCoroutine.func1(0xc000ef6cd0?)
2024-05-20T19:36:41.9299291Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:1057 +0xcf
2024-05-20T19:36:41.9300041Z created by go.temporal.io/sdk/internal.(*dispatcherImpl).NewCoroutine in goroutine 6399
2024-05-20T19:36:41.9300811Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:1048 +0x338
2024-05-20T19:36:41.9301263Z 
2024-05-20T19:36:41.9301364Z goroutine 15151 [select]:
2024-05-20T19:36:41.9301753Z go.uber.org/cadence/internal.(*localActivityTunnel).getTask(...)
2024-05-20T19:36:41.9302645Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:167
2024-05-20T19:36:41.9303488Z go.uber.org/cadence/internal.(*localActivityTaskPoller).PollTask(0xc000c5aae0?)
2024-05-20T19:36:41.9304450Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:540 +0x65
2024-05-20T19:36:41.9305239Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc900)
2024-05-20T19:36:41.9306120Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9306926Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc900)
2024-05-20T19:36:41.9307896Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9308743Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9309680Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9310410Z 
2024-05-20T19:36:41.9310547Z goroutine 6556 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9311118Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc0000f06c0, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9312122Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9312873Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9313674Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9314466Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc001177170}, 0xc0011775f0)
2024-05-20T19:36:41.9315496Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9316152Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9316911Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9318010Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0xc000a910b0?, {0x1a9bb20?, 0xc000a91390?}, 0x185a519?)
2024-05-20T19:36:41.9318994Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9319826Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func5({0x1a9bb20, 0xc000a91390})
2024-05-20T19:36:41.9320600Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:143 +0x1a9
2024-05-20T19:36:41.9321474Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc001177170})
2024-05-20T19:36:41.9322381Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9323127Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000c08c50?)
2024-05-20T19:36:41.9324094Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9324993Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 6551
2024-05-20T19:36:41.9326062Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9326636Z 
2024-05-20T19:36:41.9326729Z goroutine 15543 [select]:
2024-05-20T19:36:41.9327181Z google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000d73e60)
2024-05-20T19:36:41.9327953Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x7c
2024-05-20T19:36:41.9328673Z google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
2024-05-20T19:36:41.9329387Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346
2024-05-20T19:36:41.9330201Z google.golang.org/grpc.(*csAttempt).recvMsg(0xc00085c1a0, {0x155e160?, 0xc000f71ea8}, 0xc0008664f8?)
2024-05-20T19:36:41.9330924Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0xc9
2024-05-20T19:36:41.9331525Z google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x2712020?)
2024-05-20T19:36:41.9332120Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x1f
2024-05-20T19:36:41.9332798Z google.golang.org/grpc.(*clientStream).withRetry(0xc000d73c20, 0xc000866528, 0xc0008664f8)
2024-05-20T19:36:41.9333489Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x13a
2024-05-20T19:36:41.9334171Z google.golang.org/grpc.(*clientStream).RecvMsg(0xc000d73c20, {0x155e160?, 0xc000f71ea8?})
2024-05-20T19:36:41.9334930Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x12e
2024-05-20T19:36:41.9336343Z google.golang.org/grpc.invoke({0x1ab04b0?, 0xc000898ae0?}, {0xc0006f3c40?, 0x412a05?}, {0x155f420, 0xc000f71ef0}, {0x155e160, 0xc000f71ea8}, 0x412d85?, {0xc000dfa000, ...})
2024-05-20T19:36:41.9337310Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0xcb
2024-05-20T19:36:41.9338315Z google.golang.org/grpc.(*ClientConn).Invoke(0xc000814800, {0x1ab04b0, 0xc000898ae0}, {0xc0006f3c40, 0x32}, {0x155f420, 0xc000f71ef0}, {0x155e160, 0xc000f71ea8}, {0xc00088ce40, ...})
2024-05-20T19:36:41.9339307Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:42 +0x2cb
2024-05-20T19:36:41.9340337Z go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0xc0013b6ab0, {0x1ab0558, 0xc0003c2460}, 0xc000d44600, 0x1838f92?, 0xc0017771e8, {0xc18b07727267fbd8?, 0xc001018ea8?, 0x26e0980?})
2024-05-20T19:36:41.9341408Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x4e5
2024-05-20T19:36:41.9342370Z go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0xc0013b6ab0, {0x1ab0558, 0xc0003c2460}, 0xc000d44600)
2024-05-20T19:36:41.9343164Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0x126
2024-05-20T19:36:41.9344217Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558?, 0xc0003c2460?}, 0x184f9bf?)
2024-05-20T19:36:41.9345302Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
2024-05-20T19:36:41.9346303Z go.uber.org/yarpc/internal/observability.(*Middleware).Call(0xc00183bc80, {0x1ab0558, 0xc0003c2460}, 0xc000d44600, {0x1ab4e00, 0xc0008989f0})
2024-05-20T19:36:41.9347294Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0x162
2024-05-20T19:36:41.9348383Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc0003c2460}, 0xc0013b6ab0?)
2024-05-20T19:36:41.9349480Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9350522Z go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x17706e0?, {0x1ab0558, 0xc0003c2460}, 0xc000d44600, {0x1ab4e00?, 0xc0008989c0})
2024-05-20T19:36:41.9351624Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xa8
2024-05-20T19:36:41.9352857Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x1, 0x1}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc0003c2460}, 0xc000070c00?)
2024-05-20T19:36:41.9353933Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9354896Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
2024-05-20T19:36:41.9355597Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
2024-05-20T19:36:41.9356316Z go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
2024-05-20T19:36:41.9356990Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
2024-05-20T19:36:41.9357996Z go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x1ab4c80?, 0xc0013b6c30?}, {0x7fd73c34b008?, 0xc0013b6c30?}}, {0x1ab0558, 0xc0003c2460}, 0x1860a14?)
2024-05-20T19:36:41.9359070Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x92
2024-05-20T19:36:41.9360176Z go.uber.org/yarpc/encoding/protobuf.(*client).Call(0xc0013b7260, {0x1ab0558?, 0xc0003c2460?}, {0x18556fb?, 0x7?}, {0x1aa9be0?, 0xc000705bd0?}, 0x1903898, {0xc00067f8c0, 0x4, ...})
2024-05-20T19:36:41.9361209Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0x10c
2024-05-20T19:36:41.9362511Z github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForActivityTask(0xc0018b4c40?, {0x1ab0558?, 0xc0003c2460?}, 0xc000c36cc0?, {0xc00067f8c0?, 0xc000c36c90?, 0x184f117?})
2024-05-20T19:36:41.9364116Z    /home/runner/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220713235846-fda89e95df1e/go/proto/api/v1/service_worker.pb.yarpc.go:397 +0x66
2024-05-20T19:36:41.9365970Z go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForActivityTask({{0x1ab3c80, 0xc0010956f0}, {0x1abd1d8, 0xc001095710}, {0x1abd260, 0xc001095730}, {0x1ab7ea0, 0xc001095750}}, {0x1ab0558, 0xc0003c2460}, ...)
2024-05-20T19:36:41.9367534Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/compatibility/adapter.go:121 +0x95
2024-05-20T19:36:41.9368812Z go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForActivityTask(0xc001586fc0, {0x1ab0558, 0xc0003c2460}, 0xc0017770d8?, {0xc00067f8c0, 0x4, 0x6})
2024-05-20T19:36:41.9370259Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/metrics/service_wrapper.go:213 +0x9d
2024-05-20T19:36:41.9371231Z go.uber.org/cadence/internal.(*activityTaskPoller).poll(0xc000fce500, {0x1ab0558, 0xc0003c2460})
2024-05-20T19:36:41.9372404Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1005 +0x33e
2024-05-20T19:36:41.9373458Z go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0xc000fce500, {0x1ab0558?, 0xc0003c2460?}, 0xc000453c70?)
2024-05-20T19:36:41.9374605Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1048 +0x42
2024-05-20T19:36:41.9375939Z go.uber.org/cadence/internal.(*activityTaskPoller).PollTask.(*activityTaskPoller).pollWithMetricsFunc.func1({0x1ab0558?, 0xc0003c2460?})
2024-05-20T19:36:41.9377169Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1042 +0x2e
2024-05-20T19:36:41.9377946Z go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
2024-05-20T19:36:41.9378803Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:256 +0x4a
2024-05-20T19:36:41.9379688Z created by go.uber.org/cadence/internal.(*basePoller).doPoll in goroutine 15225
2024-05-20T19:36:41.9380651Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:255 +0x1b7
2024-05-20T19:36:41.9381247Z 
2024-05-20T19:36:41.9381370Z goroutine 6402 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9382040Z go.temporal.io/sdk/internal.(*coroutineState).initialYield(0xc000890d20, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9382934Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:945 +0x5f
2024-05-20T19:36:41.9383685Z go.temporal.io/sdk/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9384303Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:955
2024-05-20T19:36:41.9385333Z go.temporal.io/sdk/internal.(*workflowEnvironmentInterceptor).Await(0x1ab08a0?, {0x1ab08a0, 0xc000a8d9e0}, 0xc000a8df80)
2024-05-20T19:36:41.9386256Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/workflow.go:364 +0xa5
2024-05-20T19:36:41.9387009Z go.temporal.io/sdk/internal.Await({0x1ab08a0, 0xc000a8d9e0}, 0xc000a8d9e0?)
2024-05-20T19:36:41.9387738Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/workflow.go:349 +0x58
2024-05-20T19:36:41.9388261Z go.temporal.io/sdk/workflow.Await(...)
2024-05-20T19:36:41.9388893Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/workflow/deterministic_wrappers.go:84
2024-05-20T19:36:41.9390043Z github.com/indeedeng/iwf/service/interpreter/temporal.(*workflowProvider).Await(0xc0006beea0?, {0x1a9bb20?, 0xc0004cf470?}, 0x186e6a3?)
2024-05-20T19:36:41.9391120Z    /home/runner/work/iwf/iwf/service/interpreter/temporal/workflowProvider.go:203 +0x7e
2024-05-20T19:36:41.9392022Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func3({0x1a9bb20, 0xc0004cf470})
2024-05-20T19:36:41.9393025Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:99 +0x172
2024-05-20T19:36:41.9394053Z github.com/indeedeng/iwf/service/interpreter/temporal.(*workflowProvider).GoNamed.func1({0x1ab08a0?, 0xc000a8d9e0})
2024-05-20T19:36:41.9395049Z    /home/runner/work/iwf/iwf/service/interpreter/temporal/workflowProvider.go:180 +0xfd
2024-05-20T19:36:41.9395863Z go.temporal.io/sdk/internal.(*dispatcherImpl).NewCoroutine.func1(0xc000ef6df0?)
2024-05-20T19:36:41.9396681Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:1057 +0xcf
2024-05-20T19:36:41.9397509Z created by go.temporal.io/sdk/internal.(*dispatcherImpl).NewCoroutine in goroutine 6399
2024-05-20T19:36:41.9398347Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:1048 +0x338
2024-05-20T19:36:41.9398884Z 
2024-05-20T19:36:41.9398983Z goroutine 14717 [sleep]:
2024-05-20T19:36:41.9399244Z time.Sleep(0x253e07cd)
2024-05-20T19:36:41.9399712Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/time.go:195 +0x125
2024-05-20T19:36:41.9400658Z go.uber.org/cadence/internal/common/backoff.(*ConcurrentRetrier).throttleInternal(0xc0011732c0)
2024-05-20T19:36:41.9401864Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/backoff/retry.go:65 +0x8b
2024-05-20T19:36:41.9402894Z go.uber.org/cadence/internal/common/backoff.(*ConcurrentRetrier).Throttle(...)
2024-05-20T19:36:41.9403928Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/backoff/retry.go:51
2024-05-20T19:36:41.9404802Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc00161d560)
2024-05-20T19:36:41.9405780Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:291 +0x21c
2024-05-20T19:36:41.9406720Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc00161d560)
2024-05-20T19:36:41.9407729Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9408660Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 14758
2024-05-20T19:36:41.9409692Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9410366Z 
2024-05-20T19:36:41.9410473Z goroutine 15570 [chan receive]:
2024-05-20T19:36:41.9411108Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc000796540, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9412403Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9413243Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9414148Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9415013Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc000a752f0}, 0xc000a758f0)
2024-05-20T19:36:41.9416127Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9416886Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9417743Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9419018Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0xc0008c9ea0?, {0x1a9bb20?, 0xc000ff3d00?}, 0x186e6a3?)
2024-05-20T19:36:41.9420089Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9421082Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func3({0x1a9bb20, 0xc000ff3d00})
2024-05-20T19:36:41.9421996Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:99 +0x172
2024-05-20T19:36:41.9422951Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc000a752f0})
2024-05-20T19:36:41.9424117Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9424989Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000882d90?)
2024-05-20T19:36:41.9426065Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9427013Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 15556
2024-05-20T19:36:41.9428076Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9428696Z 
2024-05-20T19:36:41.9428803Z goroutine 15219 [select]:
2024-05-20T19:36:41.9429308Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000fce500, {0xe0?, 0x53?}, 0xc000858378)
2024-05-20T19:36:41.9430333Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9431184Z go.uber.org/cadence/internal.(*activityTaskPoller).PollTask(0xc000fce500)
2024-05-20T19:36:41.9432416Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1071 +0xb5
2024-05-20T19:36:41.9433248Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fc9b00)
2024-05-20T19:36:41.9434157Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9434972Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fc9b00)
2024-05-20T19:36:41.9435859Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9436705Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9437644Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9438227Z 
2024-05-20T19:36:41.9438335Z goroutine 15572 [chan receive]:
2024-05-20T19:36:41.9438931Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc000796600, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9439972Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9440722Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9441521Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9442436Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc000a75440}, 0xc000a75950)
2024-05-20T19:36:41.9443305Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9443950Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9444700Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9445802Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0xc00175c698?, {0x1a9bb20?, 0xc000ff3d40?}, 0x185a519?)
2024-05-20T19:36:41.9446786Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9447621Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func5({0x1a9bb20, 0xc000ff3d40})
2024-05-20T19:36:41.9448569Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:143 +0x1a9
2024-05-20T19:36:41.9449447Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc000a75440})
2024-05-20T19:36:41.9450345Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9451106Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc0015ac820?)
2024-05-20T19:36:41.9452084Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9453228Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 15556
2024-05-20T19:36:41.9454242Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9454822Z 
2024-05-20T19:36:41.9454943Z goroutine 6420 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9455730Z go.temporal.io/sdk/internal.(*coroutineState).initialYield(0xc0005d7ae0, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9456534Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:945 +0x5f
2024-05-20T19:36:41.9459042Z go.temporal.io/sdk/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9459690Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:955
2024-05-20T19:36:41.9460558Z go.temporal.io/sdk/internal.(*workflowEnvironmentInterceptor).Await(0x1ab08a0?, {0x1ab08a0, 0xc000ec1290}, 0xc0005d7d60)
2024-05-20T19:36:41.9461411Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/workflow.go:364 +0xa5
2024-05-20T19:36:41.9462162Z go.temporal.io/sdk/internal.Await({0x1ab08a0, 0xc000ec1290}, 0xc000ec1290?)
2024-05-20T19:36:41.9462828Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/workflow.go:349 +0x58
2024-05-20T19:36:41.9463337Z go.temporal.io/sdk/workflow.Await(...)
2024-05-20T19:36:41.9463884Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/workflow/deterministic_wrappers.go:84
2024-05-20T19:36:41.9464834Z github.com/indeedeng/iwf/service/interpreter/temporal.(*workflowProvider).Await(0x0?, {0x1a9bb20?, 0xc0000ef840?}, 0x18383c8?)
2024-05-20T19:36:41.9465819Z    /home/runner/work/iwf/iwf/service/interpreter/temporal/workflowProvider.go:203 +0x7e
2024-05-20T19:36:41.9466636Z github.com/indeedeng/iwf/service/interpreter.executeState.func2({0x1a9bb20, 0xc0000ef840})
2024-05-20T19:36:41.9467374Z    /home/runner/work/iwf/iwf/service/interpreter/workflowImpl.go:579 +0x264
2024-05-20T19:36:41.9468236Z github.com/indeedeng/iwf/service/interpreter/temporal.(*workflowProvider).GoNamed.func1({0x1ab08a0?, 0xc000ec1290})
2024-05-20T19:36:41.9469153Z    /home/runner/work/iwf/iwf/service/interpreter/temporal/workflowProvider.go:180 +0xfd
2024-05-20T19:36:41.9469925Z go.temporal.io/sdk/internal.(*dispatcherImpl).NewCoroutine.func1(0xc00046d060?)
2024-05-20T19:36:41.9470666Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:1057 +0xcf
2024-05-20T19:36:41.9471421Z created by go.temporal.io/sdk/internal.(*dispatcherImpl).NewCoroutine in goroutine 6406
2024-05-20T19:36:41.9472290Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:1048 +0x338
2024-05-20T19:36:41.9472749Z 
2024-05-20T19:36:41.9472851Z goroutine 15205 [select]:
2024-05-20T19:36:41.9473336Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000d47980, {0x1?, 0x0?}, 0xc000c18c60)
2024-05-20T19:36:41.9474373Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9475224Z go.uber.org/cadence/internal.(*workflowTaskPoller).PollTask(0xc000d47980)
2024-05-20T19:36:41.9476162Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:298 +0x65
2024-05-20T19:36:41.9477140Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc480)
2024-05-20T19:36:41.9478054Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9478965Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc480)
2024-05-20T19:36:41.9479881Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9480728Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9481842Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9482608Z 
2024-05-20T19:36:41.9482715Z goroutine 15317 [IO wait]:
2024-05-20T19:36:41.9483076Z internal/poll.runtime_pollWait(0x7fd78727ead0, 0x72)
2024-05-20T19:36:41.9483601Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/netpoll.go:343 +0x85
2024-05-20T19:36:41.9484168Z internal/poll.(*pollDesc).wait(0xc000e83e80?, 0xc000394000?, 0x0)
2024-05-20T19:36:41.9484785Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-05-20T19:36:41.9485316Z internal/poll.(*pollDesc).waitRead(...)
2024-05-20T19:36:41.9485801Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:89
2024-05-20T19:36:41.9486505Z internal/poll.(*FD).Read(0xc000e83e80, {0xc000394000, 0x1000, 0x1000})
2024-05-20T19:36:41.9487112Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_unix.go:164 +0x27a
2024-05-20T19:36:41.9487701Z net.(*netFD).Read(0xc000e83e80, {0xc000394000?, 0x7b8820?, 0x26c9ae0?})
2024-05-20T19:36:41.9488246Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/fd_posix.go:55 +0x25
2024-05-20T19:36:41.9488838Z net.(*conn).Read(0xc000ba8348, {0xc000394000?, 0x0?, 0xc00040da00?})
2024-05-20T19:36:41.9489351Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/net.go:179 +0x45
2024-05-20T19:36:41.9489947Z net/http.(*persistConn).Read(0xc0005fbb00, {0xc000394000?, 0xc001593260?, 0xc000d63d38?})
2024-05-20T19:36:41.9490598Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:1954 +0x4a
2024-05-20T19:36:41.9491075Z bufio.(*Reader).fill(0xc000608ae0)
2024-05-20T19:36:41.9491482Z    /opt/hostedtoolcache/go/1.21.10/x64/src/bufio/bufio.go:113 +0x103
2024-05-20T19:36:41.9491929Z bufio.(*Reader).Peek(0xc000608ae0, 0x1)
2024-05-20T19:36:41.9492366Z    /opt/hostedtoolcache/go/1.21.10/x64/src/bufio/bufio.go:151 +0x53
2024-05-20T19:36:41.9492829Z net/http.(*persistConn).readLoop(0xc0005fbb00)
2024-05-20T19:36:41.9493328Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:2118 +0x1b9
2024-05-20T19:36:41.9493890Z created by net/http.(*Transport).dialConn in goroutine 15265
2024-05-20T19:36:41.9494451Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/transport.go:1776 +0x169f
2024-05-20T19:36:41.9494833Z 
2024-05-20T19:36:41.9494959Z goroutine 6555 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9495695Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc0000f0640, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9496898Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9497787Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9498743Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9499664Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc0011770b0}, 0xc0011775c0)
2024-05-20T19:36:41.9500571Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9501283Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9502064Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9503274Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x7fd73c35eb38?, {0x1a9bb20?, 0xc000a91370?}, 0x186e6c3?)
2024-05-20T19:36:41.9504278Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9505120Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func4({0x1a9bb20, 0xc000a91370})
2024-05-20T19:36:41.9505906Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:123 +0x10a
2024-05-20T19:36:41.9506893Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc0011770b0})
2024-05-20T19:36:41.9507905Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9508790Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000edd830?)
2024-05-20T19:36:41.9509773Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9510685Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 6551
2024-05-20T19:36:41.9511696Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9512271Z 
2024-05-20T19:36:41.9512379Z goroutine 15216 [select]:
2024-05-20T19:36:41.9512873Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000fce580, {0xe0?, 0x53?}, 0xc001756438)
2024-05-20T19:36:41.9514093Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9515052Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask(0xc000fce580)
2024-05-20T19:36:41.9516099Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1142 +0xb5
2024-05-20T19:36:41.9517013Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc000)
2024-05-20T19:36:41.9517924Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9518734Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc000)
2024-05-20T19:36:41.9519619Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9520474Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9521426Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9522010Z 
2024-05-20T19:36:41.9522109Z goroutine 15433 [select]:
2024-05-20T19:36:41.9522576Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTunnel).getTask(...)
2024-05-20T19:36:41.9523517Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:193
2024-05-20T19:36:41.9524641Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).pollLocallyDispatchedActivity(0xc000fce580, {0xc00037c710?, 0x26b40f0?})
2024-05-20T19:36:41.9525872Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1151 +0x8b
2024-05-20T19:36:41.9526989Z go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0xc000fce580, {0x1ab0558?, 0xc00043aee0?}, 0x3?)
2024-05-20T19:36:41.9528107Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1048 +0x42
2024-05-20T19:36:41.9529310Z go.uber.org/cadence/internal.(*locallyDispatchedActivityTaskPoller).PollTask.(*activityTaskPoller).pollWithMetricsFunc.func1({0x1ab0558?, 0xc00043aee0?})
2024-05-20T19:36:41.9530716Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1042 +0x2e
2024-05-20T19:36:41.9531506Z go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
2024-05-20T19:36:41.9532374Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:256 +0x4a
2024-05-20T19:36:41.9533240Z created by go.uber.org/cadence/internal.(*basePoller).doPoll in goroutine 15238
2024-05-20T19:36:41.9534208Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:255 +0x1b7
2024-05-20T19:36:41.9534804Z 
2024-05-20T19:36:41.9534902Z goroutine 15444 [IO wait]:
2024-05-20T19:36:41.9535394Z internal/poll.runtime_pollWait(0x7fd78727eeb0, 0x72)
2024-05-20T19:36:41.9535908Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/netpoll.go:343 +0x85
2024-05-20T19:36:41.9536664Z internal/poll.(*pollDesc).wait(0xc000e82700?, 0xc0010b45b1?, 0x0)
2024-05-20T19:36:41.9537539Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-05-20T19:36:41.9538070Z internal/poll.(*pollDesc).waitRead(...)
2024-05-20T19:36:41.9538562Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:89
2024-05-20T19:36:41.9539145Z internal/poll.(*FD).Read(0xc000e82700, {0xc0010b45b1, 0x1, 0x1})
2024-05-20T19:36:41.9539710Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_unix.go:164 +0x27a
2024-05-20T19:36:41.9540314Z net.(*netFD).Read(0xc000e82700, {0xc0010b45b1?, 0xc00092bf40?, 0x470930?})
2024-05-20T19:36:41.9540874Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/fd_posix.go:55 +0x25
2024-05-20T19:36:41.9541483Z net.(*conn).Read(0xc000ba80c8, {0xc0010b45b1?, 0x0?, 0xc0010b45a8?})
2024-05-20T19:36:41.9542006Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/net.go:179 +0x45
2024-05-20T19:36:41.9542468Z net/http.(*connReader).backgroundRead(0xc0010b45a0)
2024-05-20T19:36:41.9542972Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/server.go:683 +0x37
2024-05-20T19:36:41.9543557Z created by net/http.(*connReader).startBackgroundRead in goroutine 15247
2024-05-20T19:36:41.9544224Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/server.go:679 +0xba
2024-05-20T19:36:41.9544571Z 
2024-05-20T19:36:41.9544675Z goroutine 15166 [IO wait]:
2024-05-20T19:36:41.9545009Z internal/poll.runtime_pollWait(0x7fd78727e5f8, 0x72)
2024-05-20T19:36:41.9545516Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/netpoll.go:343 +0x85
2024-05-20T19:36:41.9546062Z internal/poll.(*pollDesc).wait(0xc000d25900?, 0xc000746000?, 0x0)
2024-05-20T19:36:41.9546665Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
2024-05-20T19:36:41.9547189Z internal/poll.(*pollDesc).waitRead(...)
2024-05-20T19:36:41.9547677Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_poll_runtime.go:89
2024-05-20T19:36:41.9548271Z internal/poll.(*FD).Read(0xc000d25900, {0xc000746000, 0x8000, 0x8000})
2024-05-20T19:36:41.9548862Z    /opt/hostedtoolcache/go/1.21.10/x64/src/internal/poll/fd_unix.go:164 +0x27a
2024-05-20T19:36:41.9549637Z net.(*netFD).Read(0xc000d25900, {0xc000746000?, 0x5010100000001?, 0xc00082ed78?})
2024-05-20T19:36:41.9550312Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/fd_posix.go:55 +0x25
2024-05-20T19:36:41.9550839Z net.(*conn).Read(0xc000ba9f78, {0xc000746000?, 0xc000c880a0?, 0x0?})
2024-05-20T19:36:41.9551356Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/net.go:179 +0x45
2024-05-20T19:36:41.9551879Z bufio.(*Reader).Read(0xc00189a060, {0xc000302040, 0x9, 0xc300001bc2?})
2024-05-20T19:36:41.9552541Z    /opt/hostedtoolcache/go/1.21.10/x64/src/bufio/bufio.go:244 +0x197
2024-05-20T19:36:41.9553104Z io.ReadAtLeast({0x1a9ae40, 0xc00189a060}, {0xc000302040, 0x9, 0x9}, 0x9)
2024-05-20T19:36:41.9553636Z    /opt/hostedtoolcache/go/1.21.10/x64/src/io/io.go:335 +0x90
2024-05-20T19:36:41.9554022Z io.ReadFull(...)
2024-05-20T19:36:41.9554323Z    /opt/hostedtoolcache/go/1.21.10/x64/src/io/io.go:354
2024-05-20T19:36:41.9554969Z golang.org/x/net/http2.readFrameHeader({0xc000302040, 0x9, 0xc000c88090?}, {0x1a9ae40?, 0xc00189a060?})
2024-05-20T19:36:41.9555708Z    /home/runner/go/pkg/mod/golang.org/x/net@v0.21.0/http2/frame.go:237 +0x65
2024-05-20T19:36:41.9556263Z golang.org/x/net/http2.(*Framer).ReadFrame(0xc000302000)
2024-05-20T19:36:41.9556805Z    /home/runner/go/pkg/mod/golang.org/x/net@v0.21.0/http2/frame.go:498 +0x85
2024-05-20T19:36:41.9557509Z google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000512000, 0xc000835320?)
2024-05-20T19:36:41.9558354Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:1595 +0x245
2024-05-20T19:36:41.9559185Z created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 15190
2024-05-20T19:36:41.9560023Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/http2_client.go:397 +0x1e9e
2024-05-20T19:36:41.9560525Z 
2024-05-20T19:36:41.9560619Z goroutine 15472 [select]:
2024-05-20T19:36:41.9561146Z google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000b265a0)
2024-05-20T19:36:41.9561920Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x7c
2024-05-20T19:36:41.9562643Z google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
2024-05-20T19:36:41.9563343Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346
2024-05-20T19:36:41.9564149Z google.golang.org/grpc.(*csAttempt).recvMsg(0xc000ae7450, {0x155e160?, 0xc0004fc300}, 0xc0009c4558?)
2024-05-20T19:36:41.9564879Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0xc9
2024-05-20T19:36:41.9565470Z google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x2712020?)
2024-05-20T19:36:41.9566124Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x1f
2024-05-20T19:36:41.9566818Z google.golang.org/grpc.(*clientStream).withRetry(0xc000b26360, 0xc0009c4588, 0xc0009c4558)
2024-05-20T19:36:41.9567507Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x13a
2024-05-20T19:36:41.9568191Z google.golang.org/grpc.(*clientStream).RecvMsg(0xc000b26360, {0x155e160?, 0xc0004fc300?})
2024-05-20T19:36:41.9568940Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x12e
2024-05-20T19:36:41.9569961Z google.golang.org/grpc.invoke({0x1ab04b0?, 0xc000f18180?}, {0xc0008fce40?, 0x412a05?}, {0x155f420, 0xc0004fc348}, {0x155e160, 0xc0004fc300}, 0x412d85?, {0xc001639100, ...})
2024-05-20T19:36:41.9570886Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0xcb
2024-05-20T19:36:41.9571865Z google.golang.org/grpc.(*ClientConn).Invoke(0xc000814800, {0x1ab04b0, 0xc000f18180}, {0xc0008fce40, 0x32}, {0x155f420, 0xc0004fc348}, {0x155e160, 0xc0004fc300}, {0xc000842c10, ...})
2024-05-20T19:36:41.9572843Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:42 +0x2cb
2024-05-20T19:36:41.9573819Z go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0xc0013b6ab0, {0x1ab0558, 0xc0004ca000}, 0xc000a34d80, 0x1579640?, 0xc0015848b0, {0x76?, 0xc001019388?, 0x26e0980?})
2024-05-20T19:36:41.9574850Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x4e5
2024-05-20T19:36:41.9575772Z go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0xc0013b6ab0, {0x1ab0558, 0xc0004ca000}, 0xc000a34d80)
2024-05-20T19:36:41.9576566Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0x126
2024-05-20T19:36:41.9577609Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558?, 0xc0004ca000?}, 0x184f9bf?)
2024-05-20T19:36:41.9578794Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
2024-05-20T19:36:41.9579811Z go.uber.org/yarpc/internal/observability.(*Middleware).Call(0xc00183bc80, {0x1ab0558, 0xc0004ca000}, 0xc000a34d80, {0x1ab4e00, 0xc000f18090})
2024-05-20T19:36:41.9580810Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0x162
2024-05-20T19:36:41.9581905Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc0004ca000}, 0xc0013b6ab0?)
2024-05-20T19:36:41.9582991Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9584045Z go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x17706e0?, {0x1ab0558, 0xc0004ca000}, 0xc000a34d80, {0x1ab4e00?, 0xc000f18060})
2024-05-20T19:36:41.9585145Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xa8
2024-05-20T19:36:41.9586449Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x1, 0x1}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc0004ca000}, 0xc000070c00?)
2024-05-20T19:36:41.9587550Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9588389Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
2024-05-20T19:36:41.9589081Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
2024-05-20T19:36:41.9589809Z go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
2024-05-20T19:36:41.9590486Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
2024-05-20T19:36:41.9591509Z go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x1ab4c80?, 0xc0013b6c30?}, {0x7fd73c34b008?, 0xc0013b6c30?}}, {0x1ab0558, 0xc0004ca000}, 0xc000b1f590?)
2024-05-20T19:36:41.9592607Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x92
2024-05-20T19:36:41.9593789Z go.uber.org/yarpc/encoding/protobuf.(*client).Call(0xc0013b7260, {0x1ab0558?, 0xc0004ca000?}, {0x18556e8?, 0x4c?}, {0x1aa9d00?, 0xc000d7a420?}, 0x19038a0, {0xc000d7a3c0, 0x4, ...})
2024-05-20T19:36:41.9594851Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0x10c
2024-05-20T19:36:41.9596245Z github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForDecisionTask(0xc0004d0940?, {0x1ab0558?, 0xc0004ca000?}, 0xc000cfbd20?, {0xc000d7a3c0?, 0xc000cfbcf0?, 0x184f117?})
2024-05-20T19:36:41.9598420Z    /home/runner/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220713235846-fda89e95df1e/go/proto/api/v1/service_worker.pb.yarpc.go:361 +0x66
2024-05-20T19:36:41.9601134Z go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForDecisionTask({{0x1ab3c80, 0xc0010956f0}, {0x1abd1d8, 0xc001095710}, {0x1abd260, 0xc001095730}, {0x1ab7ea0, 0xc001095750}}, {0x1ab0558, 0xc0004ca000}, ...)
2024-05-20T19:36:41.9603034Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/compatibility/adapter.go:126 +0x95
2024-05-20T19:36:41.9604356Z go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForDecisionTask(0xc001586fc0, {0x1ab0558, 0xc0004ca000}, 0xc0004d0740?, {0xc000d7a3c0, 0x4, 0x6})
2024-05-20T19:36:41.9605776Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/metrics/service_wrapper.go:220 +0x9d
2024-05-20T19:36:41.9606763Z go.uber.org/cadence/internal.(*workflowTaskPoller).poll(0xc000d47980, {0x1ab0558, 0xc0004ca000})
2024-05-20T19:36:41.9607796Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:766 +0x25d
2024-05-20T19:36:41.9608579Z go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
2024-05-20T19:36:41.9609577Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:256 +0x4a
2024-05-20T19:36:41.9610452Z created by go.uber.org/cadence/internal.(*basePoller).doPoll in goroutine 15210
2024-05-20T19:36:41.9611417Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:255 +0x1b7
2024-05-20T19:36:41.9612014Z 
2024-05-20T19:36:41.9612119Z goroutine 15571 [chan receive]:
2024-05-20T19:36:41.9612663Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc000796580, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9613667Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9614411Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9615397Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9616216Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc000a75380}, 0xc000a75920)
2024-05-20T19:36:41.9617101Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9617752Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9618518Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9619766Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x7fd73c2b05c0?, {0x1a9bb20?, 0xc000ff3d20?}, 0x186e6c3?)
2024-05-20T19:36:41.9620764Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9621602Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func4({0x1a9bb20, 0xc000ff3d20})
2024-05-20T19:36:41.9622386Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:123 +0x10a
2024-05-20T19:36:41.9623251Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc000a75380})
2024-05-20T19:36:41.9624153Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9625012Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc001817630?)
2024-05-20T19:36:41.9626005Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9626910Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 15556
2024-05-20T19:36:41.9627997Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9628582Z 
2024-05-20T19:36:41.9628686Z goroutine 15521 [chan receive]:
2024-05-20T19:36:41.9629228Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc000796500, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9630216Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9630971Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9631980Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9632940Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc000a75230}, 0xc000a758c0)
2024-05-20T19:36:41.9633946Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9634692Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9635569Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9636733Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x10?, {0x1a9bb20?, 0xc000ff3ce0?}, 0x1865c58?)
2024-05-20T19:36:41.9637868Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9638948Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func2({0x1a9bb20, 0xc000ff3ce0})
2024-05-20T19:36:41.9639823Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:75 +0x1c4
2024-05-20T19:36:41.9640774Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc000a75230})
2024-05-20T19:36:41.9641829Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9642708Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc001427110?)
2024-05-20T19:36:41.9643727Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9644746Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 15556
2024-05-20T19:36:41.9645948Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9646532Z 
2024-05-20T19:36:41.9646691Z goroutine 15523 [select]:
2024-05-20T19:36:41.9647179Z google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000bdb200)
2024-05-20T19:36:41.9648057Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x7c
2024-05-20T19:36:41.9648888Z google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
2024-05-20T19:36:41.9649817Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346
2024-05-20T19:36:41.9650787Z google.golang.org/grpc.(*csAttempt).recvMsg(0xc001549c70, {0x155e160?, 0xc00139ed38}, 0xc00083a4f8?)
2024-05-20T19:36:41.9651591Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0xc9
2024-05-20T19:36:41.9652219Z google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x2712020?)
2024-05-20T19:36:41.9652915Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x1f
2024-05-20T19:36:41.9653696Z google.golang.org/grpc.(*clientStream).withRetry(0xc000bdafc0, 0xc00083a528, 0xc00083a4f8)
2024-05-20T19:36:41.9654481Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x13a
2024-05-20T19:36:41.9655558Z google.golang.org/grpc.(*clientStream).RecvMsg(0xc000bdafc0, {0x155e160?, 0xc00139ed38?})
2024-05-20T19:36:41.9656544Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x12e
2024-05-20T19:36:41.9657799Z google.golang.org/grpc.invoke({0x1ab04b0?, 0xc000bd3cb0?}, {0xc000ea8b40?, 0x412a05?}, {0x155f420, 0xc00139ed80}, {0x155e160, 0xc00139ed38}, 0x412d85?, {0xc000f06700, ...})
2024-05-20T19:36:41.9658859Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0xcb
2024-05-20T19:36:41.9659937Z google.golang.org/grpc.(*ClientConn).Invoke(0xc000814800, {0x1ab04b0, 0xc000bd3cb0}, {0xc000ea8b40, 0x32}, {0x155f420, 0xc00139ed80}, {0x155e160, 0xc00139ed38}, {0xc0011b7310, ...})
2024-05-20T19:36:41.9661106Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:42 +0x2cb
2024-05-20T19:36:41.9662242Z go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0xc0013b6ab0, {0x1ab0558, 0xc0004fa460}, 0xc000bdd680, 0x1838f92?, 0xc000e78230, {0xc18b07726dd78d61?, 0xc001018ea8?, 0x26e0980?})
2024-05-20T19:36:41.9663414Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x4e5
2024-05-20T19:36:41.9664224Z go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0xc0013b6ab0, {0x1ab0558, 0xc0004fa460}, 0xc000bdd680)
2024-05-20T19:36:41.9665117Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0x126
2024-05-20T19:36:41.9666352Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558?, 0xc0004fa460?}, 0x184f9bf?)
2024-05-20T19:36:41.9667545Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
2024-05-20T19:36:41.9668732Z go.uber.org/yarpc/internal/observability.(*Middleware).Call(0xc00183bc80, {0x1ab0558, 0xc0004fa460}, 0xc000bdd680, {0x1ab4e00, 0xc000bd3bc0})
2024-05-20T19:36:41.9669989Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0x162
2024-05-20T19:36:41.9671290Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc0004fa460}, 0xc0013b6ab0?)
2024-05-20T19:36:41.9672573Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9673726Z go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x17706e0?, {0x1ab0558, 0xc0004fa460}, 0xc000bdd680, {0x1ab4e00?, 0xc000bd3b90})
2024-05-20T19:36:41.9675034Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xa8
2024-05-20T19:36:41.9676274Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x1, 0x1}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc0004fa460}, 0xc0000a0000?)
2024-05-20T19:36:41.9677565Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9678396Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
2024-05-20T19:36:41.9679088Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
2024-05-20T19:36:41.9679816Z go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
2024-05-20T19:36:41.9680710Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
2024-05-20T19:36:41.9681923Z go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x1ab4c80?, 0xc0013b6c30?}, {0x7fd73c34b008?, 0xc0013b6c30?}}, {0x1ab0558, 0xc0004fa460}, 0xc00044c000?)
2024-05-20T19:36:41.9683106Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x92
2024-05-20T19:36:41.9684216Z go.uber.org/yarpc/encoding/protobuf.(*client).Call(0xc0013b7260, {0x1ab0558?, 0xc0004fa460?}, {0x18556fb?, 0x7?}, {0x1aa9be0?, 0xc00052c5a0?}, 0x1903898, {0xc000f512c0, 0x4, ...})
2024-05-20T19:36:41.9685262Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0x10c
2024-05-20T19:36:41.9686669Z github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForActivityTask(0xc00109b8c0?, {0x1ab0558?, 0xc0004fa460?}, 0xc0009bdcc0?, {0xc000f512c0?, 0xc0009bdc90?, 0x184f117?})
2024-05-20T19:36:41.9688167Z    /home/runner/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220713235846-fda89e95df1e/go/proto/api/v1/service_worker.pb.yarpc.go:397 +0x66
2024-05-20T19:36:41.9689975Z go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForActivityTask({{0x1ab3c80, 0xc0010956f0}, {0x1abd1d8, 0xc001095710}, {0x1abd260, 0xc001095730}, {0x1ab7ea0, 0xc001095750}}, {0x1ab0558, 0xc0004fa460}, ...)
2024-05-20T19:36:41.9691765Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/compatibility/adapter.go:121 +0x95
2024-05-20T19:36:41.9693213Z go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForActivityTask(0xc001586fc0, {0x1ab0558, 0xc0004fa460}, 0x6?, {0xc000f512c0, 0x4, 0x6})
2024-05-20T19:36:41.9694799Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/metrics/service_wrapper.go:213 +0x9d
2024-05-20T19:36:41.9696114Z go.uber.org/cadence/internal.(*activityTaskPoller).poll(0xc000fce500, {0x1ab0558, 0xc0004fa460})
2024-05-20T19:36:41.9697348Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1005 +0x33e
2024-05-20T19:36:41.9698501Z go.uber.org/cadence/internal.(*activityTaskPoller).pollWithMetrics(0xc000fce500, {0x1ab0558?, 0xc0004fa460?}, 0xc00044c000?)
2024-05-20T19:36:41.9699848Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1048 +0x42
2024-05-20T19:36:41.9701056Z go.uber.org/cadence/internal.(*activityTaskPoller).PollTask.(*activityTaskPoller).pollWithMetricsFunc.func1({0x1ab0558?, 0xc0004fa460?})
2024-05-20T19:36:41.9702628Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1042 +0x2e
2024-05-20T19:36:41.9703411Z go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
2024-05-20T19:36:41.9704280Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:256 +0x4a
2024-05-20T19:36:41.9705150Z created by go.uber.org/cadence/internal.(*basePoller).doPoll in goroutine 15223
2024-05-20T19:36:41.9706123Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:255 +0x1b7
2024-05-20T19:36:41.9706720Z 
2024-05-20T19:36:41.9706841Z goroutine 6567 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9707419Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc000a64700, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9708425Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9709177Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9709981Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9710885Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc0013a15c0}, 0xc000bca410)
2024-05-20T19:36:41.9712030Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9712744Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9713659Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9714897Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0xc000fa4f00?, {0x1a9bb20?, 0xc000d38510?}, 0x18383c8?)
2024-05-20T19:36:41.9716010Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9716957Z github.com/indeedeng/iwf/service/interpreter.executeState.func2({0x1a9bb20, 0xc000d38510})
2024-05-20T19:36:41.9717891Z    /home/runner/work/iwf/iwf/service/interpreter/workflowImpl.go:579 +0x264
2024-05-20T19:36:41.9718929Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc0013a15c0})
2024-05-20T19:36:41.9719952Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9720959Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000d38280?)
2024-05-20T19:36:41.9722115Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9723134Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 6558
2024-05-20T19:36:41.9724331Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9724994Z 
2024-05-20T19:36:41.9725098Z goroutine 15203 [select]:
2024-05-20T19:36:41.9725604Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000d47980, {0x1?, 0x0?}, 0xc0007ddf80)
2024-05-20T19:36:41.9726772Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9727715Z go.uber.org/cadence/internal.(*workflowTaskPoller).PollTask(0xc000d47980)
2024-05-20T19:36:41.9728753Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:298 +0x65
2024-05-20T19:36:41.9729823Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc480)
2024-05-20T19:36:41.9730748Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9731590Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc480)
2024-05-20T19:36:41.9732596Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9733442Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9734392Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9734966Z 
2024-05-20T19:36:41.9735073Z goroutine 15497 [select]:
2024-05-20T19:36:41.9735667Z google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000c77320)
2024-05-20T19:36:41.9736447Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x7c
2024-05-20T19:36:41.9737185Z google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
2024-05-20T19:36:41.9737893Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346
2024-05-20T19:36:41.9738695Z google.golang.org/grpc.(*csAttempt).recvMsg(0xc000bac340, {0x155e160?, 0xc000f708d0}, 0xc000b7c558?)
2024-05-20T19:36:41.9739432Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0xc9
2024-05-20T19:36:41.9740035Z google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x2712020?)
2024-05-20T19:36:41.9740625Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x1f
2024-05-20T19:36:41.9741314Z google.golang.org/grpc.(*clientStream).withRetry(0xc000c770e0, 0xc000b7c588, 0xc000b7c558)
2024-05-20T19:36:41.9742142Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x13a
2024-05-20T19:36:41.9742829Z google.golang.org/grpc.(*clientStream).RecvMsg(0xc000c770e0, {0x155e160?, 0xc000f708d0?})
2024-05-20T19:36:41.9743507Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x12e
2024-05-20T19:36:41.9744471Z google.golang.org/grpc.invoke({0x1ab04b0?, 0xc000ef5500?}, {0xc0006f2a40?, 0x412a05?}, {0x155f420, 0xc000f70918}, {0x155e160, 0xc000f708d0}, 0x412d85?, {0xc000e12580, ...})
2024-05-20T19:36:41.9745403Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0xcb
2024-05-20T19:36:41.9746451Z google.golang.org/grpc.(*ClientConn).Invoke(0xc000814800, {0x1ab04b0, 0xc000ef5500}, {0xc0006f2a40, 0x32}, {0x155f420, 0xc000f70918}, {0x155e160, 0xc000f708d0}, {0xc0016940f0, ...})
2024-05-20T19:36:41.9747436Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:42 +0x2cb
2024-05-20T19:36:41.9748424Z go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0xc0013b6ab0, {0x1ab0558, 0xc000452af0}, 0xc000c552c0, 0x1500222?, 0xc001776d00, {0x16e6b60?, 0xc001019388?, 0x26e0980?})
2024-05-20T19:36:41.9749528Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x4e5
2024-05-20T19:36:41.9750331Z go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0xc0013b6ab0, {0x1ab0558, 0xc000452af0}, 0xc000c552c0)
2024-05-20T19:36:41.9751303Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0x126
2024-05-20T19:36:41.9752371Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558?, 0xc000452af0?}, 0x184f9bf?)
2024-05-20T19:36:41.9753467Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
2024-05-20T19:36:41.9753983Z go.uber.org/yarpc/internal/observability.(*Middleware).Call(0xc00183bc80, {0x1ab0558, 0xc000452af0}, 0xc000c552c0, {0x1ab4e00, 0xc000ef5410})
2024-05-20T19:36:41.9754353Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0x162
2024-05-20T19:36:41.9754970Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc000452af0}, 0xc0013b6ab0?)
2024-05-20T19:36:41.9755331Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9755903Z go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x17706e0?, {0x1ab0558, 0xc000452af0}, 0xc000c552c0, {0x1ab4e00?, 0xc000ef53e0})
2024-05-20T19:36:41.9756439Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xa8
2024-05-20T19:36:41.9757057Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x1, 0x1}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc000452af0}, 0xc000814c00?)
2024-05-20T19:36:41.9757429Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9757677Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
2024-05-20T19:36:41.9758002Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
2024-05-20T19:36:41.9758278Z go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
2024-05-20T19:36:41.9758559Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
2024-05-20T19:36:41.9759160Z go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x1ab4c80?, 0xc0013b6c30?}, {0x7fd73c34b008?, 0xc0013b6c30?}}, {0x1ab0558, 0xc000452af0}, 0x0?)
2024-05-20T19:36:41.9759541Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x92
2024-05-20T19:36:41.9760162Z go.uber.org/yarpc/encoding/protobuf.(*client).Call(0xc0013b7260, {0x1ab0558?, 0xc000452af0?}, {0x18556e8?, 0x4c?}, {0x1aa9d00?, 0xc0018a7b60?}, 0x19038a0, {0xc0018a7b00, 0x4, ...})
2024-05-20T19:36:41.9760558Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0x10c
2024-05-20T19:36:41.9761431Z github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkerAPIYARPCCaller).PollForDecisionTask(0xc001563b20?, {0x1ab0558?, 0xc000452af0?}, 0xc000c36d20?, {0xc0018a7b00?, 0xc000c36cf0?, 0x184f117?})
2024-05-20T19:36:41.9762109Z    /home/runner/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220713235846-fda89e95df1e/go/proto/api/v1/service_worker.pb.yarpc.go:361 +0x66
2024-05-20T19:36:41.9762965Z go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.PollForDecisionTask({{0x1ab3c80, 0xc0010956f0}, {0x1abd1d8, 0xc001095710}, {0x1abd260, 0xc001095730}, {0x1ab7ea0, 0xc001095750}}, {0x1ab0558, 0xc000452af0}, ...)
2024-05-20T19:36:41.9763602Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/compatibility/adapter.go:126 +0x95
2024-05-20T19:36:41.9764304Z go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).PollForDecisionTask(0xc001586fc0, {0x1ab0558, 0xc000452af0}, 0xc001563540?, {0xc0018a7b00, 0x4, 0x6})
2024-05-20T19:36:41.9764976Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/metrics/service_wrapper.go:220 +0x9d
2024-05-20T19:36:41.9765327Z go.uber.org/cadence/internal.(*workflowTaskPoller).poll(0xc000d47980, {0x1ab0558, 0xc000452af0})
2024-05-20T19:36:41.9765893Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:766 +0x25d
2024-05-20T19:36:41.9766081Z go.uber.org/cadence/internal.(*basePoller).doPoll.func1()
2024-05-20T19:36:41.9766643Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:256 +0x4a
2024-05-20T19:36:41.9766921Z created by go.uber.org/cadence/internal.(*basePoller).doPoll in goroutine 15203
2024-05-20T19:36:41.9767483Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:255 +0x1b7
2024-05-20T19:36:41.9767500Z 
2024-05-20T19:36:41.9767605Z goroutine 11473 [chan receive]:
2024-05-20T19:36:41.9767944Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc00060d880, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9768485Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9768668Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9769174Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9769502Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc000ae5aa0}, 0xc000ae5f20)
2024-05-20T19:36:41.9770034Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9770159Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9770688Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9771214Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x1abd260?, {0x1a9bb20?, 0xc001190ce0?}, 0x185a519?)
2024-05-20T19:36:41.9771560Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9771935Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func5({0x1a9bb20, 0xc001190ce0})
2024-05-20T19:36:41.9772221Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:143 +0x1a9
2024-05-20T19:36:41.9772684Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc000ae5aa0})
2024-05-20T19:36:41.9773018Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9773326Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc0010e3030?)
2024-05-20T19:36:41.9773862Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9774282Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 11468
2024-05-20T19:36:41.9774834Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9774839Z 
2024-05-20T19:36:41.9774936Z goroutine 15225 [select]:
2024-05-20T19:36:41.9775414Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000fce500, {0xe0?, 0x53?}, 0xc000f71e60)
2024-05-20T19:36:41.9776000Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9776339Z go.uber.org/cadence/internal.(*activityTaskPoller).PollTask(0xc000fce500)
2024-05-20T19:36:41.9776912Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:1071 +0xb5
2024-05-20T19:36:41.9777132Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fc9b00)
2024-05-20T19:36:41.9777758Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9777990Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fc9b00)
2024-05-20T19:36:41.9778539Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9778818Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9779367Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9779372Z 
2024-05-20T19:36:41.9779470Z goroutine 15518 [select]:
2024-05-20T19:36:41.9779758Z google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000cd2480)
2024-05-20T19:36:41.9780128Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:331 +0x7c
2024-05-20T19:36:41.9780371Z google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
2024-05-20T19:36:41.9780718Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/internal/transport/transport.go:346
2024-05-20T19:36:41.9781058Z google.golang.org/grpc.(*csAttempt).recvMsg(0xc0005f3e10, {0x155e160?, 0xc0007d0018}, 0xc0009999c0?)
2024-05-20T19:36:41.9781332Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:1066 +0xc9
2024-05-20T19:36:41.9781541Z google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x2712020?)
2024-05-20T19:36:41.9781877Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:917 +0x1f
2024-05-20T19:36:41.9782186Z google.golang.org/grpc.(*clientStream).withRetry(0xc000cd2240, 0xc0009999f0, 0xc0009999c0)
2024-05-20T19:36:41.9782453Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:768 +0x13a
2024-05-20T19:36:41.9782749Z google.golang.org/grpc.(*clientStream).RecvMsg(0xc000cd2240, {0x155e160?, 0xc0007d0018?})
2024-05-20T19:36:41.9783021Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/stream.go:916 +0x12e
2024-05-20T19:36:41.9783596Z google.golang.org/grpc.invoke({0x1ab04b0?, 0xc000a74c00?}, {0xc001821d10?, 0x412a05?}, {0x155f420, 0xc0007d00c0}, {0x155e160, 0xc0007d0018}, 0x412d85?, {0xc000796240, ...})
2024-05-20T19:36:41.9783863Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:78 +0xcb
2024-05-20T19:36:41.9784483Z google.golang.org/grpc.(*ClientConn).Invoke(0xc000814800, {0x1ab04b0, 0xc000a74c00}, {0xc001821d10, 0x2e}, {0x155f420, 0xc0007d00c0}, {0x155e160, 0xc0007d0018}, {0xc000ff3870, ...})
2024-05-20T19:36:41.9784740Z    /home/runner/go/pkg/mod/google.golang.org/grpc@v1.57.0/call.go:42 +0x2cb
2024-05-20T19:36:41.9785339Z go.uber.org/yarpc/transport/grpc.(*Outbound).invoke(0xc0013b6ab0, {0x1ab0558, 0xc00043ee00}, 0xc000c598c0, 0x5?, 0xc000707368, {0x40b7e9?, 0xc0010188f8?, 0x26e0980?})
2024-05-20T19:36:41.9785655Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:210 +0x4e5
2024-05-20T19:36:41.9786099Z go.uber.org/yarpc/transport/grpc.(*Outbound).Call(0xc0013b6ab0, {0x1ab0558, 0xc00043ee00}, 0xc000c598c0)
2024-05-20T19:36:41.9786412Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/transport/grpc/outbound.go:123 +0x126
2024-05-20T19:36:41.9787029Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558?, 0xc00043ee00?}, 0x184f9bf?)
2024-05-20T19:36:41.9787395Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:103 +0x5c
2024-05-20T19:36:41.9787914Z go.uber.org/yarpc/internal/observability.(*Middleware).Call(0xc00183bc80, {0x1ab0558, 0xc00043ee00}, 0xc000c598c0, {0x1ab4e00, 0xc000a74b10})
2024-05-20T19:36:41.9788337Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/observability/middleware.go:213 +0x162
2024-05-20T19:36:41.9788953Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x0, 0x0}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc00043ee00}, 0xc0013b6ab0?)
2024-05-20T19:36:41.9789318Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9789952Z go.uber.org/yarpc/internal/firstoutboundmiddleware.(*Middleware).Call(0x17706e0?, {0x1ab0558, 0xc00043ee00}, 0xc000c598c0, {0x1ab4e00?, 0xc000a74ae0})
2024-05-20T19:36:41.9790388Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/firstoutboundmiddleware/middleware.go:51 +0xa8
2024-05-20T19:36:41.9791010Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChainExec.Call({{0xc000690f30, 0x1, 0x1}, {0x1ab3440, 0xc0013b6ab0}}, {0x1ab0558, 0xc00043ee00}, 0xc000070c00?)
2024-05-20T19:36:41.9791393Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:107 +0xf7
2024-05-20T19:36:41.9791643Z go.uber.org/yarpc/internal/outboundmiddleware.unaryChain.Call(...)
2024-05-20T19:36:41.9791973Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/outboundmiddleware/chain.go:67
2024-05-20T19:36:41.9792257Z go.uber.org/yarpc/api/middleware.unaryOutboundWithMiddleware.Call(...)
2024-05-20T19:36:41.9792537Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/api/middleware/outbound.go:113
2024-05-20T19:36:41.9793151Z go.uber.org/yarpc/internal/request.UnaryValidatorOutbound.Call({{0x1ab4c80?, 0xc0013b6c30?}, {0x7fd73c34b008?, 0xc0013b6c30?}}, {0x1ab0558, 0xc00043ee00}, 0x53336e?)
2024-05-20T19:36:41.9793514Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/internal/request/validator_outbound.go:46 +0x92
2024-05-20T19:36:41.9794208Z go.uber.org/yarpc/encoding/protobuf.(*client).Call(0xc0013b7200, {0x1ab0558?, 0xc00043ee00?}, {0x184b747?, 0x0?}, {0x1aa94f0?, 0xc000b965f0?}, 0x1903938, {0xc00183ff80, 0x4, ...})
2024-05-20T19:36:41.9794535Z    /home/runner/go/pkg/mod/go.uber.org/yarpc@v1.60.0/encoding/protobuf/outbound.go:95 +0x10c
2024-05-20T19:36:41.9795365Z github.com/uber/cadence-idl/go/proto/api/v1.(*_WorkflowAPIYARPCCaller).QueryWorkflow(0xc000a749c0?, {0x1ab0558?, 0xc00043ee00?}, 0xc000751188?, {0xc00183ff80?, 0xc00183ff80?, 0x184f117?})
2024-05-20T19:36:41.9796053Z    /home/runner/go/pkg/mod/github.com/uber/cadence-idl@v0.0.0-20220713235846-fda89e95df1e/go/proto/api/v1/service_workflow.pb.yarpc.go:445 +0x66
2024-05-20T19:36:41.9796873Z go.uber.org/cadence/internal/compatibility.thrift2protoAdapter.QueryWorkflow({{0x1ab3c80, 0xc0010956f0}, {0x1abd1d8, 0xc001095710}, {0x1abd260, 0xc001095730}, {0x1ab7ea0, 0xc001095750}}, {0x1ab0558, 0xc00043ee00}, ...)
2024-05-20T19:36:41.9797449Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/compatibility/adapter.go:131 +0x95
2024-05-20T19:36:41.9798088Z go.uber.org/cadence/internal/common/metrics.(*workflowServiceMetricsWrapper).QueryWorkflow(0xc0013b7380, {0x1ab0558, 0xc00043ee00}, 0x0?, {0xc00183ff80, 0x4, 0x6})
2024-05-20T19:36:41.9798697Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/metrics/service_wrapper.go:353 +0x9d
2024-05-20T19:36:41.9799058Z go.uber.org/cadence/internal.(*workflowClient).QueryWorkflowWithOptions.func1()
2024-05-20T19:36:41.9799641Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow_client.go:965 +0x9c
2024-05-20T19:36:41.9800120Z go.uber.org/cadence/internal/common/backoff.Retry({0x1ab04e8, 0xc000b96550}, 0xc00099b478, {0x1a9ede0?, 0xc001821cb0}, 0x1903a78)
2024-05-20T19:36:41.9800680Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/backoff/retry.go:101 +0xe5
2024-05-20T19:36:41.9801150Z go.uber.org/cadence/internal.(*workflowClient).QueryWorkflowWithOptions(0xc001000880, {0x1ab04e8?, 0xc000b96550}, 0xc00183ff20)
2024-05-20T19:36:41.9801789Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow_client.go:960 +0x3d4
2024-05-20T19:36:41.9802550Z github.com/indeedeng/iwf/service/api/cadence.queryWorkflowWithStrongConsistency(0xc001182780, {0x1ab04e8, 0xc000b96550}, {0xc000696060, 0x20}, {0xc001821c80, 0x24}, {0x1850f8a, 0x11}, {0x0, ...})
2024-05-20T19:36:41.9802793Z    /home/runner/work/iwf/iwf/service/api/cadence/client.go:233 +0x1af
2024-05-20T19:36:41.9803647Z github.com/indeedeng/iwf/service/api/cadence.(*cadenceClient).QueryWorkflow(0x75?, {0x1ab04e8?, 0xc000b96550?}, {0x1547320, 0xc0006fb810}, {0xc000696060?, 0x1001311?}, {0xc001821c80?, 0xc00010b280?}, {0x1850f8a, ...}, ...)
2024-05-20T19:36:41.9803881Z    /home/runner/work/iwf/iwf/service/api/cadence/client.go:216 +0xaa
2024-05-20T19:36:41.9804528Z github.com/indeedeng/iwf/service/api.(*serviceImpl).ApiV1WorkflowDumpPost(0xc000f08d00, {0x1ab04e8, 0xc000b96550}, {{0xc000696060, 0x20}, {0xc001821c80, 0x24}, 0x100000, 0x0})
2024-05-20T19:36:41.9804731Z    /home/runner/work/iwf/iwf/service/api/service.go:782 +0x112
2024-05-20T19:36:41.9805106Z github.com/indeedeng/iwf/service/api.(*handler).apiV1WorkflowInternalDump(0xc000691540, 0xc000c00400)
2024-05-20T19:36:41.9805309Z    /home/runner/work/iwf/iwf/service/api/handler.go:121 +0x259
2024-05-20T19:36:41.9805486Z github.com/gin-gonic/gin.(*Context).Next(...)
2024-05-20T19:36:41.9805796Z    /home/runner/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/context.go:174
2024-05-20T19:36:41.9806097Z github.com/gin-gonic/gin.CustomRecoveryWithWriter.func1(0xc000c00400)
2024-05-20T19:36:41.9806430Z    /home/runner/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/recovery.go:102 +0x7a
2024-05-20T19:36:41.9806601Z github.com/gin-gonic/gin.(*Context).Next(...)
2024-05-20T19:36:41.9806893Z    /home/runner/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/context.go:174
2024-05-20T19:36:41.9807209Z github.com/gin-gonic/gin.LoggerWithConfig.func1(0xc000c00400)
2024-05-20T19:36:41.9807531Z    /home/runner/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/logger.go:240 +0xde
2024-05-20T19:36:41.9807699Z github.com/gin-gonic/gin.(*Context).Next(...)
2024-05-20T19:36:41.9807991Z    /home/runner/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/context.go:174
2024-05-20T19:36:41.9808325Z github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xc000ba3860, 0xc000c00400)
2024-05-20T19:36:41.9808629Z    /home/runner/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:620 +0x65b
2024-05-20T19:36:41.9809052Z github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xc000ba3860, {0x1aa59b0?, 0xc0006de2a0}, 0xc000c00300)
2024-05-20T19:36:41.9809356Z    /home/runner/go/pkg/mod/github.com/gin-gonic/gin@v1.9.1/gin.go:576 +0x1dd
2024-05-20T19:36:41.9809629Z net/http.serverHandler.ServeHTTP({0x1aa1d30?}, {0x1aa59b0?, 0xc0006de2a0?}, 0x6?)
2024-05-20T19:36:41.9809864Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/server.go:2938 +0x8e
2024-05-20T19:36:41.9810049Z net/http.(*conn).serve(0xc000679950, {0x1ab04b0, 0xc001586120})
2024-05-20T19:36:41.9810279Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/server.go:2009 +0x5f4
2024-05-20T19:36:41.9810454Z created by net/http.(*Server).Serve in goroutine 15192
2024-05-20T19:36:41.9810672Z    /opt/hostedtoolcache/go/1.21.10/x64/src/net/http/server.go:3086 +0x5cb
2024-05-20T19:36:41.9810677Z 
2024-05-20T19:36:41.9810851Z goroutine 15555 [chan receive]:
2024-05-20T19:36:41.9811030Z go.uber.org/cadence/internal.(*coroutineState).call(...)
2024-05-20T19:36:41.9811555Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:839
2024-05-20T19:36:41.9811855Z go.uber.org/cadence/internal.(*dispatcherImpl).ExecuteUntilAllBlocked(0xc0017c2480)
2024-05-20T19:36:41.9812394Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:923 +0x1a6
2024-05-20T19:36:41.9812759Z go.uber.org/cadence/internal.executeDispatcher({0x1ab2890, 0xc000d7aba0}, {0x1ab27e8, 0xc0017c2480})
2024-05-20T19:36:41.9813312Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:540 +0x8a
2024-05-20T19:36:41.9813706Z go.uber.org/cadence/internal.(*syncWorkflowDefinition).OnDecisionTaskStarted(0xc000b27b00?)
2024-05-20T19:36:41.9814270Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:514 +0x26
2024-05-20T19:36:41.9814728Z go.uber.org/cadence/internal.(*workflowExecutionEventHandlerImpl).ProcessEvent(0xc0004fc7e0, 0xc000df7e00, 0x80?, 0x0)
2024-05-20T19:36:41.9815532Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_event_handlers.go:820 +0x1fb
2024-05-20T19:36:41.9815962Z go.uber.org/cadence/internal.(*workflowExecutionContextImpl).ProcessWorkflowTask(0xc000a35140, 0xc001618d80)
2024-05-20T19:36:41.9816534Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_handlers.go:895 +0x1bbc
2024-05-20T19:36:41.9816976Z go.uber.org/cadence/internal.(*workflowTaskHandlerImpl).ProcessWorkflowTask(0xc000aec000, 0xc001618d80, 0xc00101c990)
2024-05-20T19:36:41.9817546Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_handlers.go:787 +0x42a
2024-05-20T19:36:41.9817895Z go.uber.org/cadence/internal.(*workflowTaskPoller).processWorkflowTask(0xc000d47980, 0xc001618d80)
2024-05-20T19:36:41.9818465Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:341 +0x236
2024-05-20T19:36:41.9818836Z go.uber.org/cadence/internal.(*workflowTaskPoller).ProcessTask(0xc000d47980, {0x15517a0?, 0xc001618d80?})
2024-05-20T19:36:41.9819394Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:314 +0x95
2024-05-20T19:36:41.9819744Z go.uber.org/cadence/internal.(*baseWorker).processTask(0xc000fdc480, {0x1551fe0?, 0xc000f14f40?})
2024-05-20T19:36:41.9820369Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:361 +0xe2
2024-05-20T19:36:41.9820702Z created by go.uber.org/cadence/internal.(*baseWorker).runTaskDispatcher in goroutine 15211
2024-05-20T19:36:41.9821247Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:268 +0xb1
2024-05-20T19:36:41.9821254Z 
2024-05-20T19:36:41.9821354Z goroutine 15131 [semacquire]:
2024-05-20T19:36:41.9821477Z sync.runtime_Semacquire(0xc000832e80?)
2024-05-20T19:36:41.9821686Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/sema.go:62 +0x25
2024-05-20T19:36:41.9821788Z sync.(*WaitGroup).Wait(0x1551fe0?)
2024-05-20T19:36:41.9822017Z    /opt/hostedtoolcache/go/1.21.10/x64/src/sync/waitgroup.go:116 +0x48
2024-05-20T19:36:41.9822240Z go.uber.org/cadence/internal/common/util.AwaitWaitGroup.func1()
2024-05-20T19:36:41.9822773Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/util/util.go:52 +0x25
2024-05-20T19:36:41.9823086Z created by go.uber.org/cadence/internal/common/util.AwaitWaitGroup in goroutine 14758
2024-05-20T19:36:41.9823608Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/util/util.go:51 +0x7d
2024-05-20T19:36:41.9823613Z 
2024-05-20T19:36:41.9823790Z goroutine 14795 [sleep]:
2024-05-20T19:36:41.9823881Z time.Sleep(0x219beb45)
2024-05-20T19:36:41.9824096Z    /opt/hostedtoolcache/go/1.21.10/x64/src/runtime/time.go:195 +0x125
2024-05-20T19:36:41.9824470Z go.uber.org/cadence/internal/common/backoff.(*ConcurrentRetrier).throttleInternal(0xc001173320)
2024-05-20T19:36:41.9825018Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/backoff/retry.go:65 +0x8b
2024-05-20T19:36:41.9825307Z go.uber.org/cadence/internal/common/backoff.(*ConcurrentRetrier).Throttle(...)
2024-05-20T19:36:41.9825821Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/common/backoff/retry.go:51
2024-05-20T19:36:41.9826110Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc00161d9e0)
2024-05-20T19:36:41.9826675Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:291 +0x21c
2024-05-20T19:36:41.9826897Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc00161d9e0)
2024-05-20T19:36:41.9827444Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9827782Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 14758
2024-05-20T19:36:41.9828329Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9828334Z 
2024-05-20T19:36:41.9828434Z goroutine 15208 [select]:
2024-05-20T19:36:41.9828741Z go.uber.org/cadence/internal.(*basePoller).doPoll(0xc000d47980, {0x1?, 0x0?}, 0xc0011b7c20)
2024-05-20T19:36:41.9829305Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:261 +0x20a
2024-05-20T19:36:41.9829566Z go.uber.org/cadence/internal.(*workflowTaskPoller).PollTask(0xc000d47980)
2024-05-20T19:36:41.9830121Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_task_pollers.go:298 +0x65
2024-05-20T19:36:41.9830344Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc480)
2024-05-20T19:36:41.9830890Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:293 +0x284
2024-05-20T19:36:41.9831109Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc480)
2024-05-20T19:36:41.9831656Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9831994Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9832544Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9832549Z 
2024-05-20T19:36:41.9832649Z goroutine 15207 [select]:
2024-05-20T19:36:41.9833102Z golang.org/x/time/rate.(*Limiter).wait(0xc001685a90, {0x1ab04e8, 0xc0016859f0}, 0x1, {0x0?, 0xc000987c00?, 0x26e0980?}, 0x19053e0)
2024-05-20T19:36:41.9833367Z    /home/runner/go/pkg/mod/golang.org/x/time@v0.3.0/rate/rate.go:283 +0x3eb
2024-05-20T19:36:41.9833655Z golang.org/x/time/rate.(*Limiter).WaitN(0xc000c5aa80?, {0x1ab04e8, 0xc0016859f0}, 0x1?)
2024-05-20T19:36:41.9833903Z    /home/runner/go/pkg/mod/golang.org/x/time@v0.3.0/rate/rate.go:246 +0x50
2024-05-20T19:36:41.9834034Z golang.org/x/time/rate.(*Limiter).Wait(...)
2024-05-20T19:36:41.9834254Z    /home/runner/go/pkg/mod/golang.org/x/time@v0.3.0/rate/rate.go:231
2024-05-20T19:36:41.9834472Z go.uber.org/cadence/internal.(*baseWorker).pollTask(0xc000fdc480)
2024-05-20T19:36:41.9835040Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:292 +0x252
2024-05-20T19:36:41.9835260Z go.uber.org/cadence/internal.(*baseWorker).runPoller(0xc000fdc480)
2024-05-20T19:36:41.9835809Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:242 +0x8d
2024-05-20T19:36:41.9836142Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9836692Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:205 +0x85
2024-05-20T19:36:41.9836697Z 
2024-05-20T19:36:41.9836807Z goroutine 11472 [chan receive]:
2024-05-20T19:36:41.9837149Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc00060d800, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9837679Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9837870Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9838434Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9838700Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc000ae59e0}, 0xc000ae5ef0)
2024-05-20T19:36:41.9839195Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9839314Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9839924Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9840469Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x7fd73c137158?, {0x1a9bb20?, 0xc001190cc0?}, 0x186e6c3?)
2024-05-20T19:36:41.9840804Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9841185Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func4({0x1a9bb20, 0xc001190cc0})
2024-05-20T19:36:41.9841473Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:123 +0x10a
2024-05-20T19:36:41.9841936Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc000ae59e0})
2024-05-20T19:36:41.9842276Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9842580Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000c727e0?)
2024-05-20T19:36:41.9843131Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9843471Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 11468
2024-05-20T19:36:41.9844018Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9844085Z 
2024-05-20T19:36:41.9844207Z goroutine 6566 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9844547Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc000a646c0, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9845094Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9845271Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9845779Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9846039Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc0013a1470}, 0xc000bca3c0)
2024-05-20T19:36:41.9846527Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9846649Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9847174Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9847706Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0xc00180fb00?, {0x1a9bb20?, 0xc000d384e0?}, 0x18383c8?)
2024-05-20T19:36:41.9848042Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9848386Z github.com/indeedeng/iwf/service/interpreter.executeState.func2({0x1a9bb20, 0xc000d384e0})
2024-05-20T19:36:41.9848721Z    /home/runner/work/iwf/iwf/service/interpreter/workflowImpl.go:579 +0x264
2024-05-20T19:36:41.9849183Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc0013a1470})
2024-05-20T19:36:41.9849511Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9849816Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000762900?)
2024-05-20T19:36:41.9850354Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9850694Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 6558
2024-05-20T19:36:41.9851299Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9851305Z 
2024-05-20T19:36:41.9851426Z goroutine 6404 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9851765Z go.temporal.io/sdk/internal.(*coroutineState).initialYield(0xc000890dc0, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9852145Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:945 +0x5f
2024-05-20T19:36:41.9852326Z go.temporal.io/sdk/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9852631Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:955
2024-05-20T19:36:41.9853070Z go.temporal.io/sdk/internal.(*workflowEnvironmentInterceptor).Await(0x1ab08a0?, {0x1ab08a0, 0xc000a8db30}, 0xc000be4000)
2024-05-20T19:36:41.9853356Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/workflow.go:364 +0xa5
2024-05-20T19:36:41.9853616Z go.temporal.io/sdk/internal.Await({0x1ab08a0, 0xc000a8db30}, 0xc000a8db30?)
2024-05-20T19:36:41.9853886Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/workflow.go:349 +0x58
2024-05-20T19:36:41.9854007Z go.temporal.io/sdk/workflow.Await(...)
2024-05-20T19:36:41.9854328Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/workflow/deterministic_wrappers.go:84
2024-05-20T19:36:41.9854857Z github.com/indeedeng/iwf/service/interpreter/temporal.(*workflowProvider).Await(0x1547560?, {0x1a9bb20?, 0xc0004cf4e0?}, 0x185a519?)
2024-05-20T19:36:41.9855335Z    /home/runner/work/iwf/iwf/service/interpreter/temporal/workflowProvider.go:203 +0x7e
2024-05-20T19:36:41.9855714Z github.com/indeedeng/iwf/service/interpreter.NewSignalReceiver.func5({0x1a9bb20, 0xc0004cf4e0})
2024-05-20T19:36:41.9856000Z    /home/runner/work/iwf/iwf/service/interpreter/signalReceiver.go:143 +0x1a9
2024-05-20T19:36:41.9856602Z github.com/indeedeng/iwf/service/interpreter/temporal.(*workflowProvider).GoNamed.func1({0x1ab08a0?, 0xc000a8db30})
2024-05-20T19:36:41.9857067Z    /home/runner/work/iwf/iwf/service/interpreter/temporal/workflowProvider.go:180 +0xfd
2024-05-20T19:36:41.9857355Z go.temporal.io/sdk/internal.(*dispatcherImpl).NewCoroutine.func1(0xc000ef6ab0?)
2024-05-20T19:36:41.9857684Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:1057 +0xcf
2024-05-20T19:36:41.9857994Z created by go.temporal.io/sdk/internal.(*dispatcherImpl).NewCoroutine in goroutine 6399
2024-05-20T19:36:41.9858337Z    /home/runner/go/pkg/mod/go.temporal.io/sdk@v1.25.1/internal/internal_workflow.go:1048 +0x338
2024-05-20T19:36:41.9858342Z 
2024-05-20T19:36:41.9858483Z goroutine 6558 [chan receive, 4 minutes]:
2024-05-20T19:36:41.9858821Z go.uber.org/cadence/internal.(*coroutineState).initialYield(0xc0000f0780, 0x3, {0x183a2ac, 0x5})
2024-05-20T19:36:41.9859385Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:796 +0x5f
2024-05-20T19:36:41.9859564Z go.uber.org/cadence/internal.(*coroutineState).yield(...)
2024-05-20T19:36:41.9860078Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:806
2024-05-20T19:36:41.9860329Z go.uber.org/cadence/internal.Await({0x1ab0868, 0xc001177800}, 0xc0013a1620)
2024-05-20T19:36:41.9860926Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/workflow.go:605 +0xa5
2024-05-20T19:36:41.9861051Z go.uber.org/cadence/workflow.Await(...)
2024-05-20T19:36:41.9861575Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/workflow/deterministic_wrappers.go:62
2024-05-20T19:36:41.9862124Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).Await(0x15f89a0?, {0x1a9bb20?, 0xc000a91470?}, 0xc000eb7110?)
2024-05-20T19:36:41.9862463Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:165 +0x7e
2024-05-20T19:36:41.9864148Z github.com/indeedeng/iwf/service/interpreter.executeState({_, _}, {_, _}, _, {{_, _}, {_, _}}, {{{0xc0002e564e, ...}, ...}, ...}, ...)
2024-05-20T19:36:41.9864444Z    /home/runner/work/iwf/iwf/service/interpreter/workflowImpl.go:631 +0x1e1e
2024-05-20T19:36:41.9864802Z github.com/indeedeng/iwf/service/interpreter.InterpreterImpl.func2({0x1a9bb20, 0xc000a91410})
2024-05-20T19:36:41.9865074Z    /home/runner/work/iwf/iwf/service/interpreter/workflowImpl.go:183 +0x6bb
2024-05-20T19:36:41.9865599Z github.com/indeedeng/iwf/service/interpreter/cadence.(*workflowProvider).GoNamed.func1({0x1ab0868?, 0xc0011774a0})
2024-05-20T19:36:41.9865937Z    /home/runner/work/iwf/iwf/service/interpreter/cadence/workflowProvider.go:142 +0xfd
2024-05-20T19:36:41.9866240Z go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine.func1(0xc000f14400?)
2024-05-20T19:36:41.9866787Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:884 +0xcf
2024-05-20T19:36:41.9867130Z created by go.uber.org/cadence/internal.(*dispatcherImpl).newNamedCoroutine in goroutine 6551
2024-05-20T19:36:41.9867681Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_workflow.go:875 +0x2e8
2024-05-20T19:36:41.9867687Z 
2024-05-20T19:36:41.9867781Z goroutine 15152 [select]:
2024-05-20T19:36:41.9868039Z go.uber.org/cadence/internal.(*baseWorker).runTaskDispatcher(0xc000fdc900)
2024-05-20T19:36:41.9868607Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:256 +0x11d
2024-05-20T19:36:41.9868877Z created by go.uber.org/cadence/internal.(*baseWorker).Start in goroutine 15134
2024-05-20T19:36:41.9869427Z    /home/runner/go/pkg/mod/go.uber.org/cadence@v0.17.1-0.20230105221902-f50f452a8eae/internal/internal_worker_base.go:209 +0x138
2024-05-20T19:36:41.9869432Z