search

bojand / ghz

Simple gRPC benchmarking and load testing tool
https://ghz.sh
Apache License 2.0
2.99k stars 265 forks source link

concurrent map writes #449

Open dlidstrom opened 2 months ago

dlidstrom commented 2 months ago

Proto file(s)

Not sure I can share these.

Command line arguments / config

[CmdletBinding()]
param (
  [int]
  $Concurrency = 50,
  [int]
  $Total = 200, # try 10_000
  [int]
  $TimeoutSeconds = 20,
  [int]
  $Connections = 1,
  [int]
  $RPS = 200, # try 0 which means no limit
  [Uri]
  $ServerUrl = "--redacted--",
  [int]
  $ServerPort = 5672
)

$concurrencyStart = 5
$concurrencyStep = 5
$concurrencyStepDurationSeconds = 5

ghz --insecure `
  --proto .\Protos\flightsearcher.proto `
  --call FlightSearcherService.GetFlightSearches `
  --data-file=request.json `
  --metadata-file=metadata.json `
  --total $Total `
  --rps $RPS `
  --timeout "$($TimeoutSeconds)s" `
  --connections $Connections `
  --enable-compression `
    --concurrency-schedule="step" `
    --concurrency-start=$concurrencyStart `
    --concurrency-step=$concurrencyStep `
    --concurrency-end=$Concurrency `
    --concurrency-step-duration="$($concurrencyStepDurationSeconds)s" `
  "$($ServerUrl):$($ServerPort)"

Describe the bug

ghz fails after less then a minute.

Environment

Windows Server 2019

Output

fatal error: concurrent map writes
fatal error: concurrent map writes

goroutine 59 [running]:
google.golang.org/grpc/metadata.MD.Set(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/metadata/metadata.go:117
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc00042e380, {{0xc00040ff18?, 0x1c26560?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:100 +0x1af
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc00042e380)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 1 [chan receive]:
github.com/bojand/ghz/runner.(*Requester).runWorkers(0xc00005a680, {0x1c3aa80, 0xc000059860}, {0x1c380b0, 0xc00000ac00})
        D:/a/ghz/ghz/runner/requester.go:492 +0x2bb
github.com/bojand/ghz/runner.(*Requester).Run(0xc00005a680)
        D:/a/ghz/ghz/runner/requester.go:185 +0x696
github.com/bojand/ghz/runner.Run({0xc00004a2d0?, 0x0?}, {0xc00004e200?, 0x0?}, {0xc000225e80?, 0x0?, 0xc0002380c0?})
        D:/a/ghz/ghz/runner/run.go:52 +0x1e5
main.main()
        D:/a/ghz/ghz/cmd/ghz/main.go:334 +0x68c

goroutine 24 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000094d20, 0x1)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/controlbuf.go:418 +0x113
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0003cdf10)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/controlbuf.go:552 +0x86
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/http2_client.go:451 +0x85
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 38
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/http2_client.go:449 +0x21db

goroutine 34 [syscall]:
os/signal.signal_recv()
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/runtime/sigqueue.go:152 +0x29
os/signal.loop()
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/os/signal/signal_unix.go:23 +0x13
created by os/signal.Notify.func1.1 in goroutine 1
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/os/signal/signal.go:151 +0x1f

goroutine 35 [chan receive]:
github.com/bojand/ghz/runner.Run.func1()
        D:/a/ghz/ghz/runner/run.go:41 +0x25
created by github.com/bojand/ghz/runner.Run in goroutine 1
        D:/a/ghz/ghz/runner/run.go:40 +0x174

goroutine 36 [select]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000298460, {0x1c3d348, 0xc00009e230})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/grpcsync/callback_serializer.go:83 +0x112
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/grpcsync/callback_serializer.go:55 +0x11a

goroutine 37 [select]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc0002984c0, {0x1c3d348, 0xc00009e280})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/grpcsync/callback_serializer.go:83 +0x112
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/grpcsync/callback_serializer.go:55 +0x11a

goroutine 12 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0003d8240)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00005ab60, {0x1a9cf40, 0xc000058ea0}, 0xc000537a50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f1e60, 0xc000537aa0, 0xc000537a90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f1e60, {0x1a9cf40?, 0xc000058ea0?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc000058e70?}, {0xc0002a0540?, 0xc0002a0540?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc000058ea0}, 0x6?, {0xc000058ed0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc000058e70}, {0xc0002a0540, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc000058ea0}, {0xc0004082a0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc000058e70}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004082a0, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce280, 0xc0004f3e80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce280, {{0xc0004f3f18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce280)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 39 [chan receive]:
github.com/bojand/ghz/runner.(*Reporter).Run(0xc00009e410)
        D:/a/ghz/ghz/runner/reporter.go:160 +0x55
github.com/bojand/ghz/runner.(*Requester).Run.func1()
        D:/a/ghz/ghz/runner/requester.go:178 +0x1b
created by github.com/bojand/ghz/runner.(*Requester).Run in goroutine 1
        D:/a/ghz/ghz/runner/requester.go:177 +0x473

goroutine 40 [chan receive]:
github.com/bojand/ghz/load.(*StepWorkerTicker).Run(0xc000059860)
        D:/a/ghz/ghz/load/worker_ticker.go:115 +0x1a8
github.com/bojand/ghz/runner.(*Requester).runWorkers.func1()
        D:/a/ghz/ghz/runner/requester.go:361 +0x1c
created by github.com/bojand/ghz/runner.(*Requester).runWorkers in goroutine 1
        D:/a/ghz/ghz/runner/requester.go:360 +0xaf

goroutine 41 [chan receive]:
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2()
        D:/a/ghz/ghz/runner/requester.go:374 +0xcc
created by github.com/bojand/ghz/runner.(*Requester).runWorkers in goroutine 1
        D:/a/ghz/ghz/runner/requester.go:370 +0x1c5

goroutine 42 [select]:
github.com/bojand/ghz/runner.(*Requester).runWorkers.func3()
        D:/a/ghz/ghz/runner/requester.go:479 +0x255
created by github.com/bojand/ghz/runner.(*Requester).runWorkers in goroutine 1
        D:/a/ghz/ghz/runner/requester.go:446 +0x2af

goroutine 50 [chan receive]:
github.com/bojand/ghz/load.(*StepWorkerTicker).Run.func1()
        D:/a/ghz/ghz/load/worker_ticker.go:80 +0xb1
created by github.com/bojand/ghz/load.(*StepWorkerTicker).Run in goroutine 40
        D:/a/ghz/ghz/load/worker_ticker.go:79 +0x19c

goroutine 51 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00013b560)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00010f790, {0x1a9cf40, 0xc000239c50}, 0xc00042ba50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc000286000, 0xc00042baa0, 0xc00042ba90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000286000, {0x1a9cf40?, 0xc000239c50?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc000239c20?}, {0xc0002a1500?, 0xc0002a1500?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc000239c50}, 0x6?, {0xc000239c80, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc000239c20}, {0xc0002a1500, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc000239c50}, {0xc0004e6290, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc000239c20}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004e6290, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce000, 0xc000415e80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce000, {{0xc000415f18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce000)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 52 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00054d320)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00010f930, {0x1a9cf40, 0xc0004e8180}, 0xc000703a50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc00054c360, 0xc000703aa0, 0xc000703a90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00054c360, {0x1a9cf40?, 0xc0004e8180?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc0004e8150?}, {0xc0002a15c0?, 0xc0002a15c0?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0004e8180}, 0x6?, {0xc0004e81b0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc0004e8150}, {0xc0002a15c0, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0004e8180}, {0xc0004e63b0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc0004e8150}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004e63b0, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce080, 0xc00041fe80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce080, {{0xc00041ff18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce080)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 53 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0000f0120)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00010fa00, {0x1a9cf40, 0xc0004e83c0}, 0xc0006dba50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc00054d0e0, 0xc0006dbaa0, 0xc0006dba90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00054d0e0, {0x1a9cf40?, 0xc0004e83c0?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc0004e8390?}, {0xc0002a1620?, 0xc0002a1620?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0004e83c0}, 0x6?, {0xc0004e83f0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc0004e8390}, {0xc0002a1620, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0004e83c0}, {0xc0004e6440, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc0004e8390}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004e6440, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce100, 0xc000421e80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce100, {{0xc000421f18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce100)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 54 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0004f6120)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0000f41a0, {0x1a9cf40, 0xc0003e2ea0}, 0xc000427a50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc00013b320, 0xc000427aa0, 0xc000427a90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00013b320, {0x1a9cf40?, 0xc0003e2ea0?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc0003e2e70?}, {0xc0000f6090?, 0xc0000f6090?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0003e2ea0}, 0x6?, {0xc0003e2ed0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc0003e2e70}, {0xc0000f6090, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0003e2ea0}, {0xc0004081f0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc0003e2e70}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004081f0, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce180, 0xc0004f3e80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce180, {{0xc0004f3f18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce180)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 55 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0004f6240)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00010f860, {0x1a9cf40, 0xc000239e90}, 0xc00070ba50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc000286ea0, 0xc00070baa0, 0xc00070ba90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc000286ea0, {0x1a9cf40?, 0xc000239e90?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc000239e60?}, {0xc0002a1560?, 0xc0002a1560?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc000239e90}, 0x6?, {0xc000239ec0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc000239e60}, {0xc0002a1560, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc000239e90}, {0xc0004e6320, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc000239e60}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004e6320, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce200, 0xc00041de80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce200, {{0xc00041df18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce200)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 13 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0004f7560)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00035c680, {0x1a9cf40, 0xc0001073b0}, 0xc0002fba50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc0004f7320, 0xc0002fbaa0, 0xc0002fba90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0004f7320, {0x1a9cf40?, 0xc0001073b0?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc000107380?}, {0xc00004a3f0?, 0xc00004a3f0?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0001073b0}, 0x6?, {0xc0001073e0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc000107380}, {0xc00004a3f0, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0001073b0}, {0xc0001241d0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc000107380}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0001241d0, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce300, 0xc000265e80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce300, {{0xc000265f18?, 0x100000000000000?, 0x2623080?}, 0xc0001458e0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce300)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 23 [IO wait]:
internal/poll.runtime_pollWait(0x2048a0cffa0, 0x72)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/runtime/netpoll.go:345 +0x85
internal/poll.(*pollDesc).wait(0x0?, 0x0?, 0x0)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.execIO(0xc00014ef20, 0x1b0bdb0)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/internal/poll/fd_windows.go:175 +0xe6
internal/poll.(*FD).Read(0xc00014ef08, {0xc00055e000, 0x8000, 0x8000})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/internal/poll/fd_windows.go:436 +0x2b1
net.(*netFD).Read(0xc00014ef08, {0xc00055e000?, 0xc0004efcc0?, 0xc6a165?})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc000118450, {0xc00055e000?, 0x20489e8bd28?, 0xc0004340c0?})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/net/net.go:179 +0x45
bufio.(*Reader).Read(0xc000136de0, {0xc000576040, 0x9, 0xc0002b60f0?})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/bufio/bufio.go:241 +0x197
io.ReadAtLeast({0x1c24320, 0xc000136de0}, {0xc000576040, 0x9, 0x9}, 0x9)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/io/io.go:335 +0x90
io.ReadFull(...)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc000576040, 0x9, 0x1210617?}, {0x1c24320?, 0xc000136de0?})
        C:/Users/runneradmin/go/pkg/mod/golang.org/x/net@v0.24.0/http2/frame.go:237 +0x65
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000576000)
        C:/Users/runneradmin/go/pkg/mod/golang.org/x/net@v0.24.0/http2/frame.go:498 +0x85
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0003eb8c8, 0xc000136e40)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/http2_client.go:1595 +0x22d
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 38
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/http2_client.go:397 +0x1c33

goroutine 14 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0000f0fc0)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00005aa90, {0x1a9cf40, 0xc0000585d0}, 0xc0002f7a50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc0000f0d80, 0xc0002f7aa0, 0xc0002f7a90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0000f0d80, {0x1a9cf40?, 0xc0000585d0?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc0000585a0?}, {0xc0002a0300?, 0xc0002a0300?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0000585d0}, 0x6?, {0xc000058600, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc0000585a0}, {0xc0002a0300, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0000585d0}, {0xc000408130, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc0000585a0}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc000408130, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce380, 0xc00041de80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce380, {{0xc00041df18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce380)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 15 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00013b9e0)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc0000f4340, {0x1a9cf40, 0xc0003e22a0}, 0xc000063a50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc00013b7a0, 0xc000063aa0, 0xc000063a90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00013b7a0, {0x1a9cf40?, 0xc0003e22a0?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc0003e2240?}, {0xc00011e1b0?, 0xc00011e1b0?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0003e22a0}, 0x6?, {0xc0003e22d0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc0003e2240}, {0xc00011e1b0, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0003e22a0}, {0xc0004e6030, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc0003e2240}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004e6030, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce400, 0xc000415e80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce400, {{0xc000415f18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce400)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 16 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc00054cfc0)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00010e410, {0x1a9cf40, 0xc0004e8120}, 0xc0006d7a50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc00054cd80, 0xc0006d7aa0, 0xc0006d7a90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc00054cd80, {0x1a9cf40?, 0xc0004e8120?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc0004e80f0?}, {0xc0000f6030?, 0xc0000f6030?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0004e8120}, 0x6?, {0xc0004e81e0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc0004e80f0}, {0xc0000f6030, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0004e8120}, {0xc00005e140, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc0004e80f0}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc00005e140, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc0003ce480, 0xc00041fe80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc0003ce480, {{0xc00041ff18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc0003ce480)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 56 [running]:
        goroutine running on other thread; stack unavailable
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 57 [runnable, locked to thread]:
syscall.SyscallN(0x7ffffb4ee6b0?, {0xc00041d948?, 0x3?, 0x0?})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/runtime/syscall_windows.go:544 +0x107
syscall.Syscall(0xc61d7b?, 0xc00075f568?, 0x320000c00032f5b0?, 0xffffffffffffffff?, 0xa?)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/runtime/syscall_windows.go:482 +0x35
internal/syscall/windows.ProcessPrng({0xc0002b62d0?, 0x10, 0xc000100008?})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/internal/syscall/windows/zsyscall_windows.go:188 +0x68
crypto/rand.(*rngReader).Read(0x0?, {0xc0002b62d0?, 0x10, 0xc62125?})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/crypto/rand/rand_windows.go:19 +0x26
io.ReadAtLeast({0x1c25960, 0x26a93c0}, {0xc0002b62d0, 0x10, 0x10}, 0x10)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/io/io.go:335 +0x90
io.ReadFull(...)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/io/io.go:354
github.com/google/uuid.NewRandomFromReader({0x1c25960, 0x26a93c0})
        C:/Users/runneradmin/go/pkg/mod/github.com/google/uuid@v1.3.0/version4.go:49 +0x51
github.com/google/uuid.NewRandom()
        C:/Users/runneradmin/go/pkg/mod/github.com/google/uuid@v1.3.0/version4.go:41 +0x51
github.com/bojand/ghz/runner.newCallData(0xc00009e050, {0xc0002b61e5, 0x5}, 0xb, 0x1?, 0xdf?, 0x0?)
        D:/a/ghz/ghz/runner/calldata.go:84 +0x2a5
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc00042e280, {{0xc00041df18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:85 +0x85
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc00042e280)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 58 [runnable]:
compress/flate.(*compressor).reset(0x1c48b18?, {0x1c23d20?, 0xc0003e3560?})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/compress/flate/deflate.go:617 +0xd8
compress/flate.(*Writer).Reset(0xc6a43d?, {0x1c23d20?, 0xc0003e3560?})
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/compress/flate/deflate.go:744 +0x8c
compress/gzip.(*Writer).init(0xc0000e0000, {0x1c23d20, 0xc0003e3560}, 0xffffffffffffffff)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/compress/gzip/gzip.go:72 +0x65
compress/gzip.(*Writer).Reset(...)
        C:/hostedtoolcache/windows/go/1.22.2/x64/src/compress/gzip/gzip.go:89
google.golang.org/grpc/encoding/gzip.(*compressor).Compress(0x126097e?, {0x1c23d20, 0xc0003e3560})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/encoding/gzip/gzip.go:76 +0x46
google.golang.org/grpc.compress({0xc00011e270, 0x2d, 0x30}, {0x0, 0x0}, {0x1c3b590, 0xc00014c870})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/rpc_util.go:655 +0x88
google.golang.org/grpc.prepareMsg({0x1a9cf40?, 0xc000058c90?}, {0x2048a0c77a0?, 0x26a93c0?}, {0x0, 0x0}, {0x1c3b590, 0xc00014c870})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1770 +0x105
google.golang.org/grpc.(*clientStream).SendMsg(0xc0002865a0, {0x1a9cf40, 0xc000058c90})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:882 +0xf2
google.golang.org/grpc.invoke({0x1c3d310?, 0xc0003e3230?}, {0xc00011e240?, 0xc00011e240?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0003e3260}, 0x6?, {0xc0003e3290, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:75 +0x9f
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc0003e3230}, {0xc00011e240, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc0003e3260}, {0xc0004e61d0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc0003e3230}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004e61d0, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc00042e300, 0xc000717e80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc00042e300, {{0xc000717f18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc00042e300)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 60 [select]:
google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc0003d9680)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:331 +0x7c
google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/internal/transport/transport.go:346
google.golang.org/grpc.(*csAttempt).recvMsg(0xc00005ac30, {0x1a9cf40, 0xc000059500}, 0xc000533a50?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:1066 +0xc9
google.golang.org/grpc.(*clientStream).RecvMsg.func1(0x26a93c0?)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:917 +0x1f
google.golang.org/grpc.(*clientStream).withRetry(0xc0003d9440, 0xc000533aa0, 0xc000533a90)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:768 +0x13a
google.golang.org/grpc.(*clientStream).RecvMsg(0xc0003d9440, {0x1a9cf40?, 0xc000059500?})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/stream.go:916 +0x11c
google.golang.org/grpc.invoke({0x1c3d310?, 0xc000059350?}, {0xc0002a0750?, 0xc0002a0750?}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc000059500}, 0x6?, {0xc000059530, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:78 +0xcb
google.golang.org/grpc.(*ClientConn).Invoke(0xc000146008, {0x1c3d310, 0xc000059350}, {0xc0002a0750, 0x28}, {0x1a9cf40, 0xc000058c90}, {0x1a9cf40, 0xc000059500}, {0xc0004083f0, ...})
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/call.go:42 +0x2cb
github.com/jhump/protoreflect/dynamic/grpcdynamic.Stub.InvokeRpc({{0x1c380d8?, 0xc000146008?}, 0x0?}, {0x1c3d310, 0xc000059350}, 0xc00009e050, {0x1c3ab10, 0xc000058c90}, {0xc0004083f0, 0x1, ...})
        C:/Users/runneradmin/go/pkg/mod/github.com/jhump/protoreflect@v1.15.1/dynamic/grpcdynamic/stub.go:56 +0x195
github.com/bojand/ghz/runner.(*Worker).makeUnaryRequest(0xc00042e400, 0xc00040de80?, 0xc000298298, 0xc000058c90)
        D:/a/ghz/ghz/runner/worker.go:188 +0x148
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc00042e400, {{0xc00040df18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:174 +0xa5e
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc00042e400)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387

goroutine 56 [running]:
google.golang.org/grpc/metadata.MD.Set(...)
        C:/Users/runneradmin/go/pkg/mod/google.golang.org/grpc@v1.56.3/metadata/metadata.go:117
github.com/bojand/ghz/runner.(*Worker).makeRequest(0xc00042e200, {{0xc0004f3f18?, 0x0?, 0x2623080?}, 0x0?})
        D:/a/ghz/ghz/runner/worker.go:100 +0x1af
github.com/bojand/ghz/runner.(*Worker).runWorker(0xc00042e200)
        D:/a/ghz/ghz/runner/worker.go:65 +0x1aa
github.com/bojand/ghz/runner.(*Requester).runWorkers.func2.1()
        D:/a/ghz/ghz/runner/requester.go:420 +0x25
created by github.com/bojand/ghz/runner.(*Requester).runWorkers.func2 in goroutine 41
        D:/a/ghz/ghz/runner/requester.go:419 +0x387
john-tipper commented 4 days ago

I see the same error and for me it appears to be related to the —enable-compression flag. If I set that I get the same error. If I don’t enable that then everything is fine.

john-tipper commented 4 days ago

This error was introduced via https://github.com/bojand/ghz/pull/444, where a map is updated by many workers without a mutex.