search

coder / coder-logstream-kube

Stream Kubernetes Pod events to the Coder startup logs
GNU Affero General Public License v3.0
10 stars 4 forks source link

pod restarts upon workspace restarts #15

Closed ericpaulsen closed 5 months ago

ericpaulsen commented 7 months ago

i'm getting multiple restarts of the coder-logstream-kube pod when I restart my Coder workspace. i've tailed the logs and here's what's returned:

fatal error: concurrent map read and map write

goroutine 9668 [running]:
main.(*podEventLogger).sendLog(0xc000686120, {0xc00005dc60, 0x1b}, {0xc00073f6b0, 0x24}, {{0xc153aacb1a746786, 0xbfcbc6a11c, 0x358f460}, {0xc00031c660, 0x5d}, ...})
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/logger.go:303 +0x8d
created by main.(*podEventLogger).sendLog
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/logger.go:331 +0x772

goroutine 1 [select, 13 minutes]:
main.root.func1(0xc000004900, {0x21322c6?, 0x0?, 0x0?})
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/main.go:78 +0x7fc
github.com/spf13/cobra.(*Command).execute(0xc000004900, {0xc00011e210, 0x0, 0x0})
        /home/runner/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:940 +0x862
github.com/spf13/cobra.(*Command).ExecuteC(0xc000004900)
        /home/runner/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:1068 +0x3bd
github.com/spf13/cobra.(*Command).Execute(...)
        /home/runner/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:992
main.main()
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/main.go:19 +0x1f

goroutine 33 [chan receive]:
github.com/golang/glog.(*loggingT).flushDaemon(0x0?)
        /home/runner/go/pkg/mod/github.com/golang/glog@v1.0.0/glog.go:882 +0x6a
created by github.com/golang/glog.init.0
        /home/runner/go/pkg/mod/github.com/golang/glog@v1.0.0/glog.go:410 +0x1bf

goroutine 51 [chan receive, 13 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0006afdb0, 0xc00054a678?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:802 +0x58
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:55 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 19 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0001b24f8, 0xa)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/sema.go:527 +0x14c
sync.(*Cond).Wait(0xc000715f80?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/sync/cond.go:70 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0001b24d0, 0xc0005440d0)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/delta_fifo.go:575 +0x256
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000524140)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:192 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x30?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x17e2ae5?, {0x24a8400, 0xc00057f6e0}, 0x1, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005241b8?, 0x3b9aca00, 0x0, 0x0?, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:204 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*controller).Run(0xc000524140, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:163 +0x385
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0004c6840, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:503 +0x542
k8s.io/client-go/informers.(*sharedInformerFactory).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/informers/factory.go:150 +0x6b
created by k8s.io/client-go/informers.(*sharedInformerFactory).Start
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/informers/factory.go:148 +0x21b

goroutine 20 [sync.Cond.Wait, 13 minutes]:
sync.runtime_notifyListWait(0xc0001b2918, 0x2)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/sema.go:527 +0x14c
sync.(*Cond).Wait(0xc00064efe0?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/sync/cond.go:70 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0001b28f0, 0xc000544480)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/delta_fifo.go:575 +0x256
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000524960)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:192 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x30?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x17e2ae5?, {0x24a8400, 0xc00057fef0}, 0x1, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0005249d8?, 0x3b9aca00, 0x0, 0x0?, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:204 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*controller).Run(0xc000524960, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:163 +0x385
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0004c68f0, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:503 +0x542
k8s.io/client-go/informers.(*sharedInformerFactory).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/informers/factory.go:150 +0x6b
created by k8s.io/client-go/informers.(*sharedInformerFactory).Start
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/informers/factory.go:148 +0x21b

goroutine 21 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0004c6a78, 0x10)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/sema.go:527 +0x14c
sync.(*Cond).Wait(0xc000468f20?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/sync/cond.go:70 +0x8c
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0004c6a50, 0xc0006aa5e0)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/delta_fifo.go:575 +0x256
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0007112c0)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:192 +0x36
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x30?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x17e2ae5?, {0x24a8400, 0xc000520360}, 0x1, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000711338?, 0x3b9aca00, 0x0, 0x0?, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:204 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*controller).Run(0xc0007112c0, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:163 +0x385
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0004c69a0, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:503 +0x542
k8s.io/client-go/informers.(*sharedInformerFactory).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/informers/factory.go:150 +0x6b
created by k8s.io/client-go/informers.(*sharedInformerFactory).Start
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/informers/factory.go:148 +0x21b

goroutine 41 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:968 +0x57
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x30?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00067f738?, {0x24a8400, 0xc000216000}, 0x1, 0xc000732000)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0?, 0x3b9aca00, 0x0, 0x0?, 0xc00067f788?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:204 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0002adb90)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:967 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 23 [chan receive, 13 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0006afe50, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:802 +0x58
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:55 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 24 [chan receive, 13 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:133 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:132 +0xc5

goroutine 25 [select]:
k8s.io/client-go/tools/cache.watchHandler({0x0?, 0x0?, 0x358f460?}, {0x24b29a0, 0xc000134880}, {0x7fd822016c58, 0xc0004c6a50}, {0x24de330?, 0x20fb8e0}, 0x0, ...)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:712 +0x185
k8s.io/client-go/tools/cache.(*Reflector).watch(0xc0004741e0, {0x0?, 0x0?}, 0xc000114240, 0xc0006596e0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:431 +0x54e
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc0004741e0, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:356 +0x34e
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:289 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x10?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00014b4c0?, {0x24a83e0, 0xc0006afea0}, 0x1, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc0004741e0, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:288 +0x17d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:55 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 117 [select, 13 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).startResync(0xc0004741e0, 0xc000114240, 0xc000242120, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:368 +0x114
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:355 +0x32a

goroutine 52 [chan receive, 13 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:133 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:132 +0xc5

goroutine 53 [select]:
k8s.io/client-go/tools/cache.watchHandler({0x0?, 0x0?, 0x358f460?}, {0x24b29a0, 0xc000642dc0}, {0x7fd822016c58, 0xc0001b24d0}, {0x24de330?, 0x20fcea0}, 0x0, ...)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:712 +0x185
k8s.io/client-go/tools/cache.(*Reflector).watch(0xc00022e000, {0x0?, 0x0?}, 0xc000114240, 0xc000714040?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:431 +0x54e
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00022e000, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:356 +0x34e
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:289 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x10?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00046e140?, {0x24a83e0, 0xc000556050}, 0x1, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00022e000, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:288 +0x17d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:55 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 31 [select, 13 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).startResync(0xc00022e000, 0xc000114240, 0xc0001149c0, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:368 +0x114
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:355 +0x32a

goroutine 6733 [runnable]:
sync.runtime_SemacquireMutex(0x12?, 0xf0?, 0x477469?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/sema.go:77 +0x26
sync.(*Mutex).lockSlow(0xc000127df8)
        /opt/hostedtoolcache/go/1.20.7/x64/src/sync/mutex.go:171 +0x165
sync.(*Mutex).Lock(...)
        /opt/hostedtoolcache/go/1.20.7/x64/src/sync/mutex.go:90
github.com/coder/coder/codersdk/agentsdk.(*Client).QueueStartupLogs.func4()
        /home/runner/go/pkg/mod/github.com/coder/coder@v0.23.6-0.20230522192129-95839109db09/codersdk/agentsdk/agentsdk.go:679 +0x72
github.com/coder/coder/codersdk/agentsdk.closeFunc.Close(0x1e07260?)
        /home/runner/go/pkg/mod/github.com/coder/coder@v0.23.6-0.20230522192129-95839109db09/codersdk/agentsdk/agentsdk.go:722 +0x1a
main.(*podEventLogger).sendLog(0xc000686120, {0xc0006b5700, 0x1b}, {0xc00073f6b0, 0x24}, {{0xc153aacf131c00c1, 0xc37e1ada64, 0x358f460}, {0xc000516740, 0x34}, ...})
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/logger.go:330 +0x619
created by main.(*podEventLogger).sendLog
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/logger.go:331 +0x772

goroutine 58 [chan receive, 13 minutes]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0006afe00, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:802 +0x58
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:55 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 59 [chan receive, 13 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:133 +0x28
created by k8s.io/client-go/tools/cache.(*controller).Run
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/controller.go:132 +0xc5

goroutine 60 [select]:
k8s.io/client-go/tools/cache.watchHandler({0x0?, 0x0?, 0x358f460?}, {0x24b29a0, 0xc000134440}, {0x7fd822016c58, 0xc0001b28f0}, {0x24de330?, 0x20f77a0}, 0x0, ...)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:712 +0x185
k8s.io/client-go/tools/cache.(*Reflector).watch(0xc00022e0f0, {0x0?, 0x0?}, 0xc000114240, 0xc0007142a0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:431 +0x54e
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00022e0f0, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:356 +0x34e
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:289 +0x26
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x10?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00046e240?, {0x24a83e0, 0xc000556190}, 0x1, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00022e0f0, 0xc000114240)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:288 +0x17d
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:55 +0x22
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 119 [select, 13 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).startResync(0xc00022e0f0, 0xc000114240, 0xc0002426c0, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:368 +0x114
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/reflector.go:355 +0x32a

goroutine 42 [select]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0002adb90)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:938 +0x119
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 65 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:968 +0x57
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x30?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00072b738?, {0x24a8400, 0xc0003bc000}, 0x1, 0xc00031c000)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0?, 0x3b9aca00, 0x0, 0x0?, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:204 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0002adcb0)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:967 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 66 [select]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0002adcb0)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:938 +0x119
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 81 [chan receive, 13 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:968 +0x57
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x30?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:226 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0001fb738?, {0x24a8400, 0xc0003d0060}, 0x1, 0xc00038c000)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:227 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0?, 0x3b9aca00, 0x0, 0x0?, 0x0?)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:204 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*processorListener).run(0xc0002adc20)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:967 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 82 [select, 13 minutes]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0002adc20)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/tools/cache/shared_informer.go:938 +0x119
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:72 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/wait/wait.go:70 +0x85

goroutine 114 [IO wait]:
internal/poll.runtime_pollWait(0x7fd82236da18, 0x72)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc00043c100?, 0xc00078e000?, 0x0)
        /opt/hostedtoolcache/go/1.20.7/x64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /opt/hostedtoolcache/go/1.20.7/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc00043c100, {0xc00078e000, 0x5500, 0x5500})
        /opt/hostedtoolcache/go/1.20.7/x64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc00043c100, {0xc00078e000?, 0xc00078e005?, 0xc3?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000130028, {0xc00078e000?, 0x41?, 0xc0004d6530?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc00071eee8, {0xc00078e000?, 0xc00071eee8?, 0x0?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:788 +0x3d
bytes.(*Buffer).ReadFrom(0xc0004d6610, {0x24a2f40, 0xc00071eee8})
        /opt/hostedtoolcache/go/1.20.7/x64/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc0004d6380, {0x24a8d00?, 0xc000130028}, 0x5500?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:810 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc0004d6380, 0x0)
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:617 +0x116
crypto/tls.(*Conn).readRecord(...)
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:583
crypto/tls.(*Conn).Read(0xc0004d6380, {0xc000268000, 0x1000, 0xc0002a4066?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:1316 +0x16f
bufio.(*Reader).Read(0xc000266ae0, {0xc0005bc740, 0x9, 0xc000237d38?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x24a2d40, 0xc000266ae0}, {0xc0005bc740, 0x9, 0x9}, 0x9)
        /opt/hostedtoolcache/go/1.20.7/x64/src/io/io.go:332 +0x9a
io.ReadFull(...)
        /opt/hostedtoolcache/go/1.20.7/x64/src/io/io.go:351
golang.org/x/net/http2.readFrameHeader({0xc0005bc740?, 0x9?, 0xc000000000?}, {0x24a2d40?, 0xc000266ae0?})
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/frame.go:237 +0x6e
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0005bc700)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/frame.go:498 +0x95
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000237f98)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:2225 +0x12e
golang.org/x/net/http2.(*ClientConn).readLoop(0xc000244180)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:2120 +0x6f
created by golang.org/x/net/http2.(*Transport).newClientConn
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:818 +0xc1f

goroutine 146 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00087ac48, 0x7)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/sema.go:527 +0x14c
sync.(*Cond).Wait(0xc000841a70?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/sync/cond.go:70 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc00087ac30, {0xc000722001, 0x5ff, 0x5ff})
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000722001?, 0x2?, 0x213b90c?})
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:2508 +0x75
encoding/json.(*Decoder).refill(0xc000195b80)
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:165 +0x188
encoding/json.(*Decoder).readValue(0xc000195b80)
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc000195b80, {0x1e390e0, 0xc00075cb88})
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0003bce70, {0xc00075ec00, 0x800, 0xc00})
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/framer/framer.go:152 +0x1a5
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0000ae4b0, 0xc000672480?, {0x24b27e8, 0xc000134d40})
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000714380)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000134880)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/watch/streamwatcher.go:105 +0xdc
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/watch/streamwatcher.go:76 +0x130

goroutine 162 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00087a948, 0x0)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/sema.go:527 +0x14c
sync.(*Cond).Wait(0xc000279bb0?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/sync/cond.go:70 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc00087a930, {0xc0004d2c00, 0x200, 0x200})
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc0004d2c00?, 0x0?, 0x0?})
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:2508 +0x75
encoding/json.(*Decoder).refill(0xc000416780)
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:165 +0x188
encoding/json.(*Decoder).readValue(0xc000416780)
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc000416780, {0x1e390e0, 0xc00075c0c0})
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0008407e0, {0xc00029f000, 0x400, 0x400})
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/framer/framer.go:152 +0x1a5
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc000556320, 0xc000686660?, {0x24b27e8, 0xc000134480})
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000468ae0)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000134440)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/watch/streamwatcher.go:105 +0xdc
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/watch/streamwatcher.go:76 +0x130

goroutine 145 [select, 5 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc00087ac00, 0xc00037ea00)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1438 +0xb27
golang.org/x/net/http2.(*clientStream).doRequest(0xfca8e6?, 0xc000642cc0?)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1300 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1229 +0x34a

goroutine 148 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00087a7c8, 0xb)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/sema.go:527 +0x14c
sync.(*Cond).Wait(0xc0006e0150?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/sync/cond.go:70 +0x8c
golang.org/x/net/http2.(*pipe).Read(0xc00087a7b0, {0xc000246001, 0x3dff, 0x3dff})
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/pipe.go:76 +0xeb
golang.org/x/net/http2.transportResponseBody.Read({0x0?}, {0xc000246001?, 0x2?, 0x213b90c?})
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:2508 +0x75
encoding/json.(*Decoder).refill(0xc000416140)
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:165 +0x188
encoding/json.(*Decoder).readValue(0xc000416140)
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:140 +0xbb
encoding/json.(*Decoder).Decode(0xc000416140, {0x1e390e0, 0xc00071ef18})
        /opt/hostedtoolcache/go/1.20.7/x64/src/encoding/json/stream.go:63 +0x78
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000840c00, {0xc000256000, 0x4000, 0x5000})
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/util/framer/framer.go:152 +0x1a5
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc000556370, 0xc00071a6c0?, {0x24b27e8, 0xc0005bbf80})
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/runtime/serializer/streaming/streaming.go:77 +0xa7
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000468c60)
        /home/runner/go/pkg/mod/k8s.io/client-go@v0.27.1/rest/watch/decoder.go:49 +0x4f
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000642dc0)
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/watch/streamwatcher.go:105 +0xdc
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher
        /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.27.1/pkg/watch/streamwatcher.go:76 +0x130

goroutine 151 [select]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc00087a900, 0xc00037e700)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1438 +0xb27
golang.org/x/net/http2.(*clientStream).doRequest(0x0?, 0x0?)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1300 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1229 +0x34a

goroutine 9408 [runnable]:
main.(*podEventLogger).sendLog.func2()
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/logger.go:331
runtime.goexit()
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/asm_amd64.s:1598 +0x1
created by main.(*podEventLogger).sendLog
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/logger.go:331 +0x772

goroutine 183 [IO wait]:
internal/poll.runtime_pollWait(0x7fd82236d838, 0x72)
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc000756380?, 0xc000851300?, 0x0)
        /opt/hostedtoolcache/go/1.20.7/x64/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /opt/hostedtoolcache/go/1.20.7/x64/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000756380, {0xc000851300, 0x1300, 0x1300})
        /opt/hostedtoolcache/go/1.20.7/x64/src/internal/poll/fd_unix.go:167 +0x299
net.(*netFD).Read(0xc000756380, {0xc000851300?, 0xc0008513b6?, 0x1f?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc000482238, {0xc000851300?, 0x3001cc000769878?, 0xc0001208b0?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/net.go:183 +0x45
crypto/tls.(*atLeastReader).Read(0xc00071f038, {0xc000851300?, 0xc00071f038?, 0x0?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:788 +0x3d
bytes.(*Buffer).ReadFrom(0xc000120990, {0x24a2f40, 0xc00071f038})
        /opt/hostedtoolcache/go/1.20.7/x64/src/bytes/buffer.go:202 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000120700, {0x24a8d00?, 0xc000482238}, 0x124f?)
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:810 +0xe5
crypto/tls.(*Conn).readRecordOrCCS(0xc000120700, 0x0)
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:617 +0x116
crypto/tls.(*Conn).readRecord(...)
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:583
crypto/tls.(*Conn).Read(0xc000120700, {0xc000509000, 0x1000, 0x701d80?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/crypto/tls/conn.go:1316 +0x16f
bufio.(*Reader).Read(0xc000713920, {0xc0005bcac0, 0x9, 0x71fa05?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x24a2d40, 0xc000713920}, {0xc0005bcac0, 0x9, 0x9}, 0x9)
        /opt/hostedtoolcache/go/1.20.7/x64/src/io/io.go:332 +0x9a
io.ReadFull(...)
        /opt/hostedtoolcache/go/1.20.7/x64/src/io/io.go:351
net/http.http2readFrameHeader({0xc0005bcac0?, 0x9?, 0xc0006e1d70?}, {0x24a2d40?, 0xc000713920?})
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/http/h2_bundle.go:1567 +0x6e
net/http.(*http2Framer).ReadFrame(0xc0005bca80)
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/http/h2_bundle.go:1831 +0x95
net/http.(*http2clientConnReadLoop).run(0xc000769f98)
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/http/h2_bundle.go:9187 +0x12e
net/http.(*http2ClientConn).readLoop(0xc00087b380)
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/http/h2_bundle.go:9082 +0x6f
created by net/http.(*http2Transport).newClientConn
        /opt/hostedtoolcache/go/1.20.7/x64/src/net/http/h2_bundle.go:7779 +0xc3c

goroutine 147 [select, 3 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc00087a780, 0xc00037e800)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1438 +0xb27
golang.org/x/net/http2.(*clientStream).doRequest(0xfca8e6?, 0xc000642cc0?)
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1300 +0x1e
created by golang.org/x/net/http2.(*ClientConn).RoundTrip
        /home/runner/go/pkg/mod/golang.org/x/net@v0.9.0/http2/transport.go:1229 +0x34a

goroutine 169 [runnable]:
github.com/coder/coder/codersdk/agentsdk.(*Client).QueueStartupLogs.func2()
        /home/runner/go/pkg/mod/github.com/coder/coder@v0.23.6-0.20230522192129-95839109db09/codersdk/agentsdk/agentsdk.go:635
created by github.com/coder/coder/codersdk/agentsdk.(*Client).QueueStartupLogs
        /home/runner/go/pkg/mod/github.com/coder/coder@v0.23.6-0.20230522192129-95839109db09/codersdk/agentsdk/agentsdk.go:635 +0x1ea

goroutine 9570 [runnable]:
runtime.goexit1()
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/proc.go:3634 +0x54
runtime.goexit()
        /opt/hostedtoolcache/go/1.20.7/x64/src/runtime/asm_amd64.s:1599 +0x6
created by main.(*podEventLogger).sendLog
        /home/runner/work/coder-logstream-kube/coder-logstream-kube/logger.go:331 +0x772