search

PlayFab / thundernetes

Thundernetes makes it easy to run your game servers on Kubernetes
https://playfab.github.io/thundernetes
Apache License 2.0
307 stars 46 forks source link

NodeAgent crash due to TypeAssertionError #395

Closed dgkanatsios closed 1 year ago

dgkanatsios commented 1 year ago

During a test, observed a NodeAgent Pod crash with the log

E0917 13:31:46.483139       1 runtime.go:79] Observed a panic: &runtime.TypeAssertionError{_interface:(*runtime._type)(0x13590a0), concrete:(*runtime._type)(0x13c46c0), asserted:(*runtime._type)(0x14f90a0), missingMethod:""} (interface conversion: interface {} is cache.DeletedFinalStateUnknown, not *unstructured.Unstructured)
goroutine 65 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic({0x1385ea0?, 0xc000733c50})
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/runtime/runtime.go:75 +0x99
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc0007a78c0?})
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/runtime/runtime.go:49 +0x75
panic({0x1385ea0, 0xc000733c50})
    /usr/local/go/src/runtime/panic.go:838 +0x207
main.(*NodeAgentManager).gameServerDeleted(0x406b80?, {0x13c46c0?, 0xc000888da0?})
    /workspace/cmd/nodeagent/nodeagentmanager.go:285 +0x5fd
k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnDelete(...)
    /go/pkg/mod/k8s.io/client-go@v0.24.2/tools/cache/controller.go:246
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
    /go/pkg/mod/k8s.io/client-go@v0.24.2/tools/cache/shared_informer.go:820 +0xef
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x6?)
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:155 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00005b738?, {0x170b340, 0xc0000958f0}, 0x1, 0xc000048900)
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x6?, 0x3b9aca00, 0x0, 0x90?, 0x6bbc78?)
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc00044a600?)
    /go/pkg/mod/k8s.io/client-go@v0.24.2/tools/cache/shared_informer.go:812 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:71 +0x85
panic: interface conversion: interface {} is cache.DeletedFinalStateUnknown, not *unstructured.Unstructured [recovered]
    panic: interface conversion: interface {} is cache.DeletedFinalStateUnknown, not *unstructured.Unstructured

goroutine 65 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc0007a78c0?})
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/runtime/runtime.go:56 +0xd8
panic({0x1385ea0, 0xc000733c50})
    /usr/local/go/src/runtime/panic.go:838 +0x207
main.(*NodeAgentManager).gameServerDeleted(0x406b80?, {0x13c46c0?, 0xc000888da0?})
    /workspace/cmd/nodeagent/nodeagentmanager.go:285 +0x5fd
k8s.io/client-go/tools/cache.ResourceEventHandlerFuncs.OnDelete(...)
    /go/pkg/mod/k8s.io/client-go@v0.24.2/tools/cache/controller.go:246
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
    /go/pkg/mod/k8s.io/client-go@v0.24.2/tools/cache/shared_informer.go:820 +0xef
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x6?)
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:155 +0x3e
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00005b738?, {0x170b340, 0xc0000958f0}, 0x1, 0xc000048900)
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x6?, 0x3b9aca00, 0x0, 0x90?, 0x6bbc78?)
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(...)
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:90
k8s.io/client-go/tools/cache.(*processorListener).run(0xc00044a600?)
    /go/pkg/mod/k8s.io/client-go@v0.24.2/tools/cache/shared_informer.go:812 +0x6b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:73 +0x5a
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start
    /go/pkg/mod/k8s.io/apimachinery@v0.24.2/pkg/util/wait/wait.go:71 +0x85