search

clastix / kamaji

Kamaji is the Hosted Control Plane Manager for Kubernetes.
https://kamaji.clastix.io
Apache License 2.0
934 stars 81 forks source link

Panic due to a nil pointer in Datastore Certificate handler #472

Closed prometherion closed 1 month ago

prometherion commented 1 month ago

Issue started on Slack message

2024-05-29T21:32:23Z    INFO    datastore-setup has been configured {"controller": "tenantcontrolplane", "controllerGroup": "kamaji.clastix.io", "controllerKind": "TenantControlPlane", "TenantControlPlane": {"name":"tcp-capi","namespace":"default"}, "namespace": "default", "name": "tcp-capi", "reconcileID": "bfe4cb17-44d1-4678-91dd-e269e04da02d"}
E0529 21:32:23.413101       1 runtime.go:79] Observed a panic: "assignment to entry in nil map" (assignment to entry in nil map)
goroutine 370 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic({0x205a320, 0x2717a00})
    /go/pkg/mod/k8s.io/apimachinery@v0.30.1/pkg/util/runtime/runtime.go:75 +0x85
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc000204a80?})
    /go/pkg/mod/k8s.io/apimachinery@v0.30.1/pkg/util/runtime/runtime.go:49 +0x6b
panic({0x205a320?, 0x2717a00?})
    /usr/local/go/src/runtime/panic.go:770 +0x132
github.com/clastix/kamaji/internal/resources/datastore.(*Certificate).CreateOrUpdate.(*Certificate).mutate.func1()
    /workspace/internal/resources/datastore/datastore_certificate.go:91 +0x185
sigs.k8s.io/controller-runtime/pkg/controller/controllerutil.mutate(0xc0002e4640?, {{0xc000568410?, 0x0?}, {0xc000403170?, 0x273e4a0?}}, {0x275d3d0, 0xc0002e4640})
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/controller/controllerutil/controllerutil.go:426 +0x49
sigs.k8s.io/controller-runtime/pkg/controller/controllerutil.CreateOrUpdate({0x273e4a0, 0xc000432e00}, {0x274e9a0, 0xc00078f7a0}, {0x275d3d0, 0xc0002e4640}, 0xc0005635b8)
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/controller/controllerutil/controllerutil.go:292 +0x23d
github.com/clastix/kamaji/internal/utilities.CreateOrUpdateWithConflict.func1()
    /workspace/internal/utilities/create_or_update_conflict.go:27 +0x13d
k8s.io/client-go/util/retry.OnError.func1()
    /go/pkg/mod/k8s.io/client-go@v0.30.1/util/retry/util.go:51 +0x30
k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection(0x10?)
    /go/pkg/mod/k8s.io/apimachinery@v0.30.1/pkg/util/wait/wait.go:145 +0x3e
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff({0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc000563488)
    /go/pkg/mod/k8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:461 +0x5a
k8s.io/client-go/util/retry.OnError({0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0x7f29ae7aa1d8?, 0xc0002e4640?)
    /go/pkg/mod/k8s.io/client-go@v0.30.1/util/retry/util.go:50 +0xa5
k8s.io/client-go/util/retry.RetryOnConflict(...)
    /go/pkg/mod/k8s.io/client-go@v0.30.1/util/retry/util.go:104
github.com/clastix/kamaji/internal/utilities.CreateOrUpdateWithConflict({0x273e4a0?, 0xc000432e00?}, {0x274e9a0?, 0xc00078f7a0?}, {0x275d3d0?, 0xc0002e4640?}, 0xc0005635d8?)
    /workspace/internal/utilities/create_or_update_conflict.go:20 +0x115
github.com/clastix/kamaji/internal/resources/datastore.(*Certificate).CreateOrUpdate(0xc000777ad0?, {0x273e4a0?, 0xc000432e00?}, 0xc00196f060?)
    /workspace/internal/resources/datastore/datastore_certificate.go:64 +0x72
github.com/clastix/kamaji/internal/resources.createOrUpdate({0x273e4a0, 0xc000432e00}, {0x2746510, 0xc00168be00}, 0xc000e6b508)
    /workspace/internal/resources/resource.go:92 +0x43
github.com/clastix/kamaji/internal/resources.Handle({0x273e4a0, 0xc000432e00}, {0x2746510, 0xc00168be00}, 0xc000e6b508)
    /workspace/internal/resources/resource.go:67 +0xf2
github.com/clastix/kamaji/controllers.(*TenantControlPlaneReconciler).Reconcile(0xc000669380, {0x273e3f8, 0xc0008d09c0}, {{{0xc000ecab26?, 0x7?}, {0xc000ecab30?, 0xc?}}})
    /workspace/controllers/tenantcontrolplane_controller.go:178 +0xd7e
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile(0x2744490?, {0x273e3f8?, 0xc0008d09c0?}, {{{0xc000ecab26?, 0xb?}, {0xc000ecab30?, 0x0?}}})
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:123 +0xb7
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc000257340, {0x273e430, 0xc0007a8190}, {0x2157820, 0xc0002863e0})
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:320 +0x3bc
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc000257340, {0x273e430, 0xc0007a8190})
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:270 +0x1be
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2()
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:231 +0x79
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2 in goroutine 129
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:227 +0x50f
2024-05-29T21:32:23Z    INFO    Observed a panic in reconciler: assignment to entry in nil map  {"controller": "tenantcontrolplane", "controllerGroup": "kamaji.clastix.io", "controllerKind": "TenantControlPlane", "TenantControlPlane": {"name":"tcp-capi","namespace":"default"}, "namespace": "default", "name": "tcp-capi", "reconcileID": "bfe4cb17-44d1-4678-91dd-e269e04da02d"}
panic: assignment to entry in nil map [recovered]
    panic: assignment to entry in nil map [recovered]
    panic: assignment to entry in nil map

goroutine 370 [running]:
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile.func1()
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:120 +0x1e5
panic({0x205a320?, 0x2717a00?})
    /usr/local/go/src/runtime/panic.go:770 +0x132
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc000204a80?})
    /go/pkg/mod/k8s.io/apimachinery@v0.30.1/pkg/util/runtime/runtime.go:56 +0xcd
panic({0x205a320?, 0x2717a00?})
    /usr/local/go/src/runtime/panic.go:770 +0x132
github.com/clastix/kamaji/internal/resources/datastore.(*Certificate).CreateOrUpdate.(*Certificate).mutate.func1()
    /workspace/internal/resources/datastore/datastore_certificate.go:91 +0x185
sigs.k8s.io/controller-runtime/pkg/controller/controllerutil.mutate(0xc0002e4640?, {{0xc000568410?, 0x0?}, {0xc000403170?, 0x273e4a0?}}, {0x275d3d0, 0xc0002e4640})
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/controller/controllerutil/controllerutil.go:426 +0x49
sigs.k8s.io/controller-runtime/pkg/controller/controllerutil.CreateOrUpdate({0x273e4a0, 0xc000432e00}, {0x274e9a0, 0xc00078f7a0}, {0x275d3d0, 0xc0002e4640}, 0xc0005635b8)
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/controller/controllerutil/controllerutil.go:292 +0x23d
github.com/clastix/kamaji/internal/utilities.CreateOrUpdateWithConflict.func1()
    /workspace/internal/utilities/create_or_update_conflict.go:27 +0x13d
k8s.io/client-go/util/retry.OnError.func1()
    /go/pkg/mod/k8s.io/client-go@v0.30.1/util/retry/util.go:51 +0x30
k8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection(0x10?)
    /go/pkg/mod/k8s.io/apimachinery@v0.30.1/pkg/util/wait/wait.go:145 +0x3e
k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff({0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0xc000563488)
    /go/pkg/mod/k8s.io/apimachinery@v0.30.1/pkg/util/wait/backoff.go:461 +0x5a
k8s.io/client-go/util/retry.OnError({0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0}, 0x7f29ae7aa1d8?, 0xc0002e4640?)
    /go/pkg/mod/k8s.io/client-go@v0.30.1/util/retry/util.go:50 +0xa5
k8s.io/client-go/util/retry.RetryOnConflict(...)
    /go/pkg/mod/k8s.io/client-go@v0.30.1/util/retry/util.go:104
github.com/clastix/kamaji/internal/utilities.CreateOrUpdateWithConflict({0x273e4a0?, 0xc000432e00?}, {0x274e9a0?, 0xc00078f7a0?}, {0x275d3d0?, 0xc0002e4640?}, 0xc0005635d8?)
    /workspace/internal/utilities/create_or_update_conflict.go:20 +0x115
github.com/clastix/kamaji/internal/resources/datastore.(*Certificate).CreateOrUpdate(0xc000777ad0?, {0x273e4a0?, 0xc000432e00?}, 0xc00196f060?)
    /workspace/internal/resources/datastore/datastore_certificate.go:64 +0x72
github.com/clastix/kamaji/internal/resources.createOrUpdate({0x273e4a0, 0xc000432e00}, {0x2746510, 0xc00168be00}, 0xc000e6b508)
    /workspace/internal/resources/resource.go:92 +0x43
github.com/clastix/kamaji/internal/resources.Handle({0x273e4a0, 0xc000432e00}, {0x2746510, 0xc00168be00}, 0xc000e6b508)
    /workspace/internal/resources/resource.go:67 +0xf2
github.com/clastix/kamaji/controllers.(*TenantControlPlaneReconciler).Reconcile(0xc000669380, {0x273e3f8, 0xc0008d09c0}, {{{0xc000ecab26?, 0x7?}, {0xc000ecab30?, 0xc?}}})
    /workspace/controllers/tenantcontrolplane_controller.go:178 +0xd7e
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Reconcile(0x2744490?, {0x273e3f8?, 0xc0008d09c0?}, {{{0xc000ecab26?, 0xb?}, {0xc000ecab30?, 0x0?}}})
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:123 +0xb7
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc000257340, {0x273e430, 0xc0007a8190}, {0x2157820, 0xc0002863e0})
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:320 +0x3bc
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc000257340, {0x273e430, 0xc0007a8190})
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:270 +0x1be
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2()
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:231 +0x79
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2 in goroutine 129
    /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.17.1-0.20240416095710-67b27f27e514/pkg/internal/controller/controller.go:227 +0x50f