redhat-cop / cert-utils-operator

Set of functionalities around certificates packaged in a Kubernetes operator
Apache License 2.0
95 stars 35 forks source link

Pod stuck in crashloopbackoff #115

Closed jdziedzic closed 2 years ago

jdziedzic commented 2 years ago

We upgraded to 1.3.4 and are now seeing the pod constantly crash-looping. This is the last error in the log before the pod gets killed.

2022-02-15T14:48:51.250Z DPANIC controllers.secret_to_keystore_contoller odd number of arguments passed as key-value pairs for logging {"ignored key": "keystore"} github.com/go-logr/zapr.handleFields /home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:100 github.com/go-logr/zapr.(zapLogger).Error /home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:133 github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.compareKeyStoreBinary /home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:170 github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.(SecretToKeyStoreReconciler).Reconcile /home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:127 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:298 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:253 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func1.2 /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:216 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1 /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1 /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:155 k8s.io/apimachinery/pkg/util/wait.BackoffUntil /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:156 k8s.io/apimachinery/pkg/util/wait.JitterUntil /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:133 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 k8s.io/apimachinery/pkg/util/wait.UntilWithContext /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:99 E0215 14:48:51.250706 1 runtime.go:76] Observed a panic: odd number of arguments passed as key-value pairs for logging goroutine 704 [running]: k8s.io/apimachinery/pkg/util/runtime.logPanic({0x14374a0, 0xc0046d6320}) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:74 +0x85 k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc02eb69420}) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:48 +0x75 panic({0x14374a0, 0xc0046d6320}) /opt/hostedtoolcache/go/1.17.6/x64/src/runtime/panic.go:1038 +0x215 go.uber.org/zap/zapcore.(CheckedEntry).Write(0xc02e6de160, {0xc051274500, 0x1, 0x1}) /home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/entry.go:230 +0x486 go.uber.org/zap.(Logger).DPanic(0x162b539, {0x1679925, 0x14374a0}, {0xc051274500, 0x1, 0x1}) /home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/logger.go:215 +0x59 github.com/go-logr/zapr.handleFields(0xc0003a1e60, {0xc0046d62e0, 0x1, 0xe}, {0xc051274480, 0x1, 0x40e9f4}) /home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:100 +0x535 github.com/go-logr/zapr.(zapLogger).Error(0xc000532230, {0x180cdc0, 0xc0046d62d0}, {0x162e99c, 0xc01bcf4960}, {0xc0046d62e0, 0x1, 0x1}) /home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:133 +0x1eb github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.compareKeyStoreBinary({0xc054d1f400, 0x2077, 0x2077}, {0xc0522c2000, 0x1abd, 0x2cda}, {0xc0313d7b00, 0x9, 0x20}, {0x18536c0, ...}) /home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:170 +0x2ff github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.(SecretToKeyStoreReconciler).Reconcile(0xc0003a1ec0, {0x1837d38, 0xc01bcf44e0}, {{{0xc01c052d25, 0xb}, {0xc01c077830, 0x22}}}) /home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:127 +0x4cd sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler(0xc0006e6640, {0x1837c90, 0xc00086b980}, {0x14d6ac0, 0xc02eb69420}) /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:298 +0x303 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem(0xc0006e6640, {0x1837c90, 0xc00086b980}) /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:253 +0x205 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func1.2({0x1837c90, 0xc00086b980}) /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:216 +0x46 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1() /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x25 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7efeb817dfd8) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:155 +0x67 k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x0, {0x180eb80, 0xc02ff8f020}, 0x1, 0xc00091c060) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x1, 0x3b9aca00, 0x0, 0x0, 0x1f4) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext({0x1837c90, 0xc00086b980}, 0xc02ff8d1e0, 0xc0003ff560, 0xc00097f580, 0x90) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x99 k8s.io/apimachinery/pkg/util/wait.UntilWithContext({0x1837c90, 0xc00086b980}, 0xc02b771b80, 0xc0008b07a0) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:99 +0x2b created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1 /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:213 +0x356 panic: odd number of arguments passed as key-value pairs for logging [recovered] panic: odd number of arguments passed as key-value pairs for logging

goroutine 704 [running]: k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc02eb69420}) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/runtime/runtime.go:55 +0xd8 panic({0x14374a0, 0xc0046d6320}) /opt/hostedtoolcache/go/1.17.6/x64/src/runtime/panic.go:1038 +0x215 go.uber.org/zap/zapcore.(CheckedEntry).Write(0xc02e6de160, {0xc051274500, 0x1, 0x1}) /home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/zapcore/entry.go:230 +0x486 go.uber.org/zap.(Logger).DPanic(0x162b539, {0x1679925, 0x14374a0}, {0xc051274500, 0x1, 0x1}) /home/runner/go/pkg/mod/go.uber.org/zap@v1.15.0/logger.go:215 +0x59 github.com/go-logr/zapr.handleFields(0xc0003a1e60, {0xc0046d62e0, 0x1, 0xe}, {0xc051274480, 0x1, 0x40e9f4}) /home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:100 +0x535 github.com/go-logr/zapr.(zapLogger).Error(0xc000532230, {0x180cdc0, 0xc0046d62d0}, {0x162e99c, 0xc01bcf4960}, {0xc0046d62e0, 0x1, 0x1}) /home/runner/go/pkg/mod/github.com/go-logr/zapr@v0.2.0/zapr.go:133 +0x1eb github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.compareKeyStoreBinary({0xc054d1f400, 0x2077, 0x2077}, {0xc0522c2000, 0x1abd, 0x2cda}, {0xc0313d7b00, 0x9, 0x20}, {0x18536c0, ...}) /home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:170 +0x2ff github.com/redhat-cop/cert-utils-operator/controllers/secrettokeystore.(SecretToKeyStoreReconciler).Reconcile(0xc0003a1ec0, {0x1837d38, 0xc01bcf44e0}, {{{0xc01c052d25, 0xb}, {0xc01c077830, 0x22}}}) /home/runner/work/cert-utils-operator/cert-utils-operator/controllers/secrettokeystore/secret_to_keystore_controller.go:127 +0x4cd sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler(0xc0006e6640, {0x1837c90, 0xc00086b980}, {0x14d6ac0, 0xc02eb69420}) /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:298 +0x303 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem(0xc0006e6640, {0x1837c90, 0xc00086b980}) /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:253 +0x205 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func1.2({0x1837c90, 0xc00086b980}) /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:216 +0x46 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1() /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x25 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7efeb817dfd8) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:155 +0x67 k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0x0, {0x180eb80, 0xc02ff8f020}, 0x1, 0xc00091c060) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x1, 0x3b9aca00, 0x0, 0x0, 0x1f4) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext({0x1837c90, 0xc00086b980}, 0xc02ff8d1e0, 0xc0003ff560, 0xc00097f580, 0x90) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:185 +0x99 k8s.io/apimachinery/pkg/util/wait.UntilWithContext({0x1837c90, 0xc00086b980}, 0xc02b771b80, 0xc0008b07a0) /home/runner/go/pkg/mod/k8s.io/apimachinery@v0.20.2/pkg/util/wait/wait.go:99 +0x2b created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func1 /home/runner/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.8.3/pkg/internal/controller/controller.go:213 +0x356