Closed stroebitzer closed 3 years ago
On creating a user cluster with the run environment with provider Hetzner.
The user cluster gets up and running
The machine controller gets into error state with the following error log
I0415 10:13:34.478270 1 main.go:412] machine controller startup complete I0415 10:13:34.722941 1 machineset_controller.go:148] Reconcile machineset wonderful-franklin-766df4b66d I0415 10:13:34.723061 1 status.go:56] Unable to get node for machine wonderful-franklin-766df4b66d-5bl8p, machine has no node ref I0415 10:13:35.003371 1 machine_controller.go:672] Validated machine spec of wonderful-franklin-766df4b66d-5bl8p E0415 10:13:36.928668 1 runtime.go:78] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference) goroutine 388 [running]: k8s.io/apimachinery/pkg/util/runtime.logPanic(0x2e955e0, 0x5581e60) k8s.io/apimachinery@v0.19.4/pkg/util/runtime/runtime.go:74 +0xa6 k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) k8s.io/apimachinery@v0.19.4/pkg/util/runtime/runtime.go:48 +0x89 panic(0x2e955e0, 0x5581e60) runtime/panic.go:969 +0x175 github.com/hetznercloud/hcloud-go/hcloud.(ServerClient).Create(0xc0005363a0, 0x3a82ce0, 0xc000058018, 0xc000d88840, 0x23, 0xc000192900, 0xc000bed930, 0xc0002db2d0, 0x1, 0x1, ...) github.com/hetznercloud/hcloud-go@v1.23.1/hcloud/server.go:306 +0x409 github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner.(provider).Create(0xc0002da170, 0xc000560300, 0xc0004c9f20, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x0) github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner/provider.go:284 +0x8b8 github.com/kubermatic/machine-controller/pkg/cloudprovider.(cachingValidationWrapper).Create(0xc000d9c5e0, 0xc000560300, 0xc0004c9f20, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x35dc201) github.com/kubermatic/machine-controller/pkg/cloudprovider/validationwrapper.go:77 +0x5f github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).createProviderInstance(0xc000736840, 0x3aaafe0, 0xc000d9c5e0, 0xc000560300, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x0) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:329 +0x137 github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).ensureInstanceExistsForMachine(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0x3aaafe0, 0xc000d9c5e0, 0xc000560300, 0x3a23ac0, 0xc00075b960, 0xc000424000, 0x0, ...) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:706 +0x8bd github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).reconcile(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0xc000560300, 0x55e44c0, 0xc000d887b0, 0x23) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:403 +0x7ac github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).Reconcile(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0xc0008312b0, 0xb, 0xc000d887b0, 0x23, 0xc000d8c1b0, 0x40a3ff, 0xc00003a000, ...) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:357 +0x5e8 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler(0xc00036c8c0, 0x3a82ca0, 0xc00083bf40, 0x302d020, 0xc000d9e020) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:263 +0x317 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem(0xc00036c8c0, 0x3a82ca0, 0xc00083bf40, 0xc000792e00) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:235 +0x205 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func1.1(0x3a82ca0, 0xc00083bf40) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:198 +0x4a k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1() k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:185 +0x37 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000792f50) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:155 +0x5f k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000e33f50, 0x3a29520, 0xc000d8c120, 0xc00083bf01, 0xc0007fce40) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:156 +0xad k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000792f50, 0x3b9aca00, 0x0, 0x1, 0xc0007fce40) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:133 +0x98 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext(0x3a82ca0, 0xc00083bf40, 0xc000d9c550, 0x3b9aca00, 0x0, 0x1) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:185 +0xa6 k8s.io/apimachinery/pkg/util/wait.UntilWithContext(0x3a82ca0, 0xc00083bf40, 0xc000d9c550, 0x3b9aca00) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:99 +0x57 created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1 sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:195 +0x4e7 panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x22ca249] goroutine 388 [running]: k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) k8s.io/apimachinery@v0.19.4/pkg/util/runtime/runtime.go:55 +0x10c panic(0x2e955e0, 0x5581e60) runtime/panic.go:969 +0x175 github.com/hetznercloud/hcloud-go/hcloud.(ServerClient).Create(0xc0005363a0, 0x3a82ce0, 0xc000058018, 0xc000d88840, 0x23, 0xc000192900, 0xc000bed930, 0xc0002db2d0, 0x1, 0x1, ...) github.com/hetznercloud/hcloud-go@v1.23.1/hcloud/server.go:306 +0x409 github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner.(provider).Create(0xc0002da170, 0xc000560300, 0xc0004c9f20, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x0) github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner/provider.go:284 +0x8b8 github.com/kubermatic/machine-controller/pkg/cloudprovider.(cachingValidationWrapper).Create(0xc000d9c5e0, 0xc000560300, 0xc0004c9f20, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x35dc201) github.com/kubermatic/machine-controller/pkg/cloudprovider/validationwrapper.go:77 +0x5f github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).createProviderInstance(0xc000736840, 0x3aaafe0, 0xc000d9c5e0, 0xc000560300, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x0) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:329 +0x137 github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).ensureInstanceExistsForMachine(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0x3aaafe0, 0xc000d9c5e0, 0xc000560300, 0x3a23ac0, 0xc00075b960, 0xc000424000, 0x0, ...) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:706 +0x8bd github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).reconcile(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0xc000560300, 0x55e44c0, 0xc000d887b0, 0x23) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:403 +0x7ac github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).Reconcile(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0xc0008312b0, 0xb, 0xc000d887b0, 0x23, 0xc000d8c1b0, 0x40a3ff, 0xc00003a000, ...) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:357 +0x5e8 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler(0xc00036c8c0, 0x3a82ca0, 0xc00083bf40, 0x302d020, 0xc000d9e020) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:263 +0x317 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem(0xc00036c8c0, 0x3a82ca0, 0xc00083bf40, 0xc000792e00) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:235 +0x205 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func1.1(0x3a82ca0, 0xc00083bf40) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:198 +0x4a k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1() k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:185 +0x37 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000792f50) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:155 +0x5f k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000e33f50, 0x3a29520, 0xc000d8c120, 0xc00083bf01, 0xc0007fce40) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:156 +0xad k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000792f50, 0x3b9aca00, 0x0, 0x1, 0xc0007fce40) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:133 +0x98 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext(0x3a82ca0, 0xc00083bf40, 0xc000d9c550, 0x3b9aca00, 0x0, 0x1) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:185 +0xa6 k8s.io/apimachinery/pkg/util/wait.UntilWithContext(0x3a82ca0, 0xc00083bf40, 0xc000d9c550, 0x3b9aca00) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:99 +0x57 created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1 sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:195 +0x4e7
I0415 10:13:34.478270 1 main.go:412] machine controller startup complete I0415 10:13:34.722941 1 machineset_controller.go:148] Reconcile machineset wonderful-franklin-766df4b66d I0415 10:13:34.723061 1 status.go:56] Unable to get node for machine wonderful-franklin-766df4b66d-5bl8p, machine has no node ref I0415 10:13:35.003371 1 machine_controller.go:672] Validated machine spec of wonderful-franklin-766df4b66d-5bl8p E0415 10:13:36.928668 1 runtime.go:78] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference) goroutine 388 [running]: k8s.io/apimachinery/pkg/util/runtime.logPanic(0x2e955e0, 0x5581e60) k8s.io/apimachinery@v0.19.4/pkg/util/runtime/runtime.go:74 +0xa6 k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) k8s.io/apimachinery@v0.19.4/pkg/util/runtime/runtime.go:48 +0x89 panic(0x2e955e0, 0x5581e60) runtime/panic.go:969 +0x175 github.com/hetznercloud/hcloud-go/hcloud.(ServerClient).Create(0xc0005363a0, 0x3a82ce0, 0xc000058018, 0xc000d88840, 0x23, 0xc000192900, 0xc000bed930, 0xc0002db2d0, 0x1, 0x1, ...) github.com/hetznercloud/hcloud-go@v1.23.1/hcloud/server.go:306 +0x409 github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner.(provider).Create(0xc0002da170, 0xc000560300, 0xc0004c9f20, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x0) github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner/provider.go:284 +0x8b8 github.com/kubermatic/machine-controller/pkg/cloudprovider.(cachingValidationWrapper).Create(0xc000d9c5e0, 0xc000560300, 0xc0004c9f20, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x35dc201) github.com/kubermatic/machine-controller/pkg/cloudprovider/validationwrapper.go:77 +0x5f github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).createProviderInstance(0xc000736840, 0x3aaafe0, 0xc000d9c5e0, 0xc000560300, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x0) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:329 +0x137 github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).ensureInstanceExistsForMachine(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0x3aaafe0, 0xc000d9c5e0, 0xc000560300, 0x3a23ac0, 0xc00075b960, 0xc000424000, 0x0, ...) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:706 +0x8bd github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).reconcile(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0xc000560300, 0x55e44c0, 0xc000d887b0, 0x23) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:403 +0x7ac github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).Reconcile(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0xc0008312b0, 0xb, 0xc000d887b0, 0x23, 0xc000d8c1b0, 0x40a3ff, 0xc00003a000, ...) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:357 +0x5e8 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler(0xc00036c8c0, 0x3a82ca0, 0xc00083bf40, 0x302d020, 0xc000d9e020) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:263 +0x317 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem(0xc00036c8c0, 0x3a82ca0, 0xc00083bf40, 0xc000792e00) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:235 +0x205 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func1.1(0x3a82ca0, 0xc00083bf40) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:198 +0x4a k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1() k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:185 +0x37 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000792f50) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:155 +0x5f k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000e33f50, 0x3a29520, 0xc000d8c120, 0xc00083bf01, 0xc0007fce40) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:156 +0xad k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000792f50, 0x3b9aca00, 0x0, 0x1, 0xc0007fce40) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:133 +0x98 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext(0x3a82ca0, 0xc00083bf40, 0xc000d9c550, 0x3b9aca00, 0x0, 0x1) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:185 +0xa6 k8s.io/apimachinery/pkg/util/wait.UntilWithContext(0x3a82ca0, 0xc00083bf40, 0xc000d9c550, 0x3b9aca00) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:99 +0x57 created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1 sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:195 +0x4e7 panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x22ca249]
goroutine 388 [running]: k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) k8s.io/apimachinery@v0.19.4/pkg/util/runtime/runtime.go:55 +0x10c panic(0x2e955e0, 0x5581e60) runtime/panic.go:969 +0x175 github.com/hetznercloud/hcloud-go/hcloud.(ServerClient).Create(0xc0005363a0, 0x3a82ce0, 0xc000058018, 0xc000d88840, 0x23, 0xc000192900, 0xc000bed930, 0xc0002db2d0, 0x1, 0x1, ...) github.com/hetznercloud/hcloud-go@v1.23.1/hcloud/server.go:306 +0x409 github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner.(provider).Create(0xc0002da170, 0xc000560300, 0xc0004c9f20, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x0) github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner/provider.go:284 +0x8b8 github.com/kubermatic/machine-controller/pkg/cloudprovider.(cachingValidationWrapper).Create(0xc000d9c5e0, 0xc000560300, 0xc0004c9f20, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x35dc201) github.com/kubermatic/machine-controller/pkg/cloudprovider/validationwrapper.go:77 +0x5f github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).createProviderInstance(0xc000736840, 0x3aaafe0, 0xc000d9c5e0, 0xc000560300, 0xc000c68800, 0x43c6, 0x0, 0x0, 0x0, 0x0) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:329 +0x137 github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).ensureInstanceExistsForMachine(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0x3aaafe0, 0xc000d9c5e0, 0xc000560300, 0x3a23ac0, 0xc00075b960, 0xc000424000, 0x0, ...) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:706 +0x8bd github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).reconcile(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0xc000560300, 0x55e44c0, 0xc000d887b0, 0x23) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:403 +0x7ac github.com/kubermatic/machine-controller/pkg/controller/machine.(Reconciler).Reconcile(0xc000736840, 0x3a82d60, 0xc000d8c1b0, 0xc0008312b0, 0xb, 0xc000d887b0, 0x23, 0xc000d8c1b0, 0x40a3ff, 0xc00003a000, ...) github.com/kubermatic/machine-controller/pkg/controller/machine/machine_controller.go:357 +0x5e8 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler(0xc00036c8c0, 0x3a82ca0, 0xc00083bf40, 0x302d020, 0xc000d9e020) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:263 +0x317 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem(0xc00036c8c0, 0x3a82ca0, 0xc00083bf40, 0xc000792e00) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:235 +0x205 sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func1.1(0x3a82ca0, 0xc00083bf40) sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:198 +0x4a k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext.func1() k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:185 +0x37 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000792f50) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:155 +0x5f k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc000e33f50, 0x3a29520, 0xc000d8c120, 0xc00083bf01, 0xc0007fce40) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:156 +0xad k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000792f50, 0x3b9aca00, 0x0, 0x1, 0xc0007fce40) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:133 +0x98 k8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext(0x3a82ca0, 0xc00083bf40, 0xc000d9c550, 0x3b9aca00, 0x0, 0x1) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:185 +0xa6 k8s.io/apimachinery/pkg/util/wait.UntilWithContext(0x3a82ca0, 0xc00083bf40, 0xc000d9c550, 0x3b9aca00) k8s.io/apimachinery@v0.19.4/pkg/util/wait/wait.go:99 +0x57 created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1 sigs.k8s.io/controller-runtime@v0.7.0/pkg/internal/controller/controller.go:195 +0x4e7
duplicate of #943
Scenario
On creating a user cluster with the run environment with provider Hetzner.
Expected Behaviour
The user cluster gets up and running
Current Behaviour
The machine controller gets into error state with the following error log