search

flanksource / duty

Apache License 2.0
1 stars 1 forks source link

job: fatal error: concurrent map read and map write #825

Closed adityathebe closed 2 weeks ago

adityathebe commented 3 weeks ago

fatal error: concurrent map read and map write

goroutine 298 [running]: github.com/flanksource/duty/job.(Job).init(0xc000c6f4a0) /home/gunners/.local/share/go/pkg/mod/github.com/flanksource/duty@v1.0.486/job/job.go:402 +0x1291 github.com/flanksource/duty/job.(Job).Run(0xc000c6f4a0) /home/gunners/.local/share/go/pkg/mod/github.com/flanksource/duty@v1.0.486/job/job.go:290 +0x4aa github.com/robfig/cron/v3.(Cron).startJob.func1() /home/gunners/.local/share/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312 +0xad created by github.com/robfig/cron/v3.(Cron).startJob in goroutine 83 /home/gunners/.local/share/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:310 +0xc6

adityathebe commented 2 weeks ago

unable to reproduce

adityathebe commented 4 days ago 
2024-06-27T11:21:05.001 INFO    Persisting 0 config files
⇨ http server started on [::]:8085
2024-06-27T11:21:05.007 INFO    logs.disable(local)=true
2024-06-27T11:21:05.007 INFO    incidents.disable(local)=true
2024-06-27T11:21:05.007 INFO    notifications.max.window(local)=6m
2024-06-27T11:21:05.007 INFO    connection.statement_timeout(local)=2min
2024-06-27T11:21:05.007 DEBUG   ConfigScraperSync       scheduled @every 10m
2024-06-27T11:21:05.009 INFO    Cleaning up jobs every 1m0s
2024-06-27T11:21:05.011 INFO    logs.disable(local)=true
2024-06-27T11:21:05.012 INFO    incidents.disable(local)=true
2024-06-27T11:21:05.012 INFO    notifications.max.window(local)=6m
2024-06-27T11:21:05.012 INFO    connection.statement_timeout(local)=2min
2024-06-27T11:21:05.012 DEBUG   CleanupConfigAnalysis   scheduled @every 24h
2024-06-27T11:21:05.012 DEBUG   CleanupConfigChanges    scheduled @every 24h
2024-06-27T11:21:05.012 DEBUG   CleanupConfigItems      scheduled @every 24h
2024-06-27T11:21:05.012 DEBUG   Process Change Retention Rules  scheduled @every 1h
2024-06-27T11:21:05.049 DEBUG   Scraper [default/homelab]       scheduled @every 2h
2024-06-27T11:21:05.051 DEBUG   registering shared informer for: {apps/v1 DaemonSet}
2024-06-27T11:21:05.051 DEBUG   registering shared informer for: {apps/v1 Deployment}
2024-06-27T11:21:05.051 DEBUG   registering shared informer for: {apps/v1 ReplicaSet}
2024-06-27T11:21:05.051 DEBUG   registering shared informer for: {apps/v1 StatefulSet}
2024-06-27T11:21:05.051 DEBUG   registering shared informer for: {batch/v1 CronJob}
2024-06-27T11:21:05.051 DEBUG   registering shared informer for: {batch/v1 Job}
2024-06-27T11:21:05.052 DEBUG   registering shared informer for: {v1 Node}
2024-06-27T11:21:05.052 DEBUG   registering shared informer for: {v1 Pod}
2024-06-27T11:21:05.052 DEBUG   ConsumeKubernetesWatchEvents [default/homelab] scheduled @every 15s
2024-06-27T11:21:05.052 DEBUG   ConsumeKubernetesWatchResources [default/homelab]       scheduled @every 15s
2024-06-27T11:21:05.052 DEBUG   Scraper [default/aws-cluster]   scheduled @every 5h
2024-06-27T11:21:05.053 DEBUG   registering shared informer for: {apps/v1 DaemonSet}
2024-06-27T11:21:05.053 DEBUG   registering shared informer for: {apps/v1 Deployment}
2024-06-27T11:21:05.053 DEBUG   registering shared informer for: {apps/v1 ReplicaSet}
2024-06-27T11:21:05.053 DEBUG   registering shared informer for: {apps/v1 StatefulSet}
2024-06-27T11:21:05.053 DEBUG   registering shared informer for: {batch/v1 CronJob}
2024-06-27T11:21:05.054 DEBUG   registering shared informer for: {batch/v1 Job}
2024-06-27T11:21:05.055 DEBUG   registering shared informer for: {v1 Node}
2024-06-27T11:21:05.055 DEBUG   registering shared informer for: {v1 Pod}
2024-06-27T11:21:05.055 DEBUG   ConsumeKubernetesWatchEvents [default/aws-cluster]      scheduled @every 15s
2024-06-27T11:21:05.055 DEBUG   ConsumeKubernetesWatchResources [default/aws-cluster]   scheduled @every 15s
2024-06-27T11:21:05.055 DEBUG   Scraper [default/aws]   scheduled @every 5h
2024-06-27T11:21:05.094 INFO    Deleted 1 config_changes as per ChangeRetentionSpec[PodCrashLooping]
fatal error: concurrent map writes

goroutine 553 [running]:
github.com/flanksource/duty/job.(*Job).init(0xc000c46b00)
        /home/aditya/.local/share/go/pkg/mod/github.com/flanksource/duty@v1.0.500/job/job.go:403 +0x135a
github.com/flanksource/duty/job.(*Job).Run(0xc000c46b00)
        /home/aditya/.local/share/go/pkg/mod/github.com/flanksource/duty@v1.0.500/job/job.go:290 +0x4aa
github.com/robfig/cron/v3.(*Cron).startJob.func1()
        /home/aditya/.local/share/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:312 +0xad
created by github.com/robfig/cron/v3.(*Cron).startJob in goroutine 136
        /home/aditya/.local/share/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:310 +0xc6

goroutine 1 [select]:
sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start(0xc0011b41a0, {0xb97d440, 0xc0010df680})
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/internal.go:509 +0x1011
github.com/flanksource/config-db/cmd.run(0xe3eafe0, {0xc00003aaf0, 0x0, 0x7})
        /home/aditya/projects/flanksource/config-db/cmd/operator.go:106 +0xf26
github.com/spf13/cobra.(*Command).execute(0xe3eafe0, {0xc00003aa80, 0x7, 0x7})
        /home/aditya/.local/share/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:940 +0xcbb
github.com/spf13/cobra.(*Command).ExecuteC(0xe3eb2c0)
        /home/aditya/.local/share/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:1068 +0x990
github.com/spf13/cobra.(*Command).Execute(0xe3eb2c0)
        /home/aditya/.local/share/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:992 +0x32
github.com/spf13/cobra.(*Command).ExecuteContext(0xe3eb2c0, {0xb97d440, 0xc000ac9590})
        /home/aditya/.local/share/go/pkg/mod/github.com/spf13/cobra@v1.7.0/command.go:985 +0x73
main.main()
        /home/aditya/projects/flanksource/config-db/main.go:12 +0x38

goroutine 38 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc000a85000)
        /home/aditya/.local/share/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0x126
created by go.opencensus.io/stats/view.init.0 in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0xc5

goroutine 39 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc0009ea390, 0xc00096b680)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 24 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc000a88090, 0xc00016e400)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 8 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc000ac6f30, 0xc000eeef00)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 9 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc000ac6f40, 0xc000eeef40)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 10 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc000ac78b0, 0xc000eef740)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 111 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090df0, 0xc001082580)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 25 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc000a880a0, 0xc00016e440)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 110 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090db0, 0xc001082540)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 112 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090e10, 0xc0010825c0)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 114 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090e30, 0xc001082640)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 115 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090e50, 0xc001082680)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 116 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090e70, 0xc0010826c0)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 117 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090e90, 0xc001082700)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 118 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090eb0, 0xc001082740)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 119 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090ec0, 0xc001082780)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 120 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc001090ed0, 0xc0010827c0)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 122 [syscall]:
os/signal.signal_recv()
        /usr/lib/go/src/runtime/sigqueue.go:152 +0x29
os/signal.loop()
        /usr/lib/go/src/os/signal/signal_unix.go:23 +0x17
created by os/signal.Notify.func1.1 in goroutine 1
        /usr/lib/go/src/os/signal/signal.go:151 +0x26

goroutine 123 [chan receive]:
main.newCancelableContext.func1()
        /home/aditya/projects/flanksource/config-db/main.go:24 +0x2c
created by main.newCancelableContext in goroutine 1
        /home/aditya/projects/flanksource/config-db/main.go:23 +0x1d5

goroutine 124 [select]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b03de0, {0xb97d440, 0xc000d5fdb0})
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x1bc
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x15a

goroutine 125 [select]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b03e10, {0xb97d440, 0xc000d5fe00})
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x1bc
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x15a

goroutine 126 [select]:
google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run(0xc000b03e40, {0xb97d440, 0xc000d5fe50})
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:76 +0x1bc
created by google.golang.org/grpc/internal/grpcsync.NewCallbackSerializer in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/grpcsync/callback_serializer.go:52 +0x15a

goroutine 484 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000d427c8, 0x0)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc000d427b8)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
golang.org/x/net/http2.(*pipe).Read(0xc000d427b0, {0xc001d88000, 0x200, 0x200})
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/pipe.go:76 +0x3df
golang.org/x/net/http2.transportResponseBody.Read({0xc000d42780}, {0xc001d88000, 0x200, 0x200})
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:2712 +0x114
encoding/json.(*Decoder).refill(0xc000865540)
        /usr/lib/go/src/encoding/json/stream.go:165 +0x422
encoding/json.(*Decoder).readValue(0xc000865540)
        /usr/lib/go/src/encoding/json/stream.go:140 +0x429
encoding/json.(*Decoder).Decode(0xc000865540, {0xa5116c0, 0xc000708b40})
        /usr/lib/go/src/encoding/json/stream.go:63 +0xdb
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc001d86030, {0xc000fcb400, 0x400, 0x400})
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/framer/framer.go:152 +0x4f0
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc001ea2aa0, 0x0, {0xb90c160, 0xc0015a1380})
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/runtime/serializer/streaming/streaming.go:77 +0xf5
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc000d2cfe0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/rest/watch/decoder.go:49 +0xe6
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0015a1340)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/watch/streamwatcher.go:105 +0x1b3
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher in goroutine 81
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/watch/streamwatcher.go:76 +0x1d9

goroutine 128 [select]:
go.opentelemetry.io/otel/sdk/trace.(*batchSpanProcessor).processQueue(0xc000a53400)
        /home/aditya/.local/share/go/pkg/mod/go.opentelemetry.io/otel/sdk@v1.22.0/trace/batch_span_processor.go:312 +0x285
go.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor.func1()
        /home/aditya/.local/share/go/pkg/mod/go.opentelemetry.io/otel/sdk@v1.22.0/trace/batch_span_processor.go:128 +0x8c
created by go.opentelemetry.io/otel/sdk/trace.NewBatchSpanProcessor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/go.opentelemetry.io/otel/sdk@v1.22.0/trace/batch_span_processor.go:126 +0x49b

goroutine 129 [select]:
github.com/jackc/pgx/v5/pgxpool.(*Pool).backgroundHealthCheck(0xc000a534a0)
        /home/aditya/.local/share/go/pkg/mod/github.com/jackc/pgx/v5@v5.5.4/pgxpool/pool.go:400 +0x18f
github.com/jackc/pgx/v5/pgxpool.NewWithConfig.func3()
        /home/aditya/.local/share/go/pkg/mod/github.com/jackc/pgx/v5@v5.5.4/pgxpool/pool.go:264 +0x4f
created by github.com/jackc/pgx/v5/pgxpool.NewWithConfig in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/jackc/pgx/v5@v5.5.4/pgxpool/pool.go:262 +0x665

goroutine 146 [chan receive]:
github.com/flanksource/config-db/cmd.serve({0xb97d440, 0xc000ac9590}, {0xc00003aaf0, 0x0, 0x7})
        /home/aditya/projects/flanksource/config-db/cmd/server.go:102 +0x7ee
created by github.com/flanksource/config-db/cmd.run in goroutine 1
        /home/aditya/projects/flanksource/config-db/cmd/operator.go:82 +0x965

goroutine 15 [select]:
github.com/patrickmn/go-cache.(*janitor).Run(0xc000932320, 0xc000a269c0)
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1079 +0xca
created by github.com/patrickmn/go-cache.runJanitor in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/github.com/patrickmn/go-cache@v2.1.0+incompatible/cache.go:1099 +0x145

goroutine 13 [IO wait]:
internal/poll.runtime_pollWait(0x7bcac8dc0e90, 0x72)
        /usr/lib/go/src/runtime/netpoll.go:345 +0x3c
internal/poll.(*pollDesc).wait(0xc0003fc0a0, 0x72, 0x0)
        /usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x7a
internal/poll.(*pollDesc).waitRead(0xc0003fc0a0, 0x0)
        /usr/lib/go/src/internal/poll/fd_poll_runtime.go:89 +0x31
internal/poll.(*FD).Read(0xc0003fc080, {0xc000b40000, 0x8000, 0x8000})
        /usr/lib/go/src/internal/poll/fd_unix.go:164 +0x411
net.(*netFD).Read(0xc0003fc080, {0xc000b40000, 0x8000, 0x8000})
        /usr/lib/go/src/net/fd_posix.go:55 +0x73
net.(*conn).Read(0xc000d2f338, {0xc000b40000, 0x8000, 0x8000})
        /usr/lib/go/src/net/net.go:179 +0xa5
bufio.(*Reader).Read(0xc00120f680, {0xc0004b7b60, 0x9, 0x9})
        /usr/lib/go/src/bufio/bufio.go:241 +0x348
io.ReadAtLeast({0xb8edf40, 0xc00120f680}, {0xc0004b7b60, 0x9, 0x9}, 0x9)
        /usr/lib/go/src/io/io.go:335 +0x168
io.ReadFull({0xb8edf40, 0xc00120f680}, {0xc0004b7b60, 0x9, 0x9})
        /usr/lib/go/src/io/io.go:354 +0x93
golang.org/x/net/http2.readFrameHeader({0xc0004b7b60, 0x9, 0x9}, {0xb8edf40, 0xc00120f680})
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/frame.go:237 +0xbf
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0004b7b20)
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/frame.go:498 +0x116
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0006befc8, 0xc00120f6e0)
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:1620 +0x1db
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 127
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:409 +0x2918

goroutine 138 [sleep]:
time.Sleep(0xdf8475800)
        /usr/lib/go/src/runtime/time.go:195 +0x105
github.com/flanksource/duty/job.deleteEvictedJobs({{{0xb97d408, 0xc000c3ee40}, {0xb9a7350, 0xc000d1c990}, 0x0, 0x0, 0xb1bffc8, 0xb1bffd0, {0xb94bb50, 0xc000b58730}}})
        /home/aditya/.local/share/go/pkg/mod/github.com/flanksource/duty@v1.0.500/job/job.go:37 +0x259
created by github.com/flanksource/duty/job.(*Job).init in goroutine 137
        /home/aditya/.local/share/go/pkg/mod/github.com/flanksource/duty@v1.0.500/job/job.go:340 +0x145

goroutine 205 [chan receive]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc000c67310, 0xc0013502a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:803 +0x3e
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:55 +0x2a
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:72 +0x97
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 213
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:70 +0xb1

goroutine 14 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000199e50, 0x1)
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:418 +0x1bb
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000423f0)
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/transport/controlbuf.go:552 +0x134
google.golang.org/grpc/internal/transport.newHTTP2Client.func6()
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:467 +0x16b
created by google.golang.org/grpc/internal/transport.newHTTP2Client in goroutine 127
        /home/aditya/.local/share/go/pkg/mod/google.golang.org/grpc@v1.64.0/internal/transport/http2_client.go:465 +0x35ca

goroutine 134 [select]:
database/sql.(*DB).connectionOpener(0xc000aa15f0, {0xb97d440, 0xc000d783c0})
        /usr/lib/go/src/database/sql/sql.go:1246 +0xbe
created by database/sql.OpenDB in goroutine 1
        /usr/lib/go/src/database/sql/sql.go:824 +0x333

goroutine 136 [select]:
github.com/robfig/cron/v3.(*Cron).run(0xc0006fdcc0)
        /home/aditya/.local/share/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:263 +0x70c
created by github.com/robfig/cron/v3.(*Cron).Start in goroutine 163
        /home/aditya/.local/share/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:222 +0x12d

goroutine 165 [IO wait]:
internal/poll.runtime_pollWait(0x7bcac8dc0ba8, 0x72)
        /usr/lib/go/src/runtime/netpoll.go:345 +0x3c
internal/poll.(*pollDesc).wait(0xc0004a8820, 0x72, 0x0)
        /usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x7a
internal/poll.(*pollDesc).waitRead(0xc0004a8820, 0x0)
        /usr/lib/go/src/internal/poll/fd_poll_runtime.go:89 +0x31
internal/poll.(*FD).Accept(0xc0004a8800)
        /usr/lib/go/src/internal/poll/fd_unix.go:611 +0x525
net.(*netFD).accept(0xc0004a8800)
        /usr/lib/go/src/net/fd_unix.go:172 +0x69
net.(*TCPListener).accept(0xc00041ea80)
        /usr/lib/go/src/net/tcpsock_posix.go:159 +0x5a
net.(*TCPListener).AcceptTCP(0xc00041ea80)
        /usr/lib/go/src/net/tcpsock.go:314 +0x67
github.com/labstack/echo/v4.tcpKeepAliveListener.Accept({0xc00041ea80})
        /home/aditya/.local/share/go/pkg/mod/github.com/labstack/echo/v4@v4.12.0/echo.go:994 +0x45
net/http.(*Server).Serve(0xc0008dc1e0, {0xb961690, 0xc0004223c8})
        /usr/lib/go/src/net/http/server.go:3255 +0x50b
github.com/labstack/echo/v4.(*Echo).Start(0xc000d6fd48, {0xc0000146d8, 0x5})
        /home/aditya/.local/share/go/pkg/mod/github.com/labstack/echo/v4@v4.12.0/echo.go:691 +0x159
github.com/flanksource/config-db/cmd.serve.func3()
        /home/aditya/projects/flanksource/config-db/cmd/server.go:97 +0xee
created by github.com/flanksource/config-db/cmd.serve in goroutine 146
        /home/aditya/projects/flanksource/config-db/cmd/server.go:96 +0x7c7

goroutine 47 [select]:
github.com/robfig/cron/v3.(*Cron).run(0xc000826000)
        /home/aditya/.local/share/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:263 +0x70c
created by github.com/robfig/cron/v3.(*Cron).Start in goroutine 164
        /home/aditya/.local/share/go/pkg/mod/github.com/robfig/cron/v3@v3.0.1/cron.go:222 +0x12d

goroutine 166 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/manager/signals.SetupSignalHandler.func1()
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/signals/signal.go:38 +0x2c
created by sigs.k8s.io/controller-runtime/pkg/manager/signals.SetupSignalHandler in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/signals/signal.go:37 +0x169

goroutine 167 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile(0xc0011d23f0)
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:179 +0x45
created by sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).Start.func1 in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:132 +0x145

goroutine 168 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile(0xc0011d2480)
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:179 +0x45
created by sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).Start.func1 in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:132 +0x145

goroutine 169 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*Informers).Start(0xc000517a00, {0xb97d440, 0xc0010df540})
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/cache/internal/informers.go:214 +0x59
sigs.k8s.io/controller-runtime/pkg/cluster.(*cluster).Start(0xc0011d2240, {0xb97d440, 0xc0010df540})
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/cluster/internal.go:104 +0x13c
sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1(0xc00041eac0)
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:219 +0x169
created by sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile in goroutine 168
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:203 +0x1f0

goroutine 171 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile(0xc0011d25a0)
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:179 +0x45
created by sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).Start.func1 in goroutine 1
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:132 +0x145

goroutine 172 [IO wait]:
internal/poll.runtime_pollWait(0x7bcac8dc09b8, 0x72)
        /usr/lib/go/src/runtime/netpoll.go:345 +0x3c
internal/poll.(*pollDesc).wait(0xc0010f1aa0, 0x72, 0x0)
        /usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x7a
internal/poll.(*pollDesc).waitRead(0xc0010f1aa0, 0x0)
        /usr/lib/go/src/internal/poll/fd_poll_runtime.go:89 +0x31
internal/poll.(*FD).Accept(0xc0010f1a80)
        /usr/lib/go/src/internal/poll/fd_unix.go:611 +0x525
net.(*netFD).accept(0xc0010f1a80)
        /usr/lib/go/src/net/fd_unix.go:172 +0x69
net.(*TCPListener).accept(0xc0011d0320)
        /usr/lib/go/src/net/tcpsock_posix.go:159 +0x5a
net.(*TCPListener).Accept(0xc0011d0320)
        /usr/lib/go/src/net/tcpsock.go:327 +0x65
net/http.(*Server).Serve(0xc0008dc3c0, {0xb961660, 0xc0011d0320})
        /usr/lib/go/src/net/http/server.go:3255 +0x50b
sigs.k8s.io/controller-runtime/pkg/manager.(*server).Start(0xc00016ec00, {0xb97d440, 0xc0010df5e0})
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/server.go:51 +0x405
sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1(0xc00041eb20)
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:219 +0x169
created by sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile in goroutine 171
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:203 +0x1f0

goroutine 147 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:969 +0x65
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000589f20)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00191bf20, {0xb8fe220, 0xc000a5b5c0}, 0x1, 0xc0000c88a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000589f20, 0x3b9aca00, 0x0, 0x1, 0xc0000c88a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:204 +0x76
k8s.io/apimachinery/pkg/util/wait.Until(0xc000589f20, 0x3b9aca00, 0xc0000c88a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:161 +0x2d
k8s.io/client-go/tools/cache.(*processorListener).run(0xc001051b90)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:968 +0xb5
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:72 +0x97
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 185
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:70 +0xb1

goroutine 57 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile(0xc0011d2510)
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:179 +0x45
created by sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).Start.func1 in goroutine 56
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:132 +0x145

goroutine 58 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start(0xc000dc99a0, {0xb97d440, 0xc0010df590})
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:238 +0x285
sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile.func1(0xc0011d03e0)
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:219 +0x169
created by sigs.k8s.io/controller-runtime/pkg/manager.(*runnableGroup).reconcile in goroutine 57
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/runnable_group.go:203 +0x1f0

goroutine 148 [select]:
k8s.io/client-go/tools/cache.(*processorListener).pop(0xc001051b90)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:939 +0x1e5
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:72 +0x97
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 185
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:70 +0xb1

goroutine 60 [chan receive]:
k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc000f6a480)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/util/workqueue/queue.go:314 +0x12c
created by k8s.io/client-go/util/workqueue.newQueue in goroutine 58
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/util/workqueue/queue.go:106 +0x2ad

goroutine 61 [select]:
k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000f6a600)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/util/workqueue/delaying_queue.go:276 +0x657
created by k8s.io/client-go/util/workqueue.newDelayingQueue in goroutine 58
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/util/workqueue/delaying_queue.go:113 +0x365

goroutine 62 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:162 +0x47
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start in goroutine 58
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:161 +0x1e5

goroutine 264 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc000c80710, 0x1)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc000c80700)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
k8s.io/client-go/util/workqueue.(*Type).Get(0xc000f6a480)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/util/workqueue/queue.go:200 +0x167
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc000dc99a0, {0xb97d440, 0xc0010df590})
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:248 +0x5d
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2.2()
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:226 +0xd1
created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func2 in goroutine 58
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/internal/controller/controller.go:222 +0x9c7

goroutine 174 [chan receive]:
sigs.k8s.io/controller-runtime/pkg/manager.(*server).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/server.go:42 +0x86
created by sigs.k8s.io/controller-runtime/pkg/manager.(*server).Start in goroutine 172
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/manager/server.go:41 +0x395

goroutine 211 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0008145a8, 0x1)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc000814598)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc000814580, 0xc0000cfb50)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/delta_fifo.go:575 +0x1ba
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc00017cfa0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:188 +0x62
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc001215d20)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001821d20, {0xb8fe220, 0xc000fb5830}, 0x1, 0xc0010402a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001215d20, 0x3b9aca00, 0x0, 0x1, 0xc0010402a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:204 +0x76
k8s.io/apimachinery/pkg/util/wait.Until(0xc001215d20, 0x3b9aca00, 0xc0010402a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:161 +0x2d
k8s.io/client-go/tools/cache.(*controller).Run(0xc00017cfa0, 0xc0010402a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:159 +0x4af
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000c4ab00, 0xc0010402a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:504 +0x37d
github.com/flanksource/config-db/scrapers/kubernetes.(*SharedInformerManager).Register.func4()
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/informers.go:91 +0x67
created by github.com/flanksource/config-db/scrapers/kubernetes.(*SharedInformerManager).Register in goroutine 137
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/informers.go:90 +0xa38

goroutine 212 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc001234708, 0x1)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc0012346f8)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0012346e0, 0xc000d335d0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/delta_fifo.go:575 +0x1ba
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000e24820)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:188 +0x62
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000bc1d20)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0012a1d20, {0xb8fe220, 0xc0014669c0}, 0x1, 0xc0010403c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000bc1d20, 0x3b9aca00, 0x0, 0x1, 0xc0010403c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:204 +0x76
k8s.io/apimachinery/pkg/util/wait.Until(0xc000bc1d20, 0x3b9aca00, 0xc0010403c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:161 +0x2d
k8s.io/client-go/tools/cache.(*controller).Run(0xc000e24820, 0xc0010403c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:159 +0x4af
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000c4adc0, 0xc0010403c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:504 +0x37d
github.com/flanksource/config-db/scrapers/kubernetes.(*SharedInformerManager).Register.func4()
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/informers.go:91 +0x67
created by github.com/flanksource/config-db/scrapers/kubernetes.(*SharedInformerManager).Register in goroutine 137
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/informers.go:90 +0xa38

goroutine 262 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:969 +0x65
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc001505f20)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001505f20, {0xb8fe220, 0xc0020d0ea0}, 0x1, 0xc001237b60)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001505f20, 0x3b9aca00, 0x0, 0x1, 0xc001237b60)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:204 +0x76
k8s.io/apimachinery/pkg/util/wait.Until(0xc001505f20, 0x3b9aca00, 0xc001237b60)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:161 +0x2d
k8s.io/client-go/tools/cache.(*processorListener).run(0xc00147f830)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:968 +0xb5
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:72 +0x97
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 63
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:70 +0xb1

goroutine 177 [chan receive]:
k8s.io/client-go/tools/cache.(*processorListener).run.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:969 +0x65
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc000589720)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001213f20, {0xb8fe220, 0xc000a5b590}, 0x1, 0xc0000c8840)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000589720, 0x3b9aca00, 0x0, 0x1, 0xc0000c8840)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:204 +0x76
k8s.io/apimachinery/pkg/util/wait.Until(0xc000589720, 0x3b9aca00, 0xc0000c8840)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:161 +0x2d
k8s.io/client-go/tools/cache.(*processorListener).run(0xc001051b00)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:968 +0xb5
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:72 +0x97
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 49
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:70 +0xb1

goroutine 143 [IO wait]:
internal/poll.runtime_pollWait(0x7bcac8dc08c0, 0x72)
        /usr/lib/go/src/runtime/netpoll.go:345 +0x3c
internal/poll.(*pollDesc).wait(0xc0012323a0, 0x72, 0x0)
        /usr/lib/go/src/internal/poll/fd_poll_runtime.go:84 +0x7a
internal/poll.(*pollDesc).waitRead(0xc0012323a0, 0x0)
        /usr/lib/go/src/internal/poll/fd_poll_runtime.go:89 +0x31
internal/poll.(*FD).Read(0xc001232380, {0xc000908000, 0xa000, 0xa000})
        /usr/lib/go/src/internal/poll/fd_unix.go:164 +0x411
net.(*netFD).Read(0xc001232380, {0xc000908000, 0xa000, 0xa000})
        /usr/lib/go/src/net/fd_posix.go:55 +0x73
net.(*conn).Read(0xc000f68090, {0xc000908000, 0xa000, 0xa000})
        /usr/lib/go/src/net/net.go:179 +0xa5
crypto/tls.(*atLeastReader).Read(0xc000fd7800, {0xc000908000, 0xa000, 0xa000})
        /usr/lib/go/src/crypto/tls/conn.go:806 +0xc2
bytes.(*Buffer).ReadFrom(0xc000ea82b0, {0xb8f4f00, 0xc000fd7800})
        /usr/lib/go/src/bytes/buffer.go:211 +0x13c
crypto/tls.(*Conn).readFromUntil(0xc000ea8008, {0xb8f8820, 0xc000f68090}, 0x5)
        /usr/lib/go/src/crypto/tls/conn.go:828 +0x16c
crypto/tls.(*Conn).readRecordOrCCS(0xc000ea8008, 0x0)
        /usr/lib/go/src/crypto/tls/conn.go:626 +0x253
crypto/tls.(*Conn).readRecord(0xc000ea8008)
        /usr/lib/go/src/crypto/tls/conn.go:588 +0x25
crypto/tls.(*Conn).Read(0xc000ea8008, {0xc00103c000, 0x1000, 0x1000})
        /usr/lib/go/src/crypto/tls/conn.go:1370 +0x1fe
bufio.(*Reader).Read(0xc00103a9c0, {0xc000736d60, 0x9, 0x9})
        /usr/lib/go/src/bufio/bufio.go:241 +0x348
io.ReadAtLeast({0xb8edf40, 0xc00103a9c0}, {0xc000736d60, 0x9, 0x9}, 0x9)
        /usr/lib/go/src/io/io.go:335 +0x168
io.ReadFull({0xb8edf40, 0xc00103a9c0}, {0xc000736d60, 0x9, 0x9})
        /usr/lib/go/src/io/io.go:354 +0x93
golang.org/x/net/http2.readFrameHeader({0xc000736d60, 0x9, 0x9}, {0xb8edf40, 0xc00103a9c0})
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/frame.go:237 +0xbf
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000736d20)
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/frame.go:498 +0x116
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc001229fa0)
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:2429 +0x11c
golang.org/x/net/http2.(*ClientConn).readLoop(0xc001000d80)
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:2325 +0x114
created by golang.org/x/net/http2.(*ClientConn).goRun in goroutine 142
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:369 +0x4c

goroutine 195 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00131e238, 0x1)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc00131e228)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00131e210, 0xc000c62670)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/delta_fifo.go:575 +0x1ba
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000e76460)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:188 +0x62
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc001282d30)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc00135bd30, {0xb8fe220, 0xc001304810}, 0x1, 0xc0000c86c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001282d30, 0x3b9aca00, 0x0, 0x1, 0xc0000c86c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:204 +0x76
k8s.io/apimachinery/pkg/util/wait.Until(0xc001282d30, 0x3b9aca00, 0xc0000c86c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:161 +0x2d
k8s.io/client-go/tools/cache.(*controller).Run(0xc000e76460, 0xc0000c86c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:159 +0x4af
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc00131e160, 0xc0000c86c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:504 +0x37d
sigs.k8s.io/controller-runtime/pkg/cache/internal.(*Informers).startInformerLocked.func1()
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/cache/internal/informers.go:233 +0xdc
created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*Informers).startInformerLocked in goroutine 63
        /home/aditya/.local/share/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.15.0/pkg/cache/internal/informers.go:231 +0xd8

goroutine 181 [select]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc001250c00, 0xc001408fc0)
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:1608 +0x1017
golang.org/x/net/http2.(*clientStream).doRequest(0xc001250c00, 0xc001408fc0)
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:1436 +0x1d
golang.org/x/net/http2.(*ClientConn).roundTrip.func1()
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:1312 +0x2a
created by golang.org/x/net/http2.(*ClientConn).goRun in goroutine 200
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:369 +0x4c

goroutine 198 [chan receive]:
k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc001310280, 0xc0013500c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:803 +0x3e
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:55 +0x2a
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:72 +0x97
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 195
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:70 +0xb1

goroutine 199 [chan receive]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:132 +0x2c
created by k8s.io/client-go/tools/cache.(*controller).Run in goroutine 195
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:131 +0xe7

goroutine 200 [select]:
k8s.io/client-go/tools/cache.watchHandler({0xc1975c7141a9a9c8, 0x3a41f74b, 0xe595da0}, {0xb94b148, 0xc000d5c940}, {0x7bcac8d041b0, 0xc00131e210}, {0xb9c5018, 0xae18ac0}, 0x0, ...)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/reflector.go:714 +0x1af
k8s.io/client-go/tools/cache.(*Reflector).watch(0xc001352000, {0xb94b148, 0xc000d5c940}, 0xc0000c86c0, 0xc000f6ba40)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/reflector.go:433 +0x865
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc001352000, 0xc0000c86c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/reflector.go:358 +0x647
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/reflector.go:291 +0x2a
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc00121ce58)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001249e58, {0xb8fe240, 0xc001310370}, 0x1, 0xc0000c86c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc001352000, 0xc0000c86c0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/reflector.go:290 +0x285
k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:55 +0x2a
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:72 +0x97
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 195
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/wait.go:70 +0xb1

goroutine 180 [select]:
k8s.io/client-go/tools/cache.(*Reflector).startResync(0xc001352000, 0xc0000c86c0, 0xc001236960, 0xc000f6ba40)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/reflector.go:370 +0x1f4
created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch in goroutine 200
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/reflector.go:357 +0x61a

goroutine 202 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc001250c48, 0x0)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc001250c38)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
golang.org/x/net/http2.(*pipe).Read(0xc001250c30, {0xc001314400, 0x200, 0x200})
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/pipe.go:76 +0x3df
golang.org/x/net/http2.transportResponseBody.Read({0xc001250c00}, {0xc001314400, 0x200, 0x200})
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:2712 +0x114
encoding/json.(*Decoder).refill(0xc00082e780)
        /usr/lib/go/src/encoding/json/stream.go:165 +0x422
encoding/json.(*Decoder).readValue(0xc00082e780)
        /usr/lib/go/src/encoding/json/stream.go:140 +0x429
encoding/json.(*Decoder).Decode(0xc00082e780, {0xa5116c0, 0xc00130a468})
        /usr/lib/go/src/encoding/json/stream.go:63 +0xdb
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0013049f0, {0xc0001f3000, 0x400, 0x400})
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/framer/framer.go:152 +0x4f0
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc001310410, 0x0, {0xb90c160, 0xc000d5c980})
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/runtime/serializer/streaming/streaming.go:77 +0xf5
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0013082a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/rest/watch/decoder.go:49 +0xe6
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc000d5c940)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/watch/streamwatcher.go:105 +0x1b3
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher in goroutine 200
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/watch/streamwatcher.go:76 +0x1d9

goroutine 308 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc0012513c8, 0xe)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc0012513b8)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
golang.org/x/net/http2.(*pipe).Read(0xc0012513b0, {0xc000c79001, 0xdff, 0xdff})
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/pipe.go:76 +0x3df
golang.org/x/net/http2.transportResponseBody.Read({0xc001251380}, {0xc000c79001, 0xdff, 0xdff})
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:2712 +0x114
encoding/json.(*Decoder).refill(0xc000b10140)
        /usr/lib/go/src/encoding/json/stream.go:165 +0x422
encoding/json.(*Decoder).readValue(0xc000b10140)
        /usr/lib/go/src/encoding/json/stream.go:140 +0x429
encoding/json.(*Decoder).Decode(0xc000b10140, {0xa5116c0, 0xc000d37c80})
        /usr/lib/go/src/encoding/json/stream.go:63 +0xdb
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc0016021b0, {0xc001635000, 0x800, 0xc00})
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/framer/framer.go:152 +0x4f0
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc001612050, 0x0, {0xb90c160, 0xc0015a1a80})
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/runtime/serializer/streaming/streaming.go:77 +0xf5
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00161e000)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/rest/watch/decoder.go:49 +0xe6
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc00160c080)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/watch/streamwatcher.go:105 +0x1b3
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher in goroutine 145
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/watch/streamwatcher.go:76 +0x1d9

goroutine 145 [chan receive]:
github.com/flanksource/config-db/scrapers/kubernetes.WatchEvents({{{{0xb97d408, 0xc000b543c0}, {0xb9a7350, 0xc000d1c990}, 0x0, 0x0, 0xb1bffc8, 0xb1bffd0, {0xb94bb50, 0xc000b58730}}}, ...}, ...)
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/events_watch.go:118 +0x99b
github.com/flanksource/config-db/scrapers.watchKubernetesEventsWithRetry.func1({0xb97d600, 0xc001232a00})
        /home/aditya/projects/flanksource/config-db/scrapers/cron.go:101 +0xf8
github.com/sethvargo/go-retry.Do({0xb97d600, 0xc001232a00}, {0xb8f8320, 0xc000c81bc0}, 0xc0012b7bf8)
        /home/aditya/.local/share/go/pkg/mod/github.com/sethvargo/go-retry@v0.2.4/retry.go:60 +0xdc
github.com/flanksource/config-db/scrapers.watchKubernetesEventsWithRetry({{{{0xb97d408, 0xc000b543c0}, {0xb9a7350, 0xc000d1c990}, 0x0, 0x0, 0xb1bffc8, 0xb1bffd0, {0xb94bb50, 0xc000b58730}}}, ...}, ...)
        /home/aditya/projects/flanksource/config-db/scrapers/cron.go:99 +0xe5
created by github.com/flanksource/config-db/scrapers.scheduleScraperJob in goroutine 137
        /home/aditya/projects/flanksource/config-db/scrapers/cron.go:177 +0x5bb

goroutine 183 [select]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc001251380, 0xc00145b560)
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:1608 +0x1017
golang.org/x/net/http2.(*clientStream).doRequest(0xc001251380, 0xc00145b560)
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:1436 +0x1d
golang.org/x/net/http2.(*ClientConn).roundTrip.func1()
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:1312 +0x2a
created by golang.org/x/net/http2.(*ClientConn).goRun in goroutine 145
        /home/aditya/.local/share/go/pkg/mod/golang.org/x/net@v0.24.0/http2/transport.go:369 +0x4c

goroutine 213 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00131e2e8, 0x1)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc00131e2d8)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00131e2c0, 0xc000c62a50)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/delta_fifo.go:575 +0x1ba
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000e76820)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:188 +0x62
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc001282d20)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0018afd20, {0xb8fe220, 0xc001304ab0}, 0x1, 0xc0010404e0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001282d20, 0x3b9aca00, 0x0, 0x1, 0xc0010404e0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:204 +0x76
k8s.io/apimachinery/pkg/util/wait.Until(0xc001282d20, 0x3b9aca00, 0xc0010404e0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:161 +0x2d
k8s.io/client-go/tools/cache.(*controller).Run(0xc000e76820, 0xc0010404e0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:159 +0x4af
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000c4b080, 0xc0010404e0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:504 +0x37d
github.com/flanksource/config-db/scrapers/kubernetes.(*SharedInformerManager).Register.func4()
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/informers.go:91 +0x67
created by github.com/flanksource/config-db/scrapers/kubernetes.(*SharedInformerManager).Register in goroutine 137
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/informers.go:90 +0xa38

goroutine 214 [sync.Cond.Wait]:
sync.runtime_notifyListWait(0xc00128e028, 0x1)
        /usr/lib/go/src/runtime/sema.go:569 +0x152
sync.(*Cond).Wait(0xc00128e018)
        /usr/lib/go/src/sync/cond.go:70 +0x6f
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc00128e000, 0xc000d461f0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/delta_fifo.go:575 +0x1ba
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000eb00a0)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:188 +0x62
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc001286d20)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:226 +0x39
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0011e3d20, {0xb8fe220, 0xc001290030}, 0x1, 0xc001040600)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:227 +0xbb
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001286d20, 0x3b9aca00, 0x0, 0x1, 0xc001040600)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:204 +0x76
k8s.io/apimachinery/pkg/util/wait.Until(0xc001286d20, 0x3b9aca00, 0xc001040600)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/apimachinery@v0.28.2/pkg/util/wait/backoff.go:161 +0x2d
k8s.io/client-go/tools/cache.(*controller).Run(0xc000eb00a0, 0xc001040600)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/controller.go:159 +0x4af
k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc000c4b340, 0xc001040600)
        /home/aditya/.local/share/go/pkg/mod/k8s.io/client-go@v0.28.2/tools/cache/shared_informer.go:504 +0x37d
github.com/flanksource/config-db/scrapers/kubernetes.(*SharedInformerManager).Register.func4()
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/informers.go:91 +0x67
created by github.com/flanksource/config-db/scrapers/kubernetes.(*SharedInformerManager).Register in goroutine 137
        /home/aditya/projects/flanksource/config-db/scrapers/kubernetes/informers.go:90 +0xa38