Open bboreham opened 3 years ago
Seems to be looping here: https://github.com/weaveworks-experiments/kspan/blob/3e8e909978e9a07a2765aae795514f508566da2b/controllers/events/outgoing.go#L46-L54
Could a span have a parent that points at itself?
goroutine dump:
SIGQUIT: quit PC=0x46d7c1 m=0 sigcode=0 goroutine 0 [idle]: runtime.futex(0x22f5428, 0x80, 0x0, 0x0, 0x0, 0xc000208548, 0xc000042000, 0x8, 0x7ffdd21d8278, 0x40bf9f, ...) /usr/local/go/src/runtime/sys_linux_amd64.s:587 +0x21 runtime.futexsleep(0x22f5428, 0x7ffd00000000, 0xffffffffffffffff) /usr/local/go/src/runtime/os_linux.go:45 +0x46 runtime.notesleep(0x22f5428) /usr/local/go/src/runtime/lock_futex.go:159 +0x9f runtime.stoplockedm() /usr/local/go/src/runtime/proc.go:2085 +0x8d runtime.schedule() /usr/local/go/src/runtime/proc.go:2617 +0x48c runtime.park_m(0xc000217080) /usr/local/go/src/runtime/proc.go:2851 +0x9d runtime.mcall(0x0) /usr/local/go/src/runtime/asm_amd64.s:318 +0x5b goroutine 1 [select, 10213 minutes]: sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start(0xc0003e4a80, 0xc000100f60, 0x0, 0x0) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:433 +0x1dd main.main() /workspace/main.go:99 +0x548 goroutine 6 [chan receive]: k8s.io/klog.(*loggingT).flushDaemon(0x22f46e0) /go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:1010 +0x8b created by k8s.io/klog.init.0 /go/pkg/mod/k8s.io/klog@v1.0.0/klog.go:411 +0xd8 goroutine 43 [select, 10213 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000331040) /go/pkg/mod/google.golang.org/grpc@v1.32.0/balancer_conn_wrappers.go:69 +0xc8 created by google.golang.org/grpc.newCCBalancerWrapper /go/pkg/mod/google.golang.org/grpc@v1.32.0/balancer_conn_wrappers.go:60 +0x172 goroutine 44 [chan receive, 10213 minutes]: google.golang.org/grpc.(*addrConn).resetTransport(0xc0001482c0) /go/pkg/mod/google.golang.org/grpc@v1.32.0/clientconn.go:1173 +0x709 created by google.golang.org/grpc.(*addrConn).connect /go/pkg/mod/google.golang.org/grpc@v1.32.0/clientconn.go:813 +0x12a goroutine 45 [select, 10213 minutes]: go.opentelemetry.io/otel/exporters/otlp.(*Exporter).indefiniteBackgroundConnection(0xc0001d4000) /go/pkg/mod/go.opentelemetry.io/otel/exporters/otlp@v0.13.0/connection.go:81 +0x265 created by go.opentelemetry.io/otel/exporters/otlp.(*Exporter).Start.func1 /go/pkg/mod/go.opentelemetry.io/otel/exporters/otlp@v0.13.0/otlp.go:128 +0x137 goroutine 46 [chan receive, 10213 minutes]: k8s.io/apimachinery/pkg/watch.(*Broadcaster).loop(0xc000331200) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/watch/mux.go:207 +0x66 created by k8s.io/apimachinery/pkg/watch.NewBroadcaster /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/watch/mux.go:75 +0xce goroutine 26 [IO wait, 92 minutes]: internal/poll.runtime_pollWait(0x7f589ec89e88, 0x72, 0x1931580) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc00018c398, 0x72, 0x1931500, 0x226d5f0, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc00018c380, 0xc0000f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5 net.(*netFD).Read(0xc00018c380, 0xc0000f4000, 0x8000, 0x8000, 0x880fde, 0x800000601, 0xc000000000) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc00027c008, 0xc0000f4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:182 +0x8e bufio.(*Reader).Read(0xc0000941e0, 0xc000096038, 0x9, 0x9, 0xc000010e08, 0x405d49, 0xc000262360) /usr/local/go/src/bufio/bufio.go:227 +0x222 io.ReadAtLeast(0x192dd80, 0xc0000941e0, 0xc000096038, 0x9, 0x9, 0x9, 0x460940, 0xc000262360, 0xc000010e40) /usr/local/go/src/io/io.go:314 +0x87 io.ReadFull(...) /usr/local/go/src/io/io.go:333 golang.org/x/net/http2.readFrameHeader(0xc000096038, 0x9, 0x9, 0x192dd80, 0xc0000941e0, 0x0, 0xc000000000, 0x10000000040e0d8, 0xc00007e2e0) /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/frame.go:237 +0x89 golang.org/x/net/http2.(*Framer).ReadFrame(0xc000096000, 0xc00021e380, 0xc00021e380, 0x0, 0x0) /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/frame.go:492 +0xa5 google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc0000fe700) /go/pkg/mod/google.golang.org/grpc@v1.32.0/internal/transport/http2_client.go:1294 +0x179 created by google.golang.org/grpc/internal/transport.newHTTP2Client /go/pkg/mod/google.golang.org/grpc@v1.32.0/internal/transport/http2_client.go:310 +0x1071 goroutine 141 [syscall, 10213 minutes]: os/signal.signal_recv(0xc0006000c0) /usr/local/go/src/runtime/sigqueue.go:147 +0x9d os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:23 +0x25 created by os/signal.Notify.func1.1 /usr/local/go/src/os/signal/signal.go:150 +0x45 goroutine 27 [select, 92 minutes]: google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00007e2d0, 0x1, 0x0, 0x0, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.32.0/internal/transport/controlbuf.go:395 +0x125 google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000942a0, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.32.0/internal/transport/controlbuf.go:515 +0x1d3 google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc0000fe700) /go/pkg/mod/google.golang.org/grpc@v1.32.0/internal/transport/http2_client.go:356 +0x7b created by google.golang.org/grpc/internal/transport.newHTTP2Client /go/pkg/mod/google.golang.org/grpc@v1.32.0/internal/transport/http2_client.go:354 +0x123c goroutine 51 [IO wait]: internal/poll.runtime_pollWait(0x7f589ec89f70, 0x72, 0x1931580) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc000373098, 0x72, 0x1931500, 0x226d5f0, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc000373080, 0xc0005c0000, 0x4df7, 0x4df7, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5 net.(*netFD).Read(0xc000373080, 0xc0005c0000, 0x4df7, 0x4df7, 0x203000, 0x7595db, 0xc00009e160) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc00000e050, 0xc0005c0000, 0x4df7, 0x4df7, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:182 +0x8e crypto/tls.(*atLeastReader).Read(0xc0004426c0, 0xc0005c0000, 0x4df7, 0x4df7, 0x34, 0x4df2, 0xc000093730) /usr/local/go/src/crypto/tls/conn.go:779 +0x62 bytes.(*Buffer).ReadFrom(0xc00009e280, 0x192df40, 0xc0004426c0, 0x40b485, 0x15d3600, 0x171cc00) /usr/local/go/src/bytes/buffer.go:204 +0xb1 crypto/tls.(*Conn).readFromUntil(0xc00009e000, 0x192f9c0, 0xc00000e050, 0x5, 0xc00000e050, 0x0) /usr/local/go/src/crypto/tls/conn.go:801 +0xf3 crypto/tls.(*Conn).readRecordOrCCS(0xc00009e000, 0x0, 0x0, 0xc00029d6e0) /usr/local/go/src/crypto/tls/conn.go:608 +0x115 crypto/tls.(*Conn).readRecord(...) /usr/local/go/src/crypto/tls/conn.go:576 crypto/tls.(*Conn).Read(0xc00009e000, 0xc0001a5000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/crypto/tls/conn.go:1252 +0x15f bufio.(*Reader).Read(0xc000576ae0, 0xc0002e4658, 0x9, 0x9, 0xc000315200, 0xc000093d48, 0x405d49) /usr/local/go/src/bufio/bufio.go:227 +0x222 io.ReadAtLeast(0x192dd80, 0xc000576ae0, 0xc0002e4658, 0x9, 0x9, 0x9, 0x177daf0, 0x460940, 0xc00031f140) /usr/local/go/src/io/io.go:314 +0x87 io.ReadFull(...) /usr/local/go/src/io/io.go:333 golang.org/x/net/http2.readFrameHeader(0xc0002e4658, 0x9, 0x9, 0x192dd80, 0xc000576ae0, 0x0, 0xc000000000, 0xc00029d650, 0xc000328090) /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/frame.go:237 +0x89 golang.org/x/net/http2.(*Framer).ReadFrame(0xc0002e4620, 0xc00029d650, 0x0, 0x0, 0x0) /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/frame.go:492 +0xa5 golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000093fa8, 0x1, 0x0) /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/transport.go:1741 +0x8d golang.org/x/net/http2.(*ClientConn).readLoop(0xc000083b00) /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/transport.go:1669 +0x6f created by golang.org/x/net/http2.(*Transport).newClientConn /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/transport.go:674 +0x66e goroutine 223 [select, 256 minutes]: k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch.func2(0xc000688a50, 0xc000100f00, 0xc0003b2b40, 0xc000095c20) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/reflector.go:260 +0x14a created by k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/reflector.go:254 +0x25f goroutine 220 [sync.Cond.Wait, 3 minutes]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:312 sync.runtime_notifyListWait(0xc0006889c8, 0x1515) /usr/local/go/src/runtime/sema.go:513 +0xf8 sync.(*Cond).Wait(0xc0006889b8) /usr/local/go/src/sync/cond.go:56 +0x9d k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc0006889a0, 0xc0003dcb20, 0x0, 0x0, 0x0, 0x0) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/delta_fifo.go:409 +0xa5 k8s.io/client-go/tools/cache.(*controller).processLoop(0xc000620600) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/controller.go:153 +0x42 k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc000015e68) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:152 +0x5f k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000015e68, 0x3b9aca00, 0x0, 0xc000128f01, 0xc000100f00) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:153 +0x105 k8s.io/apimachinery/pkg/util/wait.Until(...) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:88 k8s.io/client-go/tools/cache.(*controller).Run(0xc000620600, 0xc000100f00) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/controller.go:125 +0x2ce k8s.io/client-go/tools/cache.(*sharedIndexInformer).Run(0xc0003ab8c0, 0xc000100f00) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/shared_informer.go:331 +0x41a created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).addInformerToMap /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/cache/internal/informers_map.go:222 +0x405 goroutine 219 [select]: k8s.io/client-go/util/workqueue.(*delayingType).waitingLoop(0xc000095800) /go/pkg/mod/k8s.io/client-go@v0.17.9/util/workqueue/delaying_queue.go:222 +0x405 created by k8s.io/client-go/util/workqueue.NewDelayingQueueWithCustomClock /go/pkg/mod/k8s.io/client-go@v0.17.9/util/workqueue/delaying_queue.go:58 +0x1c5 goroutine 176 [chan receive, 10213 minutes]: sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start(0xc0000960e0, 0xc000100f00) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/cache/internal/informers_map.go:141 +0x4c created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/cache/internal/deleg_map.go:60 +0x50 goroutine 244 [runnable]: github.com/bboreham/kspan/controllers/events.(*EventWatcher).emitSpan(0xc000392a20, 0x1965120, 0xc00004c008, 0xc00004cfe0, 0xb, 0xc0004d069c, 0x4, 0xc0004d06c0, 0x8, 0xc0000d30e0) /workspace/controllers/events/outgoing.go:47 +0x51e github.com/bboreham/kspan/controllers/events.(*EventWatcher).emitSpanFromEvent(0xc000392a20, 0x1965120, 0xc00004c008, 0x196dc40, 0xc000483480, 0xc000129900, 0x19, 0xc0006f01c0, 0x1e) /workspace/controllers/events/event_controller.go:169 +0x1f1 github.com/bboreham/kspan/controllers/events.(*EventWatcher).handleEvent(0xc000392a20, 0x1965120, 0xc00004c008, 0xc000129900, 0x196dca0, 0xc00011d080) /workspace/controllers/events/event_controller.go:111 +0x30e github.com/bboreham/kspan/controllers/events.(*EventWatcher).Reconcile(0xc000392a20, 0xc0004d066c, 0x4, 0xc000a84180, 0x19, 0x43ee500a0b7e, 0xc0006d4000, 0xc0000d2ea8, 0xc0000d2ea0) /workspace/controllers/events/event_controller.go:90 +0x3d4 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).reconcileHandler(0xc00003a780, 0x1611b60, 0xc0004832c0, 0x12a8a00) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:256 +0x166 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).processNextWorkItem(0xc00003a780, 0x0) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:232 +0xb0 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).worker(0xc00003a780) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:211 +0x2b k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00021bf60) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:152 +0x5f k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc00021bf60, 0x3b9aca00, 0x0, 0x100000000000001, 0xc000100f00) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:153 +0x105 k8s.io/apimachinery/pkg/util/wait.Until(0xc00021bf60, 0x3b9aca00, 0xc000100f00) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:88 +0x4d created by sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start.func1 /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:193 +0x32d goroutine 171 [chan receive, 10213 minutes]: sigs.k8s.io/controller-runtime/pkg/manager/signals.SetupSignalHandler.func1(0xc0002fbf80, 0xc000100f60) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/signals/signal.go:36 +0x34 created by sigs.k8s.io/controller-runtime/pkg/manager/signals.SetupSignalHandler /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/signals/signal.go:35 +0xd1 goroutine 142 [IO wait, 3 minutes]: internal/poll.runtime_pollWait(0x7f589ec89da0, 0x72, 0x0) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc0000c9198, 0x72, 0x0, 0x0, 0x1780a56) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Accept(0xc0000c9180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:394 +0x1fc net.(*netFD).accept(0xc0000c9180, 0xe2400f966b82a988, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:172 +0x45 net.(*TCPListener).accept(0xc000202560, 0x60620405, 0xc000017e18, 0x490486) /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 net.(*TCPListener).Accept(0xc000202560, 0xc000017e68, 0x18, 0xc000216780, 0x8284ec) /usr/local/go/src/net/tcpsock.go:261 +0x65 net/http.(*Server).Serve(0xc0002e4000, 0x1962ea0, 0xc000202560, 0x0, 0x0) /usr/local/go/src/net/http/server.go:2937 +0x266 sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).serveMetrics.func1(0x1781a0c, 0x8, 0xc0002e4000, 0xc0003e4a80) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:357 +0xfd created by sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).serveMetrics /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:355 +0x193 goroutine 434461 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:312 sync.runtime_notifyListWait(0xc000377a60, 0x0) /usr/local/go/src/runtime/sema.go:513 +0xf8 sync.(*Cond).Wait(0xc000377a50) /usr/local/go/src/sync/cond.go:56 +0x9d golang.org/x/net/http2.(*pipe).Read(0xc000377a48, 0xc000178a00, 0x200, 0x200, 0x0, 0x0, 0x0) /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/pipe.go:65 +0x97 golang.org/x/net/http2.transportResponseBody.Read(0xc000377a20, 0xc000178a00, 0x200, 0x200, 0x0, 0x0, 0x0) /go/pkg/mod/golang.org/x/net@v0.0.0-20200520004742-59133d7f0dd7/http2/transport.go:2027 +0xaf encoding/json.(*Decoder).refill(0xc000138b00, 0x18, 0x7f58c5956108) /usr/local/go/src/encoding/json/stream.go:165 +0xeb encoding/json.(*Decoder).readValue(0xc000138b00, 0x0, 0x0, 0x159f8e0) /usr/local/go/src/encoding/json/stream.go:140 +0x1ff encoding/json.(*Decoder).Decode(0xc000138b00, 0x15c0940, 0xc000442700, 0xffffffffffffffff, 0x0) /usr/local/go/src/encoding/json/stream.go:63 +0x79 k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc00029d770, 0xc00033e800, 0x400, 0x400, 0x3, 0x7f589e9b7e60, 0x38) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/framer/framer.go:150 +0x1a8 k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc0000b7540, 0x0, 0x193a6e0, 0xc0004b21c0, 0x0, 0xc0002997a8, 0x12f7f39, 0x1306b34, 0xc000333ef0) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/runtime/serializer/streaming/streaming.go:77 +0x89 k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc0004426e0, 0xc000200a80, 0x22fb701, 0xc000201912, 0x4, 0x0, 0xc000299758) /go/pkg/mod/k8s.io/client-go@v0.17.9/rest/watch/decoder.go:49 +0x6e k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0004b2180) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/watch/streamwatcher.go:104 +0x147 created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/watch/streamwatcher.go:71 +0xbe goroutine 144 [chan receive, 10213 minutes]: k8s.io/client-go/tools/cache.(*sharedProcessor).run(0xc0001ba850, 0xc000262480) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/shared_informer.go:529 +0x53 k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:54 +0x2e k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000b0380, 0xc000634500) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:71 +0x51 created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:69 +0x65 goroutine 172 [chan receive, 10213 minutes]: sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).serveMetrics(0xc0003e4a80, 0xc000100f00) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:364 +0x1bb created by sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:414 +0x33b goroutine 250 [IO wait]: internal/poll.runtime_pollWait(0x7f589ec89cb8, 0x72, 0x1931580) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc0000c9298, 0x72, 0x1931500, 0x226d5f0, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc0000c9280, 0xc000325000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5 net.(*netFD).Read(0xc0000c9280, 0xc000325000, 0x1000, 0x1000, 0xc00045a5c0, 0x4, 0xc0006c17d8) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc00027c120, 0xc000325000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:182 +0x8e net/http.(*connReader).Read(0xc00029d590, 0xc000325000, 0x1000, 0x1000, 0x281c, 0x0, 0x7f589ecc7208) /usr/local/go/src/net/http/server.go:798 +0x1ad bufio.(*Reader).fill(0xc0005be8a0) /usr/local/go/src/bufio/bufio.go:101 +0x105 bufio.(*Reader).ReadSlice(0xc0005be8a0, 0xc00012210a, 0x7f589ecc7208, 0xc0006c1988, 0x40d770, 0xc0001d4900, 0x100) /usr/local/go/src/bufio/bufio.go:360 +0x3d bufio.(*Reader).ReadLine(0xc0005be8a0, 0xc0001d4900, 0x476db4, 0xc000056400, 0x0, 0x16da240, 0xc000601920) /usr/local/go/src/bufio/bufio.go:389 +0x34 net/textproto.(*Reader).readLineSlice(0xc000601920, 0xc0001d4900, 0x4d850d, 0xc0000c9280, 0x463b00, 0xc0003e5b00) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(*Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc0005be8a0, 0x0, 0xc0001d4900, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1012 +0xaa net/http.(*conn).readRequest(0xc0000c6500, 0x19650e0, 0xc0006fae00, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:984 +0x19a net/http.(*conn).serve(0xc0000c6500, 0x19650e0, 0xc0006fae00) /usr/local/go/src/net/http/server.go:1851 +0x705 created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:2969 +0x36c goroutine 210 [chan receive, 10213 minutes]: sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller).Start(0xc00003a780, 0xc000100f00, 0x0, 0x0) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/internal/controller/controller.go:203 +0xd0 sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).startLeaderElectionRunnables.func1(0x192ff00, 0xc00003a780, 0xc0003e4a80) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:477 +0x4f created by sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).startLeaderElectionRunnables /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:476 +0xd7 goroutine 175 [chan receive, 10213 minutes]: sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start(0xc00000dea0, 0xc000100f00, 0x0, 0x0) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/cache/internal/deleg_map.go:62 +0x9c sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).waitForCache.func1(0xc0003e4a80) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:499 +0x39 created by sigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).waitForCache /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/manager/internal.go:498 +0x5a goroutine 217 [chan receive]: k8s.io/client-go/util/workqueue.(*Type).updateUnfinishedWorkLoop(0xc000095680) /go/pkg/mod/k8s.io/client-go@v0.17.9/util/workqueue/queue.go:198 +0xac created by k8s.io/client-go/util/workqueue.newQueue /go/pkg/mod/k8s.io/client-go@v0.17.9/util/workqueue/queue.go:58 +0x135 goroutine 193 [chan receive, 10213 minutes]: k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x193aae0, 0xc000340cc0, 0xc00045c150) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/record/event.go:291 +0xaa created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/record/event.go:289 +0x6e goroutine 96 [chan receive, 10213 minutes]: k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher.func1(0x193aae0, 0xc000340c90, 0xc000340c30) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/record/event.go:291 +0xaa created by k8s.io/client-go/tools/record.(*eventBroadcasterImpl).StartEventWatcher /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/record/event.go:289 +0x6e goroutine 194 [semacquire, 8969 minutes]: sync.runtime_SemacquireMutex(0xc0002025a4, 0x0, 0x1) /usr/local/go/src/runtime/sema.go:71 +0x47 sync.(*Mutex).lockSlow(0xc0002025a0) /usr/local/go/src/sync/mutex.go:138 +0x105 sync.(*Mutex).Lock(...) /usr/local/go/src/sync/mutex.go:81 github.com/bboreham/kspan/controllers/events.(*EventWatcher).flushOutgoing(0xc000392a20, 0x1965120, 0xc00004c008, 0xc00e918a207758f7, 0x43ec8187c172, 0x22f4460) /workspace/controllers/events/outgoing.go:58 +0x6c5 github.com/bboreham/kspan/controllers/events.(*EventWatcher).runTicker(0xc000392a20) /workspace/controllers/events/event_controller.go:269 +0x107 created by github.com/bboreham/kspan/controllers/events.(*EventWatcher).initialize /workspace/controllers/events/event_controller.go:280 +0x177 goroutine 232 [IO wait]: internal/poll.runtime_pollWait(0x7f589ec89bd0, 0x72, 0x1931580) /usr/local/go/src/runtime/netpoll.go:222 +0x55 internal/poll.(*pollDesc).wait(0xc000372298, 0x72, 0x1931500, 0x226d5f0, 0x0) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc000372280, 0xc0005f8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5 net.(*netFD).Read(0xc000372280, 0xc0005f8000, 0x1000, 0x1000, 0xc00045ad40, 0x4, 0xc00008f7d8) /usr/local/go/src/net/fd_posix.go:55 +0x4f net.(*conn).Read(0xc0000a4308, 0xc0005f8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:182 +0x8e net/http.(*connReader).Read(0xc0005f6450, 0xc0005f8000, 0x1000, 0x1000, 0x281c, 0x380, 0x7f589ecc7208) /usr/local/go/src/net/http/server.go:798 +0x1ad bufio.(*Reader).fill(0xc000576120) /usr/local/go/src/bufio/bufio.go:101 +0x105 bufio.(*Reader).ReadSlice(0xc000576120, 0xc00031440a, 0x7f589ecc7208, 0xc00008f988, 0x40d770, 0xc0001d4a00, 0x100) /usr/local/go/src/bufio/bufio.go:360 +0x3d bufio.(*Reader).ReadLine(0xc000576120, 0xc0001d4a00, 0x476db4, 0xc000080800, 0x0, 0x2, 0xc0005f9000) /usr/local/go/src/bufio/bufio.go:389 +0x34 net/textproto.(*Reader).readLineSlice(0xc000a9bf80, 0xc0001d4a00, 0x4d850d, 0xc000372280, 0x463b00, 0xc0003f1e00) /usr/local/go/src/net/textproto/reader.go:58 +0x6c net/textproto.(*Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc000576120, 0x0, 0xc0001d4a00, 0x0, 0x0) /usr/local/go/src/net/http/request.go:1012 +0xaa net/http.(*conn).readRequest(0xc0003a60a0, 0x19650e0, 0xc0005f2740, 0x0, 0x0, 0x0) /usr/local/go/src/net/http/server.go:984 +0x19a net/http.(*conn).serve(0xc0003a60a0, 0x19650e0, 0xc0005f2740) /usr/local/go/src/net/http/server.go:1851 +0x705 created by net/http.(*Server).Serve /usr/local/go/src/net/http/server.go:2969 +0x36c goroutine 209 [chan receive, 10213 minutes]: sigs.k8s.io/controller-runtime/pkg/cache/internal.(*specificInformersMap).Start(0xc0000961c0, 0xc000100f00) /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/cache/internal/informers_map.go:141 +0x4c created by sigs.k8s.io/controller-runtime/pkg/cache/internal.(*InformersMap).Start /go/pkg/mod/sigs.k8s.io/controller-runtime@v0.5.0/pkg/cache/internal/deleg_map.go:61 +0x85 goroutine 243 [select, 3 minutes]: k8s.io/client-go/tools/cache.(*processorListener).pop(0xc0006aed80) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/shared_informer.go:621 +0x157 k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ba8b0, 0xc00021be00) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:71 +0x51 created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:69 +0x65 goroutine 242 [chan receive, 3 minutes]: k8s.io/client-go/tools/cache.(*processorListener).run.func1.1(0x0, 0x0, 0x0) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/shared_informer.go:653 +0x54 k8s.io/apimachinery/pkg/util/wait.ExponentialBackoff(0x989680, 0x3ff0000000000000, 0x3fb999999999999a, 0x5, 0x0, 0xc000461e18, 0x44809f, 0xc00055b760) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:292 +0x51 k8s.io/client-go/tools/cache.(*processorListener).run.func1() /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/shared_informer.go:652 +0x79 k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00055b760) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:152 +0x5f k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc000461f60, 0xdf8475800, 0x0, 0x1, 0xc0003b2de0) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:153 +0x105 k8s.io/apimachinery/pkg/util/wait.Until(...) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:88 k8s.io/client-go/tools/cache.(*processorListener).run(0xc0006aed80) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/shared_informer.go:650 +0x9b k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0001ba8b0, 0xc00021bdf0) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:71 +0x51 created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:69 +0x65 goroutine 225 [chan receive, 10213 minutes]: k8s.io/client-go/tools/cache.(*controller).Run.func1(0xc000100f00, 0xc000620600) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/controller.go:104 +0x34 created by k8s.io/client-go/tools/cache.(*controller).Run /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/controller.go:103 +0xac goroutine 226 [select]: k8s.io/client-go/tools/cache.(*Reflector).watchHandler(0xc000688a50, 0xc0109f00a4f410a5, 0x22d4ca3a67d2e, 0x22f4460, 0x193aaa0, 0xc0004b2180, 0xc0008f5b90, 0xc000095c20, 0xc000100f00, 0x0, ...) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/reflector.go:355 +0x1a5 k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc000688a50, 0xc000100f00, 0x0, 0x0) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/reflector.go:322 +0x928 k8s.io/client-go/tools/cache.(*Reflector).Run.func1() /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/reflector.go:152 +0x33 k8s.io/apimachinery/pkg/util/wait.JitterUntil.func1(0xc00026a710) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:152 +0x5f k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc0008f5f10, 0x3b9aca00, 0x0, 0x1965101, 0xc000100f00) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:153 +0x105 k8s.io/apimachinery/pkg/util/wait.Until(...) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:88 k8s.io/client-go/tools/cache.(*Reflector).Run(0xc000688a50, 0xc000100f00) /go/pkg/mod/k8s.io/client-go@v0.17.9/tools/cache/reflector.go:151 +0x16c k8s.io/apimachinery/pkg/util/wait.(*Group).StartWithChannel.func1() /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:54 +0x2e k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1(0xc0000b0390, 0xc000634540) /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:71 +0x51 created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start /go/pkg/mod/k8s.io/apimachinery@v0.17.9/pkg/util/wait/wait.go:69 +0x65
Seems to be looping here: https://github.com/weaveworks-experiments/kspan/blob/3e8e909978e9a07a2765aae795514f508566da2b/controllers/events/outgoing.go#L46-L54
Could a span have a parent that points at itself?
goroutine dump: