search

koinos / koinos-p2p

The p2p microservice orchestrates the distribution of blocks and transactions between peers.
MIT License
6 stars 4 forks source link

Fatal error: concurrent map read and map write #114

Closed sgerbino closed 3 years ago

sgerbino commented 3 years ago

While running a p2p node, a crash may occur with the following stacktrace. This occurs on branch set_last_irr and is possibly related to that work.

2021-06-10 20:18:48.627528 (p2p.Koinos) [protocol/download_provider.go:219] <info>: Downloaded block applied - Height: 861 ID: zQmU3SPuBBMwDvbJ9BFbiW5GDgAzgPXazvrVb2fUzHSouqB Prev: zQmTwoJTMm15CXPDyAhtu6jZs7QodbCqj7LtMVXryodv68t from peer Qmf4Rz6Yg6YnaNpqQoUxDepn2ceAeEMvPv9C4R3hJVDqY3
2021-06-10 20:18:48.627634 (p2p.Koinos) [protocol/download_provider.go:399] <info>: Connecting disconnected fork head "zQmU3SPuBBMwDvbJ9BFbiW5GDgAzgPXazvrVb2fUzHSouqB"
2021-06-10 20:18:48.789442 (p2p.Koinos) [protocol/download_provider.go:219] <info>: Downloaded block applied - Height: 862 ID: zQmYtUmqCBjTgQkht8dcdEDfU2FAYCHZbekt5nAu4YpLGxY Prev: zQmU3SPuBBMwDvbJ9BFbiW5GDgAzgPXazvrVb2fUzHSouqB from peer Qmf4Rz6Yg6YnaNpqQoUxDepn2ceAeEMvPv9C4R3hJVDqY3
2021-06-10 20:18:49.159832 (p2p.Koinos) [protocol/gossip.go:187] <info>: Gossiped block applied: Height: 865 ID: zQmcESyLS6JpD3gdzNsD8oNBBjsDeftCwUHf5w9SZnhNYNb Prev: zQmSZjwFvMLKxJs9X3Qdjf3a3ArDH11mzbTHv58F4Rahuvp from peer QmSuMdPvG7vfwr69snMtRhaC5foNwc6xaLdGdETkgyNkq9
2021-06-10 20:18:50.898732 (p2p.Koinos) [protocol/gossip.go:187] <info>: Gossiped block applied: Height: 866 ID: zQmYSMRNuXPtEaSUJ16sX1ZbbB9Vvp3ohgM36fY6SjMGTSP Prev: zQmcESyLS6JpD3gdzNsD8oNBBjsDeftCwUHf5w9SZnhNYNb from peer QmSuMdPvG7vfwr69snMtRhaC5foNwc6xaLdGdETkgyNkq9
fatal error: concurrent map read and map write

goroutine 57808 [running]:
runtime.throw(0x18cc9ee, 0x21)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/panic.go:1117 +0x72 fp=0xc0003fb628 sp=0xc0003fb5f8 pc=0x1037c12
runtime.mapaccess2_fast64(0x17c6060, 0xc000308300, 0x35a, 0xc0000f9e40, 0x20)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/map_fast64.go:61 +0x1ac fp=0xc0003fb650 sp=0xc0003fb628 pc=0x10140ac
github.com/koinos/koinos-p2p/internal/protocol.(*SyncService).GetTopologyAtHeight(0xc000308510, 0x1acd878, 0xc000032840, 0x325, 0xa0, 0xc0004fe6e0, 0x0, 0x0)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/sync_service.go:186 +0x3c5 fp=0xc0003fb858 sp=0xc0003fb650 pc=0x1608c05
runtime.call64(0xc0007c9d10, 0xc000134808, 0xc000032880, 0x3000000040)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/asm_amd64.s:552 +0x3e fp=0xc0003fb8a8 sp=0xc0003fb858 pc=0x106cc9e
reflect.Value.call(0xc0001b2c00, 0xc000134808, 0x13, 0x18b55b2, 0x4, 0xc000071b90, 0x4, 0x4, 0xc000071b18, 0x1069e33, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/reflect/value.go:476 +0x8e7 fp=0xc0003fbab0 sp=0xc0003fb8a8 pc=0x10bd647
reflect.Value.Call(0xc0001b2c00, 0xc000134808, 0x13, 0xc000071b90, 0x4, 0x4, 0x10, 0xc00052c4e0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/reflect/value.go:337 +0xb9 fp=0xc0003fbb30 sp=0xc0003fbab0 pc=0x10bcb19
github.com/libp2p/go-libp2p-gorpc.(*service).svcCall(0xc0002f5cc0, 0xc0004410b0, 0xc0002e7c70, 0xc000330830, 0xb, 0xc0002142b8, 0x13, 0x181ba20, 0xc000032840, 0x16, ...)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:292 +0x152 fp=0xc0003fbc00 sp=0xc0003fbb30 pc=0x155ad52
github.com/libp2p/go-libp2p-gorpc.(*Server).handle(0xc000300960, 0xc0004410b0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:284 +0xa1a fp=0xc0003fbe88 sp=0xc0003fbc00 pc=0x155a49a
github.com/libp2p/go-libp2p-gorpc.NewServer.func1(0x1ad7eb0, 0xc0004fe680)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:173 +0xae fp=0xc0003fbf60 sp=0xc0003fbe88 pc=0x155e8ce
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1(0xc0002142a0, 0x12, 0xaa55160, 0xc0004fe680, 0x1, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:566 +0xa2 fp=0xc0003fbfb0 sp=0xc0003fbf60 pc=0x16e2942
runtime.goexit()
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc0003fbfb8 sp=0xc0003fbfb0 pc=0x106e5a1
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:416 +0x63a

goroutine 1 [chan receive, 20 minutes]:
main.main()
    /Users/sgerbino/Projects/koinos-p2p/cmd/koinos-p2p/main.go:161 +0x12e5

goroutine 9 [chan receive]:
github.com/streadway/amqp.(*consumers).buffer(0xc0002efe90, 0xc0000a0240, 0xc0000a01e0)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:61 +0x365
created by github.com/streadway/amqp.(*consumers).add
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:97 +0x165

goroutine 20 [select, 20 minutes]:
github.com/ipfs/go-log/writer.(*MirrorWriter).logRoutine(0xc0001cf320)
    /Users/sgerbino/go/pkg/mod/github.com/ipfs/go-log@v1.0.4/writer/writer.go:71 +0x106
created by github.com/ipfs/go-log/writer.NewMirrorWriter
    /Users/sgerbino/go/pkg/mod/github.com/ipfs/go-log@v1.0.4/writer/writer.go:36 +0xb9

goroutine 21 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0001b2180)
    /Users/sgerbino/go/pkg/mod/go.opencensus.io@v0.22.4/stats/view/worker.go:276 +0xcd
created by go.opencensus.io/stats/view.init.0
    /Users/sgerbino/go/pkg/mod/go.opencensus.io@v0.22.4/stats/view/worker.go:34 +0x68

goroutine 22 [chan receive, 20 minutes]:
github.com/koinos/koinos-mq-golang.(*Client).ConnectLoop(0xc000146dc0)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/client.go:139 +0x271
created by github.com/koinos/koinos-mq-golang.(*Client).Start
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/client.go:73 +0x3f

goroutine 23 [chan receive, 20 minutes]:
gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun(0xc0002cdec0)
    /Users/sgerbino/go/pkg/mod/gopkg.in/natefinch/lumberjack.v2@v2.0.0/lumberjack.go:379 +0x5d
created by gopkg.in/natefinch/lumberjack%2ev2.(*Logger).mill.func1
    /Users/sgerbino/go/pkg/mod/gopkg.in/natefinch/lumberjack.v2@v2.0.0/lumberjack.go:390 +0x75

goroutine 12 [select, 20 minutes]:
github.com/libp2p/go-libp2p-peerstore/pstoremem.(*memoryAddrBook).background(0xc0000dc000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-peerstore@v0.2.6/pstoremem/addr_book.go:93 +0x105
created by github.com/libp2p/go-libp2p-peerstore/pstoremem.NewAddrBook
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-peerstore@v0.2.6/pstoremem/addr_book.go:83 +0x18f

goroutine 7 [IO wait]:
internal/poll.runtime_pollWait(0xaa40fb8, 0x72, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc0001b2898, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0001b2880, 0xc0000a5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc0001b2880, 0xc0000a5000, 0x1000, 0x1000, 0x100ebca, 0xaa96590, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000010048, 0xc0000a5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/net.go:183 +0x91
bufio.(*Reader).Read(0xc0000885a0, 0xc00037a000, 0x7, 0x7, 0xc00024ce60, 0x100f818, 0x7)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x1abc020, 0xc0000885a0, 0xc00037a000, 0x7, 0x7, 0x7, 0xc00024ced8, 0x1006d15, 0xc000088540)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:328 +0x87
io.ReadFull(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:347
github.com/streadway/amqp.(*reader).ReadFrame(0xc00024cf40, 0xc00024cf50, 0xc0004fe720, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/read.go:49 +0x75
github.com/streadway/amqp.(*Connection).reader(0xc0000a6000, 0xaa91038, 0xc000010048)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:521 +0xdc
created by github.com/streadway/amqp.Open
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:233 +0x26c

goroutine 8 [select]:
github.com/streadway/amqp.(*Connection).heartbeater(0xc0000a6000, 0x2540be400, 0xc000088600)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:551 +0x188
created by github.com/streadway/amqp.(*Connection).openTune
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:782 +0x465

goroutine 28 [chan receive]:
github.com/koinos/koinos-mq-golang.(*Client).ConsumeRPCReturnLoop(0xc000146dc0, 0xc0000a01e0)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/client.go:304 +0x97
created by github.com/koinos/koinos-mq-golang.(*Client).ConnectLoop
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/client.go:111 +0x205

goroutine 14 [select]:
github.com/libp2p/go-libp2p-swarm.(*DialBackoff).background(0xc000001a90, 0x1acd878, 0xc000032300)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_dial.go:126 +0xee
created by github.com/libp2p/go-libp2p-swarm.(*DialBackoff).init
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_dial.go:119 +0x5d

goroutine 15 [select]:
github.com/libp2p/go-libp2p/p2p/protocol/identify.(*ObservedAddrManager).worker(0xc000088b40, 0x1acd878, 0xc0000327c0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/obsaddr.go:236 +0x20d
created by github.com/libp2p/go-libp2p/p2p/protocol/identify.NewObservedAddrManager
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/obsaddr.go:127 +0x17d

goroutine 16 [select, 8 minutes]:
github.com/libp2p/go-libp2p/p2p/protocol/identify.(*IDService).loop(0xc00009ad00)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/id.go:203 +0x56c
created by github.com/libp2p/go-libp2p/p2p/protocol/identify.NewIDService
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/id.go:150 +0x2cd

goroutine 38 [chan receive, 20 minutes]:
github.com/koinos/koinos-mq-golang.(*RequestHandler).ConsumeBroadcastLoop(0xc0002f59c0, 0xc00057a4e0, 0x18c5e38, 0x19)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:198 +0x19a
created by github.com/koinos/koinos-mq-golang.(*RequestHandler).ConnectLoop
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-g:133 +0x349

goroutine 50 [select, 20 minutes]:
github.com/libp2p/go-libp2p-circuit.(*RelayListener).Accept(0xc000088cc0, 0x0, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-circuit@v0.4.0/listen.go:27 +0x18f
github.com/libp2p/go-libp2p-transport-upgrader.(*listener).handleIncoming(0xc000088d20)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/listener.go:77 +0xf7
created by github.com/libp2p/go-libp2p-transport-upgrader.(*Upgrader).UpgradeListener
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/upgrader.go:49 +0x18d

goroutine 55 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0xaa40ed0, 0x72, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000470918, 0x72, 0x0, 0x0, 0x18b7089)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000470900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:401 +0x212
net.(*netFD).accept(0xc000470900, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/fd_unix.go:172 +0x45
net.(*TCPListener).accept(0xc00000f500, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00000f500, 0x0, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/tcpsock.go:261 +0x65
github.com/multiformats/go-multiaddr/net.(*maListener).Accept(0xc0000f3140, 0xc, 0x2170108, 0x10, 0xc000460d80)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-multiaddr@v0.3.1/net/net.go:243 +0x48
github.com/libp2p/go-tcp-transport.(*lingerListener).Accept(0xc00000f578, 0x0, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-tcp-transport@v0.2.1/tcp.go:42 +0x37
github.com/libp2p/go-libp2p-transport-upgrader.(*listener).handleIncoming(0xc000088d80)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/listener.go:77 +0xf7
created by github.com/libp2p/go-libp2p-transport-upgrader.(*Upgrader).UpgradeListener
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/upgrader.go:49 +0x18d

goroutine 54 [chan receive, 20 minutes]:
github.com/libp2p/go-libp2p-transport-upgrader.(*listener).Accept(0xc000088d20, 0x1faa570, 0xc00048b720, 0x106a69e, 0xc000460cb8)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/listener.go:155 +0x51
github.com/libp2p/go-libp2p-swarm.(*Swarm).AddListenAddr.func2(0x1acdca0, 0xc000088d20, 0xc000001980, 0x1ad78e0, 0xc00012f038)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_listen.go:84 +0x18b
created by github.com/libp2p/go-libp2p-swarm.(*Swarm).AddListenAddr
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_listen.go:69 +0x268

goroutine 30 [chan receive, 20 minutes]:
github.com/libp2p/go-libp2p-transport-upgrader.(*listener).Accept(0xc000088d80, 0x1faab70, 0xc00048c720, 0x106a69e, 0xc000460d78)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/listener.go:155 +0x51
github.com/libp2p/go-libp2p-swarm.(*Swarm).AddListenAddr.func2(0x1acdca0, 0xc000088d80, 0xc000001980, 0x1ad78e0, 0xc00000f548)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_listen.go:84 +0x18b
created by github.com/libp2p/go-libp2p-swarm.(*Swarm).AddListenAddr
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_listen.go:69 +0x268

goroutine 33 [select]:
github.com/libp2p/go-libp2p-autonat.(*AmbientAutoNAT).background(0xc0002f9110)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-autonat@v0.4.0/autonat.go:175 +0x2bc
created by github.com/libp2p/go-libp2p-autonat.New
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-autonat@v0.4.0/autonat.go:125 +0x6b5

goroutine 66 [select]:
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).background(0xc0000f4000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:520 +0x22a
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:368 +0x5c

goroutine 67 [select]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).heartbeatTimer(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:1217 +0x21d
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:422 +0x10d

goroutine 68 [select, 3 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 69 [select, 2 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 70 [select, 3 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 71 [select, 4 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 72 [select, 4 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 73 [select, 2 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 74 [select]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 75 [select]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 76 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 77 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 78 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 79 [runnable]:
reflect.(*rtype).NumField(0x1805cc0, 0xc00026e0b8)
    /usr/local/Cellar/go/1.16.4/libexec/src/reflect/type.go:974 +0xbd
encoding/asn1.parseField(0x1805cc0, 0xc00026e0b8, 0x199, 0xc0002243c2, 0x59, 0x5e, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/asn1/asn1.go:930 +0x1b7b
encoding/asn1.parseField(0x1820520, 0xc00026e0a0, 0x199, 0xc0002243c0, 0x5b, 0x60, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/asn1/asn1.go:935 +0x1d85
encoding/asn1.UnmarshalWithParams(0xc0002243c0, 0x5b, 0x60, 0x176c920, 0xc00026e0a0, 0x0, 0x0, 0x60, 0x1, 0xc0002243c0, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/asn1/asn1.go:1117 +0x2b3
encoding/asn1.Unmarshal(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/asn1/asn1.go:1090
crypto/x509.ParsePKIXPublicKey(0xc0002243c0, 0x5b, 0x60, 0x5f, 0x5f, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/crypto/x509/x509.go:58 +0x7f
github.com/libp2p/go-libp2p-core/crypto.UnmarshalECDSAPublicKey(0xc0002243c0, 0x5b, 0x60, 0xc0001ac040, 0x1, 0x100f818, 0x20)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-core@v0.8.5/crypto/ecdsa.go:92 +0x45
github.com/libp2p/go-libp2p-core/crypto.PublicKeyFromProto(0xc000150420, 0x5f, 0x5f, 0x1ac9890, 0xc000150420)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-core@v0.8.5/crypto/key.go:299 +0x93
github.com/libp2p/go-libp2p-core/crypto.UnmarshalPublicKey(0xc000224240, 0x5f, 0x5f, 0xc0004be150, 0x22, 0x30, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-core@v0.8.5/crypto/key.go:286 +0xaf
github.com/libp2p/go-libp2p-pubsub.messagePubKey(0xc0006481e0, 0x0, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/sign.go:95 +0xb9
github.com/libp2p/go-libp2p-pubsub.verifyMessageSignature(0xc0006481e0, 0xc00024af40, 0x104b3ae)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/sign.go:50 +0x45
github.com/libp2p/go-libp2p-pubsub.(*validation).validateSignature(0xc000308990, 0xc0000f1a10, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:314 +0x35
github.com/libp2p/go-libp2p-pubsub.(*validation).validate(0xc000308990, 0xc000134028, 0x1, 0x1, 0xc00003e300, 0x22, 0xc0000f1a10)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:243 +0x6dd
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:229 +0x6e
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 80 [select]:
github.com/libp2p/(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 81 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 82 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 83 [select]:
g(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 84 [select]:
github.com/libp2p/go-libp2p-pubsub.(*PubSub).processLoop(0xc0001d8ea0, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/pubsub.go:485 +0x52c
created by github.com/libp2p/go-libp2p-pubsub.NewPubSub
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/pubsub.go:297 +0xb9e

goroutine 85 [chan receive, 20 minutes]:
github.com/koinos/koinos-mq-golang.(*RequestHandler).ConnectLoop(0xc0002f59c0)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:161 +0x434
created by github.com/koinos/koinos-mq-golang.(*RequestHandler).Start
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:74 +0x3f

goroutine 86 [runnable]:
math/big.nat.make(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/nat.go:64
math/big.nat.setWord(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/nat.go:76
math/big.nat.div(0xc0006fe5a0, 0x4, 0x9, 0x0, 0x0, 0x0, 0xc0006fe5a0, 0x4, 0x9, 0xc000123000, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/nat.go:668 +0x349
math/big.(*Int).QuoRem(0xc000149510, 0xc000149510, 0xc000150680, 0xc0001494f0, 0xc000149510, 0xc0001494f0)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/int.go:239 +0xbf
math/big.(*Int).DivMod(0xc000149510, 0xc000149510, 0xc000150680, 0xc0001494f0, 0xc000149510, 0xc0001494f0)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/int.go:301 +0xe5
github.com/btcsuite/btcutil/base58.Encode(0xc0004bd260, 0x22, 0x30, 0x1fd5248, 0x100491a)
    /Users/sgerbino/go/pkg/mod/github.com/btcsuite/btcutil@v1.0.2/base58/base58.go:56 +0x19d
github.com/koinos/koinos-types-golang.Multihash.MarshalJSON(0x12, 0xc00022a900, 0x20, 0x20, 0x100d025, 0x17d9ba0, 0x181b000, 0x17c8401, 0xad65a38)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-types-golang@v0.0.0-20210601210740-559c76121b53/basetypes.go:1238 +0x88
encoding/json.marshalerEncoder(0xc000712700, 0x181b000, 0xc0006362a0, 0x99, 0x100)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:479 +0xad
encoding/json.condAddrEncoder.encode(0x1a11648, 0x1a116a0, 0xc000712700, 0x181b000, 0xc0006362a0, 0x99, 0x1810100)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:961 +0xb2
encoding/json.(*encodeState).reflectValue(0xc000712700, 0x181b000, 0xc0006362a0, 0x99, 0xc000140100)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:360 +0x82
encoding/json.(*encodeState).marshal(0xc000712700, 0x181b000, 0xc0006362a0, 0x100, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:332 +0xf9
encoding/json.Marshal(0x181b000, 0xc0006362a0, 0x181b000, 0xc0006362a0, 0x20, 0x20, 0x1)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:161 +0x52
github.com/koinos/koinos-util-golang.BlockTopologyCmpString(0xc000149c08, 0xc0001499b4, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-util-golang@v0.0.0-20210601175130-50a337109792/helpers.go:40 +0xbd
github.com/koinos/koinos-p2p/internal/protocol.(*BlockDownloadManager).downloadManagerLoop(0xc0, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_manager.go:376 +0xa5a
created by github.com/koinos/koinos-p2p/internal/protocol.(*BlockDownloadManager).Start
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_manager.go:190 +0x53

goroutine 136 [chan receive]:
github.com/koinos/koinos-mq-golang.(*RequestHandler).ConsumeBroadcastLoop(0xc0002f59c0, 0xc00026c000, 0x18c0c76, 0x12)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:198 +0x19a
created by github.com/koinos/koinos-mq-golang.(*RequestHandler).ConnectLoop
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:133 +0x349

goroutine 88 [select]:
github.com/koinos/koinos-p2p/internal/protocol.(*SyncManager).run(0xc0001b2900, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/sync_manager.go:227 +0x1b9
created by github.com/koinos/koinos-p2p/in
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/sync_manager.go:256 +0xa5

goroutine 89 [chan receive]:
github.com/koinos/koinos-p2p/internal/protocol.(*KoinosGossip).startBlockGossip.func1(0xc0002f5f00, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:156 +0x11d
created by github.com/koinos/koinos-p2p/internal/protocol.(*KoinosGossip).startBlockGossip
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:147 +0x53

goroutine 90 [chan receive, 20 minutes]:
github.com/koinos/koinos-p2p/internal/protocol.(*KoinosGossip).startTransactionGossip.func1(0xc0002f5f00, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:201 +0x11d
created by github.com/koinos/koinos-p2p/internal/protocol.(*KoinosGossip).startTransactionGossip
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:192 +0x53

goroutine 100 [runnable]:
internal/poll.runtime_pollWait(0xaa40d00, 0x72, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000470a18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000470a00, 0xc000475000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc000470a00, 0xc000475000, 0x1000, 0x1000, 0xc00026c2f8, 0xc0006aedb8, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000596030, 0xc000475000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/net.go:183 +0x91
bufio.(*Reader).Read(0xc000089560, 0xc000639339, 0x7, 0x7, 0xc0006aee60, 0x100f818, 0x7)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x1abc020, 0xc000089560, 0xc000639339, 0x7, 0x7, 0x7, 0xc0006aeed8, 0x1006d15, 0xc0005806c0)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:328 +0x87
io.ReadFull(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:347
github.com/streadway/amqp.(*reader).ReadFrame(0xc0006aef40, 0xc000279f50, 0xc000526180, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/read.go:49 +0x75
github.com/streadway/amqp.(*Connection).reader(0xc0005ba000, 0xaa91038, 0xc000596030)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:521 +0xdc
created by github.com/streadway/amqp.Open
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:233 +0x26c

goroutine 98 [select, 20 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*Subscription).Next(0xc00058e080, 0x1acd8b0, 0xc000122000, 0xc00058c000, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/subscription.go:24 +0xab
github.com/koinos/koinos-p2p/internal/protocol.(*GossipManager).readMessages(0xc000308c60, 0x1acd8b0, 0xc000122000, 0xc00060e060)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:90 +0x85
created by github.com/koinos/koinos-p2p/internal/protocol.(*GossipManager).Start
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:57 +0x10f

goroutine 99 [select]:
github.com/libp2p/go-libp2p-pubsub.(*Subscription).Next(0xc00058e040, 0x1acd8b0, 0xc000122000, 0xc000570c30, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/subscription.go:24 +0xab
github.com/koinos/koinos-p2p/internal/protocol.(*GossipManager).readMessages(0xc000308c30, 0x1acd8b0, 0xc000122000, 0xc000580120)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:90 +0x85
created by github.com/koinos/koinos-p2p/internal/protocol.(*GossipManager).Start
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:57 +0x10f

goroutine 114 [syscall, 20 minutes]:
os/signal.signal_recv(0xa9fffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/sigqueue.go:165 +0x9d
os/signal.loop()
    /usr/local/Cellar/go/1.16.4/libexec/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
    /usr/local/Cellar/go/1.16.4/libexec/src/os/signal/signal.go:151 +0x45

goroutine 64 [select]:
github.com/streadway/amqp.(*Connection).heartbeater(0xc0005ba000, 0x2540be400, 0xc000089620)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:551 +0x188
created by github.com/streadway/amqp.(*Connection).openTune
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:782 +0x465

goroutine 95 [chan receive]:
github.com/streadway/amqp.(*consumers).buffer(0xc000592390, 0xc00057a480, 0xc00057a420)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:61 +0x365
created by github.com/streadway/amqp.(*consumers).add
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:97 +0x165

goroutine 134 [select]:
github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).providerLoop(0xc0002f6600, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:443 +0x146
created by github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).Start.func1
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:501 +0x7b

goroutine 135 [select]:
github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).triggerRescanLoop(0xc0002f6600, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:469 +0x146
created by github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).Start.func1
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:502 +0xb1

goroutine 96 [runnable]:
github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).HandleBlockBroadcast(0xc0002f6600, 0x1acd8b0, 0xc000122000, 0xc000648000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provide:430 +0x156
github.com/koinos/koinos-p2p/internal/protocol.(*SyncManager).HandleBlockBroadcast(...)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/sync_manager.go:261
github.com/koinos/koinos-p2p/internal/node.(*KoinosP2PNode).handleBlockBroadcast(0xc0002e7960, 0xc00003c180, 0x13, 0xc000654240, 0x22f, 0x22f)
    /Users/sgerbino/Projects/koinos-p2p/internal/node/node.go:123 +0x365
github.com/koinos/koinos-mq-golang.(*RequestHandler).HandleBroadcastDelivery(...)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:230
github.com/koinos/koinos-mq-golang.(*RequestHandler).ConsumeBroadcastLoop(0xc0002f59c0, 0xc00057a420, 0x18c1843, 0x13)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:199 +0x17c
created by github.com/koinos/koinos-mq-golang.(*RequestHandler).ConnectLoop
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:133 +0x349

goroutine 97 [chan receive, 20 minutes]:
github.com/streadway/amqp.(*consumers).buffer(0xc000592390, 0xc00057a540, 0xc00057a4e0)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:61 +0x365
created by github.com/streadway/amqp.(*consumers).add
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:97 +0x165

goroutine 39 [chan receive]:
github.com/streadway/amqp.(*consumers).buffer(0xc000592390, 0xc00026c060, 0xc00026c000)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:61 +0x365
created by github.com/streadway/amqp.(*consumers).add
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:97 +0x165

goroutine 35342 [select]:
github.com/libp2p/go-yamux/v2.(*Stream).Read(0xc000353130, 0xc00049a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/stream.go:117 +0x197
github.com/libp2p/go-libp2p-yamux.(*stream).Read(0xc000353130, 0xc00049a000, 0x1000, 0x1000, 0xc00055e600, 0xc00030ea03, 0x99)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-yamux@v0.5.1/stream.go:14 +0x4d
github.com/libp2p/go-libp2p-swarm.(*Stream).Read(0xc0003da380, 0xc00049a000, 0x1000, 0x1000, 0x2ae, 0xc0005e3ca0, 0x101a313)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_stream.go:57 +0x62
github.com/multiformats/go-multistream.(*lazyServerConn).Read(0xc00047a2d0, 0xc00049a000, 0x1000, 0x1000, 0xaa13f50, 0xc0005e3e98, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-multistream@v0.2.0/lazyServer.go:32 +0x76
github.com/libp2p/go-libp2p/p2p/host/basic.(*streamWrapper).Read(0xc0007a2020, 0xc00049a000, 0x1000, 0x1000, 0x49, 0xaa13f50, 0x203000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:1022 +0x52
bufio.(*Reader).fill(0xc000301260)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).ReadByte(0xc000301260, 0x100f0bb, 0xc0001ac0e0, 0x70)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:253 +0x39
github.com/multiformats/go-varint.ReadUvarint(0x1abc000, 0xc000301260, 0xc0001ac0e0, 0x1aca220, 0xc0001ccd80)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-varint@v0.0.6/varint.go:80 +0x77
github.com/libp2p/go-msgio/protoio.(*uvarintReader).ReadMsg(0xc00020c140, 0x1aca220, 0xc0001ac0e0, 0x1, 0x1)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-msgio@v0.0.6/protoio/uvarint_reader.go:60 +0x3e
github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleNewStream(0xc0001d8ea0, 0x1ad7eb0, 0xc0007a2020)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:66 +0x282
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1(0xc000702030, 0xe, 0xaa55160, 0xc0007a2020, 0x1, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:566 +0xa2
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:416 +0x63a

goroutine 57809 [select]:
github.com/libp2p/go-yamux/v2.(*Stream).Read(0xc000353970, 0xc000228e0c, 0x1, 0x1, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/stream.go:117 +0x197
github.com/libp2p/go-libp2p-yamux.(*stream).Read(0xc000353970, 0xc000228e0c, 0x1, 0x1, 0x10, 0x2170f18, 0x10)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-yamux@v0.5.1/stream.go:14 +0x4d
github.com/libp2p/go-libp2p-swarm.(*Stream).Read(0xc000712500, 0xc000228e0c, 0x1, 0x1, 0x21705b8, 0xc00052c450, 0xc000282f50)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_stream.go:57 +0x62
github.com/multiformats/go-multistream.(*lazyServerConn).Read(0xc0007c9860, 0xc000228e0c, 0x1, 0x1, 0x17967e0, 0xc0004fe601, 0xc000228e0c)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-multistream@v0.2.0/lazyServer.go:32 +0x76
github.com/libp2p/go-libp2p/p2p/host/basic.(*streamWrapper).Read(0xc0004fe680, 0xc000228e0c, 0x1, 0x1, 0x106e5a1, 0xc0002142a0, 0x12)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:1022 +0x52
github.com/libp2p/go-libp2p-gorpc.(*Server).handle.func2(0xc0004410b0, 0xc00052c4e0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:277 +0x77
created by github.com/libp2p/go-libp2p-gorpc.(*Server).handle
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:275 +0x94f

goroutine 35471 [runnable]:
github.com/koinos/koinos-util-golang.MultihashToCmp(...)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-util-golang@v0.0.0-20210601175130-50a337109792/comparable.go:48
github.com/koinos/koinos-util-golang.BlockTopologyToCmp(...)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-util-golang@v0.0.0-20210601175130-50a337109792/comparable.go:63
github.com/koinos/koinos-p2p/internal/protocol.(*PeerHandler).peerHandlerCycle(0xc000096580, 0x1acd878, 0xc0000326c0, 0x0, 0x0)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/peer_handler.go:202 +0x47b
github.com/koinos/koinos-p2p/internal/protocol.(*PeerHandler).peerHandlerLoop.func1(...)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/peer_handler.go:148
github.com/koinos/koinos-p2p/internal/protocol.(*PeerHandler).peerHandlerLoop(0xc000096580, 0x1acd878, 0xc0000326c0)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/peer_handler.go:163 +0x33b
created by github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).handleNewPeer
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:277 +0x2d9

goroutine 35472 [select]:
github.com/koinos/koinos-p2p/internal/protocol.(*PeerHandler).nodeUpdateLoop(0xc000096580, 0x1acd878, 0xc0000326c0)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/peer_handler.go:132 +0x247
created by github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).handleNewPeer
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:278 +0x30f

goroutine 35364 [select]:
github.com/libp2p/go-yamux/v2.(*Session).sendLoop(0xc00007e000, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:490 +0x1e5
github.com/libp2p/go-yamux/v2.(*Session).send(0xc00007e000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:436 +0x2b
created by github.com/libp2p/go-yamux/v2.newSession
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:131 +0x410

goroutine 35361 [select, 8 minutes]:
github.com/libp2p/go-libp2p/p2p/protocol/identify.(*peerHandler).loop(0xc0002de2a0, 0x1acd878, 0xc0002f4280, 0xc000210cc0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/peer_loop.go:92 +0x105
created by github.com/libp2p/go-libp2p/p2p/protocol/identify.(*peerHandler).start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/peer_loop.go:74 +0x9c

goroutine 35117 [select, 8 minutes]:
github.com/libp2p/go-yamux/v2.(*Stream).Read(0xc0005dc2c0, 0xc000872000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/stream.go:117 +0x197
github.com/libp2p/go-libp2p-yamux.(*stream).Read(0xc0005dc2c0, 0xc000872000, 0x1000, 0x1000, 0xc0001f3500, 0x107937a, 0xc00028a004)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-yamux@v0.5.1/stream.go:14 +0x4d
github.com/libp2p/go-libp2p-swarm.(*Stream).Read(0xc00055e300, 0xc000872000, 0x1000, 0x1000, 0x203000, 0x203000, 0xb244a40)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_stream.go:57 +0x62
github.com/multiformats/go-multistream.(*lazyClientConn).Read(0xc00028a000, 0xc000872000, 0x1000, 0x1000, 0x38, 0x18421c0, 0x3230594260801)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-multistream@v0.2.0/lazyClient.go:68 +0xac
github.com/libp2p/go-libp2p/p2p/host/basic.(*streamWrapper).Read(0xc0007cc100, 0xc000872000, 0x1000, 0x1000, 0xc000032480, 0x1000000010f0345, 0x203000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:1022 +0x52
bufio.(*Reader).fill(0xc00060e660)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).ReadByte(0xc00060e660, 0x100f0bb, 0xc00028a070, 0x70)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:253 +0x39
github.com/multiformats/go-varint.ReadUvarint(0x1abc000, 0xc00060e660, 0xc00028a070, 0x1000, 0xc000600480)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-varint@v0.0.6/varint.go:80 +0x77
github.com/libp2p/go-msgio/protoio.(*uvarintReader).ReadMsg(0xc000032480, 0x1aca220, 0xc00028a070, 0x1ac59a8, 0xc000032480)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-msgio@v0.0.6/protoio/uvarint_reader.go:60 +0x3e
github.com/libp2p/go-libp2p-pubsub.(*PubSub).handlePeerEOF(0xc0001d8ea0, 0x1acd8b0, 0xc000122000, 0x1ad7eb0, 0xc0007cc100)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:122 +0x1c2
created by github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleNewPeer
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:111 +0x41e

goroutine 35363 [IO wait]:
internal/poll.runtime_pollWait(0xaa40de8, 0x72, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000402798, 0x72, 0x0, 0x2, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000402780, 0xc0005d6088, 0x2, 0x2, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc000402780, 0xc0005d6088, 0x2, 0x2, 0x1, 0xc00069cb70, 0xc000235cf8)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000506030, 0xc0005d6088, 0x2, 0x2, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/net.go:183 +0x91
io.ReadAtLeast(0xad65b00, 0xc000592c30, 0xc0005d6088, 0x2, 0x2, 0x2, 0x100, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:328 +0x87
io.ReadFull(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:347
github.com/libp2p/go-libp2p-noise.(*secureSession).readNextInsecureMsgLen(0xc0005d6000, 0x7b5b71626ddc5fe8, 0xc000235ec8, 0x16477c5)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-noise@v0.1.1/rw.go:134 +0x8c
github.com/libp2p/go-libp2p-noise.(*secureSession).Read(0xc0005d6000, 0xc000444460, 0xc, 0xc, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-noise@v0.1.1/rw.go:53 +0x1fd
io.ReadAtLeast(0xb1aa0e0, 0xc0005d6000, 0xc000444460, 0xc, 0xc, 0xc, 0xc, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:328 +0x87
io.ReadFull(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:347
github.com/libp2p/go-yamux/v2.(*Session).recvLoop(0xc00007e000, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:567 +0xda
github.com/libp2p/go-yamux/v2.(*Session).recv(0xc00007e000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:546 +0x2b
created by github.com/libp2p/go-yamux/v2.newSession
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:130 +0x3ee

goroutine 35116 [select]:
github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleSendingMessages(0xc0001d8ea0, 0x1acd8b0, 0xc000122000, 0x1ad7eb0, 0xc0007cc100, 0xc000369860)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:149 +0x24d
created by github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleNewPeer
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:110 +0x3c5

goroutine 35375 [select]:
github.com/libp2p/go-yamux/v2.(*Session).AcceptStream(0xc00007e000, 0xc000353970, 0x0, 0x1042d6e)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:229 +0xf0
github.com/libp2p/go-libp2p-yamux.(*conn).AcceptStream(0xc00007e000, 0xc0005e4f78, 0x18, 0xc00036c900, 0x17080a9)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-yamux@v0.5.1/conn.go:35 +0x2b
github.com/libp2p/go-libp2p-swarm.(*Conn).start.func1(0xc000218120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_conn.go:107 +0xe2
created by github.com/libp2p/go-libp2p-swarm.(*Conn).start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_conn.go:102 +0x3f
sgerbino commented 3 years ago

Possibly also related to #101.

sgerbino commented 3 years ago

Using #101 to handle this.