search

koinos / koinos-p2p

The p2p microservice orchestrates the distribution of blocks and transactions between peers.
MIT License
6 stars 4 forks source link

Thread safe TopologyCache #101

Closed mvandeberg closed 3 years ago

mvandeberg commented 3 years ago

The topology cache can be accessed concurrently and is not thread safe. Let's make it thread safe.

theoreticalbts commented 3 years ago

In discussion of PR #99, we originally intended this issue's scope to be limited to the cache for the local node's topology, MyTopologyCache.

However, since it's similar and relatively mechanical work, I propose expanding the scope to also include reviewing and fixing thread safety issues in TopologyCache as well. (TopologyCache is the topology cache which contains topology information from all peers.)

sgerbino commented 3 years ago

Stacktrace:

2021-06-10 20:18:48.627528 (p2p.Koinos) [protocol/download_provider.go:219] <info>: Downloaded block applied - Height: 861 ID: zQmU3SPuBBMwDvbJ9BFbiW5GDgAzgPXazvrVb2fUzHSouqB Prev: zQmTwoJTMm15CXPDyAhtu6jZs7QodbCqj7LtMVXryodv68t from peer Qmf4Rz6Yg6YnaNpqQoUxDepn2ceAeEMvPv9C4R3hJVDqY3
2021-06-10 20:18:48.627634 (p2p.Koinos) [protocol/download_provider.go:399] <info>: Connecting disconnected fork head "zQmU3SPuBBMwDvbJ9BFbiW5GDgAzgPXazvrVb2fUzHSouqB"
2021-06-10 20:18:48.789442 (p2p.Koinos) [protocol/download_provider.go:219] <info>: Downloaded block applied - Height: 862 ID: zQmYtUmqCBjTgQkht8dcdEDfU2FAYCHZbekt5nAu4YpLGxY Prev: zQmU3SPuBBMwDvbJ9BFbiW5GDgAzgPXazvrVb2fUzHSouqB from peer Qmf4Rz6Yg6YnaNpqQoUxDepn2ceAeEMvPv9C4R3hJVDqY3
2021-06-10 20:18:49.159832 (p2p.Koinos) [protocol/gossip.go:187] <info>: Gossiped block applied: Height: 865 ID: zQmcESyLS6JpD3gdzNsD8oNBBjsDeftCwUHf5w9SZnhNYNb Prev: zQmSZjwFvMLKxJs9X3Qdjf3a3ArDH11mzbTHv58F4Rahuvp from peer QmSuMdPvG7vfwr69snMtRhaC5foNwc6xaLdGdETkgyNkq9
2021-06-10 20:18:50.898732 (p2p.Koinos) [protocol/gossip.go:187] <info>: Gossiped block applied: Height: 866 ID: zQmYSMRNuXPtEaSUJ16sX1ZbbB9Vvp3ohgM36fY6SjMGTSP Prev: zQmcESyLS6JpD3gdzNsD8oNBBjsDeftCwUHf5w9SZnhNYNb from peer QmSuMdPvG7vfwr69snMtRhaC5foNwc6xaLdGdETkgyNkq9
fatal error: concurrent map read and map write

goroutine 57808 [running]:
runtime.throw(0x18cc9ee, 0x21)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/panic.go:1117 +0x72 fp=0xc0003fb628 sp=0xc0003fb5f8 pc=0x1037c12
runtime.mapaccess2_fast64(0x17c6060, 0xc000308300, 0x35a, 0xc0000f9e40, 0x20)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/map_fast64.go:61 +0x1ac fp=0xc0003fb650 sp=0xc0003fb628 pc=0x10140ac
github.com/koinos/koinos-p2p/internal/protocol.(*SyncService).GetTopologyAtHeight(0xc000308510, 0x1acd878, 0xc000032840, 0x325, 0xa0, 0xc0004fe6e0, 0x0, 0x0)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/sync_service.go:186 +0x3c5 fp=0xc0003fb858 sp=0xc0003fb650 pc=0x1608c05
runtime.call64(0xc0007c9d10, 0xc000134808, 0xc000032880, 0x3000000040)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/asm_amd64.s:552 +0x3e fp=0xc0003fb8a8 sp=0xc0003fb858 pc=0x106cc9e
reflect.Value.call(0xc0001b2c00, 0xc000134808, 0x13, 0x18b55b2, 0x4, 0xc000071b90, 0x4, 0x4, 0xc000071b18, 0x1069e33, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/reflect/value.go:476 +0x8e7 fp=0xc0003fbab0 sp=0xc0003fb8a8 pc=0x10bd647
reflect.Value.Call(0xc0001b2c00, 0xc000134808, 0x13, 0xc000071b90, 0x4, 0x4, 0x10, 0xc00052c4e0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/reflect/value.go:337 +0xb9 fp=0xc0003fbb30 sp=0xc0003fbab0 pc=0x10bcb19
github.com/libp2p/go-libp2p-gorpc.(*service).svcCall(0xc0002f5cc0, 0xc0004410b0, 0xc0002e7c70, 0xc000330830, 0xb, 0xc0002142b8, 0x13, 0x181ba20, 0xc000032840, 0x16, ...)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:292 +0x152 fp=0xc0003fbc00 sp=0xc0003fbb30 pc=0x155ad52
github.com/libp2p/go-libp2p-gorpc.(*Server).handle(0xc000300960, 0xc0004410b0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:284 +0xa1a fp=0xc0003fbe88 sp=0xc0003fbc00 pc=0x155a49a
github.com/libp2p/go-libp2p-gorpc.NewServer.func1(0x1ad7eb0, 0xc0004fe680)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:173 +0xae fp=0xc0003fbf60 sp=0xc0003fbe88 pc=0x155e8ce
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1(0xc0002142a0, 0x12, 0xaa55160, 0xc0004fe680, 0x1, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:566 +0xa2 fp=0xc0003fbfb0 sp=0xc0003fbf60 pc=0x16e2942
runtime.goexit()
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc0003fbfb8 sp=0xc0003fbfb0 pc=0x106e5a1
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:416 +0x63a

goroutine 1 [chan receive, 20 minutes]:
main.main()
    /Users/sgerbino/Projects/koinos-p2p/cmd/koinos-p2p/main.go:161 +0x12e5

goroutine 9 [chan receive]:
github.com/streadway/amqp.(*consumers).buffer(0xc0002efe90, 0xc0000a0240, 0xc0000a01e0)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:61 +0x365
created by github.com/streadway/amqp.(*consumers).add
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:97 +0x165

goroutine 20 [select, 20 minutes]:
github.com/ipfs/go-log/writer.(*MirrorWriter).logRoutine(0xc0001cf320)
    /Users/sgerbino/go/pkg/mod/github.com/ipfs/go-log@v1.0.4/writer/writer.go:71 +0x106
created by github.com/ipfs/go-log/writer.NewMirrorWriter
    /Users/sgerbino/go/pkg/mod/github.com/ipfs/go-log@v1.0.4/writer/writer.go:36 +0xb9

goroutine 21 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc0001b2180)
    /Users/sgerbino/go/pkg/mod/go.opencensus.io@v0.22.4/stats/view/worker.go:276 +0xcd
created by go.opencensus.io/stats/view.init.0
    /Users/sgerbino/go/pkg/mod/go.opencensus.io@v0.22.4/stats/view/worker.go:34 +0x68

goroutine 22 [chan receive, 20 minutes]:
github.com/koinos/koinos-mq-golang.(*Client).ConnectLoop(0xc000146dc0)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/client.go:139 +0x271
created by github.com/koinos/koinos-mq-golang.(*Client).Start
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/client.go:73 +0x3f

goroutine 23 [chan receive, 20 minutes]:
gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun(0xc0002cdec0)
    /Users/sgerbino/go/pkg/mod/gopkg.in/natefinch/lumberjack.v2@v2.0.0/lumberjack.go:379 +0x5d
created by gopkg.in/natefinch/lumberjack%2ev2.(*Logger).mill.func1
    /Users/sgerbino/go/pkg/mod/gopkg.in/natefinch/lumberjack.v2@v2.0.0/lumberjack.go:390 +0x75

goroutine 12 [select, 20 minutes]:
github.com/libp2p/go-libp2p-peerstore/pstoremem.(*memoryAddrBook).background(0xc0000dc000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-peerstore@v0.2.6/pstoremem/addr_book.go:93 +0x105
created by github.com/libp2p/go-libp2p-peerstore/pstoremem.NewAddrBook
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-peerstore@v0.2.6/pstoremem/addr_book.go:83 +0x18f

goroutine 7 [IO wait]:
internal/poll.runtime_pollWait(0xaa40fb8, 0x72, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc0001b2898, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0001b2880, 0xc0000a5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc0001b2880, 0xc0000a5000, 0x1000, 0x1000, 0x100ebca, 0xaa96590, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000010048, 0xc0000a5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/net.go:183 +0x91
bufio.(*Reader).Read(0xc0000885a0, 0xc00037a000, 0x7, 0x7, 0xc00024ce60, 0x100f818, 0x7)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x1abc020, 0xc0000885a0, 0xc00037a000, 0x7, 0x7, 0x7, 0xc00024ced8, 0x1006d15, 0xc000088540)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:328 +0x87
io.ReadFull(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:347
github.com/streadway/amqp.(*reader).ReadFrame(0xc00024cf40, 0xc00024cf50, 0xc0004fe720, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/read.go:49 +0x75
github.com/streadway/amqp.(*Connection).reader(0xc0000a6000, 0xaa91038, 0xc000010048)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:521 +0xdc
created by github.com/streadway/amqp.Open
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:233 +0x26c

goroutine 8 [select]:
github.com/streadway/amqp.(*Connection).heartbeater(0xc0000a6000, 0x2540be400, 0xc000088600)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:551 +0x188
created by github.com/streadway/amqp.(*Connection).openTune
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:782 +0x465

goroutine 28 [chan receive]:
github.com/koinos/koinos-mq-golang.(*Client).ConsumeRPCReturnLoop(0xc000146dc0, 0xc0000a01e0)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/client.go:304 +0x97
created by github.com/koinos/koinos-mq-golang.(*Client).ConnectLoop
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/client.go:111 +0x205

goroutine 14 [select]:
github.com/libp2p/go-libp2p-swarm.(*DialBackoff).background(0xc000001a90, 0x1acd878, 0xc000032300)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_dial.go:126 +0xee
created by github.com/libp2p/go-libp2p-swarm.(*DialBackoff).init
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_dial.go:119 +0x5d

goroutine 15 [select]:
github.com/libp2p/go-libp2p/p2p/protocol/identify.(*ObservedAddrManager).worker(0xc000088b40, 0x1acd878, 0xc0000327c0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/obsaddr.go:236 +0x20d
created by github.com/libp2p/go-libp2p/p2p/protocol/identify.NewObservedAddrManager
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/obsaddr.go:127 +0x17d

goroutine 16 [select, 8 minutes]:
github.com/libp2p/go-libp2p/p2p/protocol/identify.(*IDService).loop(0xc00009ad00)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/id.go:203 +0x56c
created by github.com/libp2p/go-libp2p/p2p/protocol/identify.NewIDService
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/id.go:150 +0x2cd

goroutine 38 [chan receive, 20 minutes]:
github.com/koinos/koinos-mq-golang.(*RequestHandler).ConsumeBroadcastLoop(0xc0002f59c0, 0xc00057a4e0, 0x18c5e38, 0x19)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:198 +0x19a
created by github.com/koinos/koinos-mq-golang.(*RequestHandler).ConnectLoop
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-g:133 +0x349

goroutine 50 [select, 20 minutes]:
github.com/libp2p/go-libp2p-circuit.(*RelayListener).Accept(0xc000088cc0, 0x0, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-circuit@v0.4.0/listen.go:27 +0x18f
github.com/libp2p/go-libp2p-transport-upgrader.(*listener).handleIncoming(0xc000088d20)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/listener.go:77 +0xf7
created by github.com/libp2p/go-libp2p-transport-upgrader.(*Upgrader).UpgradeListener
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/upgrader.go:49 +0x18d

goroutine 55 [IO wait, 20 minutes]:
internal/poll.runtime_pollWait(0xaa40ed0, 0x72, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000470918, 0x72, 0x0, 0x0, 0x18b7089)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000470900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:401 +0x212
net.(*netFD).accept(0xc000470900, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/fd_unix.go:172 +0x45
net.(*TCPListener).accept(0xc00000f500, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc00000f500, 0x0, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/tcpsock.go:261 +0x65
github.com/multiformats/go-multiaddr/net.(*maListener).Accept(0xc0000f3140, 0xc, 0x2170108, 0x10, 0xc000460d80)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-multiaddr@v0.3.1/net/net.go:243 +0x48
github.com/libp2p/go-tcp-transport.(*lingerListener).Accept(0xc00000f578, 0x0, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-tcp-transport@v0.2.1/tcp.go:42 +0x37
github.com/libp2p/go-libp2p-transport-upgrader.(*listener).handleIncoming(0xc000088d80)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/listener.go:77 +0xf7
created by github.com/libp2p/go-libp2p-transport-upgrader.(*Upgrader).UpgradeListener
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/upgrader.go:49 +0x18d

goroutine 54 [chan receive, 20 minutes]:
github.com/libp2p/go-libp2p-transport-upgrader.(*listener).Accept(0xc000088d20, 0x1faa570, 0xc00048b720, 0x106a69e, 0xc000460cb8)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/listener.go:155 +0x51
github.com/libp2p/go-libp2p-swarm.(*Swarm).AddListenAddr.func2(0x1acdca0, 0xc000088d20, 0xc000001980, 0x1ad78e0, 0xc00012f038)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_listen.go:84 +0x18b
created by github.com/libp2p/go-libp2p-swarm.(*Swarm).AddListenAddr
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_listen.go:69 +0x268

goroutine 30 [chan receive, 20 minutes]:
github.com/libp2p/go-libp2p-transport-upgrader.(*listener).Accept(0xc000088d80, 0x1faab70, 0xc00048c720, 0x106a69e, 0xc000460d78)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-transport-upgrader@v0.4.0/listener.go:155 +0x51
github.com/libp2p/go-libp2p-swarm.(*Swarm).AddListenAddr.func2(0x1acdca0, 0xc000088d80, 0xc000001980, 0x1ad78e0, 0xc00000f548)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_listen.go:84 +0x18b
created by github.com/libp2p/go-libp2p-swarm.(*Swarm).AddListenAddr
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_listen.go:69 +0x268

goroutine 33 [select]:
github.com/libp2p/go-libp2p-autonat.(*AmbientAutoNAT).background(0xc0002f9110)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-autonat@v0.4.0/autonat.go:175 +0x2bc
created by github.com/libp2p/go-libp2p-autonat.New
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-autonat@v0.4.0/autonat.go:125 +0x6b5

goroutine 66 [select]:
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).background(0xc0000f4000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:520 +0x22a
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:368 +0x5c

goroutine 67 [select]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).heartbeatTimer(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:1217 +0x21d
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:422 +0x10d

goroutine 68 [select, 3 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 69 [select, 2 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 70 [select, 3 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 71 [select, 4 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 72 [select, 4 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 73 [select, 2 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 74 [select]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 75 [select]:
github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).connector(0xc00030a120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:842 +0xe6
created by github.com/libp2p/go-libp2p-pubsub.(*GossipSubRouter).Attach
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/gossipsub.go:426 +0x138

goroutine 76 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 77 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 78 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 79 [runnable]:
reflect.(*rtype).NumField(0x1805cc0, 0xc00026e0b8)
    /usr/local/Cellar/go/1.16.4/libexec/src/reflect/type.go:974 +0xbd
encoding/asn1.parseField(0x1805cc0, 0xc00026e0b8, 0x199, 0xc0002243c2, 0x59, 0x5e, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/asn1/asn1.go:930 +0x1b7b
encoding/asn1.parseField(0x1820520, 0xc00026e0a0, 0x199, 0xc0002243c0, 0x5b, 0x60, 0x0, 0x0, 0x0, 0x0, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/asn1/asn1.go:935 +0x1d85
encoding/asn1.UnmarshalWithParams(0xc0002243c0, 0x5b, 0x60, 0x176c920, 0xc00026e0a0, 0x0, 0x0, 0x60, 0x1, 0xc0002243c0, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/asn1/asn1.go:1117 +0x2b3
encoding/asn1.Unmarshal(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/asn1/asn1.go:1090
crypto/x509.ParsePKIXPublicKey(0xc0002243c0, 0x5b, 0x60, 0x5f, 0x5f, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/crypto/x509/x509.go:58 +0x7f
github.com/libp2p/go-libp2p-core/crypto.UnmarshalECDSAPublicKey(0xc0002243c0, 0x5b, 0x60, 0xc0001ac040, 0x1, 0x100f818, 0x20)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-core@v0.8.5/crypto/ecdsa.go:92 +0x45
github.com/libp2p/go-libp2p-core/crypto.PublicKeyFromProto(0xc000150420, 0x5f, 0x5f, 0x1ac9890, 0xc000150420)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-core@v0.8.5/crypto/key.go:299 +0x93
github.com/libp2p/go-libp2p-core/crypto.UnmarshalPublicKey(0xc000224240, 0x5f, 0x5f, 0xc0004be150, 0x22, 0x30, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-core@v0.8.5/crypto/key.go:286 +0xaf
github.com/libp2p/go-libp2p-pubsub.messagePubKey(0xc0006481e0, 0x0, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/sign.go:95 +0xb9
github.com/libp2p/go-libp2p-pubsub.verifyMessageSignature(0xc0006481e0, 0xc00024af40, 0x104b3ae)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/sign.go:50 +0x45
github.com/libp2p/go-libp2p-pubsub.(*validation).validateSignature(0xc000308990, 0xc0000f1a10, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:314 +0x35
github.com/libp2p/go-libp2p-pubsub.(*validation).validate(0xc000308990, 0xc000134028, 0x1, 0x1, 0xc00003e300, 0x22, 0xc0000f1a10)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:243 +0x6dd
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:229 +0x6e
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 80 [select]:
github.com/libp2p/(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 81 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 82 [select]:
github.com/libp2p/go-libp2p-pubsub.(*validation).validateWorker(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 83 [select]:
g(0xc000308990)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:227 +0xf9
created by github.com/libp2p/go-libp2p-pubsub.(*validation).Start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/validation.go:120 +0x70

goroutine 84 [select]:
github.com/libp2p/go-libp2p-pubsub.(*PubSub).processLoop(0xc0001d8ea0, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/pubsub.go:485 +0x52c
created by github.com/libp2p/go-libp2p-pubsub.NewPubSub
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/pubsub.go:297 +0xb9e

goroutine 85 [chan receive, 20 minutes]:
github.com/koinos/koinos-mq-golang.(*RequestHandler).ConnectLoop(0xc0002f59c0)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:161 +0x434
created by github.com/koinos/koinos-mq-golang.(*RequestHandler).Start
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:74 +0x3f

goroutine 86 [runnable]:
math/big.nat.make(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/nat.go:64
math/big.nat.setWord(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/nat.go:76
math/big.nat.div(0xc0006fe5a0, 0x4, 0x9, 0x0, 0x0, 0x0, 0xc0006fe5a0, 0x4, 0x9, 0xc000123000, ...)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/nat.go:668 +0x349
math/big.(*Int).QuoRem(0xc000149510, 0xc000149510, 0xc000150680, 0xc0001494f0, 0xc000149510, 0xc0001494f0)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/int.go:239 +0xbf
math/big.(*Int).DivMod(0xc000149510, 0xc000149510, 0xc000150680, 0xc0001494f0, 0xc000149510, 0xc0001494f0)
    /usr/local/Cellar/go/1.16.4/libexec/src/math/big/int.go:301 +0xe5
github.com/btcsuite/btcutil/base58.Encode(0xc0004bd260, 0x22, 0x30, 0x1fd5248, 0x100491a)
    /Users/sgerbino/go/pkg/mod/github.com/btcsuite/btcutil@v1.0.2/base58/base58.go:56 +0x19d
github.com/koinos/koinos-types-golang.Multihash.MarshalJSON(0x12, 0xc00022a900, 0x20, 0x20, 0x100d025, 0x17d9ba0, 0x181b000, 0x17c8401, 0xad65a38)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-types-golang@v0.0.0-20210601210740-559c76121b53/basetypes.go:1238 +0x88
encoding/json.marshalerEncoder(0xc000712700, 0x181b000, 0xc0006362a0, 0x99, 0x100)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:479 +0xad
encoding/json.condAddrEncoder.encode(0x1a11648, 0x1a116a0, 0xc000712700, 0x181b000, 0xc0006362a0, 0x99, 0x1810100)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:961 +0xb2
encoding/json.(*encodeState).reflectValue(0xc000712700, 0x181b000, 0xc0006362a0, 0x99, 0xc000140100)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:360 +0x82
encoding/json.(*encodeState).marshal(0xc000712700, 0x181b000, 0xc0006362a0, 0x100, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:332 +0xf9
encoding/json.Marshal(0x181b000, 0xc0006362a0, 0x181b000, 0xc0006362a0, 0x20, 0x20, 0x1)
    /usr/local/Cellar/go/1.16.4/libexec/src/encoding/json/encode.go:161 +0x52
github.com/koinos/koinos-util-golang.BlockTopologyCmpString(0xc000149c08, 0xc0001499b4, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-util-golang@v0.0.0-20210601175130-50a337109792/helpers.go:40 +0xbd
github.com/koinos/koinos-p2p/internal/protocol.(*BlockDownloadManager).downloadManagerLoop(0xc0, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_manager.go:376 +0xa5a
created by github.com/koinos/koinos-p2p/internal/protocol.(*BlockDownloadManager).Start
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_manager.go:190 +0x53

goroutine 136 [chan receive]:
github.com/koinos/koinos-mq-golang.(*RequestHandler).ConsumeBroadcastLoop(0xc0002f59c0, 0xc00026c000, 0x18c0c76, 0x12)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:198 +0x19a
created by github.com/koinos/koinos-mq-golang.(*RequestHandler).ConnectLoop
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:133 +0x349

goroutine 88 [select]:
github.com/koinos/koinos-p2p/internal/protocol.(*SyncManager).run(0xc0001b2900, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/sync_manager.go:227 +0x1b9
created by github.com/koinos/koinos-p2p/in
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/sync_manager.go:256 +0xa5

goroutine 89 [chan receive]:
github.com/koinos/koinos-p2p/internal/protocol.(*KoinosGossip).startBlockGossip.func1(0xc0002f5f00, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:156 +0x11d
created by github.com/koinos/koinos-p2p/internal/protocol.(*KoinosGossip).startBlockGossip
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:147 +0x53

goroutine 90 [chan receive, 20 minutes]:
github.com/koinos/koinos-p2p/internal/protocol.(*KoinosGossip).startTransactionGossip.func1(0xc0002f5f00, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:201 +0x11d
created by github.com/koinos/koinos-p2p/internal/protocol.(*KoinosGossip).startTransactionGossip
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:192 +0x53

goroutine 100 [runnable]:
internal/poll.runtime_pollWait(0xaa40d00, 0x72, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000470a18, 0x72, 0x1000, 0x1000, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000470a00, 0xc000475000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc000470a00, 0xc000475000, 0x1000, 0x1000, 0xc00026c2f8, 0xc0006aedb8, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000596030, 0xc000475000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/net.go:183 +0x91
bufio.(*Reader).Read(0xc000089560, 0xc000639339, 0x7, 0x7, 0xc0006aee60, 0x100f818, 0x7)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x1abc020, 0xc000089560, 0xc000639339, 0x7, 0x7, 0x7, 0xc0006aeed8, 0x1006d15, 0xc0005806c0)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:328 +0x87
io.ReadFull(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:347
github.com/streadway/amqp.(*reader).ReadFrame(0xc0006aef40, 0xc000279f50, 0xc000526180, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/read.go:49 +0x75
github.com/streadway/amqp.(*Connection).reader(0xc0005ba000, 0xaa91038, 0xc000596030)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:521 +0xdc
created by github.com/streadway/amqp.Open
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:233 +0x26c

goroutine 98 [select, 20 minutes]:
github.com/libp2p/go-libp2p-pubsub.(*Subscription).Next(0xc00058e080, 0x1acd8b0, 0xc000122000, 0xc00058c000, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/subscription.go:24 +0xab
github.com/koinos/koinos-p2p/internal/protocol.(*GossipManager).readMessages(0xc000308c60, 0x1acd8b0, 0xc000122000, 0xc00060e060)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:90 +0x85
created by github.com/koinos/koinos-p2p/internal/protocol.(*GossipManager).Start
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:57 +0x10f

goroutine 99 [select]:
github.com/libp2p/go-libp2p-pubsub.(*Subscription).Next(0xc00058e040, 0x1acd8b0, 0xc000122000, 0xc000570c30, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/subscription.go:24 +0xab
github.com/koinos/koinos-p2p/internal/protocol.(*GossipManager).readMessages(0xc000308c30, 0x1acd8b0, 0xc000122000, 0xc000580120)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:90 +0x85
created by github.com/koinos/koinos-p2p/internal/protocol.(*GossipManager).Start
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/gossip.go:57 +0x10f

goroutine 114 [syscall, 20 minutes]:
os/signal.signal_recv(0xa9fffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/sigqueue.go:165 +0x9d
os/signal.loop()
    /usr/local/Cellar/go/1.16.4/libexec/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
    /usr/local/Cellar/go/1.16.4/libexec/src/os/signal/signal.go:151 +0x45

goroutine 64 [select]:
github.com/streadway/amqp.(*Connection).heartbeater(0xc0005ba000, 0x2540be400, 0xc000089620)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:551 +0x188
created by github.com/streadway/amqp.(*Connection).openTune
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/connection.go:782 +0x465

goroutine 95 [chan receive]:
github.com/streadway/amqp.(*consumers).buffer(0xc000592390, 0xc00057a480, 0xc00057a420)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:61 +0x365
created by github.com/streadway/amqp.(*consumers).add
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:97 +0x165

goroutine 134 [select]:
github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).providerLoop(0xc0002f6600, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:443 +0x146
created by github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).Start.func1
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:501 +0x7b

goroutine 135 [select]:
github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).triggerRescanLoop(0xc0002f6600, 0x1acd8b0, 0xc000122000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:469 +0x146
created by github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).Start.func1
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:502 +0xb1

goroutine 96 [runnable]:
github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).HandleBlockBroadcast(0xc0002f6600, 0x1acd8b0, 0xc000122000, 0xc000648000)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provide:430 +0x156
github.com/koinos/koinos-p2p/internal/protocol.(*SyncManager).HandleBlockBroadcast(...)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/sync_manager.go:261
github.com/koinos/koinos-p2p/internal/node.(*KoinosP2PNode).handleBlockBroadcast(0xc0002e7960, 0xc00003c180, 0x13, 0xc000654240, 0x22f, 0x22f)
    /Users/sgerbino/Projects/koinos-p2p/internal/node/node.go:123 +0x365
github.com/koinos/koinos-mq-golang.(*RequestHandler).HandleBroadcastDelivery(...)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:230
github.com/koinos/koinos-mq-golang.(*RequestHandler).ConsumeBroadcastLoop(0xc0002f59c0, 0xc00057a420, 0x18c1843, 0x13)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:199 +0x17c
created by github.com/koinos/koinos-mq-golang.(*RequestHandler).ConnectLoop
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-mq-golang@v0.0.0-20210424202816-d2bd4d1894d1/request_handler.go:133 +0x349

goroutine 97 [chan receive, 20 minutes]:
github.com/streadway/amqp.(*consumers).buffer(0xc000592390, 0xc00057a540, 0xc00057a4e0)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:61 +0x365
created by github.com/streadway/amqp.(*consumers).add
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:97 +0x165

goroutine 39 [chan receive]:
github.com/streadway/amqp.(*consumers).buffer(0xc000592390, 0xc00026c060, 0xc00026c000)
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:61 +0x365
created by github.com/streadway/amqp.(*consumers).add
    /Users/sgerbino/go/pkg/mod/github.com/streadway/amqp@v1.0.0/consumers.go:97 +0x165

goroutine 35342 [select]:
github.com/libp2p/go-yamux/v2.(*Stream).Read(0xc000353130, 0xc00049a000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/stream.go:117 +0x197
github.com/libp2p/go-libp2p-yamux.(*stream).Read(0xc000353130, 0xc00049a000, 0x1000, 0x1000, 0xc00055e600, 0xc00030ea03, 0x99)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-yamux@v0.5.1/stream.go:14 +0x4d
github.com/libp2p/go-libp2p-swarm.(*Stream).Read(0xc0003da380, 0xc00049a000, 0x1000, 0x1000, 0x2ae, 0xc0005e3ca0, 0x101a313)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_stream.go:57 +0x62
github.com/multiformats/go-multistream.(*lazyServerConn).Read(0xc00047a2d0, 0xc00049a000, 0x1000, 0x1000, 0xaa13f50, 0xc0005e3e98, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-multistream@v0.2.0/lazyServer.go:32 +0x76
github.com/libp2p/go-libp2p/p2p/host/basic.(*streamWrapper).Read(0xc0007a2020, 0xc00049a000, 0x1000, 0x1000, 0x49, 0xaa13f50, 0x203000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:1022 +0x52
bufio.(*Reader).fill(0xc000301260)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).ReadByte(0xc000301260, 0x100f0bb, 0xc0001ac0e0, 0x70)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:253 +0x39
github.com/multiformats/go-varint.ReadUvarint(0x1abc000, 0xc000301260, 0xc0001ac0e0, 0x1aca220, 0xc0001ccd80)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-varint@v0.0.6/varint.go:80 +0x77
github.com/libp2p/go-msgio/protoio.(*uvarintReader).ReadMsg(0xc00020c140, 0x1aca220, 0xc0001ac0e0, 0x1, 0x1)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-msgio@v0.0.6/protoio/uvarint_reader.go:60 +0x3e
github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleNewStream(0xc0001d8ea0, 0x1ad7eb0, 0xc0007a2020)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:66 +0x282
github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).SetStreamHandler.func1(0xc000702030, 0xe, 0xaa55160, 0xc0007a2020, 0x1, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:566 +0xa2
created by github.com/libp2p/go-libp2p/p2p/host/basic.(*BasicHost).newStreamHandler
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:416 +0x63a

goroutine 57809 [select]:
github.com/libp2p/go-yamux/v2.(*Stream).Read(0xc000353970, 0xc000228e0c, 0x1, 0x1, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/stream.go:117 +0x197
github.com/libp2p/go-libp2p-yamux.(*stream).Read(0xc000353970, 0xc000228e0c, 0x1, 0x1, 0x10, 0x2170f18, 0x10)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-yamux@v0.5.1/stream.go:14 +0x4d
github.com/libp2p/go-libp2p-swarm.(*Stream).Read(0xc000712500, 0xc000228e0c, 0x1, 0x1, 0x21705b8, 0xc00052c450, 0xc000282f50)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_stream.go:57 +0x62
github.com/multiformats/go-multistream.(*lazyServerConn).Read(0xc0007c9860, 0xc000228e0c, 0x1, 0x1, 0x17967e0, 0xc0004fe601, 0xc000228e0c)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-multistream@v0.2.0/lazyServer.go:32 +0x76
github.com/libp2p/go-libp2p/p2p/host/basic.(*streamWrapper).Read(0xc0004fe680, 0xc000228e0c, 0x1, 0x1, 0x106e5a1, 0xc0002142a0, 0x12)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:1022 +0x52
github.com/libp2p/go-libp2p-gorpc.(*Server).handle.func2(0xc0004410b0, 0xc00052c4e0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:277 +0x77
created by github.com/libp2p/go-libp2p-gorpc.(*Server).handle
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-gorpc@v0.1.2/server.go:275 +0x94f

goroutine 35471 [runnable]:
github.com/koinos/koinos-util-golang.MultihashToCmp(...)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-util-golang@v0.0.0-20210601175130-50a337109792/comparable.go:48
github.com/koinos/koinos-util-golang.BlockTopologyToCmp(...)
    /Users/sgerbino/go/pkg/mod/github.com/koinos/koinos-util-golang@v0.0.0-20210601175130-50a337109792/comparable.go:63
github.com/koinos/koinos-p2p/internal/protocol.(*PeerHandler).peerHandlerCycle(0xc000096580, 0x1acd878, 0xc0000326c0, 0x0, 0x0)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/peer_handler.go:202 +0x47b
github.com/koinos/koinos-p2p/internal/protocol.(*PeerHandler).peerHandlerLoop.func1(...)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/peer_handler.go:148
github.com/koinos/koinos-p2p/internal/protocol.(*PeerHandler).peerHandlerLoop(0xc000096580, 0x1acd878, 0xc0000326c0)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/peer_handler.go:163 +0x33b
created by github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).handleNewPeer
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:277 +0x2d9

goroutine 35472 [select]:
github.com/koinos/koinos-p2p/internal/protocol.(*PeerHandler).nodeUpdateLoop(0xc000096580, 0x1acd878, 0xc0000326c0)
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/peer_handler.go:132 +0x247
created by github.com/koinos/koinos-p2p/internal/protocol.(*BdmiProvider).handleNewPeer
    /Users/sgerbino/Projects/koinos-p2p/internal/protocol/download_provider.go:278 +0x30f

goroutine 35364 [select]:
github.com/libp2p/go-yamux/v2.(*Session).sendLoop(0xc00007e000, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:490 +0x1e5
github.com/libp2p/go-yamux/v2.(*Session).send(0xc00007e000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:436 +0x2b
created by github.com/libp2p/go-yamux/v2.newSession
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:131 +0x410

goroutine 35361 [select, 8 minutes]:
github.com/libp2p/go-libp2p/p2p/protocol/identify.(*peerHandler).loop(0xc0002de2a0, 0x1acd878, 0xc0002f4280, 0xc000210cc0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/peer_loop.go:92 +0x105
created by github.com/libp2p/go-libp2p/p2p/protocol/identify.(*peerHandler).start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/protocol/identify/peer_loop.go:74 +0x9c

goroutine 35117 [select, 8 minutes]:
github.com/libp2p/go-yamux/v2.(*Stream).Read(0xc0005dc2c0, 0xc000872000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/stream.go:117 +0x197
github.com/libp2p/go-libp2p-yamux.(*stream).Read(0xc0005dc2c0, 0xc000872000, 0x1000, 0x1000, 0xc0001f3500, 0x107937a, 0xc00028a004)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-yamux@v0.5.1/stream.go:14 +0x4d
github.com/libp2p/go-libp2p-swarm.(*Stream).Read(0xc00055e300, 0xc000872000, 0x1000, 0x1000, 0x203000, 0x203000, 0xb244a40)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_stream.go:57 +0x62
github.com/multiformats/go-multistream.(*lazyClientConn).Read(0xc00028a000, 0xc000872000, 0x1000, 0x1000, 0x38, 0x18421c0, 0x3230594260801)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-multistream@v0.2.0/lazyClient.go:68 +0xac
github.com/libp2p/go-libp2p/p2p/host/basic.(*streamWrapper).Read(0xc0007cc100, 0xc000872000, 0x1000, 0x1000, 0xc000032480, 0x1000000010f0345, 0x203000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p@v0.13.0/p2p/host/basic/basic_host.go:1022 +0x52
bufio.(*Reader).fill(0xc00060e660)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).ReadByte(0xc00060e660, 0x100f0bb, 0xc00028a070, 0x70)
    /usr/local/Cellar/go/1.16.4/libexec/src/bufio/bufio.go:253 +0x39
github.com/multiformats/go-varint.ReadUvarint(0x1abc000, 0xc00060e660, 0xc00028a070, 0x1000, 0xc000600480)
    /Users/sgerbino/go/pkg/mod/github.com/multiformats/go-varint@v0.0.6/varint.go:80 +0x77
github.com/libp2p/go-msgio/protoio.(*uvarintReader).ReadMsg(0xc000032480, 0x1aca220, 0xc00028a070, 0x1ac59a8, 0xc000032480)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-msgio@v0.0.6/protoio/uvarint_reader.go:60 +0x3e
github.com/libp2p/go-libp2p-pubsub.(*PubSub).handlePeerEOF(0xc0001d8ea0, 0x1acd8b0, 0xc000122000, 0x1ad7eb0, 0xc0007cc100)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:122 +0x1c2
created by github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleNewPeer
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:111 +0x41e

goroutine 35363 [IO wait]:
internal/poll.runtime_pollWait(0xaa40de8, 0x72, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000402798, 0x72, 0x0, 0x2, 0xffffffffffffffff)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000402780, 0xc0005d6088, 0x2, 0x2, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc000402780, 0xc0005d6088, 0x2, 0x2, 0x1, 0xc00069cb70, 0xc000235cf8)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000506030, 0xc0005d6088, 0x2, 0x2, 0x0, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/net/net.go:183 +0x91
io.ReadAtLeast(0xad65b00, 0xc000592c30, 0xc0005d6088, 0x2, 0x2, 0x2, 0x100, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:328 +0x87
io.ReadFull(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:347
github.com/libp2p/go-libp2p-noise.(*secureSession).readNextInsecureMsgLen(0xc0005d6000, 0x7b5b71626ddc5fe8, 0xc000235ec8, 0x16477c5)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-noise@v0.1.1/rw.go:134 +0x8c
github.com/libp2p/go-libp2p-noise.(*secureSession).Read(0xc0005d6000, 0xc000444460, 0xc, 0xc, 0x0, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-noise@v0.1.1/rw.go:53 +0x1fd
io.ReadAtLeast(0xb1aa0e0, 0xc0005d6000, 0xc000444460, 0xc, 0xc, 0xc, 0xc, 0x0, 0x0)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:328 +0x87
io.ReadFull(...)
    /usr/local/Cellar/go/1.16.4/libexec/src/io/io.go:347
github.com/libp2p/go-yamux/v2.(*Session).recvLoop(0xc00007e000, 0x0, 0x0)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:567 +0xda
github.com/libp2p/go-yamux/v2.(*Session).recv(0xc00007e000)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:546 +0x2b
created by github.com/libp2p/go-yamux/v2.newSession
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:130 +0x3ee

goroutine 35116 [select]:
github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleSendingMessages(0xc0001d8ea0, 0x1acd8b0, 0xc000122000, 0x1ad7eb0, 0xc0007cc100, 0xc000369860)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:149 +0x24d
created by github.com/libp2p/go-libp2p-pubsub.(*PubSub).handleNewPeer
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-pubsub@v0.4.1/comm.go:110 +0x3c5

goroutine 35375 [select]:
github.com/libp2p/go-yamux/v2.(*Session).AcceptStream(0xc00007e000, 0xc000353970, 0x0, 0x1042d6e)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-yamux/v2@v2.0.0/session.go:229 +0xf0
github.com/libp2p/go-libp2p-yamux.(*conn).AcceptStream(0xc00007e000, 0xc0005e4f78, 0x18, 0xc00036c900, 0x17080a9)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-yamux@v0.5.1/conn.go:35 +0x2b
github.com/libp2p/go-libp2p-swarm.(*Conn).start.func1(0xc000218120)
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_conn.go:107 +0xe2
created by github.com/libp2p/go-libp2p-swarm.(*Conn).start
    /Users/sgerbino/go/pkg/mod/github.com/libp2p/go-libp2p-swarm@v0.4.0/swarm_conn.go:102 +0x3f