search

QuarkChain / goquarkchain

QuarkChain implemented in Go
https://quarkchain.io
GNU Lesser General Public License v3.0
53 stars 39 forks source link

Cluster restart failure #549

Open syntrust opened 4 years ago

syntrust commented 4 years ago

Report from customer:

I made many transactions with 20ms interval, but the network shut down again. so I restart whole network and I got the error from slave node. I attached the logs following :

=== log start ===

INFO [01-22|07:55:12.156] db-operation Write RootBlock height=4620 hash=48175b…f71946 INFO [01-22|07:55:12.156] shard:1 tipMinor=9215 hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 mete.root=0xdbcd9a42a7b78a68a05418db0499a91144c4bfd1431dc64643d6ff4fa6b20f6a rootBlock=4620 rootTip=4620 WARN [01-22|07:55:12.156] shard:1 miss trie block=9215 block.hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 currNumber=9215 currHash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 INFO [01-22|07:55:12.377] reRunBlockWithState blockchain to past state number=8704 hash=0xc761a2c5db8fbf7524950c6158a9d44154bfda14ff3e3ad3555f78a60edbdbce INFO [01-22|07:55:15.147] slave ping response request op=2 INFO [01-22|07:56:00.195] shard:1 commit trie number=8704 hash=0xc761a2c5db8fbf7524950c6158a9d44154bfda14ff3e3ad3555f78a60edbdbce root=0xe1d70098901cf2e24b6990bc7b6168aad1bd32ccd2964b41849763f47c5d8233 WARN [01-22|07:56:37.854] shard:1 miss trie reRun time=85.698 currentBlock=9215 currHash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 WARN [01-22|07:56:37.854] reorg same chain curr=9215 curr.Hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 newBlock=9215 newBlock's hash=1bf806…9dfa26 newBlock=false WARN [01-22|07:56:37.854] Rewinding blockchain target=9215 WARN [01-22|07:56:37.854] Rewinding blockchain-end target number=9215 INFO [01-22|07:56:37.855] shard:1 init from root block end=9215 INFO [01-22|07:56:37.855] shard:1 InitFromRootBlock=end INFO [01-22|07:56:37.855] slave conn manager, add connect to slave add target=172.16.1.8:48000 INFO [01-22|07:56:37.856] slave conn manager, add connect to slave add target=172.16.1.8:48000 INFO [01-22|07:56:37.856] slave conn manager, add connect to slave add target=172.16.1.39:48000 INFO [01-22|07:56:37.857] slave ping response request op=2 INFO [01-22|07:56:37.857] slave conn manager, add connect to slave add target=172.16.1.39:48000 INFO [01-22|07:56:37.857] slave master info response master endpoint=172.16.1.17:38191 INFO [01-22|07:59:46.708] db-operation Write MinorBlock branch=1 height=9216 hash=0x247e8a40197fee5ea333abd69eb9cbb8344bce0356012b3f4c1ec0fc433f24fd len(tx)=0 fatal error: unexpected signal during runtime execution [signal SIGSEGV: segmentation violation code=0x1 addr=0x38 pc=0x7fa1f48f9e4e]

runtime stack: runtime.throw(0xe5793f, 0x2a) /usr/local/go/src/runtime/panic.go:617 +0x72 runtime.sigpanic() /usr/local/go/src/runtime/signal_unix.go:374 +0x4a9

goroutine 36 [syscall]: runtime.cgocall(0xc1eab5, 0xc008c69c60, 0x7fa1d8238c70) /usr/local/go/src/runtime/cgocall.go:128 +0x5b fp=0xc008c69c30 sp=0xc008c69bf8 pc=0x4151bb github.com/tecbot/gorocksdb._Cfunc_rocksdb_writebatch_clear(0x0) _cgo_gotypes.go:4312 +0x41 fp=0xc008c69c60 sp=0xc008c69c30 pc=0xa82aa1 github.com/tecbot/gorocksdb.(WriteBatch).Clear.func1(0xc00e0d64e8) /go/pkg/mod/github.com/tecbot/gorocksdb@v0.0.0-20181010114359-8752a9433481/write_batch.go:94 +0x5e fp=0xc008c69ca0 sp=0xc008c69c60 pc=0xa9b3be github.com/tecbot/gorocksdb.(WriteBatch).Clear(0xc00e0d64e8) /go/pkg/mod/github.com/tecbot/gorocksdb@v0.0.0-20181010114359-8752a9433481/write_batch.go:94 +0x2b fp=0xc008c69cb8 sp=0xc008c69ca0 pc=0xa8ee6b github.com/QuarkChain/goquarkchain/qkcdb.(rdbBatch).Reset(0xc00d8fbee0) /go/src/github.com/QuarkChain/goquarkchain/qkcdb/database.go:170 +0x2f fp=0xc008c69cd0 sp=0xc008c69cb8 pc=0xa9d2cf github.com/ethereum/go-ethereum/trie.(Database).commit(0xc0002ae1e0, 0x8a85952d68a3096e, 0xfb7ea0d8da8e4e86, 0x4e92eee4ebefcec1, 0xea6e8ba2589eaa17, 0xfdd0c0, 0xc00d8fbee0, 0x0, 0x0) /go/pkg/mod/github.com/ethereum/go-ethereum@v1.8.20/trie/database.go:749 +0x298 fp=0xc008c69db0 sp=0xc008c69cd0 pc=0x6f5548 ... github.com/ethereum/go-ethereum/trie.(Database).Commit(0xc0002ae1e0, 0x47ffdcbd38722293, 0xa042b6592f067cab, 0xde765f7a2e66c7db, 0x827d02b6cb5d0bf7, 0x0, 0x0, 0x0) /go/pkg/mod/github.com/ethereum/go-ethereum@v1.8.20/trie/database.go:689 +0x52b fp=0xc008c6a8a8 sp=0xc008c6a670 pc=0x6f481b github.com/QuarkChain/goquarkchain/core.(MinorBlockChain).WriteBlockWithState(0xc0026ea000, 0xc002d8be60, 0xc00ed8d8e0, 0x2, 0x2, 0xc00b7a42c0, 0xc00ed8d870, 0x2, 0x2, 0x1, ...) /go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:921 +0xbbd fp=0xc008c6ab70 sp=0xc008c6a8a8 pc=0xb2866d github.com/QuarkChain/goquarkchain/core.(MinorBlockChain).insertChain(0xc0026ea000, 0xc00ed8d2a0, 0x1, 0x1, 0x1, 0x0, 0xc005b37200, 0x42942f, 0x7fa1e55c3ce0, 0x0, ...) /go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1151 +0x9d4 fp=0xc008c6b170 sp=0xc008c6ab70 pc=0xb2a8e4 github.com/QuarkChain/goquarkchain/core.(MinorBlockChain).InsertChainForDeposits(0xc0026ea000, 0xc00ed8d2a0, 0x1, 0x1, 0x3f2b015603ce4b00, 0xfd243f43fcc01e4c, 0x8163e3432ee33100, 0xca30ca728342d1e9, 0xf1c97c8f62b9495d, 0x4b19f0d1d1709210, ...) /go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1015 +0x9cf fp=0xc008c6b590 sp=0xc008c6b170 pc=0xb298bf github.com/QuarkChain/goquarkchain/cluster/shard.(ShardBackend).AddMinorBlock(0xc0002ae000, 0xc002d8be60, 0x0, 0x0) /go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:275 +0x264 fp=0xc008c6b8b0 sp=0xc008c6b590 pc=0xbd5964 github.com/QuarkChain/goquarkchain/cluster/shard.(ShardBackend).NewMinorBlock(0xc0002ae000, 0x0, 0x0, 0xc002d8be60, 0x41, 0x81) /go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:262 +0xa8d fp=0xc008c6bc90 sp=0xc008c6b8b0 pc=0xbd4ded github.com/QuarkChain/goquarkchain/cluster/shard.(ShardBackend).InsertMinedBlock(0xc0002ae000, 0xfe56a0, 0xc002d8be60, 0x0, 0x0) /go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:409 +0xa5 fp=0xc008c6bcd8 sp=0xc008c6bc90 pc=0xbd7605 github.com/QuarkChain/goquarkchain/cluster/miner.(Miner).mainLoop(0xc00010a400) /go/src/github.com/QuarkChain/goquarkchain/cluster/miner/miner.go:130 +0x339 fp=0xc008c6bfd8 sp=0xc008c6bcd8 pc=0xb80829 runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc008c6bfe0 sp=0xc008c6bfd8 pc=0x46f2c1 created by github.com/QuarkChain/goquarkchain/cluster/miner.New /go/src/github.com/QuarkChain/goquarkchain/cluster/miner/miner.go:59 +0x1c9

goroutine 1 [chan receive, 5 minutes]: github.com/QuarkChain/goquarkchain/cluster/service.(Node).Wait(0xc0000fc780) /go/src/github.com/QuarkChain/goquarkchain/cluster/service/node.go:498 +0x77 main.cluster(0xc0000dd4a0, 0x18, 0xc000219c50) /go/src/github.com/QuarkChain/goquarkchain/cmd/cluster/main.go:130 +0x144 gopkg.in/urfave/cli%2ev1.HandleAction(0xce2120, 0xe72840, 0xc0000dd4a0, 0xc0000dd4a0, 0xc000219f58) /go/pkg/mod/gopkg.in/urfave/cli.v1@v1.20.0/app.go:490 +0xc8 gopkg.in/urfave/cli%2ev1.(App).Run(0xc0000c5520, 0xc0000bc000, 0x5, 0x5, 0x0, 0x0) /go/pkg/mod/gopkg.in/urfave/cli.v1@v1.20.0/app.go:264 +0x590 main.main() /go/src/github.com/QuarkChain/goquarkchain/cmd/cluster/main.go:115 +0x55

goroutine 7 [syscall, 5 minutes]: os/signal.signal_recv(0x0) /usr/local/go/src/runtime/sigqueue.go:139 +0x9c os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:23 +0x22 created by os/signal.init.0 /usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 8 [chan receive, 4 minutes]: github.com/QuarkChain/goquarkchain/core.(*txSenderCacher).cache(0xc0000447d0) /go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:62 +0x9b created by github.com/QuarkChain/goquarkchain/core.newTxSenderCacher /go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:54 +0x9f

goroutine 9 [chan receive, 4 minutes]: github.com/QuarkChain/goquarkchain/core.(*txSenderCacher).cache(0xc0000447d0) /go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:62 +0x9b created by github.com/QuarkChain/goquarkchain/core.newTxSenderCacher /go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:54 +0x9f

goroutine 34 [select]: github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).update(0xc0026ea000) /go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1478 +0xfc created by github.com/QuarkChain/goquarkchain/core.NewMinorBlockChain /go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:236 +0xa33

goroutine 25 [IO wait, 3 minutes]: internal/poll.runtime_pollWait(0x7fa1f4f2c2d8, 0x72, 0x0) /usr/local/go/src/runtime/netpoll.go:182 +0x56 internal/poll.(pollDesc).wait(0xc00010a018, 0x72, 0x0, 0x0, 0xe36df3) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Accept(0xc00010a000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1ba net.(netFD).accept(0xc00010a000, 0xc00db8e001, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:238 +0x42 net.(TCPListener).accept(0xc0000b6108, 0xc000069ec0, 0xc000069ec8, 0x18) /usr/local/go/src/net/tcpsock_posix.go:139 +0x32 net.(TCPListener).Accept(0xc0000b6108, 0xe72360, 0xc000001e00, 0xfe3200, 0xc00db8e0a0) /usr/local/go/src/net/tcpsock.go:260 +0x48 google.golang.org/grpc.(*Server).Serve(0xc000001e00, 0xfd7400, 0xc0000b6108, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:561 +0x1e9 created by github.com/QuarkChain/goquarkchain/cluster/rpc.StartGRPCServer /go/src/github.com/QuarkChain/goquarkchain/cluster/rpc/grpc_server.go:38 +0x161

goroutine 26 [chan receive, 5 minutes]: github.com/QuarkChain/goquarkchain/cmd/utils.StartService.func1(0xc0000fc780) /go/src/github.com/QuarkChain/goquarkchain/cmd/utils/cmd.go:43 +0xbf created by github.com/QuarkChain/goquarkchain/cmd/utils.StartService /go/src/github.com/QuarkChain/goquarkchain/cmd/utils/cmd.go:40 +0xad

goroutine 14 [select]: google.golang.org/grpc/internal/transport.(controlBuffer).get(0xc00014a100, 0x1, 0x0, 0x0, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104 google.golang.org/grpc/internal/transport.(loopyWriter).run(0xc000082900, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6 google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000506180) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:276 +0xcb created by google.golang.org/grpc/internal/transport.newHTTP2Server /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:273 +0xfba

goroutine 15 [select, 5 minutes]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000506180) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:921 +0x1ed created by google.golang.org/grpc/internal/transport.newHTTP2Server /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:282 +0xfdf

goroutine 16 [IO wait]: internal/poll.runtime_pollWait(0x7fa1f4f2c208, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:182 +0x56 internal/poll.(pollDesc).wait(0xc0001c0198, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc0001c0180, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc0001c0180, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x800010601, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc000010008, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:177 +0x69 bufio.(Reader).Read(0xc0000828a0, 0xc000366038, 0x9, 0x9, 0xc0001aee00, 0x7fa1f4f836d0, 0x0) /usr/local/go/src/bufio/bufio.go:223 +0x23e io.ReadAtLeast(0xfc2580, 0xc0000828a0, 0xc000366038, 0x9, 0x9, 0x9, 0xbf821e1caa354c46, 0x45f1e84e85, 0x19067a0) /usr/local/go/src/io/io.go:310 +0x88 io.ReadFull(...) /usr/local/go/src/io/io.go:329 golang.org/x/net/http2.readFrameHeader(0xc000366038, 0x9, 0x9, 0xfc2580, 0xc0000828a0, 0x0, 0x0, 0xc0000bc0a0, 0x7070e0910100402) /go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88 golang.org/x/net/http2.(Framer).ReadFrame(0xc000366000, 0xc027eb11a0, 0xc027eb11a0, 0x0, 0x0) /go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1 google.golang.org/grpc/internal/transport.(http2Server).HandleStreams(0xc000506180, 0xc000144600, 0xe723c0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:429 +0x7c google.golang.org/grpc.(Server).serveStreams(0xc000001e00, 0xfe43a0, 0xc000506180) /go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:686 +0xdd google.golang.org/grpc.(Server).handleRawConn.func1(0xc000001e00, 0xfe43a0, 0xc000506180) /go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:648 +0x43 created by google.golang.org/grpc.(*Server).handleRawConn /go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:647 +0x583

goroutine 107 [select]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00ad64a40) /go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:122 +0x110 created by google.golang.org/grpc.newCCBalancerWrapper /go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:113 +0x14f

goroutine 39 [select, 3 minutes]: google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00014ab40) /go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:122 +0x110 created by google.golang.org/grpc.newCCBalancerWrapper /go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:113 +0x14f

goroutine 35 [select, 4 minutes]: github.com/QuarkChain/goquarkchain/cluster/sync.(*synchronizer).loop(0xc0000cd380) /go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:105 +0x19c created by github.com/QuarkChain/goquarkchain/cluster/sync.NewSynchronizer /go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:138 +0xf7

goroutine 31 [chan receive, 4 minutes]: github.com/QuarkChain/goquarkchain/cluster/sync.(synchronizer).loop.func1(0xc0000cd380) /go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:82 +0x104 created by github.com/QuarkChain/goquarkchain/cluster/sync.(synchronizer).loop /go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:80 +0x57

goroutine 54 [select, 3 minutes]: google.golang.org/grpc/internal/transport.(controlBuffer).get(0xc00007e080, 0x1, 0x0, 0x0, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104 google.golang.org/grpc/internal/transport.(loopyWriter).run(0xc00e52e3c0, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6 google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00ea4c000) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:330 +0x7b created by google.golang.org/grpc/internal/transport.newHTTP2Client /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:328 +0xeb2

goroutine 33 [select]: google.golang.org/grpc/internal/transport.(controlBuffer).get(0xc00ad65240, 0x1, 0x0, 0x0, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104 google.golang.org/grpc/internal/transport.(loopyWriter).run(0xc0000831a0, 0x0, 0x0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6 google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000507b00) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:276 +0xcb created by google.golang.org/grpc/internal/transport.newHTTP2Server /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:273 +0xfba

goroutine 50 [select, 4 minutes]: google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000507b00) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:921 +0x1ed created by google.golang.org/grpc/internal/transport.newHTTP2Server /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:282 +0xfdf

goroutine 51 [IO wait]: internal/poll.runtime_pollWait(0x7fa1f4f2c138, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:182 +0x56 internal/poll.(pollDesc).wait(0xc00a70dd18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc00a70dd00, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc00a70dd00, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x8, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00ac97b48, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:177 +0x69 bufio.(Reader).Read(0xc000083140, 0xc000366738, 0x9, 0x9, 0x18, 0x18, 0xc00d974320) /usr/local/go/src/bufio/bufio.go:223 +0x23e io.ReadAtLeast(0xfc2580, 0xc000083140, 0xc000366738, 0x9, 0x9, 0x9, 0x46b6f0, 0x20, 0xc0006c2d98) /usr/local/go/src/io/io.go:310 +0x88 io.ReadFull(...) /usr/local/go/src/io/io.go:329 golang.org/x/net/http2.readFrameHeader(0xc000366738, 0x9, 0x9, 0xfc2580, 0xc000083140, 0x0, 0x0, 0xbf821e1caa344ce2, 0x45f1e73803) /go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88 golang.org/x/net/http2.(Framer).ReadFrame(0xc000366700, 0xc027eb10e0, 0xc027eb10e0, 0x0, 0x0) /go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1 google.golang.org/grpc/internal/transport.(http2Server).HandleStreams(0xc000507b00, 0xc00adea870, 0xe723c0) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:429 +0x7c google.golang.org/grpc.(Server).serveStreams(0xc000001e00, 0xfe43a0, 0xc000507b00) /go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:686 +0xdd google.golang.org/grpc.(Server).handleRawConn.func1(0xc000001e00, 0xfe43a0, 0xc000507b00) /go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:648 +0x43 created by google.golang.org/grpc.(*Server).handleRawConn /go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:647 +0x583

goroutine 37 [select, 3 minutes]: github.com/QuarkChain/goquarkchain/core.(*TxPool).scheduleReorgLoop(0xc01fb712c0) /go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:813 +0x227 created by github.com/QuarkChain/goquarkchain/core.NewTxPool /go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:216 +0x5c3

goroutine 38 [select]: github.com/QuarkChain/goquarkchain/core.(*TxPool).loop(0xc01fb712c0) /go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:244 +0x250 created by github.com/QuarkChain/goquarkchain/core.NewTxPool /go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:221 +0x651

goroutine 40 [chan receive, 3 minutes]: google.golang.org/grpc.(addrConn).resetTransport(0xc0001e6280) /go/pkg/mod/google.golang.org/grpc@v1.19.1/clientconn.go:1067 +0x9ec created by google.golang.org/grpc.(addrConn).connect /go/pkg/mod/google.golang.org/grpc@v1.19.1/clientconn.go:684 +0xb6

goroutine 53 [IO wait, 3 minutes]: internal/poll.runtime_pollWait(0x7fa1f4f2c068, 0x72, 0xffffffffffffffff) /usr/local/go/src/runtime/netpoll.go:182 +0x56 internal/poll.(pollDesc).wait(0xc001b9a298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b internal/poll.(pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(FD).Read(0xc001b9a280, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b net.(netFD).Read(0xc001b9a280, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x800010601, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x4f net.(conn).Read(0xc00db8e000, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/net/net.go:177 +0x69 bufio.(Reader).Read(0xc00e52e360, 0xc00e97c038, 0x9, 0x9, 0xc0001aea80, 0x7fa1f4f836d0, 0x0) /usr/local/go/src/bufio/bufio.go:223 +0x23e io.ReadAtLeast(0xfc2580, 0xc00e52e360, 0xc00e97c038, 0x9, 0x9, 0x9, 0x9a1375, 0xc00a8ac4cc, 0xc00006de38) /usr/local/go/src/io/io.go:310 +0x88 io.ReadFull(...) /usr/local/go/src/io/io.go:329 golang.org/x/net/http2.readFrameHeader(0xc00e97c038, 0x9, 0x9, 0xfc2580, 0xc00e52e360, 0x0, 0xc000000000, 0x19f9802abc, 0x19067a0) /go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88 golang.org/x/net/http2.(Framer).ReadFrame(0xc00e97c000, 0xc00a8ac4c0, 0xc00a8ac4c0, 0x0, 0x0) /go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1 google.golang.org/grpc/internal/transport.(http2Client).reader(0xc00ea4c000) /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:1223 +0x168 created by google.golang.org/grpc/internal/transport.newHTTP2Client /go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:286 +0xd15

syntrust commented 4 years ago

Possibly related error in a snapshot:

quark-error quark-error-2nd_cluster_when_connect_to_1st_cluster