search

kaspanet / kaspad

Kaspad was the reference full node Kaspa implementation written in Go (golang), now rewritten in Rust: https://github.com/kaspanet/rusty-kaspa
ISC License
457 stars 231 forks source link

Leveldb crash on exit #1494

Open aspect opened 3 years ago

aspect commented 3 years ago 
2021-01-27 18:02:12.824 [INF] KASD: Received signal (interrupt). Shutting down...
2021-01-27 18:02:12.824 [INF] KASD: Gracefully shutting down kaspad...
2021-01-27 18:02:12.824 [WRN] KASD: Kaspad shutting down
2021-01-27 18:02:12.825 [INF] KASD: Kaspad shutdown complete
2021-01-27 18:02:12.825 [INF] KASD: Gracefully shutting down the database...
2021-01-27 18:02:12.830 [INF] PROT: IBD finished
2021-01-27 18:02:12.833 [CRT] PROT: Exiting: Fatal error in goroutine `flow-HandleRelayInvs 77`: leveldb: closed
github.com/kaspanet/kaspad/infrastructure/db/database/ldb.(*LevelDB).Has
        /home/aspect/releases/kaspad/infrastructure/db/database/ldb/leveldb.go:78
github.com/kaspanet/kaspad/domain/consensus/database.(*dbManager).Has
        /home/aspect/releases/kaspad/domain/consensus/database/dbmanager.go:17
github.com/kaspanet/kaspad/domain/consensus/datastructures/pruningstore.(*pruningStore).HadStartedUpdatingPruningPointUTXOSet
        /home/aspect/releases/kaspad/domain/consensus/datastructures/pruningstore/pruningstore.go:266
github.com/kaspanet/kaspad/domain/consensus/processes/pruningmanager.(*pruningManager).UpdatePruningPointUTXOSetIfRequired
        /home/aspect/releases/kaspad/domain/consensus/processes/pruningmanager/pruningmanager.go:503
github.com/kaspanet/kaspad/domain/consensus/processes/blockprocessor.(*blockProcessor).validateAndInsertBlock
        /home/aspect/releases/kaspad/domain/consensus/processes/blockprocessor/validateandinsertblock.go:120
github.com/kaspanet/kaspad/domain/consensus/processes/blockprocessor.(*blockProcessor).ValidateAndInsertBlock
        /home/aspect/releases/kaspad/domain/consensus/processes/blockprocessor/blockprocessor.go:137
github.com/kaspanet/kaspad/domain/consensus.(*consensus).ValidateAndInsertBlock
        /home/aspect/releases/kaspad/domain/consensus/consensus.go:68
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).processHeader
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/ibd.go:278
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).downloadHeaders
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/ibd.go:225
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).syncHeaders
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/ibd.go:68
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).runIBDIfNotRunning
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/ibd.go:31
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).processOrphan
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:257
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).start
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:118
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.HandleRelayInvs
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:56
github.com/kaspanet/kaspad/app/protocol.(*Manager).registerBlockRelayFlows.func1
        /home/aspect/releases/kaspad/app/protocol/protocol.go:144
github.com/kaspanet/kaspad/app/protocol.(*Manager).registerFlow.func1
        /home/aspect/releases/kaspad/app/protocol/protocol.go:261
github.com/kaspanet/kaspad/app/protocol.(*Manager).runFlows.func1
        /home/aspect/releases/kaspad/app/protocol/manager.go:60
github.com/kaspanet/kaspad/util/panics.handleSpawnedFunction
        /home/aspect/releases/kaspad/util/panics/panics.go:84
github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1.1
        /home/aspect/releases/kaspad/util/panics/panics.go:32
runtime.goexit
        /usr/lib/go-1.13/src/runtime/asm_amd64.s:1357
failed to process header 283ff03541eaa3ec447ea211f8389aac7ec408056c5c04bd07222e4f0c70a0e2 during IBD
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).processHeader
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/ibd.go:281
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).downloadHeaders
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/ibd.go:225
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).syncHeaders
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/ibd.go:68
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).runIBDIfNotRunning
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/ibd.go:31
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).processOrphan
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:257
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(*handleRelayInvsFlow).start
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:118
github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.HandleRelayInvs
        /home/aspect/releases/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:56
github.com/kaspanet/kaspad/app/protocol.(*Manager).registerBlockRelayFlows.func1
        /home/aspect/releases/kaspad/app/protocol/protocol.go:144
github.com/kaspanet/kaspad/app/protocol.(*Manager).registerFlow.func1
        /home/aspect/releases/kaspad/app/protocol/protocol.go:261
github.com/kaspanet/kaspad/app/protocol.(*Manager).runFlows.func1
        /home/aspect/releases/kaspad/app/protocol/manager.go:60
github.com/kaspanet/kaspad/util/panics.handleSpawnedFunction
        /home/aspect/releases/kaspad/util/panics/panics.go:84
github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1.1
        /home/aspect/releases/kaspad/util/panics/panics.go:32
runtime.goexit
        /usr/lib/go-1.13/src/runtime/asm_amd64.s:1357
2021-01-27 18:02:12.834 [CRT] PROT: Goroutine stack trace: goroutine 80 [running]:
runtime/debug.Stack(0x4d9894, 0xc0000a24e0, 0xc00013e200)
        /usr/lib/go-1.13/src/runtime/debug/stack.go:24 +0x9d
github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1(0xc000037b20, 0x14, 0xc0082a2400)
        /home/aspect/releases/kaspad/util/panics/panics.go:30 +0x2f
github.com/kaspanet/kaspad/app/protocol.(*Manager).runFlows(0xc0000b6160, 0xc007b61220, 0xf, 0x14, 0xc000116960, 0xc00008fe00, 0xc000116960, 0x0)
        /home/aspect/releases/kaspad/app/protocol/manager.go:59 +0x49
github.com/kaspanet/kaspad/app/protocol.(*Manager).routerInitializer.func1()
        /home/aspect/releases/kaspad/app/protocol/protocol.go:68 +0x38c
github.com/kaspanet/kaspad/util/panics.handleSpawnedFunction(0xc0000ac280, 0xc007fef000, 0xa04, 0x1000, 0xcc1e0d, 0x1a, 0xc008266900)
        /home/aspect/releases/kaspad/util/panics/panics.go:84 +0x306
github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1.1(0xc0000ac280, 0xc007fef000, 0xa04, 0x1000, 0xcc1e0d, 0x1a, 0xc008266900)
        /home/aspect/releases/kaspad/util/panics/panics.go:32 +0x67
created by github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1
        /home/aspect/releases/kaspad/util/panics/panics.go:31 +0x8c
oudeis commented 2 years ago

This is still happening, now consistently on one mac machine on kaspad startup:

➜ kaspad git:(v0.11.3-dev) ~/go/bin/kaspad --utxoindex 2021-11-16 15:06:41.478 [INF] KASD: Version 0.11.3 2021-11-16 15:06:41.478 [INF] KASD: Loading database from '/Users/denis/Library/Application Support/Kaspad/kaspa-mainnet/data' 2021-11-16 15:06:41.571 [INF] PRFX: Deleting database prefix &{1} 2021-11-16 15:06:41.725 [INF] ADXR: Loaded 633 addresses and 155 banned addresses 2021-11-16 15:06:43.911 [INF] KASD: UTXO index started 2021-11-16 15:06:43.911 [INF] TXMP: P2P Server listening on [::]:16111 2021-11-16 15:06:43.911 [INF] TXMP: RPC Server listening on [::]:16110 2021-11-16 15:06:43.911 [INF] CMGR: Connecting to 139.59.159.116:16111 2021-11-16 15:06:44.913 [INF] CMGR: Couldn't connect to 139.59.159.116:16111: P2P error connecting to 139.59.159.116:16111: context deadline exceeded 2021-11-16 15:06:44.914 [INF] CMGR: Connecting to 65.21.228.92:16111 2021-11-16 15:06:45.090 [INF] TXMP: P2P Connected to 65.21.228.92:16111 2021-11-16 15:06:45.090 [INF] CMGR: Connecting to [2a00:a040:183:2ed2:2c69:1325:659d:5a62]:16111 2021-11-16 15:06:45.485 [INF] PROT: IBD started 2021-11-16 15:06:45.596 [INF] PROT: Starting IBD with headers proof 2021-11-16 15:06:45.624 [INF] PROT: Downloading the pruning point proof from <4a309475d6a3b1ffdd1f1ea32b722672: 65.21.228.92:16111> 2021-11-16 15:06:46.094 [INF] CMGR: Couldn't connect to [2a00:a040:183:2ed2:2c69:1325:659d:5a62]:16111: P2P error connecting to [2a00:a040:183:2ed2:2c69:1325:659d:5a62]:16111: context dead line exceeded 2021-11-16 15:06:46.094 [INF] CMGR: Connecting to [2a02:6680:2130:6a8:c40b:b675:7bc4:abca]:16111 2021-11-16 15:06:47.094 [INF] CMGR: Couldn't connect to [2a02:6680:2130:6a8:c40b:b675:7bc4:abca]:16111: P2P error connecting to [2a02:6680:2130:6a8:c40b:b675:7bc4:abca]:16111: context dead line exceeded 2021-11-16 15:06:47.094 [INF] CMGR: Connecting to [2a02:1205:34e5:d230:b3a2:805d:af86:e5ba]:16111 2021-11-16 15:06:48.097 [INF] CMGR: Couldn't connect to [2a02:1205:34e5:d230:b3a2:805d:af86:e5ba]:16111: P2P error connecting to [2a02:1205:34e5:d230:b3a2:805d:af86:e5ba]:16111: context de adline exceeded 2021-11-16 15:06:48.097 [INF] CMGR: Connecting to [2a00:7c40:c2e0:33a:497:b453:fc1e:ab93]:16111 2021-11-16 15:06:49.099 [INF] CMGR: Couldn't connect to [2a00:7c40:c2e0:33a:497:b453:fc1e:ab93]:16111: P2P error connecting to [2a00:7c40:c2e0:33a:497:b453:fc1e:ab93]:16111: context deadli ne exceeded 2021-11-16 15:06:49.100 [INF] CMGR: Connecting to [2a02:6680:2131:67f6:d8e3:980d:c54b:eb05]:16111 2021-11-16 15:06:50.104 [INF] CMGR: Couldn't connect to [2a02:6680:2131:67f6:d8e3:980d:c54b:eb05]:16111: P2P error connecting to [2a02:6680:2131:67f6:d8e3:980d:c54b:eb05]:16111: context de adline exceeded 2021-11-16 15:06:50.105 [INF] CMGR: Connecting to [2a00:a040:196:22f9:2406:cbc8:2252:e98b]:16111 2021-11-16 15:06:51.105 [INF] CMGR: Couldn't connect to [2a00:a040:196:22f9:2406:cbc8:2252:e98b]:16111: P2P error connecting to [2a00:a040:196:22f9:2406:cbc8:2252:e98b]:16111: context dead line exceeded 2021-11-16 15:07:13.912 [INF] CMGR: Connecting to [2a10:8008:6aef:0:3411:3d6a:98fc:7647]:16111 2021-11-16 15:07:14.913 [INF] CMGR: Couldn't connect to [2a10:8008:6aef:0:3411:3d6a:98fc:7647]:16111: P2P error connecting to [2a10:8008:6aef:0:3411:3d6a:98fc:7647]:16111: context deadline exceeded 2021-11-16 15:07:14.913 [INF] CMGR: Connecting to 65.108.46.173:16111 2021-11-16 15:07:15.111 [INF] TXMP: P2P Connected to 65.108.46.173:16111 2021-11-16 15:07:15.111 [INF] CMGR: Connecting to [2a00:a040:196:22f9:e8d3:c1cb:ee7f:66fc]:16111 2021-11-16 15:07:16.112 [INF] CMGR: Couldn't connect to [2a00:a040:196:22f9:e8d3:c1cb:ee7f:66fc]:16111: P2P error connecting to [2a00:a040:196:22f9:e8d3:c1cb:ee7f:66fc]:16111: context dead line exceeded 2021-11-16 15:07:16.112 [INF] CMGR: Connecting to 18.169.157.75:16111 2021-11-16 15:07:17.113 [INF] CMGR: Couldn't connect to 18.169.157.75:16111: P2P error connecting to 18.169.157.75:16111: context deadline exceeded 2021-11-16 15:07:17.113 [INF] CMGR: Connecting to 18.130.104.86:16111 2021-11-16 15:07:18.113 [INF] CMGR: Couldn't connect to 18.130.104.86:16111: P2P error connecting to 18.130.104.86:16111: context deadline exceeded 2021-11-16 15:07:18.113 [INF] CMGR: Connecting to 161.97.184.17:16111 2021-11-16 15:07:18.269 [INF] TXMP: P2P Connected to 161.97.184.17:16111 2021-11-16 15:07:18.269 [INF] CMGR: Connecting to 66.23.200.117:16111 2021-11-16 15:07:18.552 [INF] TXMP: P2P Connected to 66.23.200.117:16111 2021-11-16 15:07:18.909 [INF] PROT: IBD finished (interrupted) 2021-11-16 15:07:18.909 [CRT] PROT: Exiting: Fatal error in goroutine flow-HandleRelayInvs 13: not found key 002f062f626c6f636b2d72656c6174696f6e732f8ec994401f1c721b756b66765567593b6b8afb3bd47d2919d513e562720a1305 not found github.com/kaspanet/kaspad/infrastructure/db/database/ldb.(LevelDB).Get /Users/denis/go/src/github.com/kaspanet/kaspad/infrastructure/db/database/ldb/leveldb.go:69 github.com/kaspanet/kaspad/domain/consensus/database.(dbManager).Get /Users/denis/go/src/github.com/kaspanet/kaspad/domain/consensus/database/dbmanager.go:13 github.com/kaspanet/kaspad/domain/consensus/datastructures/blockrelationstore.(blockRelationStore).BlockRelation /Users/denis/go/src/github.com/kaspanet/kaspad/domain/consensus/datastructures/blockrelationstore/block_relation_store.go:51 github.com/kaspanet/kaspad/domain/consensus/processes/dagtopologymanager.(dagTopologyManager).Parents /Users/denis/go/src/github.com/kaspanet/kaspad/domain/consensus/processes/dagtopologymanager/dagtopologymanager.go:35 github.com/kaspanet/kaspad/domain/consensus/processes/pruningproofmanager.(pruningProofManager).ValidatePruningPointProof /Users/denis/go/src/github.com/kaspanet/kaspad/domain/consensus/processes/pruningproofmanager/pruningproofmanager.go:444 github.com/kaspanet/kaspad/domain/consensus.(consensus).ValidatePruningPointProof /Users/denis/go/src/github.com/kaspanet/kaspad/domain/consensus/consensus.go:749 github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(handleRelayInvsFlow).syncAndValidatePruningPointProof /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/flows/blockrelay/ibd_with_headers_proof.go:95 github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(handleRelayInvsFlow).downloadHeadersAndPruningUTXOSet /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/flows/blockrelay/ibd_with_headers_proof.go:112 github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(handleRelayInvsFlow).ibdWithHeadersProof /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/flows/blockrelay/ibd_with_headers_proof.go:20 github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(handleRelayInvsFlow).runIBDIfNotRunning /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/flows/blockrelay/ibd.go:53 github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(handleRelayInvsFlow).processOrphan /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:278 github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.(handleRelayInvsFlow).start /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:130 github.com/kaspanet/kaspad/app/protocol/flows/blockrelay.HandleRelayInvs /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/flows/blockrelay/handle_relay_invs.go:58 github.com/kaspanet/kaspad/app/protocol.(Manager).registerBlockRelayFlows.func2 /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/protocol.go:179 github.com/kaspanet/kaspad/app/protocol.(Manager).registerFlowForRoute.func1 /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/protocol.go:321 github.com/kaspanet/kaspad/app/protocol.(Manager).runFlows.func1 /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/manager.go:79 github.com/kaspanet/kaspad/util/panics.handleSpawnedFunction /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:83 github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1.1 /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:32 runtime.goexit /usr/local/go/src/runtime/asm_amd64.s:1371 2021-11-16 15:07:18.909 [CRT] PROT: Goroutine stack trace: goroutine 12 [running]: runtime/debug.Stack(0xc008466380, 0xc0003d8000, 0xc000088d80) /usr/local/go/src/runtime/debug/stack.go:24 +0x9f github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1(0xc013f6a060, 0x14, 0xc008466380) /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:30 +0x2f github.com/kaspanet/kaspad/app/protocol.(Manager).runFlows(0xc004f6d260, 0xc0002383c0, 0x11, 0x18, 0xc01482a140, 0xc000376ba0, 0xc000614640, 0x0, 0x0) /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/manager.go:78 +0x67 github.com/kaspanet/kaspad/app/protocol.(*Manager).routerInitializer.func1() /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/protocol.go:90 +0x64b github.com/kaspanet/kaspad/util/panics.handleSpawnedFunction(0xc00026d680, 0xc00021f000, 0xadb, 0x1000, 0x4933175, 0x1a, 0xc0001118c0) /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:83 +0x2b4 github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1.1(0xc00026d680, 0xc00021f000, 0xadb, 0x1000, 0x4933175, 0x1a, 0xc0001118c0) /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:32 +0x67 created by github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1 /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:31 +0x8c

2021-11-16 15:07:18.911 [CRT] PROT: Stack trace: goroutine 71 [running]: runtime/debug.Stack(0x4939dc2, 0x22, 0xc0bed1fb38) /usr/local/go/src/runtime/debug/stack.go:24 +0x9f github.com/kaspanet/kaspad/util/panics.HandlePanic(0xc00026d680, 0xc004f68cd8, 0x17, 0xc01416c800, 0x515, 0x800) /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:22 +0x10b panic(0x4865420, 0xc040afdec0) /usr/local/go/src/runtime/panic.go:965 +0x1b9 github.com/kaspanet/kaspad/app/protocol/flowcontext.(FlowContext).HandleError(0xc0002ba360, 0x4a11280, 0xc040afdec0, 0x4929d40, 0xf, 0xc0006cb538, 0xc000376ba0) /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/flowcontext/errors.go:22 +0x1e5 github.com/kaspanet/kaspad/app/protocol.(Manager).registerFlowForRoute.func1(0xc01482a140) /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/protocol.go:323 +0xbf github.com/kaspanet/kaspad/app/protocol.(*Manager).runFlows.func1() /Users/denis/go/src/github.com/kaspanet/kaspad/app/protocol/manager.go:79 +0x37 github.com/kaspanet/kaspad/util/panics.handleSpawnedFunction(0xc00026d680, 0xc01416c800, 0x515, 0x800, 0xc013f6a060, 0x14, 0xc008466380) /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:83 +0x2b4 github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1.1(0xc00026d680, 0xc01416c800, 0x515, 0x800, 0xc013f6a060, 0x14, 0xc008466380) /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:32 +0x67 created by github.com/kaspanet/kaspad/util/panics.GoroutineWrapperFunc.func1 /Users/denis/go/src/github.com/kaspanet/kaspad/util/panics/panics.go:31 +0x8c