search

sei-protocol / sei-chain

Apache License 2.0
2.67k stars 797 forks source link

[BUG] Unable to use snapshot recovery nodes #1707

Closed YuXiaoCoder closed 5 days ago

YuXiaoCoder commented 1 month ago

Seid version Show us output of seid version --long | head

root@sei-1-0:/opt/seimain/core# ./seid version --long | head
name: sei
server_name: <appd>
version: v5.5.2
commit: c92dc5fb3b15e8a62a3fcf8b7c53cfb3ce940119
build_tags: netgo ledger,
go: go version go1.21.8 linux/amd64
build_deps:
- cosmossdk.io/errors@v1.0.0
- filippo.io/edwards25519@v1.0.0-rc.1
- github.com/99designs/keyring@v1.2.1

Chain ID pacific-1

Describe the bug I've tried downloading snapshots from Polkachu and kjnodes to quickly restore node sync, but both report the same error!

/opt/seimain/core/seid start --x-crisis-skip-assert-invariants --home=/mnt/seimain/node
11:12PM INF --tracing not passed in, tracing is not enabled
11:12PM INF Creating node metrics provider
11:12PM INF Starting Process
11:12PM INF SeiDB is disabled, falling back to IAVL
11:12PM INF starting node with ABCI Tendermint in-process
11:12PM INF Adding new peer 20e1000e88125698264454a884812746c2eb4807 with address mconn://20e1000e88125698264454a884812746c2eb4807@seeds.lavenderfive.com:11956 to peer store
 module=p2p
11:12PM INF Adding new peer 400f3d9e30b69e78a7fb891f60d76fa3c73f0ecc with address mconn://400f3d9e30b69e78a7fb891f60d76fa3c73f0ecc@sei.rpc.kjnodes.com:16859 to peer store
 module=p2p
11:12PM INF Adding new peer 8542cd7e6bf9d260fef543bc49e59be5a3fa9074 with address mconn://8542cd7e6bf9d260fef543bc49e59be5a3fa9074@seed.publicnode.com:26656 to peer store
 module=p2p
11:12PM INF Found local state with non-zero height, skipping state sync
11:12PM INF starting service impl=Node service=Node
11:12PM INF starting service impl=proxyClient module=proxy service=proxyClient
11:12PM INF starting service impl=localClient service=localClient
11:12PM INF starting service impl=EventBus module=eventbus service=EventBus
11:12PM INF starting service impl=PubSub module=events service=PubSub
11:12PM INF starting service impl=IndexerService module=txindex service=IndexerService
11:12PM INF ABCI Handshake App Info hash= height=0 module=handshaker protocol-version=0 software-version=v5.5.2
11:12PM INF ABCI Replay Blocks appHeight=0 module=handshaker stateHeight=79120066 storeHeight=79120067
11:12PM INF Current epoch 0
panic: recovered: runtime error: integer divide by zero
stack:
goroutine 1 [running]:
runtime/debug.Stack()
    /opt/seimain/goenv/go/src/runtime/debug/stack.go:24 +0x5e
github.com/cosmos/cosmos-sdk/baseapp.NewBaseApp.newDefaultRecoveryMiddleware.func1({0x2598460?, 0x4b2b5d0})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/recovery.go:91 +0x26
github.com/cosmos/cosmos-sdk/baseapp.NewBaseApp.newDefaultRecoveryMiddleware.newRecoveryMiddleware.func2({0x2598460?, 0x4b2b5d0?})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/recovery.go:40 +0x2d
github.com/cosmos/cosmos-sdk/baseapp.processRecovery({0x2598460, 0x4b2b5d0}, 0xc000bfea50?)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/recovery.go:29 +0x2f
github.com/cosmos/cosmos-sdk/baseapp.processRecovery({0x2598460, 0x4b2b5d0}, 0x1?)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/recovery.go:34 +0x53
github.com/cosmos/cosmos-sdk/baseapp.processRecovery({0x2598460, 0x4b2b5d0}, 0x65?)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/recovery.go:34 +0x53
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).runTx.func1()
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/baseapp.go:880 +0x325
panic({0x2598460?, 0x4b2b5d0?})
    /opt/seimain/goenv/go/src/runtime/panic.go:920 +0x270
github.com/cosmos/cosmos-sdk/x/auth/ante.SetUpContextDecorator.AnteHandle.func1()
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/auth/ante/setup.go:69 +0x1d8
panic({0x2598460?, 0x4b2b5d0?})
    /opt/seimain/goenv/go/src/runtime/panic.go:914 +0x21f
github.com/cosmos/cosmos-sdk/store/types.(*infiniteMultiplierGasMeter).adjustGas(...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/store/types/gas.go:289
github.com/cosmos/cosmos-sdk/store/types.(*infiniteMultiplierGasMeter).ConsumeGas(0x18?, 0x1?, {0x29477e7?, 0x15?})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/store/types/gas.go:293 +0x50
github.com/cosmos/cosmos-sdk/store/gaskv.(*Store).Get(0xc000b56720, {0xc000584420, 0x15, 0x18})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/store/gaskv/store.go:42 +0x4a
github.com/cosmos/cosmos-sdk/x/auth/keeper.AccountKeeper.GetAccount({{0x35b35b8, 0xc000ad7240}, {0x35e3890, 0xc0009a3820}, {{0x35e3890, 0xc0009a3820}, 0xc0000c0fd8, {0x35b35b8, 0xc000ad72c0}, {0x35b35e0, ...}, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/auth/keeper/account.go:37 +0x126
github.com/cosmos/cosmos-sdk/x/auth/ante.DeductFeeDecorator.checkDeductFee({{0x35ced78, 0xc00092e6e0}, {0x7f3438221bd8, 0xc000f01a00}, {0x35abc60, 0xc00092ac00}, {{0x35e3890, 0xc0009a3820}, 0xc0000c0fd8, {0x35b35b8, ...}, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/auth/ante/fee.go:177 +0x3bf
github.com/cosmos/cosmos-sdk/x/auth/ante.DeductFeeDecorator.AnteHandle({{0x35ced78, 0xc00092e6e0}, {0x7f3438221bd8, 0xc000f01a00}, {0x35abc60, 0xc00092ac00}, {{0x35e3890, 0xc0009a3820}, 0xc0000c0fd8, {0x35b35b8, ...}, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/auth/ante/fee.go:139 +0x15b
github.com/sei-protocol/sei-chain/app/antedecorators.GaslessDecorator.handleWrapped({{0xc000be7c00, 0x1, 0x1}, {{0x35e3890, 0xc0009a3820}, {0x35b35b8, 0xc000ad7330}, {0x35b3608, 0xc000ad73f0}, {{0x35e3890, ...}, ...}, ...}}, ...)
    /opt/seimain/core/sei-chain/app/antedecorators/gasless.go:61 +0x147
github.com/sei-protocol/sei-chain/app/antedecorators.GaslessDecorator.AnteHandle({{0xc000be7c00, 0x1, 0x1}, {{0x35e3890, 0xc0009a3820}, {0x35b35b8, 0xc000ad7330}, {0x35b3608, 0xc000ad73f0}, {{0x35e3890, ...}, ...}, ...}}, ...)
    /opt/seimain/core/sei-chain/app/antedecorators/gasless.go:48 +0x390
github.com/cosmos/cosmos-sdk/types.chainAnteDecoratorHandlers.func1({{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000afa580}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, {0x0, ...}, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/types/handler.go:81 +0x136
github.com/cosmos/cosmos-sdk/x/auth/ante.SetUpContextDecorator.AnteHandle({_}, {{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000afa580}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/auth/ante/setup.go:74 +0x210
github.com/cosmos/cosmos-sdk/types.WrappedAnteDecorator.AnteHandle(...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/types/handler.go:122
github.com/cosmos/cosmos-sdk/types.chainAnteDecoratorHandlers.func1({{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000afa580}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, {0x0, ...}, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/types/handler.go:81 +0x136
github.com/sei-protocol/sei-chain/x/evm/ante.EVMRouterDecorator.AnteHandle({_, _, _, _}, {{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000afa580}, {{0x0, 0x0}, ...}, ...}, ...)
    /opt/seimain/core/sei-chain/x/evm/ante/router.go:40 +0x1d7
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).runTx(_, {{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000277740}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/baseapp.go:918 +0xbd3
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).DeliverTx(_, {{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000277740}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/abci.go:299 +0x34d
github.com/cosmos/cosmos-sdk/x/genutil.DeliverGenTxs({{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000277740}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, {0x0, ...}, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/genutil/gentx.go:113 +0x20a
github.com/cosmos/cosmos-sdk/x/genutil.InitGenesis(...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/genutil/genesis.go:18
github.com/cosmos/cosmos-sdk/x/genutil.AppModule.InitGenesis({{}, {_, _}, {_, _}, _, {_, _}}, {{0x35cf108, 0xc0008a2000}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/genutil/module.go:99 +0x13e
github.com/cosmos/cosmos-sdk/types/module.(*Manager).InitGenesis(_, {{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000277740}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/types/module/module.go:341 +0x21d
github.com/sei-protocol/sei-chain/app.(*App).InitChainer(_, {{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000277740}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, ...}, ...}, ...)
    /opt/seimain/core/sei-chain/app/app.go:1053 +0x2ae
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).InitChain(0xc000137180, {0x40?, 0xc000208800?}, 0xc000277700?)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/abci.go:73 +0x559
github.com/tendermint/tendermint/internal/proxy.(*proxyClient).InitChain(0xc00012c2a0, {0x35cf140, 0xc00069a140}, 0xc0008b1e60?)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/internal/proxy/client.go:129 +0x16e
github.com/tendermint/tendermint/internal/consensus.(*Handshaker).ReplayBlocks(_, {_, _}, {{{0xb, 0x0}, {0xc000bce048, 0x11}}, {0xc000a83a50, 0x9}, 0x1, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/internal/consensus/replay.go:312 +0xf45
github.com/tendermint/tendermint/internal/consensus.(*Handshaker).Handshake(0xc000969d18, {0x35cf140, 0xc00069a140}, {0x35f1660?, 0xc00012c2a0?})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/internal/consensus/replay.go:268 +0x3db
github.com/tendermint/tendermint/node.(*nodeImpl).OnStart(0xc000934e00, {0x35cf140, 0xc00069a140})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/node/node.go:510 +0x2d5
github.com/tendermint/tendermint/libs/service.(*BaseService).Start(0xc000934e00, {0x35cf140?, 0xc00069a140})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/libs/service/service.go:117 +0x203
github.com/cosmos/cosmos-sdk/server.startInProcess(_, {{0x0, 0x0, 0x0}, {0x35f81d0, 0xc000bae900}, {0xc000c1b780, 0x9}, {0x35d6d48, 0xc000520410}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/server/start.go:402 +0x967
github.com/cosmos/cosmos-sdk/server.StartCmd.func2(0xc0002dfe00?, {0xc00053a300?, 0x0?, 0x2?})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/server/start.go:207 +0x8d9
github.com/spf13/cobra.(*Command).execute(0xc000380900, {0xc00053a2c0, 0x2, 0x2})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916 +0x87c
github.com/spf13/cobra.(*Command).ExecuteC(0xc0002f6c00)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044 +0x3a5
github.com/spf13/cobra.(*Command).Execute(...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968
github.com/spf13/cobra.(*Command).ExecuteContext(...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:961
github.com/cosmos/cosmos-sdk/server/cmd.Execute(0x4b6f358?, {0xc0004b65d0, 0xa})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/server/cmd/execute.go:35 +0x145
main.main()
    /opt/seimain/core/sei-chain/cmd/seid/main.go:16 +0x2b
: panic

goroutine 1 [running]:
github.com/cosmos/cosmos-sdk/x/genutil.DeliverGenTxs({{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000277740}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, {0x0, ...}, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/genutil/gentx.go:115 +0x2f7
github.com/cosmos/cosmos-sdk/x/genutil.InitGenesis(...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/genutil/genesis.go:18
github.com/cosmos/cosmos-sdk/x/genutil.AppModule.InitGenesis({{}, {_, _}, {_, _}, _, {_, _}}, {{0x35cf108, 0xc0008a2000}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/x/genutil/module.go:99 +0x13e
github.com/cosmos/cosmos-sdk/types/module.(*Manager).InitGenesis(_, {{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000277740}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, ...}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/types/module/module.go:341 +0x21d
github.com/sei-protocol/sei-chain/app.(*App).InitChainer(_, {{0x35cf108, 0xc0008a2000}, {0x35ecb40, 0xc000277740}, {{0x0, 0x0}, {0xc000a83390, 0x9}, 0x0, ...}, ...}, ...)
    /opt/seimain/core/sei-chain/app/app.go:1053 +0x2ae
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).InitChain(0xc000137180, {0x40?, 0xc000208800?}, 0xc000277700?)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/baseapp/abci.go:73 +0x559
github.com/tendermint/tendermint/internal/proxy.(*proxyClient).InitChain(0xc00012c2a0, {0x35cf140, 0xc00069a140}, 0xc0008b1e60?)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/internal/proxy/client.go:129 +0x16e
github.com/tendermint/tendermint/internal/consensus.(*Handshaker).ReplayBlocks(_, {_, _}, {{{0xb, 0x0}, {0xc000bce048, 0x11}}, {0xc000a83a50, 0x9}, 0x1, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/internal/consensus/replay.go:312 +0xf45
github.com/tendermint/tendermint/internal/consensus.(*Handshaker).Handshake(0xc000969d18, {0x35cf140, 0xc00069a140}, {0x35f1660?, 0xc00012c2a0?})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/internal/consensus/replay.go:268 +0x3db
github.com/tendermint/tendermint/node.(*nodeImpl).OnStart(0xc000934e00, {0x35cf140, 0xc00069a140})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/node/node.go:510 +0x2d5
github.com/tendermint/tendermint/libs/service.(*BaseService).Start(0xc000934e00, {0x35cf140?, 0xc00069a140})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-tendermint@v0.3.3/libs/service/service.go:117 +0x203
github.com/cosmos/cosmos-sdk/server.startInProcess(_, {{0x0, 0x0, 0x0}, {0x35f81d0, 0xc000bae900}, {0xc000c1b780, 0x9}, {0x35d6d48, 0xc000520410}, ...}, ...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/server/start.go:402 +0x967
github.com/cosmos/cosmos-sdk/server.StartCmd.func2(0xc0002dfe00?, {0xc00053a300?, 0x0?, 0x2?})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/server/start.go:207 +0x8d9
github.com/spf13/cobra.(*Command).execute(0xc000380900, {0xc00053a2c0, 0x2, 0x2})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:916 +0x87c
github.com/spf13/cobra.(*Command).ExecuteC(0xc0002f6c00)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:1044 +0x3a5
github.com/spf13/cobra.(*Command).Execute(...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:968
github.com/spf13/cobra.(*Command).ExecuteContext(...)
    /opt/seimain/goenv/gopath/pkg/mod/github.com/spf13/cobra@v1.6.1/command.go:961
github.com/cosmos/cosmos-sdk/server/cmd.Execute(0x4b6f358?, {0xc0004b65d0, 0xa})
    /opt/seimain/goenv/gopath/pkg/mod/github.com/sei-protocol/sei-cosmos@v0.3.13/server/cmd/execute.go:35 +0x145
main.main()
    /opt/seimain/core/sei-chain/cmd/seid/main.go:16 +0x2b

To Reproduce Steps to reproduce the behavior: stop sei curl -L https://snapshots.kjnodes.com/sei/snapshot_latest.tar.lz4 | tar -Ilz4 -xf - -C $HOME/.sei start sei

Expected behavior Nodes can be synchronised normally

Screenshots image

philipsu522 commented 1 week ago

can you try running with https://www.docs.sei.io/dev-node/quickstart and letting us knowif you still have issues?

yzang2019 commented 6 days ago

The reason why you are not able to recover from the downloaded snapshot is because the storage config mismatch with the one used by the snapshot.

You can fix it by following this guide to add the configuration to enable SeiDB: https://github.com/sei-protocol/sei-chain/blob/main/docs/migration/seidb_migration.md#step-1-add-configurations