search

cosmos / gaia

Cosmos Hub
https://hub.cosmos.network
Apache License 2.0
456 stars 666 forks source link

[Question]: Atom Node Not Syncing with Gaia 17.1.0 #3142

Open mayank-daga opened 3 weeks ago

mayank-daga commented 3 weeks ago

Is there an existing issue for this?

What happened?

Hi, I am getting following error logs :

6:28PM INF starting node with ABCI Tendermint in-process module=server ,6:28PM INF service start impl=multiAppConn module=proxy msg="Starting multiAppConn service" ,6:28PM INF service start connection=query impl=localClient module=abci-client msg="Starting localClient service" ,6:28PM INF service start connection=snapshot impl=localClient module=abci-client msg="Starting localClient service" ,6:28PM INF service start connection=mempool impl=localClient module=abci-client msg="Starting localClient service" ,6:28PM INF service start connection=consensus impl=localClient module=abci-client msg="Starting localClient service" ,6:28PM INF service start impl=EventBus module=events msg="Starting EventBus service" ,6:28PM INF service start impl=PubSub module=pubsub msg="Starting PubSub service" ,6:40PM INF service start impl=IndexerService module=txindex msg="Starting IndexerService service" ,6:40PM INF ABCI Handshake App Info hash="X��\x04�Fn{:2<\x0fV��\x03\x18�x\fN�\u05feS�\x7f���U�" height=20740969 module=consensus protocol-version=0 software-version=v17.1.0 ,6:40PM INF ABCI Replay Blocks appHeight=20740969 module=consensus stateHeight=20740969 storeHeight=20740970 ,6:40PM INF Replay last block using real app module=consensus ,6:40PM INF minted coins from module account amount=9195619uatom from=mint module=x/bank ,6:40PM INF burned tokens from module account amount=1790000000000000000000ibc/49ACCECDED4A6A2B5DD683FA048235F4F66B65F58CD558C9A441195502191D01 from=transfer module=x/bank ,6:40PM INF packet sent dst_channel=channel-1 dst_port=transfer module=x/ibc/channel sequence=14209 src_channel=channel-794 srcport=transfer ,6:40PM INF IBC fungible token transfer amount=1790000000000000000000 module=x/ibc-transfer receiver=dym178scrs2mqwffhnkjdelyw7ywdlfh5q5zypdp8l sender=cosmos1r2v0lfzjrqusfugg3s9w56zznyugezsc6zuc9y token=ibc/49ACCECDED4A6A2B5DD683FA048235F4F66B65F58CD558C9A441195502191D01 ,6:40PM INF validator un-jailed module=x/staking validator=cosmosvalcons1eemgqv7hylr29zvq4mmnwtgjgvnusy9g4nlklf ,panic: more validators than maxValidators found , ,goroutine 50 [running]: ,github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.GetLastValidators({{0x35d37d0, 0xc0a8bc4720}, {0x360a430, 0xc001042e30}, {0x35f6b20, 0xc09a59c230}, {0x7ff848d79b38, 0xc000fb1340}, {0x360b5b0, 0xc09adf4108}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/x/staking/keeper/validator.go:326 +0x471 ,github.com/cosmos/interchain-security/v4/x/ccv/provider/keeper.Keeper.GetAllConsumerChains({{0x35d37d0, 0xc0a8bc4850}, {0x360a430, 0xc001042e30}, {{0x360a430, 0xc001042e30}, 0xc000f990a0, {0x35d37d0, 0xc0a8bc4780}, {0x35d3848, ...}, ...}, ...}, ...) , github.com/cosmos/interchain-security/v4@v4.2.0/x/ccv/provider/keeper/keeper.go:260 +0x2d5 ,github.com/cosmos/interchain-security/v4/x/ccv/provider/keeper.Hooks.AfterUnbondingInitiated({}, {{0x35f4d80, 0x4dd3060}, {0x360aee0, 0xc9c3298800}, {{0xb, 0x0}, {0xc9c2ee0c40, 0xb}, 0x13c7b6a, ...}, ...}, ...) , github.com/cosmos/interchain-security/v4@v4.2.0/x/ccv/provider/keeper/hooks.go:93 +0x7db ,github.com/cosmos/cosmos-sdk/x/staking/types.MultiStakingHooks.AfterUnbondingInitiated({, , _}, {{0x35f4d80, 0x4dd3060}, {0x360aee0, 0xc9c3298800}, {{0xb, 0x0}, {0xc9c2ee0c40, ...}, ...}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/x/staking/types/hooks.go:109 +0xa5 ,github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.BeginUnbondingValidator({{0x35d37d0, 0xc0a8bc4720}, {0x360a430, 0xc001042e30}, {0x35f6b20, 0xc09a59c230}, {0x7ff848d79b38, 0xc000fb1340}, {0x360b5b0, 0xc09adf4108}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/x/staking/keeper/val_state_change.go:354 +0x806 ,github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.bondedToUnbonding({{0x35d37d0, 0xc0a8bc4720}, {0x360a430, 0xc001042e30}, {0x35f6b20, 0xc09a59c230}, {0x7ff848d79b38, 0xc000fb1340}, {0x360b5b0, 0xc09adf4108}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/x/staking/keeper/val_state_change.go:236 +0x158 ,github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.ApplyAndReturnValidatorSetUpdates({{0x35d37d0, 0xc0a8bc4720}, {0x360a430, 0xc001042e30}, {0x35f6b20, 0xc09a59c230}, {0x7ff848d79b38, 0xc000fb1340}, {0x360b5b0, 0xc09adf4108}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/x/staking/keeper/val_state_change.go:194 +0xf78 ,github.com/cosmos/cosmos-sdk/x/staking/keeper.Keeper.BlockValidatorUpdates({{0x35d37d0, 0xc0a8bc4720}, {0x360a430, 0xc001042e30}, {0x35f6b20, 0xc09a59c230}, {0x7ff848d79b38, 0xc000fb1340}, {0x360b5b0, 0xc09adf4108}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/x/staking/keeper/val_statechange.go:29 +0x65 ,github.com/cosmos/cosmos-sdk/x/staking.EndBlocker({{0x35f4d80, 0x4dd3060}, {0x360aee0, 0xc9c3298800}, {{0xb, 0x0}, {0xc9c2ee0c40, 0xb}, 0x13c7b6a, {0x3135aa43, ...}, ...}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/x/staking/abci.go:27 +0x118 ,github.com/cosmos/cosmos-sdk/x/staking.AppModule.EndBlock(...) , github.com/cosmos/cosmos-sdk@v0.47.11/x/staking/module.go:192 ,github.com/cosmos/cosmos-sdk/types/module.(*Manager).EndBlock(, {{0x35f4d80, 0x4dd3060}, {0x360aee0, 0xc9c3298800}, {{0xb, 0x0}, {0xc9c2ee0c40, 0xb}, 0x13c7b6a, ...}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/types/module/module.go:583 +0x1c9 ,github.com/cosmos/gaia/v17/app.(GaiaApp).EndBlocker(...) , github.com/cosmos/gaia/v17/app/app.go:266 ,github.com/cosmos/cosmos-sdk/baseapp.(BaseApp).EndBlock(0xc0003ba5a0, {0x40?}) , github.com/cosmos/cosmos-sdk@v0.47.11/baseapp/abci.go:223 +0x165 ,github.com/cometbft/cometbft/abci/client.(localClient).EndBlockSync(0xc0652ff800, {0xc556276c80?}) , github.com/cometbft/cometbft@v0.37.6/abci/client/local_client.go:291 +0xd3 ,github.com/cometbft/cometbft/proxy.(appConnConsensus).EndBlockSync(0xc067b56ff0, {0xc556276c80?}) , github.com/cometbft/cometbft@v0.37.6/proxy/appconn.go:111 +0x14c ,github.com/cometbft/cometbft/state.execBlockOnProxyApp({0x35f4d48?, 0xc9c2eddd70}, {0x36070c8, 0xc067b56ff0}, 0xc0003bad20, {0x360b4c0, 0xc361c7fc50}, 0x13c7b69?) , github.com/cometbft/cometbft@v0.37.6/state/execution.go:383 +0x6fc ,github.com/cometbft/cometbft/state.(*BlockExecutor).ApplyBlock(, {{{0xb, 0x0}, {0xc0575abe68, 0x7}}, {0xc0575abe90, 0xb}, 0x4f5b97, 0x13c7b69, {{0xc165ea47e0, ...}, ...}, ...}, ...) , github.com/cometbft/cometbft@v0.37.6/state/execution.go:197 +0x165 ,github.com/cometbft/cometbft/consensus.(Handshaker).replayBlock(_, {{{0xb, 0x0}, {0xc0575abe68, 0x7}}, {0xc0575abe90, 0xb}, 0x4f5b97, 0x13c7b69, {{0xc165ea47e0, ...}, ...}, ...}, ...) , github.com/cometbft/cometbft@v0.37.6/consensus/replay.go:527 +0x217 ,github.com/cometbft/cometbft/consensus.(Handshaker).ReplayBlocksWithContext(, {, }, {{{0xb, 0x0}, {0xc0575abe68, 0x7}}, {0xc0575abe90, 0xb}, 0x4f5b97, ...}, ...) , github.com/cometbft/cometbft@v0.37.6/consensus/replay.go:433 +0x6ee ,github.com/cometbft/cometbft/consensus.(*Handshaker).HandshakeWithContext(0xca6daee080, {0x35f5138, 0x4dd3060}, {0x360d410?, 0xc000c37500?}) , github.com/cometbft/cometbft@v0.37.6/consensus/replay.go:274 +0x3b8 ,github.com/cometbft/cometbft/node.doHandshake({, }, {, }, {{{0xb, 0x0}, {0xc0575abe68, 0x7}}, {0xc0575abe90, 0xb}, ...}, ...) , github.com/cometbft/cometbft@v0.37.6/node/node.go:449 +0x1a8 ,github.com/cometbft/cometbft/node.NewNodeWithContext({0x35f5138, 0x4dd3060}, 0xc000a1be40, {0x35d7f20, 0xc000f76fa0}, 0xc1edc41800, {0x35c6ea0, 0xc121e0f560}, 0x1?, 0x30a9ab0, ...) , github.com/cometbft/cometbft@v0.37.6/node/node.go:929 +0x55b ,github.com/cometbft/cometbft/node.NewNode(0x0?, {0x35d7f20?, 0xc000f76fa0?}, 0x0?, {0x35c6ea0?, 0xc121e0f560?}, 0x1?, 0x4dd3060?, 0x0?, {0x35f4d48, ...}, ...) , github.com/cometbft/cometbft@v0.37.6/node/node.go:849 +0xa9 ,github.com/cosmos/cosmos-sdk/server.startInProcess(, {{0x0, 0x0, 0x0}, {0x360bf00, 0xc0013d0bd0}, 0x0, {0x0, 0x0}, {0x361e400, ...}, ...}, ...) , github.com/cosmos/cosmos-sdk@v0.47.11/server/start.go:372 +0x6f7 ,github.com/cosmos/cosmos-sdk/server.StartCmd.func2.2() , github.com/cosmos/cosmos-sdk@v0.47.11/server/start.go:153 +0x45 ,github.com/cosmos/cosmos-sdk/server.wrapCPUProfile.func2() , github.com/cosmos/cosmos-sdk@v0.47.11/server/start.go:617 +0x22 ,created by github.com/cosmos/cosmos-sdk/server.wrapCPUProfile in goroutine 1 , github.com/cosmos/cosmos-sdk@v0.47.11/server/start.go:616 +0x22f

Few days ago node was running fine on gaia v16.0.0 but then got same issue so upgraded to gaia v17.1.0, but the issue persists

Gaia Version

v17.1.0

How to reproduce?

No response

MSalopek commented 2 weeks ago

Please run: https://github.com/cosmos/gaia/releases/tag/v17.2.0.

Are you a member of the cosmos-hub discord channels? All communication is available there.

MSalopek commented 1 week ago

@mayank-daga Has the advice above resolved your issue?