Closed jeremyletang closed 10 months ago
CONSENSUS FAILURE!!! {"err": "decimal division by 0", "stack": "goroutine 11464 [running]:\nruntime/debug.Stack()\n\t/usr/local/go/src/runtime/debug/stack.go:24 +0x65\ngithub.com/tendermint/tendermint/consensus.(*State).receiveRoutine.func2()\n\t/jenkins/GOPATH/pkg/mod/github.com/vegaprotocol/cometbft@v0.34.28\ -0.20230322133204-3d8588de736e/consensus/state.go:732 +0x4c\npanic({0x2f41b80, 0x3eb5d70})\n\t/usr/local/go/src/runtime/panic.go:884 +0x213\ngithub.com/shopspring/decimal.Decimal.QuoRem({0xc006a4c720, 0x0}, {0xc006a4c760, 0x0}, 0x10)\n\t/jenkins/GOPATH/pkg/mod/github.com/vegaprotocol/decimal@v1.3.1-uint256/decimal.go:590 +0x2d8\ngithub.com/shopspring/decimal.Decimal.DivRound({0xc006a4c720?, 0x17a4a90?}, {0xc006a4c760?, 0x40?}, 0x10)\n\t/jenkins/GOPATH/pkg/mod/github.com/vegaprotocol/decimal@v1.3.1-uint256/decimal.go:632 +0x59\ngithub.com/shopspring/decimal.Decimal.Div(...)\n\t/jenkins/GOPATH/pkg/mod/github.com/vegaprotocol/decimal@v1.3.1-uint256/decimal.go:577\ncode.v\ egaprotocol.io/vega/core/execution/common.(*MarketLiquidity).applyPendingProvisions(0xc00052bc20, {0x3eed930, 0xc0065c82a0}, {0x0?, 0xc00280c390?, 0x0?}, 0xc006fbe860)\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/execution/common/liquidity_provision.go:171 +0x726\ncode.vegaprotocol.io/vega/core/execution/common.(*MarketLiquidity).OnEpochStart(0xc00052bc20, {0x3eed930, 0xc0065c82a0}, {0x0?, 0x0?, 0x0?}, 0x0?, 0x0?, 0x0?, {0xc0028042a0, ...})\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/execution/common/liquidity_provision.go:293 +0xaa\ncode.vegaprotocol.io/vega/core/execution/future.(*Market).OnEpochEvent(0xc0027f26c0, {0x3eed930, 0xc0065c82a0}, {0x10, {0xe0a3fe9, 0xedc799791, 0x0}, {0xe0a3fe9, 0xedc7997cd, 0x0}, ...})\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/execution/future/market.go:339 +0x16e\ncode.vegaprotocol.io/vega/core/execution.(*Engine).OnEpochEvent(...)\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/\ core/execution/engine.go:226\ncode.vegaprotocol.io/vega/core/epochtime.(*Svc).notify(0xc001532fc0, {0x3eed930, 0xc0065c82a0}, {0x10, {0xe0a3fe9, 0xedc799791, 0x0}, {0xe0a3fe9, 0xedc7997cd, 0x0}, ...})\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/epochtime/service.go:201 +0xe5\ncode.vegaprotocol.io/vega/core/epochtime.(*Svc).OnTick(0xc001532fc0, {0x3eed930, 0xc0065c82a0}, {0x1?, 0x3f07a10?, 0x0?})\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/epochtime/service.go:142 +0x854\ncode.vegaprotocol.io/vega/core/vegatime.(*Svc).notify(...)\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/vegat\ ime/service.go:104\ncode.vegaprotocol.io/vega/core/vegatime.(*Svc).SetTimeNow(0xc000db4380, {0x3eed930, 0xc0065c82a0}, {0x40?, 0x2f41200?, 0x0?})\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/vegatime/service.go:77 +0x24f\ncode.vegaprotocol.io/vega/core/processor.(*App).OnBeginBlock(_, {{0xc0034ad7a0, 0x20, 0x20}, {{0xb, 0x1}, {0xc0057a3d60, 0x19}, 0x3b3, {0xe0a3fe9, ...}, ...}, ...})\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/processor/abci.go:804 +0x747\ncode.vegaprotocol.io/vega/core/blockchain/abci.(*App).BeginBlock(_, {{0xc0034ad7a0, 0x20, 0x20}, {{0xb, 0x1}, {0xc0057a3d60, 0x19}, 0x3b3, {0xe0a3fe9, ...}, ...}, ...})\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/core/blockchain/abci/abci.go:46 +0x75\ncode.vegaprotocol.io/vega/cmd/vega/commands/node.(*appW).BeginBlock(_, {{0xc0034ad7a0, 0x20, 0x20}, {{0xb, 0x1}, {0xc0057a3d60, 0x19}, 0x3b3, {0xe0a3fe9, ...}, ...}, ...})\n\t/jenkins/workspace/private/Deployments/Publish-vega-dev-releases/vega/cmd/vega/commands/node/app_wrapper.go:64 +0x55\ngithub.com/tendermint/tendermint/abci/client.(*localClient).BeginBlockSync(_, {{0xc0034ad7a0, 0x20, 0x20}, {{0xb, 0x1}, {0xc0057a3d60, 0x19}, 0x3b3, {0xe0a3fe9, ...}, ...}, ...})\n\t/jenkins/GOPATH/pkg/mod/github.com/vegaprotocol/cometbft@v0.34.28-0.20230322133204-3d8588de736e/abci/client/local_client.go:280 +0x118\ngithub.com/tendermint/tendermint/proxy.(*appConnConsensus).BeginBlockSync(_, {{0xc0034ad7a0, 0x20, 0x20}, {{0xb, 0x1}, {0xc0057a3d60, 0x19}, 0x3b3, {0xe0a3fe9, ...}, ...}, ...})\n\t/jenkins/GOPATH/pkg/mod/github.com/vegaprotocol/cometbft@v0.34.28-0.20230322133204-3d8588de736e/proxy/app_conn.go:81 +0x55\ngithub.com/tendermint/tendermint/state.execBlockOnProxyApp({0x3eeba90?, 0xc001e4b788}, {0x3efef60, 0xc002c3b020}, 0xc0021aad20, {0x3f100f8, 0xc00256e648}, 0x3b2?)\n\t/jenkins/GOPATH/pkg/mod/github.com/vegaprotocol/cometbft@v0.34.28-0.20230322133204-3d8588de736e/state/execution.go:307 +0x51d\ngithub.com/tendermint/tendermint/state.(*BlockExecutor).ApplyBlock(_, {{{0xb, 0x1}, {0xc005679ef0, 0x7}}, {0xc0057a2040, 0x19}, 0x1, 0x3b2, {{0xc0071f7a60, ...}, ...}, ...},
The Devnet state is backed up on n03.devnet1.vega.rocks in /home/vega/bk_2023_08_24_panic_division_by_0
n03.devnet1.vega.rocks
/home/vega/bk_2023_08_24_panic_division_by_0