Closed wwestgarth closed 10 months ago
A replay file can be found here for a similar market-sim failure, which ran on commit eb2cd3e8996723d37d8fcddabfdfec55e9c8c776
with genesis file:
{
"genesis_time": "2023-05-09T10:00:00.000000000Z",
"chain_id": "CUSTOM",
"initial_height": "0",
"consensus_params": {
"block": {
"max_bytes": "22020096",
"max_gas": "-1",
"time_iota_ms": "1"
},
"evidence": {
"max_age_num_blocks": "100000",
"max_age_duration": "172800000000000",
"max_bytes": "1048576"
},
"validator": {
"pub_key_types": ["ed25519"]
},
"version": {}
},
"validators": [
{
"address": "7DFD3744A773C166B36F170E4F5EF3E998703F4E",
"pub_key": {
"type": "tendermint/PubKeyEd25519",
"value": "2lPO6I0p5uCBZzeoA2+ECJ2+57LtLVMjXpxH5za9TvE="
},
"power": "10",
"name": ""
}
],
"app_hash": "",
"app_state": {
"assets": {
"VOTE": {
"name": "VOTE",
"symbol": "VOTE",
"decimals": 5,
"min_lp_stake": "1",
"source": {
"builtin_asset": {
"max_faucet_amount_mint": "100000000000"
}
}
}
},
"validators": {
"6NemqXSfN4zfZvAQeIANflEFqQ98izX9C0KLI2dITSY=": {
"id": "c2b0ab69fb1bd19862c04df401920ea62d3e0bce41a50148240f42017824d17a",
"vega_pub_key": "dfcf0fa8d67cfa11b3a511e320f3c47807a5e0735cc06dd9c2bc00f7ccc67117",
"vega_pub_key_index": 1,
"ethereum_address": "0x9e5BEEC6E56B28cCbd02864840B0f1e0125e42Ce",
"tm_pub_key": "6NemqXSfN4zfZvAQeIANflEFqQ98izX9C0KLI2dITSY=",
"info_url": "",
"country": "",
"name": "",
"avatar_url": ""
}
},
"network": {
"replay_attack_threshold": 150
},
"network_parameters": {
"blockchains.ethereumConfig": "{\"network_id\": \"3\", \"chain_id\": \"3\", \"collateral_bridge_contract\": { \"address\": \"0xa6F1E140daC13002Dfd9789D6dBA59117c717D7a\" }, \"confirmations\": 50, \"staking_bridge_contract\": { \"address\": \"0xfce2CC92203A266a9C8e67461ae5067c78f67235\", \"deployment_block_height\": 11001702}, \"multisig_control_contract\": {\"address\": \"0xCF6d41235911184fe6F35D47207813bFF3B91601\", \"deployment_block_height\": 12710009 } }",
"governance.proposal.asset.maxClose": "8760h0m0s",
"governance.proposal.asset.maxEnact": "8760h0m0s",
"governance.proposal.asset.minClose": "1s",
"governance.proposal.asset.minEnact": "1s",
"governance.proposal.asset.minProposerBalance": "1",
"governance.proposal.asset.minVoterBalance": "1",
"governance.proposal.asset.requiredMajority": "0.66",
"governance.proposal.asset.requiredParticipation": "0.00001",
"governance.proposal.freeform.maxClose": "8760h0m0s",
"governance.proposal.freeform.minClose": "1s",
"governance.proposal.freeform.minProposerBalance": "1",
"governance.proposal.freeform.minVoterBalance": "1",
"governance.proposal.freeform.requiredMajority": "0.66",
"governance.proposal.freeform.requiredParticipation": "0.00001",
"governance.proposal.market.maxClose": "8760h0m0s",
"governance.proposal.market.maxEnact": "8760h0m0s",
"governance.proposal.market.minClose": "1s",
"governance.proposal.market.minEnact": "2s",
"governance.proposal.market.minProposerBalance": "1",
"governance.proposal.market.minVoterBalance": "1",
"governance.proposal.market.requiredMajority": "0.66",
"governance.proposal.market.requiredParticipation": "0.00001",
"governance.proposal.updateAsset.maxClose": "720h",
"governance.proposal.updateAsset.maxEnact": "720h",
"governance.proposal.updateAsset.minClose": "1s",
"governance.proposal.updateAsset.minEnact": "1s",
"governance.proposal.updateAsset.minProposerBalance": "1",
"governance.proposal.updateAsset.minVoterBalance": "1",
"governance.proposal.updateAsset.requiredMajority": "0.66",
"governance.proposal.updateAsset.requiredParticipation": "0.09",
"governance.proposal.updateMarket.maxClose": "8760h0m0s",
"governance.proposal.updateMarket.maxEnact": "8760h0m0s",
"governance.proposal.updateMarket.minClose": "1s",
"governance.proposal.updateMarket.minEnact": "1s",
"governance.proposal.updateMarket.minProposerBalance": "1",
"governance.proposal.updateMarket.minProposerEquityLikeShare": "0",
"governance.proposal.updateMarket.minVoterBalance": "1",
"governance.proposal.updateMarket.requiredMajority": "0.66",
"governance.proposal.updateMarket.requiredMajorityLP": "0.66",
"governance.proposal.updateMarket.requiredParticipation": "0.00001",
"governance.proposal.updateMarket.requiredParticipationLP": "0",
"governance.proposal.updateNetParam.maxClose": "8760h0m0s",
"governance.proposal.updateNetParam.maxEnact": "8760h0m0s",
"governance.proposal.updateNetParam.minClose": "1s",
"governance.proposal.updateNetParam.minEnact": "1s",
"governance.proposal.updateNetParam.minProposerBalance": "1",
"governance.proposal.updateNetParam.minVoterBalance": "1",
"governance.proposal.updateNetParam.requiredMajority": "0.5",
"governance.proposal.updateNetParam.requiredParticipation": "0.00001",
"governance.proposal.referralProgram.maxClose": "8760h0m0s",
"governance.proposal.referralProgram.maxEnact": "8760h0m0s",
"governance.proposal.referralProgram.minClose": "1s",
"governance.proposal.referralProgram.minEnact": "1s",
"governance.proposal.referralProgram.minProposerBalance": "1",
"governance.proposal.referralProgram.minVoterBalance": "1",
"governance.proposal.referralProgram.requiredMajority": "0.5",
"governance.proposal.referralProgram.requiredParticipation": "0.00001",
"governance.proposal.VolumeDiscountProgram.maxClose": "8760h0m0s",
"governance.proposal.VolumeDiscountProgram.maxEnact": "8760h0m0s",
"governance.proposal.VolumeDiscountProgram.minClose": "1s",
"governance.proposal.VolumeDiscountProgram.minEnact": "1s",
"governance.proposal.VolumeDiscountProgram.minProposerBalance": "1",
"governance.proposal.VolumeDiscountProgram.minVoterBalance": "1",
"governance.proposal.VolumeDiscountProgram.requiredMajority": "0.5",
"governance.proposal.VolumeDiscountProgram.requiredParticipation": "0.00001",
"governance.proposal.transfer.maxClose": "8760h0m0s",
"governance.proposal.transfer.maxEnact": "8760h0m0s",
"governance.proposal.transfer.minClose": "1s",
"governance.proposal.transfer.minEnact": "1s",
"governance.proposal.transfer.minProposerBalance": "1",
"governance.proposal.transfer.minVoterBalance": "1",
"governance.proposal.transfer.requiredMajority": "0.5",
"governance.proposal.transfer.requiredParticipation": "0.00001",
"limits.markets.maxPeggedOrders": "1500",
"limits.markets.proposePerpetualEnabled": "1",
"market.auction.maximumDuration": "168h",
"market.auction.minimumDuration": "1s",
"market.fee.factors.infrastructureFee": "0.0005",
"market.fee.factors.makerFee": "0.0002",
"market.liquidity.bondPenaltyParameter": "0.1",
"market.liquidity.earlyExitPenalty": "0.1",
"market.liquidity.maximumLiquidityFeeFactorLevel": "0.03",
"market.liquidity.minimum.probabilityOfTrading.lpOrders": "1e-6",
"market.liquidity.probabilityOfTrading.tau.scaling": "10.0",
"market.liquidity.providersFeeCalculationTimeStep": "1s",
"market.liquidity.sla.nonPerformanceBondPenaltyMax": "0.5",
"market.liquidity.sla.nonPerformanceBondPenaltySlope": "2",
"market.liquidity.stakeToCcyVolume": "1.0",
"market.liquidity.targetstake.triggering.ratio": "0.25",
"market.liquidityProvision.minLpStakeQuantumMultiple": "5000",
"market.liquidityProvision.shapes.maxSize": "100",
"market.margin.scalingFactors": "{\"search_level\": 1.1, \"initial_margin\": 1.5, \"collateral_release\": 1.7}",
"market.monitor.price.defaultParameters": "{\"triggers\": [{\"auction_extension\": 300, \"horizon\": 43200, \"probability\": \"0.9999999\"}] }",
"market.stake.target.scalingFactor": "0.0001",
"market.stake.target.timeWindow": "1h",
"market.value.windowLength": "2h0m0s",
"network.checkpoint.timeElapsedBetweenCheckpoints": "5m",
"network.floatingPointUpdates.delay": "5m",
"network.markPriceUpdateMaximumFrequency": "5s",
"network.transaction.defaultgas": "1",
"network.transactions.maxgasperblock": "3000000",
"network.transactions.minBlockCapacity": "32",
"network.validators.ersatz.multipleOfTendermintValidators": "0",
"network.validators.ersatz.rewardFactor": "0.85",
"network.validators.incumbentBonus": "0.05",
"network.validators.minimumEthereumEventsForNewValidator": "3",
"network.validators.multisig.numberOfSigners": "13",
"network.validators.tendermint.number": "13",
"reward.asset": "VOTE",
"reward.staking.delegation.competitionLevel": "1.3",
"reward.staking.delegation.delegatorShare": "0.8",
"reward.staking.delegation.maxPayoutPerEpoch": "9890000000000000000000",
"reward.staking.delegation.maxPayoutPerParticipant": "700000000000000000000",
"reward.staking.delegation.minimumValidatorStake": "3000000000000000000000",
"reward.staking.delegation.minValidators": "5",
"reward.staking.delegation.optimalStakeMultiplier": "3.0",
"reward.staking.delegation.payoutDelay": "0h",
"reward.staking.delegation.payoutFraction": "1.0",
"rewards.marketCreationQuantumMultiple": "1",
"snapshot.interval.length": "300",
"spam.pow.difficulty": "15",
"spam.pow.hashFunction": "sha3_24_rounds",
"spam.pow.increaseDifficulty": "0",
"spam.pow.numberOfPastBlocks": "100",
"spam.pow.numberOfTxPerBlock": "1000",
"spam.protection.delegation.min.tokens": "100000000000000000",
"spam.protection.max.batchSize": "30",
"spam.protection.max.delegations": "360",
"spam.protection.max.proposals": "300",
"spam.protection.max.votes": "300",
"spam.protection.maxUserTransfersPerEpoch": "10000",
"spam.protection.minimumWithdrawalQuantumMultiple": "10",
"spam.protection.minMultisigUpdates": "100000000000000000000",
"spam.protection.proposal.min.tokens": "2000000000000000000000",
"spam.protection.voting.min.tokens": "1000000000000000000",
"spam.protection.applyReferral.min.funds": "0",
"transfer.fee.factor": "0.001",
"transfer.minTransferQuantumMultiple": "100",
"validator.performance.scaling.factor": "0",
"validators.delegation.minAmount": "1",
"validators.epoch.length": "2m",
"validators.vote.required": "0.67",
"referralProgram.maxReferralTiers": "10",
"referralProgram.maxReferralRewardFactor": "1",
"referralProgram.maxReferralDiscountFactor": "1",
"referralProgram.maxPartyNotionalVolumeByQuantumPerEpoch": "10000",
"referralProgram.minStakedVegaTokens": "0",
"referralProgram.maxReferralRewardProportion": "1"
},
"network_limits": {
"propose_market_enabled": true,
"propose_asset_enabled": true,
"bootstrap_block_count": 0
},
"checkpoint": {
"load_hash": ""
}
}
}
Problem encountered
Both the overnight system test run and the overnight market-simulation run have seen core panic:
This is likely due to the recent code change relating to the new liquidation engine.
Failing overnight system tests: https://jenkins.vega.rocks/blue/organizations/jenkins/common%2Fsystem-tests-nightly/detail/system-tests-nightly/102/tests
There is a high chance that
test_position_resolution()
was the test that caused it.Failing market sim case: https://jenkins.vega.rocks/job/common/job/vega-market-sim-reinforcement/116/artifact/network_home/vega-sim-m0fv8j4h/node.err/*view*
Observed behaviour
A panic has occurred during win-socialisation.
Expected behaviour
Win-socialisation occurs without a panic.
Steps to reproduce
Software version
develop
Failing test
test_position_resolution
Jenkins run
https://jenkins.vega.rocks/blue/organizations/jenkins/common%2Fsystem-tests-nightly/detail/system-tests-nightly/102/tests
Configuration used
No response
Relevant log output
No response