search

QuarkChain / pyquarkchain

Python implementation of QuarkChain
https://quarkchain.io
MIT License
225 stars 114 forks source link

Flaky xshard integration test #900

Open ninjaahhh opened 4 years ago

ninjaahhh commented 4 years ago

seeing flaky tests (like this run) sometimes. @qizhou can you take a look?

2020-07-15T01:03:14.5971924Z =================================== FAILURES ===================================
2020-07-15T01:03:14.5972127Z ___________ TestCluster.test_broadcast_cross_shard_transactions_2x1 ____________
2020-07-15T01:03:14.5972214Z 
2020-07-15T01:03:14.5972435Z self = <quarkchain.cluster.tests.test_cluster.TestCluster testMethod=test_broadcast_cross_shard_transactions_2x1>
2020-07-15T01:03:14.5972588Z 
2020-07-15T01:03:14.5972718Z     def test_broadcast_cross_shard_transactions_2x1(self):
2020-07-15T01:03:14.5973590Z         """ Test the cross shard transactions are broadcasted to the destination shards """
2020-07-15T01:03:14.5973860Z         id1 = Identity.create_random_identity()
2020-07-15T01:03:14.5974051Z         id2 = Identity.create_random_identity()
2020-07-15T01:03:14.5974240Z         acc1 = Address.create_from_identity(id1, full_shard_key=0)
2020-07-15T01:03:14.5974452Z         acc2 = Address.create_from_identity(id2, full_shard_key=1 << 16)
2020-07-15T01:03:14.5974647Z         acc3 = Address.create_random_account(full_shard_key=2 << 16)
2020-07-15T01:03:14.5974838Z         acc4 = Address.create_random_account(full_shard_key=1 << 16)
2020-07-15T01:03:14.5975027Z         acc5 = Address.create_random_account(full_shard_key=0)
2020-07-15T01:03:14.5975150Z     
2020-07-15T01:03:14.5975504Z         with ClusterContext(
2020-07-15T01:03:14.5975701Z             1, acc1, chain_size=8, shard_size=1, mblock_coinbase_amount=1000000
2020-07-15T01:03:14.5975886Z         ) as clusters:
2020-07-15T01:03:14.5976083Z             master = clusters[0].master
2020-07-15T01:03:14.5976260Z             slaves = clusters[0].slave_list
2020-07-15T01:03:14.5976428Z     
2020-07-15T01:03:14.5976553Z             # Add a root block first so that later minor blocks referring to this root
2020-07-15T01:03:14.5976751Z             # can be broadcasted to other shards
2020-07-15T01:03:14.5976928Z             root_block = call_async(
2020-07-15T01:03:14.5977106Z                 master.get_next_block_to_mine(
2020-07-15T01:03:14.5977301Z                     Address.create_empty_account(), branch_value=None
2020-07-15T01:03:14.5977496Z                 )
2020-07-15T01:03:14.5977658Z             )
2020-07-15T01:03:14.5977827Z             call_async(master.add_root_block(root_block))
2020-07-15T01:03:14.5977951Z     
2020-07-15T01:03:14.5978109Z             b0 = (
2020-07-15T01:03:14.5978273Z                 clusters[0]
2020-07-15T01:03:14.5978443Z                 .get_shard_state((1 << 16) + 1)
2020-07-15T01:03:14.5978747Z                 .create_block_to_mine(address=acc2)
2020-07-15T01:03:14.5978951Z             )
2020-07-15T01:03:14.5979126Z             call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b0))
2020-07-15T01:03:14.5979253Z     
2020-07-15T01:03:14.5979421Z             self.assert_balance(master, [acc1, acc2], [1000000, 500000])
2020-07-15T01:03:14.5979596Z     
2020-07-15T01:03:14.5979760Z             tx1 = create_transfer_transaction(
2020-07-15T01:03:14.5979942Z                 shard_state=clusters[0].get_shard_state(1),
2020-07-15T01:03:14.5980185Z                 key=id1.get_key(),
2020-07-15T01:03:14.5980354Z                 from_address=acc1,
2020-07-15T01:03:14.5980550Z                 to_address=acc3,
2020-07-15T01:03:14.5980667Z                 value=54321,
2020-07-15T01:03:14.5980847Z                 gas=opcodes.GTXXSHARDCOST + opcodes.GTXCOST,
2020-07-15T01:03:14.5981021Z                 gas_price=1,
2020-07-15T01:03:14.5981186Z             )
2020-07-15T01:03:14.5981360Z             self.assertTrue(slaves[0].add_tx(tx1))
2020-07-15T01:03:14.5981535Z             tx2 = create_transfer_transaction(
2020-07-15T01:03:14.5981714Z                 shard_state=clusters[0].get_shard_state(1),
2020-07-15T01:03:14.5981838Z                 key=id1.get_key(),
2020-07-15T01:03:14.5982019Z                 from_address=acc1,
2020-07-15T01:03:14.5982199Z                 to_address=acc3,
2020-07-15T01:03:14.5982368Z                 value=5555,
2020-07-15T01:03:14.5982543Z                 gas=opcodes.GTXXSHARDCOST + opcodes.GTXCOST,
2020-07-15T01:03:14.5982723Z                 nonce=tx1.tx.to_evm_tx().nonce + 1,
2020-07-15T01:03:14.5982893Z                 gas_price=3,
2020-07-15T01:03:14.5983054Z             )
2020-07-15T01:03:14.5983170Z             self.assertTrue(slaves[0].add_tx(tx2))
2020-07-15T01:03:14.5983346Z     
2020-07-15T01:03:14.5983522Z             tx3 = create_transfer_transaction(
2020-07-15T01:03:14.5983715Z                 shard_state=clusters[0].get_shard_state(1),
2020-07-15T01:03:14.5983891Z                 key=id2.get_key(),
2020-07-15T01:03:14.5984058Z                 from_address=acc2,
2020-07-15T01:03:14.5984224Z                 to_address=acc3,
2020-07-15T01:03:14.5984339Z                 value=7787,
2020-07-15T01:03:14.5984512Z                 gas=opcodes.GTXXSHARDCOST + opcodes.GTXCOST,
2020-07-15T01:03:14.5984685Z                 gas_price=2,
2020-07-15T01:03:14.5984859Z             )
2020-07-15T01:03:14.5985026Z             self.assertTrue(slaves[1].add_tx(tx3))
2020-07-15T01:03:14.5985206Z     
2020-07-15T01:03:14.5985378Z             b1 = clusters[0].get_shard_state(1).create_block_to_mine(address=acc5)
2020-07-15T01:03:14.5985560Z             b2 = (
2020-07-15T01:03:14.5985672Z                 clusters[0]
2020-07-15T01:03:14.5985842Z                 .get_shard_state((1 << 16) + 1)
2020-07-15T01:03:14.5986084Z                 .create_block_to_mine(address=acc4)
2020-07-15T01:03:14.5986327Z             )
2020-07-15T01:03:14.5986485Z     
2020-07-15T01:03:14.5986655Z             call_async(clusters[0].get_shard(1).add_block(b1))
2020-07-15T01:03:14.5986857Z             call_async(clusters[0].get_shard((1 << 16) + 1).add_block(b2))
2020-07-15T01:03:14.5986981Z     
2020-07-15T01:03:14.5987141Z             self.assert_balance(
2020-07-15T01:03:14.5987309Z                 master,
2020-07-15T01:03:14.5987486Z                 [acc1, acc1.address_in_shard(2 << 16), acc2, acc4, acc5],
2020-07-15T01:03:14.5987665Z                 [
2020-07-15T01:03:14.5989154Z                     1000000
2020-07-15T01:03:14.5989981Z                     - 54321
2020-07-15T01:03:14.5990238Z                     - 5555
2020-07-15T01:03:14.5990603Z                     - (opcodes.GTXXSHARDCOST + opcodes.GTXCOST) * 4,
2020-07-15T01:03:14.5990818Z                     1000000,
2020-07-15T01:03:14.5991210Z                     500000 - 7787 - (opcodes.GTXXSHARDCOST + opcodes.GTXCOST) * 2,
2020-07-15T01:03:14.5991401Z                     500000 + opcodes.GTXCOST,
2020-07-15T01:03:14.5991681Z                     500000 + opcodes.GTXCOST * 2,
2020-07-15T01:03:14.5991878Z                 ],
2020-07-15T01:03:14.5992040Z             )
2020-07-15T01:03:14.5992151Z             self.assertEqual(
2020-07-15T01:03:14.5992320Z                 call_async(
2020-07-15T01:03:14.5992495Z                     master.get_primary_account_data(acc3)
2020-07-15T01:03:14.5992694Z                 ).token_balances.balance_map,
2020-07-15T01:03:14.5992862Z                 {},
2020-07-15T01:03:14.5993021Z             )
2020-07-15T01:03:14.5993191Z     
2020-07-15T01:03:14.5993311Z             # expect chain 2 got the CrossShardTransactionList of b1
2020-07-15T01:03:14.5993488Z             xshard_tx_list = (
2020-07-15T01:03:14.5993655Z                 clusters[0]
2020-07-15T01:03:14.5993824Z                 .get_shard_state((2 << 16) | 1)
2020-07-15T01:03:14.5994011Z                 .db.get_minor_block_xshard_tx_list(b1.header.get_hash())
2020-07-15T01:03:14.5994203Z             )
2020-07-15T01:03:14.5994382Z             self.assertEqual(len(xshard_tx_list.tx_list), 2)
2020-07-15T01:03:14.5994573Z             self.assertEqual(xshard_tx_list.tx_list[0].tx_hash, tx1.get_hash())
2020-07-15T01:03:14.5994714Z             self.assertEqual(xshard_tx_list.tx_list[1].tx_hash, tx2.get_hash())
2020-07-15T01:03:14.5994907Z     
2020-07-15T01:03:14.5995133Z             xshard_tx_list = (
2020-07-15T01:03:14.5995296Z                 clusters[0]
2020-07-15T01:03:14.5995465Z                 .get_shard_state((2 << 16) | 1)
2020-07-15T01:03:14.5995647Z                 .db.get_minor_block_xshard_tx_list(b2.header.get_hash())
2020-07-15T01:03:14.5995840Z             )
2020-07-15T01:03:14.5995961Z             self.assertEqual(len(xshard_tx_list.tx_list), 1)
2020-07-15T01:03:14.5996148Z             self.assertEqual(xshard_tx_list.tx_list[0].tx_hash, tx3.get_hash())
2020-07-15T01:03:14.5996349Z     
2020-07-15T01:03:14.5996508Z             root_block = call_async(
2020-07-15T01:03:14.5996690Z                 master.get_next_block_to_mine(address=acc1, branch_value=None)
2020-07-15T01:03:14.5996870Z             )
2020-07-15T01:03:14.5997042Z             call_async(master.add_root_block(root_block))
2020-07-15T01:03:14.5997214Z     
2020-07-15T01:03:14.5997328Z             # b3 should include the deposits of tx1, t2, t3
2020-07-15T01:03:14.5998197Z             b3 = (
2020-07-15T01:03:14.5998384Z                 clusters[0]
2020-07-15T01:03:14.5998556Z                 .get_shard_state((2 << 16) | 1)
2020-07-15T01:03:14.5998686Z                 .create_block_to_mine(address=acc1.address_in_shard(2 << 16))
2020-07-15T01:03:14.5998870Z             )
2020-07-15T01:03:14.5999032Z             self.assertTrue(
2020-07-15T01:03:14.5999214Z                 call_async(master.add_raw_minor_block(b3.header.branch, b3.serialize()))
2020-07-15T01:03:14.5999403Z             )
2020-07-15T01:03:14.5999675Z             self.assert_balance(
2020-07-15T01:03:14.5999856Z                 master,
2020-07-15T01:03:14.5999983Z                 [acc1, acc1.address_in_shard(2 << 16), acc2, acc3, acc4, acc5],
2020-07-15T01:03:14.6000169Z                 [
2020-07-15T01:03:14.6000341Z                     1000000  # minior block reward
2020-07-15T01:03:14.6000753Z                     - 54321
2020-07-15T01:03:14.6001053Z                     - 5555
2020-07-15T01:03:14.6001676Z                     - (opcodes.GTXXSHARDCOST + opcodes.GTXCOST) * 4,
2020-07-15T01:03:14.6001869Z                     1500000 + opcodes.GTXXSHARDCOST * 3,
2020-07-15T01:03:14.6002292Z                     500000 - 7787 - (opcodes.GTXXSHARDCOST + opcodes.GTXCOST) * 2,
2020-07-15T01:03:14.6002432Z                     54321 + 5555 + 7787,
2020-07-15T01:03:14.6002609Z                     500000 + opcodes.GTXCOST,
2020-07-15T01:03:14.6002783Z >                   500000 + opcodes.GTXCOST * 2,
2020-07-15T01:03:14.6003003Z                 ],
2020-07-15T01:03:14.6003172Z             )
2020-07-15T01:03:14.6003239Z 
2020-07-15T01:03:14.6141761Z quarkchain/cluster/tests/test_cluster.py:1146: 
2020-07-15T01:03:14.6142207Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
2020-07-15T01:03:14.6142370Z quarkchain/cluster/tests/test_cluster.py:1001: in assert_balance
2020-07-15T01:03:14.6142518Z     {genesis_token: balance_list[idx]},
2020-07-15T01:03:14.6142647Z E   AssertionError: {35760: 1518000} != {35760: 1527000}
2020-07-15T01:03:14.6143321Z E   - {35760: 1518000}
2020-07-15T01:03:14.6143453Z E   ?           ^^
2020-07-15T01:03:14.6143576Z E   
2020-07-15T01:03:14.6143695Z E   + {35760: 1527000}
2020-07-15T01:03:14.6143820Z E   ?           ^^
2020-07-15T01:03:14.6143963Z ===================== 1 failed, 31 passed in 69.45 seconds =====================
qizhou commented 4 years ago

Looks like it is reproducible via: while [[ $? == 0 ]]; do pytest tests/test_cluster.py -k test_broadcast_cross_shard_transactions_2x1; done

qizhou commented 4 years ago

The cause is that the root block only contains 2 minor blocks (expected 3, miss b2), and thus the corresponding xshard deposit is not processed.

Interesting why there is a race that a root block does not contain all minor blocks that are successfully added.