search

dusk-network / dusk-blockchain

Reference implementation of the DUSK Network node, written in Golang
MIT License
102 stars 47 forks source link

Investigate a lost quorum at first iteration when network is busy #1444

Closed goshawk-3 closed 2 years ago

goshawk-3 commented 2 years ago

Describe the bug In rare cases, it was noticed that there could be a sudden increase in consensus time due to a failure to reach quorum at iteration 1. At first glance, it seems like part of the nodes does timeout at Selection step while other does timeout at 1th Reduction. This is a situation that could happen if the candidate block of the iteration is not propagated to all provisioners.

To Reproduce Send a batch of 90 transfer transactions 5 times in a row.

Expected behaviour We should see all provisioners collecting the candidate block and reaching quorum for up to 14s.

Logs/Screenshot


{"block_time":56,"cert_step":6,"curr_h":124642,"e_prov":95,"event":"accept_block","hash":"6ee65d623f...8a62970dcd","height":124643,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:35:53Z","txs_count":9}

{"block_time":14,"cert_step":3,"curr_h":124643,"e_prov":95,"event":"accept_block","hash":"b3a405b59a...3dcd402e89","height":124644,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:36:06Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":124644,"e_prov":95,"event":"accept_block","hash":"a4a5441865...c43cad95e1","height":124645,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:36:20Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":124645,"e_prov":95,"event":"accept_block","hash":"f0762fa2ee...10a7ee6200","height":124646,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:36:29Z","txs_count":9}
{"block_time":12,"cert_step":3,"curr_h":124646,"e_prov":95,"event":"accept_block","hash":"d593cb6d54...5b45d72f06","height":124647,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:36:39Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":124647,"e_prov":95,"event":"accept_block","hash":"d93bf8493c...b7154e058c","height":124648,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:36:50Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":124648,"e_prov":95,"event":"accept_block","hash":"4dcda3a844...826f5cf691","height":124649,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:37:05Z","txs_count":9}
{"block_time":14,"cert_step":3,"curr_h":124649,"e_prov":95,"event":"accept_block","hash":"913e23bf42...86b2dd89a3","height":124650,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:37:16Z","txs_count":8}
{"block_time":10,"cert_step":3,"curr_h":124650,"e_prov":95,"event":"accept_block","hash":"0ed4de6c74...4fd7deefda","height":124651,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:37:22Z","txs_count":0}
...
{"block_time":11,"cert_step":3,"curr_h":124826,"e_prov":95,"event":"accept_block","hash":"ea50ecb0c2...5db9a6f18b","height":124827,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:53:07Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":124827,"e_prov":95,"event":"accept_block","hash":"eb06ad2da5...78f2a33b16","height":124828,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:53:18Z","txs_count":9}
{"block_time":12,"cert_step":3,"curr_h":124828,"e_prov":95,"event":"accept_block","hash":"be536cc82a...ec982fe7f8","height":124829,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:53:31Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":124829,"e_prov":95,"event":"accept_block","hash":"6591d3e164...2367f58c7d","height":124830,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:53:41Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":124830,"e_prov":95,"event":"accept_block","hash":"d5d1b263ed...7f6cce02ec","height":124831,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:53:52Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":124831,"e_prov":95,"event":"accept_block","hash":"931fbf5608...65c377cecd","height":124832,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:54:02Z","txs_count":9}
{"block_time":12,"cert_step":3,"curr_h":124832,"e_prov":95,"event":"accept_block","hash":"10e4b17cba...5220b60f36","height":124833,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:54:13Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":124833,"e_prov":95,"event":"accept_block","hash":"70b25c1872...232cfeb1d0","height":124834,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:54:24Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":124834,"e_prov":95,"event":"accept_block","hash":"30ef4d6abb...e6462dfe1f","height":124835,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:54:39Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":124835,"e_prov":95,"event":"accept_block","hash":"29167d22c7...9c70d3ca14","height":124836,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T12:54:49Z","txs_count":6}
...
{"block_time":8,"cert_step":3,"curr_h":124950,"e_prov":95,"event":"accept_block","hash":"b68dec4b03...0914c6b6bb","height":124951,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:05:01Z","txs_count":4}
{"block_time":11,"cert_step":3,"curr_h":124951,"e_prov":95,"event":"accept_block","hash":"1cfacb6828...a6b6d5e701","height":124952,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:05:14Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":124952,"e_prov":95,"event":"accept_block","hash":"b9c4eea8a5...ac25a0e4fa","height":124953,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:05:25Z","txs_count":9}
{"block_time":14,"cert_step":3,"curr_h":124953,"e_prov":95,"event":"accept_block","hash":"d00aa38cf5...47e1a51e36","height":124954,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:05:40Z","txs_count":9}
{"block_time":12,"cert_step":3,"curr_h":124954,"e_prov":95,"event":"accept_block","hash":"355f922dd0...3f950a53a9","height":124955,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:05:49Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":124955,"e_prov":95,"event":"accept_block","hash":"9f8026ea38...ad8ba0004c","height":124956,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:06:00Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":124956,"e_prov":95,"event":"accept_block","hash":"4b26ea615d...c365b5d1dd","height":124957,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:06:11Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":124957,"e_prov":95,"event":"accept_block","hash":"3f3694de1a...0cfdd5f334","height":124958,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:06:24Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":124958,"e_prov":95,"event":"accept_block","hash":"37a37574e9...e1d1287a0f","height":124959,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:06:36Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":124959,"e_prov":95,"event":"accept_block","hash":"bf5a063145...fd2d39b6c5","height":124960,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:06:50Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":124960,"e_prov":95,"event":"accept_block","hash":"f210d08cac...86ec9fbcce","height":124961,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-07-27T13:07:01Z","txs_count":5}

Platform Devnet, consensustimeout=20s PR #1442

Additional context This could be due to dropped udp packages on propagating candidate block.

goshawk-3 commented 2 years ago

From most recent devnet.

{
  "block_time": 35,
  "cert_step": 6,
  "curr_h": 6800,
  "e_prov": 95,
  "event": "accept_block",
  "hash": "d76be27fab...0eb779d6b5",
  "height": 6801,
  "level": "info",
  "msg": "run state transition",
  "process": "chain",
  "prov": 95,
  "time": "2022-08-01T18:08:16Z",
  "txs_count": 0
}
goshawk-3 commented 2 years ago

As per increasing udp recv buffer size (after round 14229) and enabling multiple core profile, we have a steady consensus time - 6 sec on empty blocks, 9s-15s on full blocks (BlockGasLimit reached).

Logged results from devnet for blocks with 9 transfer txns

{"block_time":8,"cert_step":3,"curr_h":13297,"e_prov":95,"event":"accept_block","hash":"fb1c4296b9...74fea22643","height":13298,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:07:57Z","txs_count":9}
{"block_time":14,"cert_step":3,"curr_h":13298,"e_prov":95,"event":"accept_block","hash":"9c4821354d...8cc43743d3","height":13299,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:08:10Z","txs_count":9}
{"block_time":34,"cert_step":6,"curr_h":13299,"e_prov":95,"event":"accept_block","hash":"70854b2afc...9523123e70","height":13300,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:08:58Z","txs_count":9}
{"block_time":22,"cert_step":3,"curr_h":13300,"e_prov":95,"event":"accept_block","hash":"7acb9cdba1...90420f5a69","height":13301,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:09:09Z","txs_count":9}
{"block_time":12,"cert_step":3,"curr_h":13301,"e_prov":95,"event":"accept_block","hash":"9d7a4d213d...12bfc3bdfb","height":13302,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:09:18Z","txs_count":9}
{"block_time":33,"cert_step":6,"curr_h":13302,"e_prov":95,"event":"accept_block","hash":"151bbdf9b1...81f25fd0d0","height":13303,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:10:06Z","txs_count":9}
{"block_time":29,"cert_step":3,"curr_h":13303,"e_prov":95,"event":"accept_block","hash":"6e89b5a8a3...422944ea8b","height":13304,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:10:20Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":13304,"e_prov":95,"event":"accept_block","hash":"fb89270c78...fc00a6ef7d","height":13305,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:10:30Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":13305,"e_prov":95,"event":"accept_block","hash":"0bea723508...75543393da","height":13306,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:10:38Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":13306,"e_prov":95,"event":"accept_block","hash":"5c6e72b394...a0ae3fe951","height":13307,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T11:10:47Z","txs_count":9}

{"block_time":9,"cert_step":3,"curr_h":14229,"e_prov":95,"event":"accept_block","hash":"5fe455f8cb...cc1347cfe5","height":14230,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:45:12Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14230,"e_prov":95,"event":"accept_block","hash":"25ce7076ce...3a57fd35c8","height":14231,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:45:22Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14231,"e_prov":95,"event":"accept_block","hash":"1fec04306e...b620531f3b","height":14232,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:45:31Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":14232,"e_prov":95,"event":"accept_block","hash":"f9c61088ab...c81c3e9de7","height":14233,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:45:43Z","txs_count":9}
{"block_time":12,"cert_step":3,"curr_h":14233,"e_prov":95,"event":"accept_block","hash":"acff5e0099...09fc81933c","height":14234,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:45:55Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14234,"e_prov":95,"event":"accept_block","hash":"3fa59c02cc...fb898f2071","height":14235,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:46:03Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14235,"e_prov":95,"event":"accept_block","hash":"d49daf35ca...0410c6899f","height":14236,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:46:12Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14236,"e_prov":95,"event":"accept_block","hash":"3c4cd14645...d51c178513","height":14237,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:46:22Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14237,"e_prov":95,"event":"accept_block","hash":"0d339a2f08...43d437aabb","height":14238,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:46:31Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14238,"e_prov":95,"event":"accept_block","hash":"7136ef6a90...62920aec0b","height":14239,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:46:40Z","txs_count":9}
{"block_time":7,"cert_step":3,"curr_h":14239,"e_prov":95,"event":"accept_block","hash":"81b5b24f82...877f3a8ac1","height":14240,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T12:46:47Z","txs_count":9}
{"block_time":6,"cert_step":3,"curr_h":14464,"e_prov":95,"event":"accept_block","hash":"54ab7c1947...0fd6619d12","height":14465,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:09:20Z","txs_count":9}
{"block_time":12,"cert_step":3,"curr_h":14465,"e_prov":95,"event":"accept_block","hash":"e44c5adfc9...f0f2986289","height":14466,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:09:33Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":14466,"e_prov":95,"event":"accept_block","hash":"ccc2e21b12...0cb418054b","height":14467,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:09:42Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14467,"e_prov":95,"event":"accept_block","hash":"a67a0f8c17...6b6bee9355","height":14468,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:09:51Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14468,"e_prov":95,"event":"accept_block","hash":"26a0564d00...f4eddf12ea","height":14469,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:10:00Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14469,"e_prov":95,"event":"accept_block","hash":"66f4cbbe1f...3097407967","height":14470,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:10:11Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14470,"e_prov":95,"event":"accept_block","hash":"e268871641...e325446146","height":14471,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:10:23Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14471,"e_prov":95,"event":"accept_block","hash":"1aadc66b38...f977ae6593","height":14472,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:10:30Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":14472,"e_prov":95,"event":"accept_block","hash":"072bafd454...f1cb3cc319","height":14473,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:10:42Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":14473,"e_prov":95,"event":"accept_block","hash":"7afe0f5c90...a19275eb81","height":14474,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:10:53Z","txs_count":9}
{"block_time":5,"cert_step":3,"curr_h":14518,"e_prov":95,"event":"accept_block","hash":"161875ffde...c21ac60bff","height":14519,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:15:30Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":14519,"e_prov":95,"event":"accept_block","hash":"09aa22ed61...34cc45d2ec","height":14520,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:15:41Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14520,"e_prov":95,"event":"accept_block","hash":"edf2d13c6d...333d17b685","height":14521,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:15:51Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14521,"e_prov":95,"event":"accept_block","hash":"456f816e8f...561af5b5b1","height":14522,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:16:00Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14522,"e_prov":95,"event":"accept_block","hash":"7176af9def...7eea2ae827","height":14523,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:16:09Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14523,"e_prov":95,"event":"accept_block","hash":"da0cc92bde...787a2df398","height":14524,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:16:19Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14524,"e_prov":95,"event":"accept_block","hash":"3c1a2218f4...c59c154370","height":14525,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:16:28Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14525,"e_prov":95,"event":"accept_block","hash":"f1d48c37ec...ca0de896c2","height":14526,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:16:36Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14526,"e_prov":95,"event":"accept_block","hash":"a76ce2b156...49114eeca3","height":14527,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:16:45Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14527,"e_prov":95,"event":"accept_block","hash":"8e0f2077a2...9cfd5c3ad3","height":14528,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:16:56Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14528,"e_prov":95,"event":"accept_block","hash":"a96fe1c6f2...be9c0f1065","height":14529,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:17:05Z","txs_count":9}
{"block_time":6,"cert_step":3,"curr_h":14574,"e_prov":95,"event":"accept_block","hash":"5799522964...b78700cc4a","height":14575,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:21:49Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14575,"e_prov":95,"event":"accept_block","hash":"bef8050bfd...fe989cbef5","height":14576,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:21:59Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14576,"e_prov":95,"event":"accept_block","hash":"c83022fbaa...543159c4f5","height":14577,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:22:07Z","txs_count":9}
{"block_time":15,"cert_step":3,"curr_h":14577,"e_prov":95,"event":"accept_block","hash":"06defd7df0...98022bc1eb","height":14578,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:22:21Z","txs_count":9}
{"block_time":13,"cert_step":3,"curr_h":14578,"e_prov":95,"event":"accept_block","hash":"60c54de861...58bfbb8194","height":14579,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:22:35Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14579,"e_prov":95,"event":"accept_block","hash":"762397f5da...d2a9d1d106","height":14580,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:22:45Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14580,"e_prov":95,"event":"accept_block","hash":"0f35099ef7...828a987c68","height":14581,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:22:52Z","txs_count":9}
{"block_time":7,"cert_step":3,"curr_h":14581,"e_prov":95,"event":"accept_block","hash":"e5cd80df85...bbc34dbe1a","height":14582,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:23:00Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14582,"e_prov":95,"event":"accept_block","hash":"3c92d33a1b...3e7a290975","height":14583,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:23:10Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14583,"e_prov":95,"event":"accept_block","hash":"d653b56f82...e4fe6b04cb","height":14584,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:23:19Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14584,"e_prov":95,"event":"accept_block","hash":"f37d67f9cb...c34af4d4dc","height":14585,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:23:28Z","txs_count":9}
{"block_time":7,"cert_step":3,"curr_h":14622,"e_prov":95,"event":"accept_block","hash":"cfd56ef856...b17723f5d7","height":14623,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:27:21Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14623,"e_prov":95,"event":"accept_block","hash":"5f94f0dedd...259b83d3a4","height":14624,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:27:33Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14624,"e_prov":95,"event":"accept_block","hash":"f6c6449c3d...8dcbfd94ac","height":14625,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:27:41Z","txs_count":9}
{"block_time":12,"cert_step":3,"curr_h":14625,"e_prov":95,"event":"accept_block","hash":"970ab4c4e2...f55a442898","height":14626,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:27:52Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14626,"e_prov":95,"event":"accept_block","hash":"df959b27a1...02c8df441a","height":14627,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:28:00Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":14627,"e_prov":95,"event":"accept_block","hash":"7908f2f6e6...8ee9110712","height":14628,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:28:12Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":14628,"e_prov":95,"event":"accept_block","hash":"3348bb3af4...f5a5cf0104","height":14629,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:28:22Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14629,"e_prov":95,"event":"accept_block","hash":"810822e69a...2ac2a51679","height":14630,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:28:32Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14630,"e_prov":95,"event":"accept_block","hash":"72fe0f64b3...a1fa14d812","height":14631,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:28:42Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14631,"e_prov":95,"event":"accept_block","hash":"e3f2557709...60e92f3848","height":14632,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:28:48Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14632,"e_prov":95,"event":"accept_block","hash":"2987c65815...75e89f35f8","height":14633,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:28:57Z","txs_count":9}
{"block_time":6,"cert_step":3,"curr_h":14679,"e_prov":95,"event":"accept_block","hash":"c0863f10b5...41fba4ef16","height":14680,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:33:43Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14680,"e_prov":95,"event":"accept_block","hash":"6e35adec30...7dc84260ac","height":14681,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:33:52Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14681,"e_prov":95,"event":"accept_block","hash":"e1038b7cb0...925e5a23f4","height":14682,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:34:00Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14682,"e_prov":95,"event":"accept_block","hash":"6460211c6c...d8228771e0","height":14683,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:34:08Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14683,"e_prov":95,"event":"accept_block","hash":"3df98ce015...2de94f2de3","height":14684,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:34:18Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14684,"e_prov":95,"event":"accept_block","hash":"096768494f...d9b344cc3e","height":14685,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:34:27Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14685,"e_prov":95,"event":"accept_block","hash":"9630982bf4...4e572c98a6","height":14686,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:34:37Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14686,"e_prov":95,"event":"accept_block","hash":"b263308c56...3389a12066","height":14687,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:34:45Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14687,"e_prov":95,"event":"accept_block","hash":"3a4c12482f...4fd7155ce2","height":14688,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:34:54Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14688,"e_prov":95,"event":"accept_block","hash":"ab57c9b1e0...30c1e26f49","height":14689,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:35:03Z","txs_count":9}
{"block_time":6,"cert_step":3,"curr_h":14714,"e_prov":95,"event":"accept_block","hash":"f1971e171b...d6d10f37e2","height":14715,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:37:45Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14715,"e_prov":95,"event":"accept_block","hash":"05974c359b...484cdcbcdd","height":14716,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:37:54Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":14716,"e_prov":95,"event":"accept_block","hash":"9e0b199f51...3bd94268fb","height":14717,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:38:05Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14717,"e_prov":95,"event":"accept_block","hash":"56e32c6e92...f629db7cf6","height":14718,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:38:13Z","txs_count":9}
{"block_time":10,"cert_step":3,"curr_h":14718,"e_prov":95,"event":"accept_block","hash":"4dedf0b34b...07c23b87fd","height":14719,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:38:24Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14719,"e_prov":95,"event":"accept_block","hash":"26a16b4669...6313a72d6f","height":14720,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:38:32Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14720,"e_prov":95,"event":"accept_block","hash":"2abf16e3b0...d0928105ad","height":14721,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:38:42Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14721,"e_prov":95,"event":"accept_block","hash":"dbfa26d300...f8d8167837","height":14722,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:38:49Z","txs_count":9}
{"block_time":9,"cert_step":3,"curr_h":14722,"e_prov":95,"event":"accept_block","hash":"b82e88d120...306d6c114c","height":14723,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:38:58Z","txs_count":9}
{"block_time":8,"cert_step":3,"curr_h":14723,"e_prov":95,"event":"accept_block","hash":"af08fe68f7...c1257a9b8a","height":14724,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:39:07Z","txs_count":9}
{"block_time":11,"cert_step":3,"curr_h":14724,"e_prov":95,"event":"accept_block","hash":"3bbeb1f105...e446ac8eef","height":14725,"level":"info","msg":"run state transition","process":"chain","prov":95,"time":"2022-08-02T13:39:20Z","txs_count":9}

Changes made on all nodes