search

vegaprotocol / vegatools

A go command line utility providing a bunch of tools to use with a Vega network
MIT License
3 stars 4 forks source link

vegatools - difftool reporting mismatches for liquidityProvisions #284

Closed MuthuVega closed 1 year ago

MuthuVega commented 1 year ago

The vegatools - difftool is reporting mismatches for liquidityProvisions table.

The actual difference being reported is:

key=liquidityProvisions, matchResult=mismatching number of elements, coreLength=10, datanodeLength=9,
coreResult=id:"1647af9912f9a1c7092ed19a354022681449c0dad96321ee301697a646ef7add" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768634661556000 updated_at:1683768634661556000 market_id:"3c60f99f29c38ad3be07b55b049ae7fcb510541466e84859e6e006eccd44b2a4" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"686891cb74858a4b554245c659be56834c5771d8514246ada43d8eb2abd69332" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6385c50dc9efbb769b9a3cbfc31248ed55f2a8cccc7242f00bb2ee73155fc8f2" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[40]-276c9291"\nid:"346e3f6271453f149cfd8df2aff5996b8bcde173d473db6f0a3096f3d81c5c8f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767578347485000 updated_at:1683767578347485000 market_id:"50ea1a1b4c65ac836b1145f690b1965ca0d00294792bac96728959e975097191" commitment_amount:"390500000000000000" fee:"0.3" sells:{order_id:"9465e4e578ea31cf8478fd158d7aca6312f5dcce51a7e6fa1d57e74cef50729e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"1000000000000"}} buys:{order_id:"478ba09795193a8e05dfbfc0174c0fbbc9c2229aed1cd7a60012dc04738cf837" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"1000000000000"}} version:1 status:STATUS_PENDING reference:"LIQPROV-test_List_Markets-1dc663e9"\nid:"4cda01385931005d3cc96781bc5b82ef845afccfea39ad587095d9e9a6ab3552" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768857409212000 updated_at:1683768857409212000 market_id:"ac586c1ad1d11735593d97c127c00d64837c6d5220da9b2844eddda07ff8f895" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2ac1b51f300615b181b359c5c81fc81d3cfd389f0d3aa17b4aa822e41916f524" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"4db83fc6389fa2a8a50cfa1c7b2309ed362b6d7af5a7b7eb76576b55af30fd1b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_behavior-81ac4f0c"\nid:"59a4669eb5335f3b8fa6c68d91cbae1e62ca7484256d93a4d9e9d5f586196e45" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770758299333000 updated_at:1683770758299333000 market_id:"9d8ac718dfa66c734e2774b2760fc2f52401bc3dc5dfa30ec3100507fae74046" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2d763e1540ef4e078f798382d57d29d0666dd8a44b6fb70d946cc7de8202b33a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"a2b9f1acf79e8d7a785d4ef241bfb816b2ccd172c9a1e0fcef6bb1a1f24628fe" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_reject_duplicated_or_filtered_out_messages-9c0800e3"\nid:"63e3054e90ee097257126cadc486f39ca383faf8184f9ff291f378fe5adcc7fa" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683772691201778000 updated_at:1683772691201778000 market_id:"dddd2690b14764ab3005aa41e50b61340489e16795a9ae4cb9678370f3ab7a27" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"75d9e6f11f1cd1e7ba3953648910e9b1a2cee3ddff5dd275ec475e7e29c2987d" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"c97ca893092dd3ea26ffe96229d952a66aa71b641db84cdd6beca8260b08900a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_disconnection-3c69bcdf"\nid:"6966f0a86498d41010af317ecb7db8a9321455f5c4958d7f64234c98c6a01190" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767647338681000 updated_at:1683767647338681000 market_id:"366db27b1cf8cf55a6ec0b45989056b8582d61f5dad471441cbfccd9ba966483" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"b27799f64df4074c22a0f126d7bbfb26550c55a8713076445ac3a6dc1eb22025" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6d9c420881367d8fdacbb4afc834d1df17262d045bfedb359e42f551d6c10cb8" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-tests/API/rest_endpoints_test.py-0060ddda"\nid:"6edde00f797a48073c93f248fc5d8c3784590679ebe5e95875088deffb8850c9" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683773056704914000 updated_at:1683773056704914000 market_id:"b8c0068c169a783e9a6ee73dfc512e089855f00d9b20a78ef6f3f23e2f48faeb" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"68e79f532062af699d94ae3969e268ef195c7f113862bd4ad906c7df30762b92" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"ef67ef53e3cfc7240a1da3242320b8cc13a8d78fa71a52e81864134a6312819e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_start_stop_restart_from_network_history-93619c69"\nid:"8b69b6a80506f8502899b5a74e25de92eccbafc63a4f60a4cd719ee64ff52dc9" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768736333362000 updated_at:1683768736333362000 market_id:"b3ea38f67a55d19ee231998f3db2d57d26b99510e2aa7700a362f26e9d8800a9" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"9bba65e0b4d462864300000fb5067fb2325dc822c2d6d95b7759d684cb6e1f34" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"9fcc0b65dd4b3a47e3b507eeea78b149a48be4843b3fd66dafa9feb546bf5ab4" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[60]-e484782f"\nid:"abcdaafc4effb5343c89613bdbb0eff70e75f845d99c01b42da56f328fe0650f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768525052419000 updated_at:1683768525052419000 market_id:"ed701eb422ba98e93fe87adba49e75b5e3b53655a56e89af9bf2bf0cf07fcb81" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"cc0e37d496e43b38e9abff3f754f3bb1f7a9b907cd3ef17e999b46b235b2fa2b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"f4714eb302b231a9264240b7a46085c2e6d6ffce6e2009dbe86614c8ecad06e6" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mark_to_market_settlement-8f7d0153"\nid:"dd03e23234cc8ee4d0aa3fd4160290157a6e79b0a6f1f39f39b436bb0ecf6f7b" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770738987019000 updated_at:1683770738987019000 market_id:"aeb9a27a135b82b3963ec53aea230848a6b896fdd231fa9e40839a544ad1beef" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"766edf5aaabbb04fe7da1ed106d69ac652e0aa050cbeb7ebc1d1ef4fad777972" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"2d4234579f5e8a60dfee98d5cede8f81f1136ec8276c6fe73c8af7a5a483e085" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_oracle_messages_signed_with_wrong_key_are_rejected-7d081bfb"\n, 
datanodeResult=id:"1647af9912f9a1c7092ed19a354022681449c0dad96321ee301697a646ef7add" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768634661556000 updated_at:1683768634661556000 market_id:"3c60f99f29c38ad3be07b55b049ae7fcb510541466e84859e6e006eccd44b2a4" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"686891cb74858a4b554245c659be56834c5771d8514246ada43d8eb2abd69332" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6385c50dc9efbb769b9a3cbfc31248ed55f2a8cccc7242f00bb2ee73155fc8f2" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[40]-276c9291"\nid:"4cda01385931005d3cc96781bc5b82ef845afccfea39ad587095d9e9a6ab3552" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768857409212000 updated_at:1683768857409212000 market_id:"ac586c1ad1d11735593d97c127c00d64837c6d5220da9b2844eddda07ff8f895" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2ac1b51f300615b181b359c5c81fc81d3cfd389f0d3aa17b4aa822e41916f524" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"4db83fc6389fa2a8a50cfa1c7b2309ed362b6d7af5a7b7eb76576b55af30fd1b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_behavior-81ac4f0c"\nid:"59a4669eb5335f3b8fa6c68d91cbae1e62ca7484256d93a4d9e9d5f586196e45" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770758299333000 updated_at:1683770758299333000 market_id:"9d8ac718dfa66c734e2774b2760fc2f52401bc3dc5dfa30ec3100507fae74046" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2d763e1540ef4e078f798382d57d29d0666dd8a44b6fb70d946cc7de8202b33a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"a2b9f1acf79e8d7a785d4ef241bfb816b2ccd172c9a1e0fcef6bb1a1f24628fe" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_reject_duplicated_or_filtered_out_messages-9c0800e3"\nid:"63e3054e90ee097257126cadc486f39ca383faf8184f9ff291f378fe5adcc7fa" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683772691201778000 updated_at:1683772691201778000 market_id:"dddd2690b14764ab3005aa41e50b61340489e16795a9ae4cb9678370f3ab7a27" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"75d9e6f11f1cd1e7ba3953648910e9b1a2cee3ddff5dd275ec475e7e29c2987d" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"c97ca893092dd3ea26ffe96229d952a66aa71b641db84cdd6beca8260b08900a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_disconnection-3c69bcdf"\nid:"6966f0a86498d41010af317ecb7db8a9321455f5c4958d7f64234c98c6a01190" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767647338681000 updated_at:1683767647338681000 market_id:"366db27b1cf8cf55a6ec0b45989056b8582d61f5dad471441cbfccd9ba966483" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"b27799f64df4074c22a0f126d7bbfb26550c55a8713076445ac3a6dc1eb22025" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6d9c420881367d8fdacbb4afc834d1df17262d045bfedb359e42f551d6c10cb8" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-tests/API/rest_endpoints_test.py-0060ddda"\nid:"6edde00f797a48073c93f248fc5d8c3784590679ebe5e95875088deffb8850c9" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683773056704914000 updated_at:1683773056704914000 market_id:"b8c0068c169a783e9a6ee73dfc512e089855f00d9b20a78ef6f3f23e2f48faeb" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"68e79f532062af699d94ae3969e268ef195c7f113862bd4ad906c7df30762b92" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"ef67ef53e3cfc7240a1da3242320b8cc13a8d78fa71a52e81864134a6312819e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_start_stop_restart_from_network_history-93619c69"\nid:"8b69b6a80506f8502899b5a74e25de92eccbafc63a4f60a4cd719ee64ff52dc9" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768736333362000 updated_at:1683768736333362000 market_id:"b3ea38f67a55d19ee231998f3db2d57d26b99510e2aa7700a362f26e9d8800a9" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"9bba65e0b4d462864300000fb5067fb2325dc822c2d6d95b7759d684cb6e1f34" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"9fcc0b65dd4b3a47e3b507eeea78b149a48be4843b3fd66dafa9feb546bf5ab4" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[60]-e484782f"\nid:"abcdaafc4effb5343c89613bdbb0eff70e75f845d99c01b42da56f328fe0650f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768525052419000 updated_at:1683768525052419000 market_id:"ed701eb422ba98e93fe87adba49e75b5e3b53655a56e89af9bf2bf0cf07fcb81" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"cc0e37d496e43b38e9abff3f754f3bb1f7a9b907cd3ef17e999b46b235b2fa2b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"f4714eb302b231a9264240b7a46085c2e6d6ffce6e2009dbe86614c8ecad06e6" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mark_to_market_settlement-8f7d0153"\nid:"dd03e23234cc8ee4d0aa3fd4160290157a6e79b0a6f1f39f39b436bb0ecf6f7b" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770738987019000 updated_at:1683770738987019000 market_id:"aeb9a27a135b82b3963ec53aea230848a6b896fdd231fa9e40839a544ad1beef" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"766edf5aaabbb04fe7da1ed106d69ac652e0aa050cbeb7ebc1d1ef4fad777972" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"2d4234579f5e8a60dfee98d5cede8f81f1136ec8276c6fe73c8af7a5a483e085" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_oracle_messages_signed_with_wrong_key_are_rejected-7d081bfb"\n\n

Full output of the vegatools difftool command:

Datanode data diff output for /jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/vega/node3: 
CompletedProcess(args=['/jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/../vegatools', 'difftool', '-s', '/jenkins/workspace/common/system-tests-wrapper/networkdata/testnet/vega/node3/state/node/snapshots/', '-d', '127.0.0.1:3047'], returncode=1, stdout='found snapshot for block-height 9680\nsnapshot payloads written to: /tmp/snapshot.dat\nmismatch between core and datanode: success=false, 
report:\n
key=accounts, matchResult=full match, coreLength=405, datanodeLength=323, coreResult=, datanodeResult=\n
key=orders, matchResult=full match, coreLength=58, datanodeLength=58, coreResult=, datanodeResult=\n
key=markets, matchResult=full match, coreLength=12, datanodeLength=12, coreResult=, datanodeResult=\n
key=parties, matchResult=full match, coreLength=134, datanodeLength=134, coreResult=, datanodeResult=\n
key=limits, matchResult=full match, coreLength=1, datanodeLength=1, coreResult=, datanodeResult=\n
key=assets, matchResult=full match, coreLength=21, datanodeLength=21, coreResult=, datanodeResult=\n
key=delegations, matchResult=full match, coreLength=2, datanodeLength=2, coreResult=, datanodeResult=\n
key=epoch, matchResult=full match, coreLength=1, datanodeLength=1, coreResult=, datanodeResult=\n
key=vegaTime, matchResult=mismatching values, coreLength=1, datanodeLength=1, coreResult=1683773419795709000, datanodeResult=1683773421351244000\n
key=nodes, matchResult=full match, coreLength=2, datanodeLength=2, coreResult=, datanodeResult=\n
key=netparams, matchResult=full match, coreLength=116, datanodeLength=116, coreResult=, datanodeResult=\n
key=proposals, matchResult=full match, coreLength=94, datanodeLength=94, coreResult=, datanodeResult=\n
key=deposits, matchResult=full match, coreLength=0, datanodeLength=0, coreResult=, datanodeResult=\n
key=withdrawals, matchResult=full match, coreLength=10, datanodeLength=0, coreResult=, datanodeResult=\n
key=liquidityProvisions, matchResult=mismatching number of elements, coreLength=10, datanodeLength=9, coreResult=id:"1647af9912f9a1c7092ed19a354022681449c0dad96321ee301697a646ef7add" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768634661556000 updated_at:1683768634661556000 market_id:"3c60f99f29c38ad3be07b55b049ae7fcb510541466e84859e6e006eccd44b2a4" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"686891cb74858a4b554245c659be56834c5771d8514246ada43d8eb2abd69332" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6385c50dc9efbb769b9a3cbfc31248ed55f2a8cccc7242f00bb2ee73155fc8f2" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[40]-276c9291"\nid:"346e3f6271453f149cfd8df2aff5996b8bcde173d473db6f0a3096f3d81c5c8f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767578347485000 updated_at:1683767578347485000 market_id:"50ea1a1b4c65ac836b1145f690b1965ca0d00294792bac96728959e975097191" commitment_amount:"390500000000000000" fee:"0.3" sells:{order_id:"9465e4e578ea31cf8478fd158d7aca6312f5dcce51a7e6fa1d57e74cef50729e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"1000000000000"}} buys:{order_id:"478ba09795193a8e05dfbfc0174c0fbbc9c2229aed1cd7a60012dc04738cf837" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"1000000000000"}} version:1 status:STATUS_PENDING reference:"LIQPROV-test_List_Markets-1dc663e9"\nid:"4cda01385931005d3cc96781bc5b82ef845afccfea39ad587095d9e9a6ab3552" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768857409212000 updated_at:1683768857409212000 market_id:"ac586c1ad1d11735593d97c127c00d64837c6d5220da9b2844eddda07ff8f895" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2ac1b51f300615b181b359c5c81fc81d3cfd389f0d3aa17b4aa822e41916f524" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"4db83fc6389fa2a8a50cfa1c7b2309ed362b6d7af5a7b7eb76576b55af30fd1b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_behavior-81ac4f0c"\nid:"59a4669eb5335f3b8fa6c68d91cbae1e62ca7484256d93a4d9e9d5f586196e45" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770758299333000 updated_at:1683770758299333000 market_id:"9d8ac718dfa66c734e2774b2760fc2f52401bc3dc5dfa30ec3100507fae74046" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2d763e1540ef4e078f798382d57d29d0666dd8a44b6fb70d946cc7de8202b33a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"a2b9f1acf79e8d7a785d4ef241bfb816b2ccd172c9a1e0fcef6bb1a1f24628fe" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_reject_duplicated_or_filtered_out_messages-9c0800e3"\nid:"63e3054e90ee097257126cadc486f39ca383faf8184f9ff291f378fe5adcc7fa" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683772691201778000 updated_at:1683772691201778000 market_id:"dddd2690b14764ab3005aa41e50b61340489e16795a9ae4cb9678370f3ab7a27" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"75d9e6f11f1cd1e7ba3953648910e9b1a2cee3ddff5dd275ec475e7e29c2987d" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"c97ca893092dd3ea26ffe96229d952a66aa71b641db84cdd6beca8260b08900a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_disconnection-3c69bcdf"\nid:"6966f0a86498d41010af317ecb7db8a9321455f5c4958d7f64234c98c6a01190" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767647338681000 updated_at:1683767647338681000 market_id:"366db27b1cf8cf55a6ec0b45989056b8582d61f5dad471441cbfccd9ba966483" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"b27799f64df4074c22a0f126d7bbfb26550c55a8713076445ac3a6dc1eb22025" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6d9c420881367d8fdacbb4afc834d1df17262d045bfedb359e42f551d6c10cb8" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-tests/API/rest_endpoints_test.py-0060ddda"\nid:"6edde00f797a48073c93f248fc5d8c3784590679ebe5e95875088deffb8850c9" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683773056704914000 updated_at:1683773056704914000 market_id:"b8c0068c169a783e9a6ee73dfc512e089855f00d9b20a78ef6f3f23e2f48faeb" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"68e79f532062af699d94ae3969e268ef195c7f113862bd4ad906c7df30762b92" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"ef67ef53e3cfc7240a1da3242320b8cc13a8d78fa71a52e81864134a6312819e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_start_stop_restart_from_network_history-93619c69"\nid:"8b69b6a80506f8502899b5a74e25de92eccbafc63a4f60a4cd719ee64ff52dc9" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768736333362000 updated_at:1683768736333362000 market_id:"b3ea38f67a55d19ee231998f3db2d57d26b99510e2aa7700a362f26e9d8800a9" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"9bba65e0b4d462864300000fb5067fb2325dc822c2d6d95b7759d684cb6e1f34" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"9fcc0b65dd4b3a47e3b507eeea78b149a48be4843b3fd66dafa9feb546bf5ab4" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[60]-e484782f"\nid:"abcdaafc4effb5343c89613bdbb0eff70e75f845d99c01b42da56f328fe0650f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768525052419000 updated_at:1683768525052419000 market_id:"ed701eb422ba98e93fe87adba49e75b5e3b53655a56e89af9bf2bf0cf07fcb81" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"cc0e37d496e43b38e9abff3f754f3bb1f7a9b907cd3ef17e999b46b235b2fa2b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"f4714eb302b231a9264240b7a46085c2e6d6ffce6e2009dbe86614c8ecad06e6" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mark_to_market_settlement-8f7d0153"\nid:"dd03e23234cc8ee4d0aa3fd4160290157a6e79b0a6f1f39f39b436bb0ecf6f7b" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770738987019000 updated_at:1683770738987019000 market_id:"aeb9a27a135b82b3963ec53aea230848a6b896fdd231fa9e40839a544ad1beef" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"766edf5aaabbb04fe7da1ed106d69ac652e0aa050cbeb7ebc1d1ef4fad777972" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"2d4234579f5e8a60dfee98d5cede8f81f1136ec8276c6fe73c8af7a5a483e085" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_oracle_messages_signed_with_wrong_key_are_rejected-7d081bfb"\n, datanodeResult=id:"1647af9912f9a1c7092ed19a354022681449c0dad96321ee301697a646ef7add" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768634661556000 updated_at:1683768634661556000 market_id:"3c60f99f29c38ad3be07b55b049ae7fcb510541466e84859e6e006eccd44b2a4" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"686891cb74858a4b554245c659be56834c5771d8514246ada43d8eb2abd69332" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6385c50dc9efbb769b9a3cbfc31248ed55f2a8cccc7242f00bb2ee73155fc8f2" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[40]-276c9291"\nid:"4cda01385931005d3cc96781bc5b82ef845afccfea39ad587095d9e9a6ab3552" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768857409212000 updated_at:1683768857409212000 market_id:"ac586c1ad1d11735593d97c127c00d64837c6d5220da9b2844eddda07ff8f895" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2ac1b51f300615b181b359c5c81fc81d3cfd389f0d3aa17b4aa822e41916f524" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"4db83fc6389fa2a8a50cfa1c7b2309ed362b6d7af5a7b7eb76576b55af30fd1b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_behavior-81ac4f0c"\nid:"59a4669eb5335f3b8fa6c68d91cbae1e62ca7484256d93a4d9e9d5f586196e45" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770758299333000 updated_at:1683770758299333000 market_id:"9d8ac718dfa66c734e2774b2760fc2f52401bc3dc5dfa30ec3100507fae74046" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2d763e1540ef4e078f798382d57d29d0666dd8a44b6fb70d946cc7de8202b33a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"a2b9f1acf79e8d7a785d4ef241bfb816b2ccd172c9a1e0fcef6bb1a1f24628fe" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_reject_duplicated_or_filtered_out_messages-9c0800e3"\nid:"63e3054e90ee097257126cadc486f39ca383faf8184f9ff291f378fe5adcc7fa" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683772691201778000 updated_at:1683772691201778000 market_id:"dddd2690b14764ab3005aa41e50b61340489e16795a9ae4cb9678370f3ab7a27" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"75d9e6f11f1cd1e7ba3953648910e9b1a2cee3ddff5dd275ec475e7e29c2987d" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"c97ca893092dd3ea26ffe96229d952a66aa71b641db84cdd6beca8260b08900a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_disconnection-3c69bcdf"\nid:"6966f0a86498d41010af317ecb7db8a9321455f5c4958d7f64234c98c6a01190" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767647338681000 updated_at:1683767647338681000 market_id:"366db27b1cf8cf55a6ec0b45989056b8582d61f5dad471441cbfccd9ba966483" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"b27799f64df4074c22a0f126d7bbfb26550c55a8713076445ac3a6dc1eb22025" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6d9c420881367d8fdacbb4afc834d1df17262d045bfedb359e42f551d6c10cb8" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-tests/API/rest_endpoints_test.py-0060ddda"\nid:"6edde00f797a48073c93f248fc5d8c3784590679ebe5e95875088deffb8850c9" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683773056704914000 updated_at:1683773056704914000 market_id:"b8c0068c169a783e9a6ee73dfc512e089855f00d9b20a78ef6f3f23e2f48faeb" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"68e79f532062af699d94ae3969e268ef195c7f113862bd4ad906c7df30762b92" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"ef67ef53e3cfc7240a1da3242320b8cc13a8d78fa71a52e81864134a6312819e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_start_stop_restart_from_network_history-93619c69"\nid:"8b69b6a80506f8502899b5a74e25de92eccbafc63a4f60a4cd719ee64ff52dc9" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768736333362000 updated_at:1683768736333362000 market_id:"b3ea38f67a55d19ee231998f3db2d57d26b99510e2aa7700a362f26e9d8800a9" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"9bba65e0b4d462864300000fb5067fb2325dc822c2d6d95b7759d684cb6e1f34" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"9fcc0b65dd4b3a47e3b507eeea78b149a48be4843b3fd66dafa9feb546bf5ab4" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[60]-e484782f"\nid:"abcdaafc4effb5343c89613bdbb0eff70e75f845d99c01b42da56f328fe0650f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768525052419000 updated_at:1683768525052419000 market_id:"ed701eb422ba98e93fe87adba49e75b5e3b53655a56e89af9bf2bf0cf07fcb81" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"cc0e37d496e43b38e9abff3f754f3bb1f7a9b907cd3ef17e999b46b235b2fa2b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"f4714eb302b231a9264240b7a46085c2e6d6ffce6e2009dbe86614c8ecad06e6" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mark_to_market_settlement-8f7d0153"\nid:"dd03e23234cc8ee4d0aa3fd4160290157a6e79b0a6f1f39f39b436bb0ecf6f7b" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770738987019000 updated_at:1683770738987019000 market_id:"aeb9a27a135b82b3963ec53aea230848a6b896fdd231fa9e40839a544ad1beef" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"766edf5aaabbb04fe7da1ed106d69ac652e0aa050cbeb7ebc1d1ef4fad777972" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"2d4234579f5e8a60dfee98d5cede8f81f1136ec8276c6fe73c8af7a5a483e085" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_oracle_messages_signed_with_wrong_key_are_rejected-7d081bfb"\n\n
key=stake, matchResult=full match, coreLength=17, datanodeLength=17, coreResult=, datanodeResult=\n
key=transfers, matchResult=full match, coreLength=0, datanodeLength=0, coreResult=, datanodeResult=\n\n', 

stderr='/tmp/\n
completed diff for accounts with status 0\n
completed diff for orders with status 0\n
completed diff for markets with status 0\n
completed diff for parties with status 0\n
completed diff for limits with status 0\n
completed diff for assets with status 0\n
completed diff for delegations with status 0\n
completed diff for epoch with status 0\n
completed diff for vegaTime with status 2\n
completed diff for nodes with status 0\n
completed diff for netparams with status 0\n
completed diff for proposals with status 0\n
completed diff for deposits with status 0\n
completed diff for withdrawals with status 0\n
completed diff for liquidityProvisions with status 1\n
completed diff for stake with status 0\n
completed diff for transfers with status 0\n
completed diff with success? false\nError: mismatch between core and datanode: success=false, 

report:\n
key=accounts, matchResult=full match, coreLength=405, datanodeLength=323, coreResult=, datanodeResult=\n
key=orders, matchResult=full match, coreLength=58, datanodeLength=58, coreResult=, datanodeResult=\n
key=markets, matchResult=full match, coreLength=12, datanodeLength=12, coreResult=, datanodeResult=\n
key=parties, matchResult=full match, coreLength=134, datanodeLength=134, coreResult=, datanodeResult=\n
key=limits, matchResult=full match, coreLength=1, datanodeLength=1, coreResult=, datanodeResult=\n
key=assets, matchResult=full match, coreLength=21, datanodeLength=21, coreResult=, datanodeResult=\n
key=delegations, matchResult=full match, coreLength=2, datanodeLength=2, coreResult=, datanodeResult=\n
key=epoch, matchResult=full match, coreLength=1, datanodeLength=1, coreResult=, datanodeResult=\n
key=vegaTime, matchResult=mismatching values, coreLength=1, datanodeLength=1, coreResult=1683773419795709000, datanodeResult=1683773421351244000\n
key=nodes, matchResult=full match, coreLength=2, datanodeLength=2, coreResult=, datanodeResult=\n
key=netparams, matchResult=full match, coreLength=116, datanodeLength=116, coreResult=, datanodeResult=\n
key=proposals, matchResult=full match, coreLength=94, datanodeLength=94, coreResult=, datanodeResult=\n
key=deposits, matchResult=full match, coreLength=0, datanodeLength=0, coreResult=, datanodeResult=\n
key=withdrawals, matchResult=full match, coreLength=10, datanodeLength=0, coreResult=, datanodeResult=\n
key=liquidityProvisions, matchResult=mismatching number of elements, coreLength=10, datanodeLength=9, coreResult=id:"1647af9912f9a1c7092ed19a354022681449c0dad96321ee301697a646ef7add" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768634661556000 updated_at:1683768634661556000 market_id:"3c60f99f29c38ad3be07b55b049ae7fcb510541466e84859e6e006eccd44b2a4" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"686891cb74858a4b554245c659be56834c5771d8514246ada43d8eb2abd69332" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6385c50dc9efbb769b9a3cbfc31248ed55f2a8cccc7242f00bb2ee73155fc8f2" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[40]-276c9291"\nid:"346e3f6271453f149cfd8df2aff5996b8bcde173d473db6f0a3096f3d81c5c8f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767578347485000 updated_at:1683767578347485000 market_id:"50ea1a1b4c65ac836b1145f690b1965ca0d00294792bac96728959e975097191" commitment_amount:"390500000000000000" fee:"0.3" sells:{order_id:"9465e4e578ea31cf8478fd158d7aca6312f5dcce51a7e6fa1d57e74cef50729e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"1000000000000"}} buys:{order_id:"478ba09795193a8e05dfbfc0174c0fbbc9c2229aed1cd7a60012dc04738cf837" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"1000000000000"}} version:1 status:STATUS_PENDING reference:"LIQPROV-test_List_Markets-1dc663e9"\nid:"4cda01385931005d3cc96781bc5b82ef845afccfea39ad587095d9e9a6ab3552" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768857409212000 updated_at:1683768857409212000 market_id:"ac586c1ad1d11735593d97c127c00d64837c6d5220da9b2844eddda07ff8f895" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2ac1b51f300615b181b359c5c81fc81d3cfd389f0d3aa17b4aa822e41916f524" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"4db83fc6389fa2a8a50cfa1c7b2309ed362b6d7af5a7b7eb76576b55af30fd1b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_behavior-81ac4f0c"\nid:"59a4669eb5335f3b8fa6c68d91cbae1e62ca7484256d93a4d9e9d5f586196e45" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770758299333000 updated_at:1683770758299333000 market_id:"9d8ac718dfa66c734e2774b2760fc2f52401bc3dc5dfa30ec3100507fae74046" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2d763e1540ef4e078f798382d57d29d0666dd8a44b6fb70d946cc7de8202b33a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"a2b9f1acf79e8d7a785d4ef241bfb816b2ccd172c9a1e0fcef6bb1a1f24628fe" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_reject_duplicated_or_filtered_out_messages-9c0800e3"\nid:"63e3054e90ee097257126cadc486f39ca383faf8184f9ff291f378fe5adcc7fa" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683772691201778000 updated_at:1683772691201778000 market_id:"dddd2690b14764ab3005aa41e50b61340489e16795a9ae4cb9678370f3ab7a27" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"75d9e6f11f1cd1e7ba3953648910e9b1a2cee3ddff5dd275ec475e7e29c2987d" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"c97ca893092dd3ea26ffe96229d952a66aa71b641db84cdd6beca8260b08900a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_disconnection-3c69bcdf"\nid:"6966f0a86498d41010af317ecb7db8a9321455f5c4958d7f64234c98c6a01190" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767647338681000 updated_at:1683767647338681000 market_id:"366db27b1cf8cf55a6ec0b45989056b8582d61f5dad471441cbfccd9ba966483" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"b27799f64df4074c22a0f126d7bbfb26550c55a8713076445ac3a6dc1eb22025" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6d9c420881367d8fdacbb4afc834d1df17262d045bfedb359e42f551d6c10cb8" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-tests/API/rest_endpoints_test.py-0060ddda"\nid:"6edde00f797a48073c93f248fc5d8c3784590679ebe5e95875088deffb8850c9" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683773056704914000 updated_at:1683773056704914000 market_id:"b8c0068c169a783e9a6ee73dfc512e089855f00d9b20a78ef6f3f23e2f48faeb" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"68e79f532062af699d94ae3969e268ef195c7f113862bd4ad906c7df30762b92" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"ef67ef53e3cfc7240a1da3242320b8cc13a8d78fa71a52e81864134a6312819e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_start_stop_restart_from_network_history-93619c69"\nid:"8b69b6a80506f8502899b5a74e25de92eccbafc63a4f60a4cd719ee64ff52dc9" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768736333362000 updated_at:1683768736333362000 market_id:"b3ea38f67a55d19ee231998f3db2d57d26b99510e2aa7700a362f26e9d8800a9" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"9bba65e0b4d462864300000fb5067fb2325dc822c2d6d95b7759d684cb6e1f34" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"9fcc0b65dd4b3a47e3b507eeea78b149a48be4843b3fd66dafa9feb546bf5ab4" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[60]-e484782f"\nid:"abcdaafc4effb5343c89613bdbb0eff70e75f845d99c01b42da56f328fe0650f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768525052419000 updated_at:1683768525052419000 market_id:"ed701eb422ba98e93fe87adba49e75b5e3b53655a56e89af9bf2bf0cf07fcb81" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"cc0e37d496e43b38e9abff3f754f3bb1f7a9b907cd3ef17e999b46b235b2fa2b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"f4714eb302b231a9264240b7a46085c2e6d6ffce6e2009dbe86614c8ecad06e6" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mark_to_market_settlement-8f7d0153"\nid:"dd03e23234cc8ee4d0aa3fd4160290157a6e79b0a6f1f39f39b436bb0ecf6f7b" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770738987019000 updated_at:1683770738987019000 market_id:"aeb9a27a135b82b3963ec53aea230848a6b896fdd231fa9e40839a544ad1beef" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"766edf5aaabbb04fe7da1ed106d69ac652e0aa050cbeb7ebc1d1ef4fad777972" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"2d4234579f5e8a60dfee98d5cede8f81f1136ec8276c6fe73c8af7a5a483e085" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_oracle_messages_signed_with_wrong_key_are_rejected-7d081bfb"\n, datanodeResult=id:"1647af9912f9a1c7092ed19a354022681449c0dad96321ee301697a646ef7add" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768634661556000 updated_at:1683768634661556000 market_id:"3c60f99f29c38ad3be07b55b049ae7fcb510541466e84859e6e006eccd44b2a4" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"686891cb74858a4b554245c659be56834c5771d8514246ada43d8eb2abd69332" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6385c50dc9efbb769b9a3cbfc31248ed55f2a8cccc7242f00bb2ee73155fc8f2" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[40]-276c9291"\nid:"4cda01385931005d3cc96781bc5b82ef845afccfea39ad587095d9e9a6ab3552" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768857409212000 updated_at:1683768857409212000 market_id:"ac586c1ad1d11735593d97c127c00d64837c6d5220da9b2844eddda07ff8f895" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2ac1b51f300615b181b359c5c81fc81d3cfd389f0d3aa17b4aa822e41916f524" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"4db83fc6389fa2a8a50cfa1c7b2309ed362b6d7af5a7b7eb76576b55af30fd1b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_behavior-81ac4f0c"\nid:"59a4669eb5335f3b8fa6c68d91cbae1e62ca7484256d93a4d9e9d5f586196e45" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770758299333000 updated_at:1683770758299333000 market_id:"9d8ac718dfa66c734e2774b2760fc2f52401bc3dc5dfa30ec3100507fae74046" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"2d763e1540ef4e078f798382d57d29d0666dd8a44b6fb70d946cc7de8202b33a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"a2b9f1acf79e8d7a785d4ef241bfb816b2ccd172c9a1e0fcef6bb1a1f24628fe" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_reject_duplicated_or_filtered_out_messages-9c0800e3"\nid:"63e3054e90ee097257126cadc486f39ca383faf8184f9ff291f378fe5adcc7fa" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683772691201778000 updated_at:1683772691201778000 market_id:"dddd2690b14764ab3005aa41e50b61340489e16795a9ae4cb9678370f3ab7a27" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"75d9e6f11f1cd1e7ba3953648910e9b1a2cee3ddff5dd275ec475e7e29c2987d" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"c97ca893092dd3ea26ffe96229d952a66aa71b641db84cdd6beca8260b08900a" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_disconnection-3c69bcdf"\nid:"6966f0a86498d41010af317ecb7db8a9321455f5c4958d7f64234c98c6a01190" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767647338681000 updated_at:1683767647338681000 market_id:"366db27b1cf8cf55a6ec0b45989056b8582d61f5dad471441cbfccd9ba966483" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"b27799f64df4074c22a0f126d7bbfb26550c55a8713076445ac3a6dc1eb22025" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"6d9c420881367d8fdacbb4afc834d1df17262d045bfedb359e42f551d6c10cb8" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-tests/API/rest_endpoints_test.py-0060ddda"\nid:"6edde00f797a48073c93f248fc5d8c3784590679ebe5e95875088deffb8850c9" party_id:"317cb4b11454c20779db9b14f1a47b046ea86607ff4e1efca2a16d2604d6072a" created_at:1683773056704914000 updated_at:1683773056704914000 market_id:"b8c0068c169a783e9a6ee73dfc512e089855f00d9b20a78ef6f3f23e2f48faeb" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"68e79f532062af699d94ae3969e268ef195c7f113862bd4ad906c7df30762b92" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"ef67ef53e3cfc7240a1da3242320b8cc13a8d78fa71a52e81864134a6312819e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_datanode_start_stop_restart_from_network_history-93619c69"\nid:"8b69b6a80506f8502899b5a74e25de92eccbafc63a4f60a4cd719ee64ff52dc9" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768736333362000 updated_at:1683768736333362000 market_id:"b3ea38f67a55d19ee231998f3db2d57d26b99510e2aa7700a362f26e9d8800a9" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"9bba65e0b4d462864300000fb5067fb2325dc822c2d6d95b7759d684cb6e1f34" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"9fcc0b65dd4b3a47e3b507eeea78b149a48be4843b3fd66dafa9feb546bf5ab4" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mtm_frequency[60]-e484782f"\nid:"abcdaafc4effb5343c89613bdbb0eff70e75f845d99c01b42da56f328fe0650f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683768525052419000 updated_at:1683768525052419000 market_id:"ed701eb422ba98e93fe87adba49e75b5e3b53655a56e89af9bf2bf0cf07fcb81" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"cc0e37d496e43b38e9abff3f754f3bb1f7a9b907cd3ef17e999b46b235b2fa2b" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"f4714eb302b231a9264240b7a46085c2e6d6ffce6e2009dbe86614c8ecad06e6" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_mark_to_market_settlement-8f7d0153"\nid:"dd03e23234cc8ee4d0aa3fd4160290157a6e79b0a6f1f39f39b436bb0ecf6f7b" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683770738987019000 updated_at:1683770738987019000 market_id:"aeb9a27a135b82b3963ec53aea230848a6b896fdd231fa9e40839a544ad1beef" commitment_amount:"390500000000000000000000" fee:"0.3" sells:{order_id:"766edf5aaabbb04fe7da1ed106d69ac652e0aa050cbeb7ebc1d1ef4fad777972" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"100000"}} buys:{order_id:"2d4234579f5e8a60dfee98d5cede8f81f1136ec8276c6fe73c8af7a5a483e085" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"100000"}} version:1 status:STATUS_ACTIVE reference:"LIQPROV-test_oracle_messages_signed_with_wrong_key_are_rejected-7d081bfb"\n\n
key=stake, matchResult=full match, coreLength=17, datanodeLength=17, coreResult=, datanodeResult=\n
key=transfers, matchResult=full match, coreLength=0, datanodeLength=0, coreResult=, datanodeResult=\n\n

Usage:\n  vegatools difftool [flags]\n\nFlags:\n  -r, --block-height int      block-height of the snapshot to dump\n  -d, --datanode string       datanode url\n  -h, --help                  help for difftool\n  -s, --snap-db-path string   path to the goleveldb database folder\n\n')
[2023-05-11T02:50:32.065Z] FAILED
MuthuVega commented 1 year ago

This issue happens only during the full run where there are a lot of markets and liquidity provisions. If this test is run on its own , the test will pass.

Jenkins Failure:

https://jenkins.ops.vega.xyz/job/common/job/system-tests-wrapper/69948/testReport/junit/tests.datanode/datanode_test/test_datanode_compare_restored_datanode_data_against_core/

Jenkins Logs:

https://jenkins.ops.vega.xyz/job/common/job/system-tests-wrapper/69948/consoleText

In the above logs search for Datanode data diff output for to find the output.

System Tests Test name - test_datanode_compare_restored_datanode_data_against_core System Tests Marker - datanode_core_datadiff

ze97286 commented 1 year ago

this is an issue with the tool. The LP missing from datanode results (hence causing the count to mismatch) is a PENDING LP. This is caused by the tool excluding pending LPs from datanode result before the comparison. The ticket should be moved to vegatools.

MuthuVega commented 1 year ago

Thanks @ze97286

Some additional info

the one that's missing from datanode is this one:
id:"346e3f6271453f149cfd8df2aff5996b8bcde173d473db6f0a3096f3d81c5c8f" party_id:"de57914db91d4bdcb31ac406895da4d91d0c01ab652e81f1b97844c347ce5080" created_at:1683767578347485000 updated_at:1683767578347485000 market_id:"50ea1a1b4c65ac836b1145f690b1965ca0d00294792bac96728959e975097191" commitment_amount:"390500000000000000" fee:"0.3" sells:{order_id:"9465e4e578ea31cf8478fd158d7aca6312f5dcce51a7e6fa1d57e74cef50729e" liquidity_order:{reference:PEGGED_REFERENCE_BEST_ASK proportion:13 offset:"1000000000000"}} buys:{order_id:"478ba09795193a8e05dfbfc0174c0fbbc9c2229aed1cd7a60012dc04738cf837" liquidity_order:{reference:PEGGED_REFERENCE_BEST_BID proportion:2 offset:"1000000000000"}} version:1 status:STATUS_PENDING reference:"LIQPROV-test_List_Markets-1dc663e9"
[11:12](https://vegaprotocol.slack.com/archives/D02QGGS6TKK/p1684231948591849)
which is the only one that's pending. need to look at the API if it's expected to return LPs with status pending or if it doesn't is it a matter of passing different parameters to the API
[11:14](https://vegaprotocol.slack.com/archives/D02QGGS6TKK/p1684232041364109)
we filter them out in the tool, out of the result returned from datanode:
func (dnc *dataNodeClient) listLiquidityProvisions(market string) ([]*vega.LiquidityProvision, error) {
    live := true
    resp, err := dnc.datanode.ListLiquidityProvisions(context.Background(), &dn.ListLiquidityProvisionsRequest{MarketId: &market, Live: &live})
    if err != nil {
        return nil, err
    }
    lps := make([]*vega.LiquidityProvision, 0, len(resp.LiquidityProvisions.Edges))
    for _, lpe := range resp.LiquidityProvisions.Edges {
        if lpe.Node.Status == vega.LiquidityProvision_STATUS_ACTIVE || lpe.Node.Status == vega.LiquidityProvision_STATUS_UNDEPLOYED {
            lps = append(lps, lpe.Node)
        }
    }
    return lps, nil
}
gordsport commented 1 year ago

closed by:

https://github.com/vegaprotocol/vegatools/pull/283