search

attestantio / vouch

Apache License 2.0
112 stars 28 forks source link

Missing attestations without obvious reason #8

Closed stefa2k closed 3 years ago

stefa2k commented 3 years ago

As discussed with @mcdee on Discord there are some missed attestations on https://pyrmont.beaconcha.in/validator/100100#attestations which I can't explain. They start after updating vouch to 1.0.0 and in the case of the missed attestation on slot 74144 here are the logs: vouch.log

I think the relevant part is this:

vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","slot":74144,"time":"2020-11-28T19:08:57Z","message":"Received beacon chain head"}
vouch         | {"level":"trace","service":"controller","impl":"standard","slot":74144,"time":"2020-11-28T19:08:57Z","message":"Received head event"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","time":"2020-11-28T19:08:58Z","message":"Calling GetDuties()"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","epoch":2317,"time":"2020-11-28T19:08:58Z","message":"Fetching epoch validators"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","time":"2020-11-28T19:08:58Z","message":"Calling ListValidators()"}
vouch         | {"level":"trace","service":"controller","impl":"standard","elapsed":2775.238476,"duties":0,"time":"2020-11-28T19:08:58Z","message":"Fetched proposer duties"}
vouch         | {"level":"trace","service":"controller","impl":"standard","elapsed":2775.936604,"duties":0,"time":"2020-11-28T19:08:58Z","message":"Filtered proposer duties"}
vouch         | {"level":"trace","service":"controller","impl":"standard","elapsed":2779.136312,"time":"2020-11-28T19:08:58Z","message":"Scheduled beacon block proposals"}
vouch         | {"level":"trace","service":"controller","impl":"standard","slot":74144,"time":"2020-11-28T19:08:58Z","message":"Kicking off attestations for slot early due to receiving relevant block"}
vouch         | {"level":"trace","service":"scheduler","impl":"basic","job":"Beacon block attestations for slot 74144","scheduled":"2020-11-28 19:08:59.2 +0000 UTC","time":"2020-11-28T19:08:58Z","message":"Run triggered; job running"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"25a7f01d","endpoint":"/eth/v1/validator/attestation_data?slot=74144&committee_index=15","time":"2020-11-28T19:08:58Z","message":"GET request"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","time":"2020-11-28T19:08:58Z","message":"Calling GetAttestationData()"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","time":"2020-11-28T19:08:58Z","message":"Calling GetAttestationData()"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"6fdc67d2","endpoint":"/eth/v1/validator/attestation_data?slot=74144&committee_index=15","time":"2020-11-28T19:08:58Z","message":"GET request"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"25a7f01d","response":"{\"data\":{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x705fa1416f14273d6683834392c9ab962cba11bfd218dd53a871a954eca9e22b\",\"source\":{\"epoch\":\"2316\",\"root\":\"0x4ed29ff3ac49bdcd4efbb72e3741a3e150dc8bcca5289b0c8a706c3490cd117a\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x705fa1416f14273d6683834392c9ab962cba11bfd218dd53a871a954eca9e22b\"}}}","time":"2020-11-28T19:08:58Z","message":"GET response"}
vouch         | {"level":"trace","strategy":"attestationdata","impl":"best","elapsed":27.888959,"time":"2020-11-28T19:08:58Z","message":"Obtained attestation data"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"651fe010","endpoint":"/eth/v1/beacon/headers/0x705fa1416f14273d6683834392c9ab962cba11bfd218dd53a871a954eca9e22b","time":"2020-11-28T19:08:58Z","message":"GET request"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","attestation_data":"{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x705fa1416f14273d6683834392c9ab962cba11bfd218dd53a871a954eca9e22b\",\"source\":{\"epoch\":\"2316\",\"root\":\"0x4ed29ff3ac49bdcd4efbb72e3741a3e150dc8bcca5289b0c8a706c3490cd117a\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x705fa1416f14273d6683834392c9ab962cba11bfd218dd53a871a954eca9e22b\"}}","time":"2020-11-28T19:08:58Z","message":"Attestation data"}
vouch         | {"level":"trace","strategy":"attestationdata","impl":"best","elapsed":28.906216,"time":"2020-11-28T19:08:58Z","message":"Obtained attestation data"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","attestation_data":"{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x705fa1416f14273d6683834392c9ab962cba11bfd218dd53a871a954eca9e22b\",\"source\":{\"epoch\":\"2316\",\"root\":\"0x4ed29ff3ac49bdcd4efbb72e3741a3e150dc8bcca5289b0c8a706c3490cd117a\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x705fa1416f14273d6683834392c9ab962cba11bfd218dd53a871a954eca9e22b\"}}","time":"2020-11-28T19:08:58Z","message":"Attestation data"}
vouch         | {"level":"trace","strategy":"attestationdata","impl":"best","elapsed":38.723279,"time":"2020-11-28T19:08:58Z","message":"Obtained attestation data"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"651fe010","response":"{\"data\":{\"root\":\"0x705fa1416f14273d6683834392c9ab962cba11bfd218dd53a871a954eca9e22b\",\"canonical\":true,\"header\":{\"message\":{\"slot\":\"74144\",\"proposer_index\":\"100912\",\"parent_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"state_root\":\"0x6cbf169947dadceaeb4190ad3ff54a152fd486b9485b1f7e2e932132033bb10c\",\"body_root\":\"0x160ec54a886cc2f3771eefe6f3a851904b56f28d34b0818e436bfa65034773e5\"},\"signature\":\"0x83f13700a368731e25659013a1e6667589ef5239d39a15f485e0fd63dfc9bd3833a049300d46eb7f4f4dd5d531e9e6241032d655023e623e6ebae06f1eac1f646638568310e1ccbab3cc8dfbd4492e290f3b329ec9151144a7c2daede6ce5662\"}}}","time":"2020-11-28T19:08:58Z","message":"GET response"}
vouch         | {"level":"trace","strategy":"attestationdata","impl":"best","provider":"eth2-test-node-b:5052","attestation_slot":74144,"head_slot":74144,"score":1,"time":"2020-11-28T19:08:58Z","message":"Scored attestation data"}
vouch         | {"level":"trace","strategy":"attestationdata","impl":"best","provider":"eth2-test-node-b:4000","attestation_slot":74144,"head_slot":74144,"score":1,"time":"2020-11-28T19:08:58Z","message":"Scored attestation data"}
vouch         | {"level":"trace","strategy":"attestationdata","impl":"best","provider":"eth2-test-node-b:4001","attestation_slot":74144,"head_slot":74144,"score":1,"time":"2020-11-28T19:08:58Z","message":"Scored attestation data"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"6fdc67d2","response":"{\"data\":{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"source\":{\"epoch\":\"2315\",\"root\":\"0x718041e14c75905b86384733d8923d18b3b70c695a93a0890b06625782b81775\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\"}}}","time":"2020-11-28T19:08:58Z","message":"GET response"}
vouch         | {"level":"trace","strategy":"attestationdata","impl":"best","elapsed":171.57718,"time":"2020-11-28T19:08:58Z","message":"Obtained attestation data"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"2c0b394a","endpoint":"/eth/v1/beacon/headers/0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2","time":"2020-11-28T19:08:58Z","message":"GET request"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"2c0b394a","response":"{\"data\":{\"root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"canonical\":true,\"header\":{\"message\":{\"slot\":\"74143\",\"proposer_index\":\"40987\",\"parent_root\":\"0x6c8092950b8b6c0391381e9ca3dad40d00cedaa11f3b08569bf1d6091f9626cf\",\"state_root\":\"0xf7911f2e0933e40ae3c4f0943ff5f67319a48a9d85aba0f90bdec3461a5584b4\",\"body_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\"},\"signature\":\"0xafb170f54503f799b2b3bb3a085c1c3d5ffe76a2bd646387d1f8e25e9746a99f9355ad048c3873a809b0216076108d4e11109935f0359acfb5b7d56ec7484edab9a1701e692458bb439cdc2482f45ae3ccaa1874e81223b289108197e7344e30\"}}}","time":"2020-11-28T19:08:58Z","message":"GET response"}
vouch         | {"level":"trace","strategy":"attestationdata","impl":"best","provider":"eth2-test-node-b:5051","attestation_slot":74144,"head_slot":74143,"score":0.5,"time":"2020-11-28T19:08:58Z","message":"Scored attestation data"}
vouch         | {"level":"trace","service":"attester","impl":"standard","slot":74144,"validator_indices":[100100],"elapsed":210.815401,"time":"2020-11-28T19:08:58Z","message":"Obtained attestation data"}
vouch         | {"level":"trace","service":"attester","impl":"standard","slot":74144,"validator_indices":[100100],"elapsed":210.950987,"time":"2020-11-28T19:08:58Z","message":"Obtained validating accounts"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","time":"2020-11-28T19:08:58Z","message":"Calling DomainData()"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","time":"2020-11-28T19:08:58Z","message":"Calling ListValidators()"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","epoch":2317,"domain":"0x01000000","signature_domain":"0x010000003b088795c48775dffda8f6bacceaa9a46df68510af3846589abd56bb","time":"2020-11-28T19:08:58Z","message":"Signature domain obtained"}
vouch         | {"level":"trace","service":"attester","impl":"standard","elapsed":247.0744,"time":"2020-11-28T19:08:58Z","message":"Signed"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"6de1542c","endpoint":"/eth/v1/beacon/pool/attestations","body":"[{\"aggregation_bits\":\"0x0000000000080000000000000000000010\",\"data\":{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"source\":{\"epoch\":\"2315\",\"root\":\"0x718041e14c75905b86384733d8923d18b3b70c695a93a0890b06625782b81775\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\"}},\"signature\":\"0xa9c45377569af80392d5ce1caa07c2b875d794ecd79c676b1b609a4e72919796f089eb1995756f2f3292632e4957388e09b1790f1b9e9233002896278ffe460b0b8f5a628dd713eb14880782b4a6394f3d804d6c675754f8eec95873242e1976\"}]","time":"2020-11-28T19:08:58Z","message":"POST request"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"6a965449","endpoint":"/eth/v1/beacon/pool/attestations","body":"[{\"aggregation_bits\":\"0x0000000000080000000000000000000010\",\"data\":{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"source\":{\"epoch\":\"2315\",\"root\":\"0x718041e14c75905b86384733d8923d18b3b70c695a93a0890b06625782b81775\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\"}},\"signature\":\"0xa9c45377569af80392d5ce1caa07c2b875d794ecd79c676b1b609a4e72919796f089eb1995756f2f3292632e4957388e09b1790f1b9e9233002896278ffe460b0b8f5a628dd713eb14880782b4a6394f3d804d6c675754f8eec95873242e1976\"}]","time":"2020-11-28T19:08:58Z","message":"POST request"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","time":"2020-11-28T19:08:58Z","message":"Calling ProposeAttestation()"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"6a965449","response":"","time":"2020-11-28T19:08:58Z","message":"GET response"}
vouch         | {"level":"trace","strategy":"submitter","impl":"multinode","beacon_node_address":"eth2-test-node-b:5051","slot":74144,"data":"[{\"aggregation_bits\":\"0x0000000000080000000000000000000010\",\"data\":{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"source\":{\"epoch\":\"2315\",\"root\":\"0x718041e14c75905b86384733d8923d18b3b70c695a93a0890b06625782b81775\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\"}},\"signature\":\"0xa9c45377569af80392d5ce1caa07c2b875d794ecd79c676b1b609a4e72919796f089eb1995756f2f3292632e4957388e09b1790f1b9e9233002896278ffe460b0b8f5a628dd713eb14880782b4a6394f3d804d6c675754f8eec95873242e1976\"}]","time":"2020-11-28T19:08:58Z","message":"Submitted attestations"}
vouch         | {"level":"trace","service":"client","impl":"prysmgrpc","time":"2020-11-28T19:08:58Z","message":"Calling ProposeAttestation()"}
vouch         | {"level":"trace","service":"client","impl":"standardv1","id":"6de1542c","response":"null","time":"2020-11-28T19:08:58Z","message":"GET response"}
vouch         | {"level":"trace","strategy":"submitter","impl":"multinode","beacon_node_address":"eth2-test-node-b:5052","slot":74144,"data":"[{\"aggregation_bits\":\"0x0000000000080000000000000000000010\",\"data\":{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"source\":{\"epoch\":\"2315\",\"root\":\"0x718041e14c75905b86384733d8923d18b3b70c695a93a0890b06625782b81775\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\"}},\"signature\":\"0xa9c45377569af80392d5ce1caa07c2b875d794ecd79c676b1b609a4e72919796f089eb1995756f2f3292632e4957388e09b1790f1b9e9233002896278ffe460b0b8f5a628dd713eb14880782b4a6394f3d804d6c675754f8eec95873242e1976\"}]","time":"2020-11-28T19:08:58Z","message":"Submitted attestations"}
vouch         | {"level":"trace","strategy":"submitter","impl":"multinode","beacon_node_address":"eth2-test-node-b:4000","slot":74144,"data":"[{\"aggregation_bits\":\"0x0000000000080000000000000000000010\",\"data\":{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"source\":{\"epoch\":\"2315\",\"root\":\"0x718041e14c75905b86384733d8923d18b3b70c695a93a0890b06625782b81775\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\"}},\"signature\":\"0xa9c45377569af80392d5ce1caa07c2b875d794ecd79c676b1b609a4e72919796f089eb1995756f2f3292632e4957388e09b1790f1b9e9233002896278ffe460b0b8f5a628dd713eb14880782b4a6394f3d804d6c675754f8eec95873242e1976\"}]","time":"2020-11-28T19:08:58Z","message":"Submitted attestations"}
vouch         | {"level":"trace","strategy":"submitter","impl":"multinode","beacon_node_address":"eth2-test-node-b:4001","slot":74144,"data":"[{\"aggregation_bits\":\"0x0000000000080000000000000000000010\",\"data\":{\"slot\":\"74144\",\"index\":\"15\",\"beacon_block_root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\",\"source\":{\"epoch\":\"2315\",\"root\":\"0x718041e14c75905b86384733d8923d18b3b70c695a93a0890b06625782b81775\"},\"target\":{\"epoch\":\"2317\",\"root\":\"0x84f9acdba5492ffae6981b6c2ab478f8c4f5519bba46fe025ced90e9c8b3bbc2\"}},\"signature\":\"0xa9c45377569af80392d5ce1caa07c2b875d794ecd79c676b1b609a4e72919796f089eb1995756f2f3292632e4957388e09b1790f1b9e9233002896278ffe460b0b8f5a628dd713eb14880782b4a6394f3d804d6c675754f8eec95873242e1976\"}]","time":"2020-11-28T19:08:58Z","message":"Submitted attestations"}
vouch         | {"level":"trace","service":"attester","impl":"standard","elapsed":304.807387,"time":"2020-11-28T19:08:58Z","message":"Submitted attestations"}
vouch         | {"level":"trace","service":"controller","impl":"standard","elapsed":304.902045,"time":"2020-11-28T19:08:58Z","message":"Attested"}
vouch         | {"level":"trace","service":"scheduler","impl":"basic","job":"Beacon block attestations for slot 74144","scheduled":"2020-11-28 19:08:59.2 +0000 UTC","time":"2020-11-28T19:08:58Z","message":"Job complete"}