bug(bmr): error while comparing receipt hash

[manishbista28]

Overview

Current implementation of snow bmr receiver computes receipt hash and compares it with the value received on block header. These values should be equal but, as observed, these values can be unequal. Until the block receipt hash verification succeeds, relay won't be able to process further blocks and will become stuck.

Steps to Reproduce

Steps to reproduce the behavior:

1. On Snow mainnet, process block receipts for height 238516. Hash comparison works
2. On Snow mainnet, process block receipts for height 244743 and 244834. Hash comparison fails.

Expected Behavior

Block receipts hash comparison should work as expected to verify that the receipts of a BTP message when hashed is equal to the receiptHash value observed on a verified block header.

Additional Context

Error Triggered on:

q.v.Receipts, q.err = r.client().GetBlockReceiptsFromHeight(q.v.Height)
if q.err == nil {
    receiptsRoot := ethTypes.DeriveSha(q.v.Receipts, trie.NewStackTrie(nil))
    if !bytes.Equal(receiptsRoot.Bytes(), q.v.Header.ReceiptHash.Bytes()) {
        q.err = fmt.Errorf(
            "invalid receipts: remote=%v, local=%v",
            q.v.Header.ReceiptHash, receiptsRoot)
    }
}

Unit Test Code to Verify Error

func TestReceiver_GetReceiptProofs(t *testing.T) {
    cl := newTestClient(t, "")
    h, err := cl.GetHeaderByHeight(big.NewInt(244743))
    require.NoError(t, err)
    fmt.Println("Header ", h)
    receipts, err := cl.GetBlockReceiptsFromHeight(big.NewInt(244743))
    require.NoError(t, err)
    fmt.Println(receipts)
    receiptsRoot := ethTypes.DeriveSha(receipts, trie.NewStackTrie(nil))
    if !bytes.Equal(receiptsRoot.Bytes(), h.ReceiptHash.Bytes()) {
        err = fmt.Errorf(
            "invalid receipts: remote=%v, local=%v",
            h.ReceiptHash, receiptsRoot)
        require.NoError(t, err)
    }
}

func newTestClient(t *testing.T, bmcAddr string) IClient {
    url := ""  // use valid RPC endpoint
    cls, _, err := newClients([]string{url}, "", log.New())
    require.NoError(t, err)
    return cls[0]
}

Error Log:

D|17:02:47.363806|hx20|snow|s2i|rx_receiver.go:343 receiveLoop: bnq: h=244743:0xd1cc1f9591de7ba963832f1668fe12e7dcdf22f19ee6e7c38c316e0eceed3c8f, GetBlockReceipts: invalid receipts: remote=0x87586f27b559afaf3cf30d09711485c95cf6c07782e355fdd4fb2af2f9588dc5, local=0xaca3f4847473788316c81ccb20cdb9189eaa863e1daa35871b0ff93d369075f3: invalid receipts: remote=0x87586f27b559afaf3cf30d09711485c95cf6c07782e355fdd4fb2af2f9588dc5, local=0xaca3f4847473788316c81ccb20cdb9189eaa863e1daa35871b0ff93d369075f3

D|17:02:40.233149|hx20|snow|s2i|rx_receiver.go:343 receiveLoop: bnq: h=244834:0x24d7febab05494d2a4e4f1cb2bb6d8cc719848ee88817edd68ea6bb0f320074b, GetBlockReceipts: invalid receipts: remote=0xc345a0b647242c98e54e7fcdef7a42b6a63ba6772da51c9e44d3e13939b2694c, local=0x65dd8875de7cefd02ea048a2e1939f3f1952269a794b2ed741e6ed4204dd63f9: invalid receipts: remote=0xc345a0b647242c98e54e7fcdef7a42b6a63ba6772da51c9e44d3e13939b2694c, local=0x65dd8875de7cefd02ea048a2e1939f3f1952269a794b2ed741e6ed4204dd63f9

[manishbista28]

@pragyanshrestha-ibriz -- please check.

[manishbista28]

List of Transaction Hash that passed

0x016a15a6f2110f3ee12cf585fd365312f93264a40863406aae684bea37a6421e
0x7760223b3d7e8400387bf6e7502edd2018366ad93d44ded08f15c8bc0d11f5b3
0x13491d3d80c8f82c1af50014cff9b2d3580e07e36c438727cd4f7f4a7e1079c6
0xcd29b03de77a67e337c538db32ad840f87e78ac6b88e816f26e78f9c1f9f39fe
0x52fa3c4e771d5b6a515d8a42650bae298c025e1bd5809f46e4407fd66ff62e14
0x99ff240c46d2222d508d68cfaa69b234d273d4192835d90373c2b7742acbf29d
0xfafd7287572db921981922599964c9601d460ebc2eabfaa41e8ceb3cc7c120ca
0x634f0ee5afc2f21af30fcaa59dff3db8b675f0fe174c32002b7d9e95e21dff50
0x7e2848b0179c3f76ed6a14fb873db04eb169bb7da3a93de627e245cae8eff365
0x05d9047f046f0ebcd306470ac81a4f05f96576825faedae0aee4cdcaa7daeef1
0xdebc9ab030ba79d1a5b57cc972e17280f6e622e72396d2e8426a5e621f410601
0x2bb819fb1b89edca9882e6531d7bd0e074d5be4cf6da68b46f5430b0807dc46a
0x2f3f3763bc21eca71fefd35143fa5b066e9f64cd5770a697bf2b4c871e75cd82
0xc8833c6b3cfc425ca1a494fb7005f7fc52a12e13c5cae276be56e1ef1ac050f4

[manishbista28]

The following script fetches blocks and displays result where if the receipt hash comparison is not equal, then transaction type (EIP559 => 2, Legacy => 0) and block height is displayed

func TestParallelFetch(t *testing.T) {
    start := 0
    end := 245210 //latest block
    concurrency := 250
    err := parallelFetch(start, end, concurrency)
    if err != nil {
        t.Fatal(fmt.Errorf("parallelTransfers %v", err))
    }
}

func parallelFetch(start, end, concurrency int) error {
    url := "" //enter valid rpc endpoint here
    cls, _, err := newClients([]string{url}, "", log.New())
    if err != nil {
        return err
    }
    cl := cls[0]
    zeroInt := 0
    reqCursor := 0
    lenRequests := end - start
    type req struct {
        height  int
        err     error
        res     *int
        txnType int
    }

    for reqCursor < lenRequests {
        rqch := make(chan *req, concurrency)
        for i := reqCursor; len(rqch) < cap(rqch) && i < lenRequests; i++ {
            rqch <- &req{height: start + i, err: nil, res: nil}
            reqCursor++
        }
        sres := make([]*req, 0, len(rqch))
        for q := range rqch {
            switch {
            case q.err != nil || q.res != nil:
                sres = append(sres, q)
                if len(sres) == cap(sres) {
                    close(rqch)
                }
            default:
                go func(q *req) {
                    defer func() {
                        time.Sleep(time.Millisecond * 20)
                        //fmt.Println("reqCursor ", q.height, q.err, *q.res, q.txnType)
                        rqch <- q
                    }()
                    q.res = &zeroInt
                    h, err := cl.GetHeaderByHeight(big.NewInt(int64(q.height)))
                    if err != nil {
                        q.err = fmt.Errorf("GetHeaderByHeight %v err %v", q.height, err)
                        return
                    }
                    rcpts, err := cl.GetBlockReceiptsFromHeight(big.NewInt(int64(q.height)))
                    if err != nil {
                        q.err = fmt.Errorf("GetBlockReceiptsFromHeight %v err %v", q.height, err)
                        return
                    }
                    receiptsRoot := ethTypes.DeriveSha(rcpts, trie.NewStackTrie(nil))
                    if receiptsRoot != h.ReceiptHash {
                        q.res = &q.height
                        body, err := cl.GetEthClient().BlockByNumber(context.TODO(), big.NewInt(int64(q.height)))
                        if err != nil {
                            q.err = fmt.Errorf("BlockByNumber %v err %v", q.height, err)
                            return
                        }
                        for _, txn := range body.Transactions() {
                            if txn.Type() != 0 {
                                q.txnType = int(txn.Type())
                                break
                            }
                        }
                    }
                }(q)
            }
        }
        for _, sr := range sres {
            if sr.err != nil || *sr.res != 0 {
                fmt.Println("Mark ", sr.txnType, sr.height, sr.err)
            }
        }
        if reqCursor%1000 == 0 {
            fmt.Println("cursor ", reqCursor)
        }
    }
    return nil
}

[manishbista28]

Out of blocks 1 through 247550 (latest at the time of observation), the problem is present in the following blocks. All of the following blocks include a type 2(EIP-1559) transaction.

154975,154991,155018,155022,160373,160379,160593,160591,160672,160524,160653,161314,161315,161306,161394,161304,161310,161312,161324,161537,161534,161986,177865,177867,224226,224131,224079,224145,224087,224306,242942,243134,243068,244741,244743,244834,244826,245179,245141,245204,245207,245138

The script above was used to fetch the information

[manishbista28]

A workaround has been placed to perform receipt hash comparison only if the block does not include an EIP-1559 transaction.