QSCC GetBlockByHash fails with marshal error "string field contains invalid UTF-8"

[samuelvenzi]

Description

Using qscc to get block by hash using the GetBlockByHash transaction. I am getting the following error as logged by the peer.

failed to invoke chaincode qscc, error: transaction returned with failure: failed to marshal response: string field contains invalid UTF-8

I'm querying it using the Gateway SDK for Go like this (some syntax supressed for clarity):

hashBytes, _ := hex.DecodeString(hash)
args := []string{channelName, string(hashBytes)}

contract.EvaluateTransaction("GetBlockByHash", args...)

Before the error the peer also logs:

2024-10-02 19:05:55.754 UTC 80fa DEBU [blkstorage] retrieveBlockByHash -> retrieveBlockByHash() - blockHash = [[]byte{0x38, 0xe5, 0x3a, 0xba, 0x44, 0x71, 0xa1, 0x1b, 0x78, 0xd7, 0x21, 0x1c, 0xc2, 0x73, 0x56, 0x45, 0xe9, 0x96, 0xee, 0xd8, 0xaf, 0x41, 0xca, 0xf, 0x3, 0x7, 0xc2, 0xb9, 0xa1, 0x1f, 0xbe, 0x70}]
2024-10-02 19:05:55.754 UTC 80fb DEBU [chaincode] handleMessage -> [448dfb03] Fabric side handling ChaincodeMessage of type: ERROR in state ready

One thing to note is that GetBlockByNumber works fine.

This has been observed in my testing of v2.5.3 and v2.5.10.

Steps to reproduce

This has been reproducible locally with Fabric Samples' test-network, on Hyperledger Labs CC-Tools and in production environments. To reproduce it with the test-network, follow the steps.

1. Deploy the test network in Fabric Samples

cd test-network
./network.sh up createChannel 
./network.sh deployCC

2. Assuming you are on Fabric Samples, replace asset-transfer-basic/application-gateway-go/assetTransfer.go with the following content:

/*
Copyright 2021 IBM All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package main

import (
    "crypto/x509"
    "encoding/hex"
    "fmt"
    "os"
    "path"
    "time"

    "github.com/hyperledger/fabric-gateway/pkg/client"
    "github.com/hyperledger/fabric-gateway/pkg/identity"
    protos "github.com/hyperledger/fabric-protos-go-apiv2/common"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials"
    "google.golang.org/protobuf/proto"
)

const (
    mspID        = "Org1MSP"
    cryptoPath   = "../../test-network/organizations/peerOrganizations/org1.example.com"
    certPath     = cryptoPath + "/users/User1@org1.example.com/msp/signcerts"
    keyPath      = cryptoPath + "/users/User1@org1.example.com/msp/keystore"
    tlsCertPath  = cryptoPath + "/peers/peer0.org1.example.com/tls/ca.crt"
    peerEndpoint = "dns:///localhost:7051"
    gatewayPeer  = "peer0.org1.example.com"
)

func main() {
    // The gRPC client connection should be shared by all Gateway connections to this endpoint
    clientConnection := newGrpcConnection()
    defer clientConnection.Close()

    id := newIdentity()
    sign := newSign()

    // Create a Gateway connection for a specific client identity
    gw, err := client.Connect(
        id,
        client.WithSign(sign),
        client.WithClientConnection(clientConnection),
        // Default timeouts for different gRPC calls
        client.WithEvaluateTimeout(5*time.Second),
        client.WithEndorseTimeout(15*time.Second),
        client.WithSubmitTimeout(5*time.Second),
        client.WithCommitStatusTimeout(1*time.Minute),
    )
    if err != nil {
        panic(err)
    }
    defer gw.Close()

    // Override default values for chaincode and channel name as they may differ in testing contexts.
    chaincodeName := "qscc"
    if ccname := os.Getenv("CHAINCODE_NAME"); ccname != "" {
        chaincodeName = ccname
    }

    channelName := "mychannel"
    if cname := os.Getenv("CHANNEL_NAME"); cname != "" {
        channelName = cname
    }

    network := gw.GetNetwork(channelName)
    contract := network.GetContract(chaincodeName)

    block, _ := contract.EvaluateTransaction("GetBlockByNumber", "mychannel", "3")

    blockMap, _ := decodeBlock(block)

    blockHash := blockMap["header"].(map[string]interface{})["data_hash"].(string)

    fmt.Println("Block hash:", blockHash)

    hashBytes, _ := hex.DecodeString(blockHash)

    _, err = contract.EvaluateTransaction("GetBlockByHash", "mychannel", string(hashBytes))
    if err != nil {
        panic(fmt.Errorf("failed to evaluate transaction: %w", err))
    }

}

func decodeBlock(b []byte) (map[string]interface{}, error) {
    var block protos.Block

    err := proto.Unmarshal(b, &block)
    if err != nil {
        return nil, err
    }

    blockMap := map[string]interface{}{
        "header": map[string]interface{}{
            "number":        block.Header.Number,
            "previous_hash": hex.EncodeToString(block.Header.PreviousHash),
            "data_hash":     hex.EncodeToString(block.Header.DataHash),
        },
        "metadata": block.Metadata,
    }
    return blockMap, nil
}

// newGrpcConnection creates a gRPC connection to the Gateway server.
func newGrpcConnection() *grpc.ClientConn {
    certificatePEM, err := os.ReadFile(tlsCertPath)
    if err != nil {
        panic(fmt.Errorf("failed to read TLS certifcate file: %w", err))
    }

    certificate, err := identity.CertificateFromPEM(certificatePEM)
    if err != nil {
        panic(err)
    }

    certPool := x509.NewCertPool()
    certPool.AddCert(certificate)
    transportCredentials := credentials.NewClientTLSFromCert(certPool, gatewayPeer)

    connection, err := grpc.NewClient(peerEndpoint, grpc.WithTransportCredentials(transportCredentials))
    if err != nil {
        panic(fmt.Errorf("failed to create gRPC connection: %w", err))
    }

    return connection
}

// newIdentity creates a client identity for this Gateway connection using an X.509 certificate.
func newIdentity() *identity.X509Identity {
    certificatePEM, err := readFirstFile(certPath)
    if err != nil {
        panic(fmt.Errorf("failed to read certificate file: %w", err))
    }

    certificate, err := identity.CertificateFromPEM(certificatePEM)
    if err != nil {
        panic(err)
    }

    id, err := identity.NewX509Identity(mspID, certificate)
    if err != nil {
        panic(err)
    }

    return id
}

// newSign creates a function that generates a digital signature from a message digest using a private key.
func newSign() identity.Sign {
    privateKeyPEM, err := readFirstFile(keyPath)
    if err != nil {
        panic(fmt.Errorf("failed to read private key file: %w", err))
    }

    privateKey, err := identity.PrivateKeyFromPEM(privateKeyPEM)
    if err != nil {
        panic(err)
    }

    sign, err := identity.NewPrivateKeySign(privateKey)
    if err != nil {
        panic(err)
    }

    return sign
}

func readFirstFile(dirPath string) ([]byte, error) {
    dir, err := os.Open(dirPath)
    if err != nil {
        return nil, err
    }

    fileNames, err := dir.Readdirnames(1)
    if err != nil {
        return nil, err
    }

    return os.ReadFile(path.Join(dirPath, fileNames[0]))
}

3. Under asset-transfer-basic/application-gateway-go/ run

go mod tidy
go mod vendor

4. Run the application

go run .

The application will GetBlockByNumber with 3 as the number for the block, then it will decode it partially so as to get the hash of the block, and then attempt to fetch the block by hash. This should throw a panic.

panic: failed to evaluate transaction: rpc error: code = Aborted desc = failed to evaluate transaction, see attached details for more info

6. Check the peer logs

2024-10-02 22:25:40.176 UTC 0113 ERRO [endorser] simulateProposal -> failed to invoke chaincode qscc, error: transaction returned with failure: failed to marshal response: string field contains invalid UTF-8
github.com/hyperledger/fabric/core/chaincode.processChaincodeExecutionResult
        /core/chaincode/chaincode_support.go:188
github.com/hyperledger/fabric/core/chaincode.(*ChaincodeSupport).Execute
        /core/chaincode/chaincode_support.go:162
github.com/hyperledger/fabric/core/endorser.(*SupportImpl).Execute
        /core/endorser/support.go:126
github.com/hyperledger/fabric/core/endorser.(*Endorser).callChaincode
        /core/endorser/endorser.go:120
github.com/hyperledger/fabric/core/endorser.(*Endorser).simulateProposal
        /core/endorser/endorser.go:187
github.com/hyperledger/fabric/core/endorser.(*Endorser).ProcessProposalSuccessfullyOrError
        /core/endorser/endorser.go:409
github.com/hyperledger/fabric/core/endorser.(*Endorser).ProcessProposal
        /core/endorser/endorser.go:350
github.com/hyperledger/fabric/core/handlers/auth/filter.(*expirationCheckFilter).ProcessProposal
        /core/handlers/auth/filter/expiration.go:61
github.com/hyperledger/fabric/core/handlers/auth/filter.(*filter).ProcessProposal
        /core/handlers/auth/filter/filter.go:32
github.com/hyperledger/fabric/internal/pkg/gateway.(*EndorserServerAdapter).ProcessProposal
        /internal/pkg/gateway/gateway.go:43
github.com/hyperledger/fabric/internal/pkg/gateway.(*Server).Evaluate.func1
        /internal/pkg/gateway/evaluate.go:64
runtime.goexit
        /usr/local/go/src/runtime/asm_arm64.s:1223 channel=mychannel txID=54e6bb0b

[samuelvenzi]

Upon further investigation @Tubar2 has discovered that the data_hash that we were using to try and get the block is not the hash needed for the function.

We got the function to work correctly by going inside the peer chains folder, opening up the LevelDB data that indexes each block with the hash. With that hash (which is not the data_hash), it works.

The question is: how do I get the correct hash since it doesn't seem to be in the block and taking the SHA256 of the block bytes does not return an existing hash. The answer is that the hash should be taken from the header of the block. Like this:

func BlockHeaderHash(b *protos.BlockHeader) []byte {
    sum := sha256.Sum256(BlockHeaderBytes(b))
    return sum[:]
}

type asn1Header struct {
    Number       *big.Int
    PreviousHash []byte
    DataHash     []byte
}

func BlockHeaderBytes(b *protos.BlockHeader) []byte {
    asn1Header := asn1Header{
        PreviousHash: b.PreviousHash,
        DataHash:     b.DataHash,
        Number:       new(big.Int).SetUint64(b.Number),
    }
    result, err := asn1.Marshal(asn1Header)
    if err != nil {
        // Errors should only arise for types which cannot be encoded, since the
        // BlockHeader type is known a-priori to contain only encodable types, an
        // error here is fatal and should not be propagated
        panic(err)
    }
    return result
}

Because of that, I'll close the issue. Hopefully, this serves for anyone that thinks that data_hash is the actual hash the block is indexed by.