feature request - construct block from parameters

[kumavis]

would like to construct blocks from an options hash, not just raw params Edit: you can do this but not for the blockHeader

[kumavis]

something like

var block = new Block({
 coinbase: '...',
 blockNumber: '...',
 timeStamp: '...',
})

[kumavis]

rpc serializes blocks this way

curl -H "Content-Type: application/json" -X POST -d '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["pending", true],"id":1}' https://rpc.metamask.io | jsonogram

├─ id: 1
├─ jsonrpc: 2.0
└─ result
   ├─ number: 0x9b924
   ├─ hash: 0x12b82561ed2fe63f306b5178a545622512d74e3aed9fd53500484bb3e2c8335b
   ├─ parentHash: 0x5b374e2ef631678e6d7bc6d299c307c69c9d16c672e926d94144a06936c81ff9
   ├─ nonce: 0x0000000000000000
   ├─ sha3Uncles: 0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347
   ├─ logsBloom: 0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
   ├─ transactionsRoot: 0x859cc732cb10d167b5785a6b6bad2027b8c95a29a2cd9b4634a134f4dffc4954
   ├─ stateRoot: 0x0000000000000000000000000000000000000000000000000000000000000000
   ├─ receiptRoot: 0xe79abd911261842da202ca39dd506add61a29c4b9498c06e8442a313f864eef6
   ├─ miner: 0x0000000000000000000000000000000000000000
   ├─ difficulty: 0x6f028493870
   ├─ totalDifficulty: 0x0
   ├─ size: 0x291
   ├─ extraData: 0xd783010400844765746887676f312e352e31856c696e7578
   ├─ gasLimit: 0x2fefd8
   ├─ gasUsed: 0x5208
   ├─ timestamp: 0x5660d1a7
   ├─ transactions
   │  └─ 0
   │     ├─ hash: 0xf365d4ddd7bf0c527bee73676df632cf738d43b287865424dee226e9d9794e3d
   │     ├─ nonce: 0xbcbc
   │     ├─ blockHash: 0x12b82561ed2fe63f306b5178a545622512d74e3aed9fd53500484bb3e2c8335b
   │     ├─ blockNumber: 0x9b924
   │     ├─ transactionIndex: 0x0
   │     ├─ from: 0x2a65aca4d5fc5b5c859090a6c34d164135398226
   │     ├─ to: 0xc65e24227e40a6a0be5c5cf1f0f120c2310e9f7a
   │     ├─ value: 0x43f33b195cdbf000
   │     ├─ gas: 0x15f90
   │     ├─ gasPrice: 0xba43b7400
   │     └─ input: 0x
   └─ uncles

[kumavis]

for the block header the mapping looks like this:

blockHeader.number = blockParams.number
blockHeader.mixHash = blockParams.hash
blockHeader.parentHash = blockParams.parentHash
blockHeader.nonce = blockParams.nonce
blockHeader.uncleHash = blockParams.sha3Uncles
blockHeader.bloom = blockParams.logsBloom
blockHeader.transactionsTrie = blockParams.transactionsRoot
blockHeader.stateRoot = blockParams.stateRoot
blockHeader.receiptTrie = blockParams.receiptRoot
blockHeader.coinbase = blockParams.miner
blockHeader.difficulty = blockParams.difficulty
blockHeader.extraData = blockParams.extraData
blockHeader.gasLimit = blockParams.gasLimit
blockHeader.gasUsed = blockParams.gasUsed
blockHeader.timestamp = blockParams.timestamp

[kumavis]

heres a thing for getting the blockheader from the jsonrpc

const request = require('request')
const BlockHeader = require('ethereumjs-block/header')

request({
  method: 'POST',
  uri: 'https://rpc.metamask.io',
  body: '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params": ["latest", false], "id":1}',
}, function(err, res){
  if (err) throw err
  var response = JSON.parse(res.body)
  var blockParams = response.result
  var blockHeader = new BlockHeader()

  blockHeader.number = blockParams.number
  blockHeader.mixHash = blockParams.hash
  blockHeader.parentHash = blockParams.parentHash
  blockHeader.nonce = blockParams.nonce
  blockHeader.uncleHash = blockParams.sha3Uncles
  blockHeader.bloom = blockParams.logsBloom
  blockHeader.transactionsTrie = blockParams.transactionsRoot
  blockHeader.stateRoot = blockParams.stateRoot
  blockHeader.receiptTrie = blockParams.receiptRoot
  blockHeader.coinbase = blockParams.miner
  blockHeader.difficulty = blockParams.difficulty
  blockHeader.extraData = blockParams.extraData
  blockHeader.gasLimit = blockParams.gasLimit
  blockHeader.gasUsed = blockParams.gasUsed
  blockHeader.timestamp = blockParams.timestamp

  console.log(blockHeader)
  console.log(blockHeader.toJSON())
  // console.log(blockParams)
  console.log(`size: ${blockHeader.serialize().length} bytes`)
})

[kumavis]

heres another snippet for the whole block -- but need to look up uncles separately : /

function rpcToBlock(blockParams){
  var blockHeader = rpcToBlockHeader(blockParams)
  if (blockParams.sha3Uncles !== '0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347') throw new Error('Missing ommers....')
  return new Block({
    header: blockHeader.toJSON(),
    transactions: blockParams.transactions,
    uncleHeaders: [],
  })
}

function rpcToBlockHeader(blockParams){
  var blockHeader = new BlockHeader()

  blockHeader.number = blockParams.number
  blockHeader.mixHash = blockParams.hash
  blockHeader.parentHash = blockParams.parentHash
  blockHeader.nonce = blockParams.nonce
  blockHeader.uncleHash = blockParams.sha3Uncles
  blockHeader.bloom = blockParams.logsBloom
  blockHeader.transactionsTrie = blockParams.transactionsRoot
  blockHeader.stateRoot = blockParams.stateRoot
  blockHeader.receiptTrie = blockParams.receiptRoot
  blockHeader.coinbase = blockParams.miner
  blockHeader.difficulty = blockParams.difficulty
  blockHeader.extraData = blockParams.extraData
  blockHeader.gasLimit = blockParams.gasLimit
  blockHeader.gasUsed = blockParams.gasUsed
  blockHeader.timestamp = blockParams.timestamp

  return blockHeader
}

[kumavis]

update! so i wasn't familiar with the mixHash property and was confusing it with the block hash (its actually related to EthHash. The RPC does not supply the mixHash and is seemingly the only information you can't get from the blockchain via the RPC. We don't really need it unless we're checking PoW, but it causes the hash generation to be incorrect.

so we cheat and override block.header.hash()

function rpcToBlock(blockParams){
  if (blockParams.sha3Uncles !== '0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347') throw new Error('Missing ommers....')
  var block = new Block({
    transactions: blockParams.transactions,
    uncleHeaders: [],
  })
  var blockHeader = block.header
  blockHeader.number = blockParams.number
  blockHeader.parentHash = blockParams.parentHash
  blockHeader.nonce = blockParams.nonce
  blockHeader.uncleHash = blockParams.sha3Uncles
  blockHeader.bloom = blockParams.logsBloom
  blockHeader.transactionsTrie = blockParams.transactionsRoot
  blockHeader.stateRoot = blockParams.stateRoot
  blockHeader.receiptTrie = blockParams.receiptRoot
  blockHeader.coinbase = blockParams.miner
  blockHeader.difficulty = blockParams.difficulty
  blockHeader.extraData = blockParams.extraData
  blockHeader.gasLimit = blockParams.gasLimit
  blockHeader.gasUsed = blockParams.gasUsed
  blockHeader.timestamp = blockParams.timestamp
  blockHeader.hash = function () {
    return ethUtil.toBuffer(blockParams.hash)
  }
  return block
}

[kumavis]

latest version can handle some uncles

const Block = require('ethereumjs-block')
const ethUtil = require('ethereumjs-util')

module.exports = materializeBlock

function materializeBlock(blockParams, uncles){
  console.log(blockParams)
  var block = new Block({
    transactions: blockParams.transactions || [],
    uncleHeaders: uncles.map(function(uncle){ return materializeBlock(uncle, []).header.toJSON() }),
  })
  var blockHeader = block.header
  blockHeader.number = blockParams.number
  blockHeader.parentHash = blockParams.parentHash
  blockHeader.nonce = blockParams.nonce
  blockHeader.uncleHash = blockParams.sha3Uncles
  blockHeader.bloom = blockParams.logsBloom
  blockHeader.transactionsTrie = blockParams.transactionsRoot
  blockHeader.stateRoot = blockParams.stateRoot
  blockHeader.receiptTrie = blockParams.receiptRoot || ethUtil.SHA3_NULL
  blockHeader.coinbase = blockParams.miner
  blockHeader.difficulty = blockParams.difficulty
  blockHeader.extraData = blockParams.extraData
  blockHeader.gasLimit = blockParams.gasLimit
  blockHeader.gasUsed = blockParams.gasUsed
  blockHeader.timestamp = blockParams.timestamp
  blockHeader.hash = function () {
    return ethUtil.toBuffer(blockParams.hash)
  }
  return block
}

[wanderer]

@kumavis agg so the mixhash is a problem... i can't believe the rpc don't give that back... are you sure there is no way to get it from geth?

[wanderer]

https://github.com/ethereum/go-ethereum/issues/2508

[kumavis]

latest version overrides the hash look ups to get past the mixhash issue

const Block = require('ethereumjs-block')
const Transaction = require('ethereumjs-tx')
const ethUtil = require('ethereumjs-util')

module.exports = materializeBlock

function materializeBlock(blockParams, uncles){
  // console.log(blockParams)
  var block = new Block({
    transactions: [],
    uncleHeaders: [],
  })
  var blockHeader = block.header
  blockHeader.number = blockParams.number
  blockHeader.parentHash = blockParams.parentHash
  blockHeader.nonce = blockParams.nonce
  blockHeader.uncleHash = blockParams.sha3Uncles
  blockHeader.bloom = blockParams.logsBloom
  blockHeader.transactionsTrie = blockParams.transactionsRoot
  blockHeader.stateRoot = blockParams.stateRoot
  blockHeader.receiptTrie = blockParams.receiptRoot || ethUtil.SHA3_NULL
  blockHeader.coinbase = blockParams.miner
  blockHeader.difficulty = blockParams.difficulty
  blockHeader.extraData = blockParams.extraData
  blockHeader.gasLimit = blockParams.gasLimit
  blockHeader.gasUsed = blockParams.gasUsed
  blockHeader.timestamp = blockParams.timestamp
  blockHeader.hash = function () {
    return ethUtil.toBuffer(blockParams.hash)
  }

  block.transactions = (blockParams.transactions || []).map(function(txParams){
    // hot fix for https://github.com/ethereumjs/ethereumjs-util/issues/40
    txParams.gasLimit = (txParams.gasLimit === undefined)? txParams.gas : txParams.gasLimit
    txParams.data = (txParams.data === undefined)? txParams.input : txParams.data
    var tx = new Transaction(txParams)
    // override from address
    tx._from = ethUtil.toBuffer(txParams.from)
    // override hash
    tx.hash = function(){ return ethUtil.toBuffer(txParams.hash) }
    return tx
  })
  block.uncleHeaders = (uncles || []).map(function(uncleParams){
    return materializeBlock(uncleParams).header
  })

  return block
}

[axic]

@kumavis did you manage to finish implementing this?

[kumavis]

haha this thread - im like reimplenting git on github comments @axic The latest version is here https://github.com/kumavis/eth-tx-summary/blob/master/materialize-blocks.js

[axic]

:rocket:

Can we include it in this repo?

[axic]

I could replace this manual code here if it merged: https://github.com/ethereum/browser-solidity/blob/master/src/universal-dapp.js#L750-L757

[kumavis]

sure, if you make the PR 😸

[axic]

I can help with cheering from the sidelines :v:

[kumavis]

const blockFromRpc = require('ethereumjs-block/from-rpc')
blockFromRpc(blockJson, unclesJson)

[k06a]

@kumavis is it possible to recalculate block hash? Wanna serialize header but this method is missing for BlockHeader.

[k06a]

Having problem trying to get serialized BlockHash by encoding it in RLP.

const block = blockFromRpc(blockData);
const blockHash = '0x' + block.header.hash().toString('hex');
blockHash.should.be.equal(blockData.hash); // <--- No problem

const rawHash = '0x' + utils.keccak256(utils.rlp.encode(block.header.raw)).toString('hex');
rawHash.should.be.equal(blockData.hash); // <--- Problem 1

const rawHash2 = '0x' + utils.rlphash(block.header.raw).toString('hex');
rawHash2.should.be.equal(blockData.hash); // <--- Problem 2

[k06a]

// 1
const blockData = await (new Promise(done =>
    web3.eth.getBlock(blockNumber, (err,res) => done(res))
));
blockData.difficulty = blockData.difficulty.toNumber();
blockData.totalDifficulty = blockData.totalDifficulty.toNumber();
blockData.uncleHash = blockData.sha3Uncles;
blockData.coinbase = blockData.miner;
blockData.transactionTrie = blockData.transactionsRoot;
blockData.receiptTrie = blockData.receiptsRoot;
blockData.bloom = blockData.logsBloom;

// 2
// https://ethereum.stackexchange.com/questions/31314/block-header-format
// https://ethereum.stackexchange.com/questions/268/ethereum-block-architecture
const rawHeader = [
    blockData.parentHash,
    blockData.sha3Uncles,
    blockData.miner,
    blockData.stateRoot,
    blockData.transactionsRoot,
    blockData.receiptsRoot,
    blockData.logsBloom,
    blockData.difficulty,
    blockData.number,
    blockData.gasLimit,
    blockData.gasUsed,
    blockData.timestamp,
    blockData.extraData,
    blockData.mixHash,
    blockData.nonce
];

// 3
const block = blockFromRpc(blockData);

console.log(blockData);
console.log(rawHeader);
console.log(block.header.raw);

Web3 block header:

{ number: 200,
  hash: '0xc9e562b66933fb390bc935c046eac4223c47f338bc666c4debde94778c2c7534',
  parentHash: '0x51c9c3fb1d99d7fe9a66e57235983ac5e4d0954032ace9408b28a6754d5efd63',
  mixHash: '0x0000000000000000000000000000000000000000000000000000000000000000',
  nonce: '0x0000000000000000',
  sha3Uncles: '0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347',
  logsBloom: '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000',
  transactionsRoot: '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421',
  stateRoot: '0x2037312fee597a6aef9529de0bbf7c00a2688994d23530c4669fe4a18fe3522a',
  receiptsRoot: '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421',
  miner: '0x0000000000000000000000000000000000000000',
  difficulty: 0,
  totalDifficulty: 0,
  extraData: '0x',
  size: 1000,
  gasLimit: 17592186044415,
  gasUsed: 0,
  timestamp: 1543577953,
  transactions: [],
  uncles: [],
  uncleHash: '0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347',
  coinbase: '0x0000000000000000000000000000000000000000',
  transactionTrie: '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421',
  receiptTrie: '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421',
  bloom: '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' }

Manually formed array of values:

[ '0x51c9c3fb1d99d7fe9a66e57235983ac5e4d0954032ace9408b28a6754d5efd63',
  '0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347',
  '0x0000000000000000000000000000000000000000',
  '0x2037312fee597a6aef9529de0bbf7c00a2688994d23530c4669fe4a18fe3522a',
  '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421',
  '0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421',
  '0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000',
  0,
  200,
  17592186044415,
  0,
  1543577953,
  '0x',
  '0x0000000000000000000000000000000000000000000000000000000000000000',
  '0x0000000000000000' ]

ethereumjs-block constructed with blockFromRpc with some fields renamed previously:

[ <Buffer 51 c9 c3 fb 1d 99 d7 fe 9a 66 e5 72 35 98 3a c5 e4 d0 95 40 32 ac e9 40 8b 28 a6 75 4d 5e fd 63>,
  <Buffer 1d cc 4d e8 de c7 5d 7a ab 85 b5 67 b6 cc d4 1a d3 12 45 1b 94 8a 74 13 f0 a1 42 fd 40 d4 93 47>,
  <Buffer 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00>,
  <Buffer 20 37 31 2f ee 59 7a 6a ef 95 29 de 0b bf 7c 00 a2 68 89 94 d2 35 30 c4 66 9f e4 a1 8f e3 52 2a>,
  <Buffer 56 e8 1f 17 1b cc 55 a6 ff 83 45 e6 92 c0 f8 6e 5b 48 e0 1b 99 6c ad c0 01 62 2f b5 e3 63 b4 21>,
  <Buffer 56 e8 1f 17 1b cc 55 a6 ff 83 45 e6 92 c0 f8 6e 5b 48 e0 1b 99 6c ad c0 01 62 2f b5 e3 63 b4 21>,
  <Buffer 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ... >,
  <Buffer >,
  <Buffer c8>,
  <Buffer 0f ff ff ff ff ff>,
  <Buffer >,
  <Buffer 5c 01 21 61>,
  <Buffer >,
  <Buffer 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00>,
  <Buffer 00 00 00 00 00 00 00 00> ]

utils.rlphash(block.header.raw):

0xf579b522fd8e0f024333c6af3df45061501dd5efa37b553c532cfd5ed89d0035

is not equal to original block hash:

0xc9e562b66933fb390bc935c046eac4223c47f338bc666c4debde94778c2c7534

[holgerd77]

Wonder if this is also related to https://github.com/ethereumjs/rlp/issues/28.

[k06a]

@holgerd77 tried to replace gasLimit and difficulty empty buffers with <Buffer 00> - not solved a problem.

[vpulim]

@k06a I'm having a hard time reproducing this using rpc calls to infura servers on mainnet and rinkeby. Are you testing against a private testnet? Can you please post the raw blockData object returned from the call to web3.eth.getBlock (before you modify the fields)?

[vpulim]

This is the test script I used (NOTE: I had to replace blockData.difficulty.toNumber() with parseInt(blockData.difficulty, 10) since toNumber fails on difficulty values that are decimal strings):

const Web3 = require('web3');
const blockFromRpc = require('ethereumjs-block/from-rpc');
const utils = require('ethereumjs-util');
const web3 = new Web3('xxxxxxxxxx'); // replace xxxxxxxxxx with your provider

async function run (blockNumber) {
  const blockData = await (new Promise(done =>
      web3.eth.getBlock(blockNumber, (err,res) => done(res))
  ));
  blockData.difficulty = parseInt(blockData.difficulty, 10);
  blockData.totalDifficulty = parseInt(blockData.totalDifficulty, 10);
  blockData.uncleHash = blockData.sha3Uncles;
  blockData.coinbase = blockData.miner;
  blockData.transactionTrie = blockData.transactionsRoot;
  blockData.receiptTrie = blockData.receiptsRoot;
  blockData.bloom = blockData.logsBloom;

  const block = blockFromRpc(blockData);
  console.log(blockData.hash);
  console.log('0x' + block.header.hash().toString('hex'));
  console.log('0x' + utils.keccak256(utils.rlp.encode(block.header.raw)).toString('hex'));
  console.log('0x' + utils.rlphash(block.header.raw).toString('hex'));
}

run(200);

Output:

0x75208f95cc7ef5829df543ec3189b18d57d582e27636c46a7f889070ad15e768
0x75208f95cc7ef5829df543ec3189b18d57d582e27636c46a7f889070ad15e768
0x75208f95cc7ef5829df543ec3189b18d57d582e27636c46a7f889070ad15e768
0x75208f95cc7ef5829df543ec3189b18d57d582e27636c46a7f889070ad15e768

[k06a]

@vpulim thanks for this! For some reason, this code doesn't work for ganache-cli:

0x9bcf1168175dd96f135d7ece919138686c792668dd7aec38bf1dcc99ec30960b
0x9bcf1168175dd96f135d7ece919138686c792668dd7aec38bf1dcc99ec30960b
0x0973440ac23ce325e76644ed39e2cfdab17228d2ce1898bd8a875f39068acd71
0x0973440ac23ce325e76644ed39e2cfdab17228d2ce1898bd8a875f39068acd71

blockFromRpc also replaces hash method to return hash from data, not being calculated.

[vpulim]

Interesting... in my test script, I was making RPC calls to Infura servers. When I switched to the ganache-cli provider, I was also getting incorrect hashes like you. Perhaps ganache is generating an incorrect hash? I'll take a closer look at the ganache code and try to figure out what's going on.

[davidmurdoch]

@vpulim, possibly related to https://github.com/trufflesuite/ganache-cli/issues/387

Have you seen the hash inconsistency using ganache-cli 6.2.1 or earlier?

[vpulim]

@davidmurdoch, tried with 6.2.1 and 6.1.0 and they also have the same hash issue.

[vpulim]

I submitted a fix to ganache-core: https://github.com/trufflesuite/ganache-core/pull/238. Once this fix is released, ganache-cli needs to be updated to reference the latest version of ganache-core with this fix.

I don't think there is anything more to be done on our end within ethereumjs-block

[k06a]

@vpulim thank you for your help!