bitcoinjs / bitcoinjs-lib

A javascript Bitcoin library for node.js and browsers.
MIT License
5.66k stars 2.09k forks source link

How can I create a wallet from a mnemonic and import xpub to bitcoin-core? #1677

Closed j3ko closed 3 years ago

j3ko commented 3 years ago

Sorry if this has been covered but I can't seem to figure it out:

I have bitcoin-core running on testnet (p2sh-segwit) and the command at https://developer.bitcoin.org/reference/rpc/importpubkey.html expects a hex encoded public key. How can I go from a BIP39 mnemonic to a public key bitcoin-core requires for the command?

Is something like this far off from what I should be doing? The command runs and imports the xpubs I generate but I do not believe they are correct since the watch balances are 0 (even after rescan).

import * as HDNode from 'bip32';
import * as bip39 from 'bip39';

const seed = bip39.mnemonicToSeedSync('mnemonic here');
const node = HDNode.fromSeed(seed, this.bitcoinJsNetwork);
return node.publicKey.toString('hex');

// or
// const path = `m/0'/1'`;
// const child = node.derivePath(path).neutered();
// return child.publicKey.toString('hex');
junderw commented 3 years ago

You might want to use importmulti instead. You can use output descriptors and bitcoin core should generate tons of keys for you.

https://developer.bitcoin.org/reference/rpc/importmulti.html

I've never used it before, but this should work. (Let me know if it works)

import * as HDNode from 'bip32';
import * as bip39 from 'bip39';

const seed = bip39.mnemonicToSeedSync('mnemonic here');
const root = HDNode.fromSeed(seed, this.bitcoinJsNetwork);

const masterFingerPrint = root.fingerprint.toString('hex');
const pathBelowRootToXpub = "/49'/0'/0'"
const bip49Xpub = root.derivePath(`m${pathBelowRootToXpub}`);
const xpubString = bip49Xpub.neutered().toBase58();

// See descriptorChecksum function implementation below
const receiveDescriptor = descriptorChecksum(`sh(wpkh([${masterFingerPrint}${pathBelowRootToXpub}]${xpubString}/0/*))`);
const changeDescriptor = descriptorChecksum(`sh(wpkh([${masterFingerPrint}${pathBelowRootToXpub}]${xpubString}/1/*))`);

const importMultiBase = {
  // Uses output descriptor syntax
  // Looks something like this
  // sh(wpkh([d34db33f/49'/0'/0']xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhkkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/0/*))
  desc: receiveDescriptor,
  // The date/time when the keys were generated (so we know it's impossible there will be coins received before this date)
  timestamp: Math.floor(new Date('2020-10-23T12:08:35.605Z').getTime() / 1000),
  // This will generate scripts for the * part of the path above from index 0 to 999 inclusive
  range: 1000,
  internal: false,
  watchonly: true,
  label: 'myWallet',
};

return JSON.stringify([
  importMultiBase,
  { ...importMultiBase, ...{ desc: changeDescriptor, internal: true, label: undefined } },
]);

/*
 * input: "wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)"
 * output: "wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)#qwlqgth7"
 * (This has been checked to match bitcoin-core)
 */
function descriptorChecksum(desc) {
  if (!(typeof desc === 'string' || desc instanceof String)) throw new Error('desc must be string')

  const descParts = desc.match(/^(.*?)(?:#([qpzry9x8gf2tvdw0s3jn54khce6mua7l]{8}))?$/);
  if (descParts[1] === '') throw new Error('desc string must not be empty')

  const INPUT_CHARSET = '0123456789()[],\'/*abcdefgh@:$%{}IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~ijklmnopqrstuvwxyzABCDEFGH`#"\\ ';
  const CHECKSUM_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l';
  const MOD_CONSTS = [
    parseInt('f5dee51989', 16),
    parseInt('a9fdca3312', 16),
    parseInt('1bab10e32d', 16),
    parseInt('3706b1677a', 16),
    parseInt('644d626ffd', 16),
  ];
  const BIT35 = Math.pow(2, 35)
  const BIT31 = Math.pow(2, 31)
  const BIT5 = Math.pow(2, 5)

  function polyMod(c, val) {
    const c0 = Math.floor(c / BIT35);
    let ret = xor5Byte((c % BIT35) * BIT5, val)
    if (c0 & 1) ret = xor5Byte(ret, MOD_CONSTS[0])
    if (c0 & 2) ret = xor5Byte(ret, MOD_CONSTS[1])
    if (c0 & 4) ret = xor5Byte(ret, MOD_CONSTS[2])
    if (c0 & 8) ret = xor5Byte(ret, MOD_CONSTS[3])
    if (c0 & 16) ret = xor5Byte(ret, MOD_CONSTS[4])
    return ret
  }

  function xor5Byte(a, b) {
    const a1 = Math.floor(a / BIT31);
    const a2 = a % BIT31
    const b1 = Math.floor(b / BIT31);
    const b2 = b % BIT31
    return (a1 ^ b1) * BIT31 + (a2 ^ b2)
  }

  let c = 1
  let cls = 0
  let clscount = 0
  for (const ch of descParts[1]) {
    const pos = INPUT_CHARSET.indexOf(ch)
    if (pos === -1) return ''
    c = polyMod(c, pos & 31)
    cls = cls * 3 + (pos >> 5)
    clscount++
    if (clscount === 3) {
      c = polyMod(c, cls)
      cls = 0
      clscount = 0
    }
  }
  if (clscount > 0) {
    c = polyMod(c, cls)
  }
  for (let i = 0; i < 8; i++) {
    c = polyMod(c, 0)
  }
  c = xor5Byte(c, 1)

  const arr = []
  for (let i = 0; i < 8; i++) {
    arr.push(CHECKSUM_CHARSET.charAt(Math.floor(c / Math.pow(2, (5 * (7 - i)))) % BIT5))
  }
  const checksum = arr.join('')
  if (descParts[2] !== undefined && descParts[2] !== checksum) throw new Error('Checksum Mismatch')

  return `${descParts[1]}#${checksum}`
}
junderw commented 3 years ago

A bonus of using this approach is that you can now make PSBTs using bitcoin core RPC and it will include the BIP32 path information so other wallets can sign easier (in bitcoinjs-lib signHD takes the bip32 root object and signs using the path info inside the PSBT... so you don't need to code path logic in your signing app.

j3ko commented 3 years ago

Thanks @junderw I really appreciate the help, this approach would definitely solve a lot of problems for me. The code above (with a few tweaks) appears to import successfully. Unfortunately, like the importpubkey command it doesn't look like it is the correct wallet.

I have sent coins to legacy, segwit and native-segwit addresses tied to the wallet and only when I import the addresses directly via importaddress do they show up.

The tweaks I made were:

  1. removed label: 'myWallet', (for an "Internal addresses should not have a label" error)
  2. ran the resulting descriptor through getdescriptorinfo to get a checksum (for a "Missing checksum" error)
  3. attempted both const pathBelowRootToXpub = "/49'/0'/0'" and const pathBelowRootToXpub = "/49'/1'/0'" since I am trying to do it on testnet

Is there anything else you can suggest I try?

junderw commented 3 years ago

The above code will not generate legacy or native-segwit addresses.

for native-segwit, remove the sh() and for legacy remove the sh() and change wpkh() to pkh()

native-segwit uses BIP84 path, so change the 49 path to 84 legacy uses BIP44, so change the 49 to 44

However.

That being said, judging from your original question, you probably generated the original wallet in a strange way that was not done in a BIP standardized method.

You need to modify the path and descriptors to match your script.

I don't know what else to tell you.

Where / how did you generate your wallet? Can you show the code you used to generate it?

j3ko commented 3 years ago

ah I see, I generated my wallet from https://iancoleman.io/bip39/ using the default 15 word setting. Is that the problem?

junderw commented 3 years ago

No.

The problem is you don't understand how to match the addresses on that site with your output descriptor.

I don't know what options you picked on that site (there are tons of tabs and entry boxes which you could have entered any variety of values into.

If you can find your addresses and let me know what tabs you are looking at / what values you have changed from the default state of the site (after entering your mnemonic, of course... but don't tell me your mnemonic)

j3ko commented 3 years ago

ah ok. The steps I took to generate the wallet:

  1. Leave all settings on the site at their default values, click "Generate"
  2. Halfway down the page, under the Coin dropdown choose BTC - Bitcoin Testnet
  3. Under each 3 tabs a bit lower on the page: BIP44, BIP49 and BIP84. Choose the first address generated.
  4. Send coins to each address

The three addresses are:

Tab Path Address Value
BIP44 m/44'/1'/0'/0/0 muLgskW5vrNNqw1K2ouyNz4PS1NZCsfLao 0.1 mBTC
BIP49 m/49'/1'/0'/0/0 2N97i76ewGe85A1vFc9JnwrJfQkhfvHL6CZ 17.12862 mBTC (3 txs)
BIP84 m/84'/1'/0'/0/0 tb1q6t9q9m54gng69mc27gc6c7xvzxp66g3t5a2hhr 0.1 mBTC
junderw commented 3 years ago

If you do the above, it should work.

I was able to get it to work with a different mnemonic using the same addresses you are showing.

junderw commented 3 years ago

I made a function that calculates the checksum of the descriptor so you don't need to check with bitcoin core:

(JavaScript only does bitwise operations & | ^ >> << reliably up to 32 bits (4 bytes) and this checksum is based on 40 bit integers (5 bytes)... so while JS can handle up to 53 bits fine, the bitwise operations need to be substituted with equivalent Math ops)

/*
 * input: "wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)"
 * output: "wpkh([d34db33f/84h/0h/0h]0279be667ef9dcbbac55a06295Ce870b07029Bfcdb2dce28d959f2815b16f81798)#qwlqgth7"
 * (This has been checked to match bitcoin-core)
 */
function descriptorChecksum(desc) {
  if (!(typeof desc === 'string' || desc instanceof String)) throw new Error('desc must be string')

  const descParts = desc.match(/^(.*?)(?:#([qpzry9x8gf2tvdw0s3jn54khce6mua7l]{8}))?$/);
  if (descParts[1] === '') throw new Error('desc string must not be empty')

  const INPUT_CHARSET = '0123456789()[],\'/*abcdefgh@:$%{}IJKLMNOPQRSTUVWXYZ&+-.;<=>?!^_|~ijklmnopqrstuvwxyzABCDEFGH`#"\\ ';
  const CHECKSUM_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l';
  const MOD_CONSTS = [
    parseInt('f5dee51989', 16),
    parseInt('a9fdca3312', 16),
    parseInt('1bab10e32d', 16),
    parseInt('3706b1677a', 16),
    parseInt('644d626ffd', 16),
  ];
  const BIT35 = Math.pow(2, 35)
  const BIT31 = Math.pow(2, 31)
  const BIT5 = Math.pow(2, 5)

  function polyMod(c, val) {
    const c0 = Math.floor(c / BIT35);
    let ret = xor5Byte((c % BIT35) * BIT5, val)
    if (c0 & 1) ret = xor5Byte(ret, MOD_CONSTS[0])
    if (c0 & 2) ret = xor5Byte(ret, MOD_CONSTS[1])
    if (c0 & 4) ret = xor5Byte(ret, MOD_CONSTS[2])
    if (c0 & 8) ret = xor5Byte(ret, MOD_CONSTS[3])
    if (c0 & 16) ret = xor5Byte(ret, MOD_CONSTS[4])
    return ret
  }

  function xor5Byte(a, b) {
    const a1 = Math.floor(a / BIT31);
    const a2 = a % BIT31
    const b1 = Math.floor(b / BIT31);
    const b2 = b % BIT31
    return (a1 ^ b1) * BIT31 + (a2 ^ b2)
  }

  let c = 1
  let cls = 0
  let clscount = 0
  for (const ch of descParts[1]) {
    const pos = INPUT_CHARSET.indexOf(ch)
    if (pos === -1) return ''
    c = polyMod(c, pos & 31)
    cls = cls * 3 + (pos >> 5)
    clscount++
    if (clscount === 3) {
      c = polyMod(c, cls)
      cls = 0
      clscount = 0
    }
  }
  if (clscount > 0) {
    c = polyMod(c, cls)
  }
  for (let i = 0; i < 8; i++) {
    c = polyMod(c, 0)
  }
  c = xor5Byte(c, 1)

  const arr = []
  for (let i = 0; i < 8; i++) {
    arr.push(CHECKSUM_CHARSET.charAt(Math.floor(c / Math.pow(2, (5 * (7 - i)))) % BIT5))
  }
  const checksum = arr.join('')
  if (descParts[2] !== undefined && descParts[2] !== checksum) throw new Error('Checksum Mismatch')

  return `${descParts[1]}#${checksum}`
}
j3ko commented 3 years ago

Awesome! I am able to see the balances now.

I ran through everything with your updated steps/checksum and initially I was getting the same result. I'm not sure why, but I forced a full rescan with rescanblockchain and it worked! I went back and re-ran my previous attempts and they all work now as well.

I really appreciate your time on this 🍻

junderw commented 3 years ago

I think you didn't set the timestamp early enough? Not sure.

Either way, it's working now, and that's what counts.