building `wasm32-unknown-unknown` fails

[realeinherjar]

I am trying to setup some CI checks in https://github.com/bitcoindevkit/bdk/pull/1165. Here's the important flake.nix parts:

outputs = {...}:
  flake-utils.lib.eachDefaultSystem (system:
    let
      rustWASMTarget = pkgs.rust-bin.stable.latest.default.override {
        targets = [ "wasm32-unknown-unknown" ];
      };
      WASMArgs = {
        CARGO_BUILD_TARGET = "wasm32-unknown-unknown";
        buildInputs = buildInputs ++ WASMInputs;
        inherit nativeBuildInputs;
        # crane tries to run the Wasm file as if it were a binary
        doCheck = false;
        # stdenv = pkgs.llvmPackages_14.stdenv;
        CC = "${stdenv.cc.nativePrefix}cc";
        AR = "${stdenv.cc.nativePrefix}ar";
        CC_wasm32_unknown_unknown = "${pkgs.llvmPackages_14.clang-unwrapped}/bin/clang-14";
        CFLAGS_wasm32_unknown_unknown = "-I ${pkgs.llvmPackages_14.libclang.lib}/lib/clang/14.0.6/include/";
        AR_wasm32_unknown_unknown = "${pkgs.llvmPackages_14.llvm}/bin/llvm-ar";
      };
      # craneUtils needs to be built using Rust latest (not WASM)
      # check https://github.com/ipetkov/crane/issues/422
      craneWASMLib = ((crane.mkLib pkgs).overrideToolchain rustWASMTarget).overrideScope' (final: prev: { inherit (craneLib) craneUtils; });
      buildInputs = [...];
      nativeBuildInputs = [...];
      WASMInputs = [...];
      commonArgs = {
        # When filtering sources, we want to allow assets other than .rs files
        src = lib.cleanSourceWith {
          src = ./.; # The original, unfiltered source
          filter = path: type:
            # esplora uses `.md` in the source code
            (lib.hasSuffix "\.md" path) ||
            # bitcoin_rpc uses `.db` in the source code
            (lib.hasSuffix "\.db" path) ||
            # Default filter from crane (allow .rs files)
            (craneLib.filterCargoSources path type);
        };

        # Fixing name/version here to avoid warnings
        # This does not interact with the versioning
        # in any of bdk crates' Cargo.toml
        pname = "crates";
        version = "0.1.0";

        inherit buildInputs;
        inherit nativeBuildInputs;
      };
      # WASM derivation arguments
      WASMArgs = {
        CARGO_BUILD_TARGET = "wasm32-unknown-unknown";
        buildInputs = buildInputs ++ WASMInputs;
        inherit nativeBuildInputs;
        # crane tries to run the Wasm file as if it were a binary
        doCheck = false;
        # don't test
        cargoTestCommand = "";
        # just build bdk for now
        cargoExtraArgs = "--locked --lib --target wasm32-unknown-unknown -p bdk --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm";
        # env vars
        CC = "${stdenv.cc.nativePrefix}cc";
        AR = "${stdenv.cc.nativePrefix}ar";
        CC_wasm32_unknown_unknown = "${pkgs.llvmPackages_14.clang-unwrapped}/bin/clang-14";
        CFLAGS_wasm32_unknown_unknown = "-I ${pkgs.llvmPackages_14.libclang.lib}/lib/clang/14.0.6/include/";
        AR_wasm32_unknown_unknown = "${pkgs.llvmPackages_14.llvm}/bin/llvm-ar";
      };
      cargoArtifactsWASM = craneWASMLib.buildDepsOnly (commonArgs // WASMArgs);
    in
    rec {
      checks = {
        # WASM
        WASMBdk = craneWASMLib.buildPackage (commonArgs // WASMArgs // {
          cargoArtifacts = cargoArtifactsWASM;
        });
        WASMEsplora = craneWASMLib.cargoBuild (commonArgs // WASMArgs // {
          cargoArtifacts = cargoArtifactsWASM;
          cargoExtraArgs = "--locked -p bdk_esplora --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,async";
        });
      };

      legacyPackages = {
        ci = {
          WASM = {
            bdk = checks.WASMBdk;
            esplora = checks.WASMEsplora;
          };
        };
      };
      devShells = {
        WASM = craneWASMLib.devShell {
          # dependencies
          packages = buildInputs ++ WASMInputs ++ [
            pkgs.bashInteractive
            pkgs.git
            pkgs.ripgrep
            rustWASMTarget
          ];

          CARGO_BUILD_TARGET = WASMArgs.CARGO_BUILD_TARGET;
          CC = WASMArgs.CC;
          AR = WASMArgs.AR;
          CC_wasm32_unknown_unknown = WASMArgs.CC_wasm32_unknown_unknown;
          CFLAGS_wasm32_unknown_unknown = WASMArgs.CFLAGS_wasm32_unknown_unknown;
          AR_wasm32_unknown_unknown = WASMArgs.AR_wasm32_unknown_unknown;
        };
      };
    }
  );
}

then I get an erroar while building #ci.WASM.bdk:

❱ nix build -L .#ci.WASM.bdk
warning: Git tree '/Users/einherjar/git/btc/bdk' is dirty
crates> cargoArtifacts not set, will not reuse any cargo artifacts
crates> unpacking sources
crates> unpacking source archive /nix/store/fdncw00w7k2r17bh2gr8x6rx5z992kcy-source
crates> source root is source
crates> patching sources
crates> Executing configureCargoCommonVars
crates> updateAutotoolsGnuConfigScriptsPhase
crates> configuring
crates> will append /private/tmp/nix-build-crates-0.1.0.drv-0/source/.cargo-home/config.toml with contents of /nix/store/n2rs6cia0gxvdakw59zdhb13r6dmjwza-vendor-cargo-deps/config.toml
crates> default configurePhase, nothing to do
crates> building
crates> ++ command cargo --version
crates> cargo 1.73.0 (9c4383fb5 2023-08-26)
crates> ++ command cargo build --release --message-format json-render-diagnostics --locked -p bdk --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm
crates> error: no matching package found
crates> searched package name: `core2`
crates> perhaps you meant: ctrlc, home or ureq
crates> location searched: registry `crates-io`
crates> required by package `bitcoin v0.30.1`
crates> ... which satisfies dependency `bitcoin = "^0.30.0"` (locked to 0.30.1) of package `bdk v1.0.0-alpha.2 (/private/tmp/nix-build-crates-0.1.0.drv-0/source/crates/bdk)`
error: builder for '/nix/store/5sihawqk9rdm1cllhqwwbwjbbay6qjhx-crates-0.1.0.drv' failed with exit code 101;
last 10 log lines:
> building
> ++ command cargo --version
> cargo 1.73.0 (9c4383fb5 2023-08-26)
> ++ command cargo build --release --message-format json-render-diagnostics --locked -p bdk --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm
> error: no matching package found
> searched package name: `core2`
> perhaps you meant: ctrlc, home or ureq
> location searched: registry `crates-io`
> required by package `bitcoin v0.30.1`
> ... which satisfies dependency `bitcoin = "^0.30.0"` (locked to 0.30.1) of package `bdk v1.0.0-alpha.2 (/private/tmp/nix-build-crates-0.1.0.drv-0/source/crates/bdk)`

However the WASM devShell executing the same command works:

bash-5.2$ cargo check --release --locked -p bdk --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm
+ command cargo check --release --locked -p bdk --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm
Compiling proc-macro2 v1.0.69
Compiling unicode-ident v1.0.12
Compiling wasm-bindgen-shared v0.2.87
Compiling log v0.4.20
Compiling once_cell v1.18.0
Compiling bumpalo v3.14.0
Compiling wasm-bindgen v0.2.87
Checking cfg-if v1.0.0
Compiling libc v0.2.149
Compiling serde v1.0.189
Compiling bitcoin-private v0.1.0
Checking ppv-lite86 v0.2.17
Compiling version_check v0.9.4
Checking memchr v2.6.4
Compiling ahash v0.7.6
Checking core2 v0.3.3
Compiling hex_lit v0.1.1
Compiling bitcoin v0.30.1
Compiling quote v1.0.33
Compiling syn v2.0.38
Checking bech32 v0.9.1
Checking base64 v0.13.1
Compiling serde_json v1.0.107
Checking ryu v1.0.15
Checking itoa v1.0.9
Compiling cc v1.0.83
Compiling secp256k1-sys v0.8.1
Compiling wasm-bindgen-backend v0.2.87
Compiling serde_derive v1.0.189
Compiling wasm-bindgen-macro-support v0.2.87
Compiling wasm-bindgen-macro v0.2.87
Checking js-sys v0.3.64
Checking getrandom v0.2.10
Checking rand_core v0.6.4
Checking rand_chacha v0.3.1
Checking rand v0.8.5
Checking bitcoin_hashes v0.12.0
Checking hashbrown v0.11.2
Checking secp256k1 v0.27.0
Checking miniscript v10.0.0
Checking bdk_chain v0.6.0 (/Users/einherjar/git/btc/bdk/crates/chain)
Checking bdk v1.0.0-alpha.2 (/Users/einherjar/git/btc/bdk/crates/bdk)
Finished release [optimized] target(s) in 8.55s

[dpc]

I did a quick check and looks like core2 didn't get vendored?

09:24:32 /tmp/nix-build-crates-deps-0.1.0.drv-0/source🔒 🦀v1.73.0 ❄️                       r                               u                                   > cat .cargo-home/config.toml                                                                                                                                   
[source.nix-sources-c19b7c6f923b580ac259164a89f2577984ad5ab09ee9d583b888f934adbbe8d0]                                                      
directory = "/nix/store/yfbswavryrlhq35v7xpmb2dsm0ik0aan-vendor-cargo-deps/c19b7c6f923b580ac259164a89f2577984ad5ab09ee9d583b888f934adbbe8d0"          
[source.crates-io]                                                                                                                                              
registry = "https://github.com/rust-lang/crates.io-index"                                                                                                       
replace-with = "nix-sources-c19b7c6f923b580ac259164a89f2577984ad5ab09ee9d583b888f934adbbe8d0"                         
> ls /nix/store/yfbswavryrlhq35v7xpmb2dsm0ik0aan-vendor-cargo-deps/c19b7c6f923b580ac259164a89f2577984ad5ab09ee9d583b888f934adbbe8d0 | grep core
bitcoincore-rpc-0.17.0
bitcoincore-rpc-json-0.17.0
core-foundation-0.9.3
core-foundation-sys-0.8.4
futures-core-0.3.28
parking_lot_core-0.8.6
rand_core-0.4.2
rand_core-0.6.4
tracing-core-0.1.32

[realeinherjar]

This is fixed by adding core2 to bdk's Cargo.toml, since it is used in no-std builds of rust-bitcoin.

Don't know why in a devShell running the cargo check --release --locked --target wasm32-unknown-unknown -p bdk --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm is not modifying Cargo.lock

[dpc]

It looks like Cargo.lock does not contain core2, (which is a conditional dependency of bitcoin, and no matter what is behing called cargo update, etc. cargo will just not add it there, yet things are working fine. It's only when building in crane where this missing Cargo.lock entry causes core2 not being vendored.

I don't understand that cargo's behavior. Bug? Or that's how it works?

[ipetkov]

I don't understand that cargo's behavior. Bug? Or that's how it works?

I'm pretty surprised by this behavior, I'd consider this a cargo bug (Cargo.lock already contains all possible deps across targets, it should be containing it for all possible features too)

---

I'm not entirely sure if --features bitcoin/no-std is a "supported" way of toggling downstream features or if its a happenstance that it works. I think the "correct" way to solve this is to add a

[target.'cfg(target = "wasm32-unknown-unknown').dependencies]
bitcoin = { version = "...", features = ["no-std"] }

which should convince cargo to add it to the Cargo.lock.

---

Don't think there's anything for us to do here since we cannot vendor dependencies if they don't show up in Cargo.lock (but at least you have a workaround at least!)

[realeinherjar]

Thank you, this works :)