Trouble compiling with -Z build_std

[benwis]

I've got a web crate I'm trying to build with Crane, and it has been working great. But I was hoping to use the unstable feature build_std to compile the std crate, and chop off about 100kb in WASM size.

I'm having trouble getting it to work. If I try cargo build --release --target=x86_64-linux-gnu outside Nix in the dev shell it works fine, but if I try it inside Nix with Crane I get

       > ++ command cargo --version
       > cargo 1.69.0-nightly (9d5b32f50 2023-02-22) > ++ command cargo --version
       > cargo 1.69.0-nightly (9d5b32f50 2023-02-22)
       > ++ command cargo check --release --target=x86_64-unknown-linux-gnu
       > error: no matching package named `compiler_builtins` found
       > ++ command cargo check --release --target=x86_64-unknown-linux-gnu
       > error: no matching package named `compiler_builtins` found

This is my hacky flake.nix with Crane, any chance you might know how I can provide compiler_builtins or what I'm missing here. I realize this might be a bit outside your purview, but I've been stuck for quite a while. Thanks!

{
  description = "Build the Benwis Blog!";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";

    crane = {
      url = "github:ipetkov/crane";
      inputs.nixpkgs.follows = "nixpkgs";
    };

    flake-utils.url = "github:numtide/flake-utils";

    cargo-leptos = {
      #url= "github:leptos-rs/cargo-leptos/v1.7";
      url = "github:benwis/cargo-leptos";
      flake = false;
    };

    rust-overlay = {
      url = "github:oxalica/rust-overlay";
      inputs = {
        nixpkgs.follows = "nixpkgs";
        flake-utils.follows = "flake-utils";
      };
    };
    advisory-db = {
      url = "github:rustsec/advisory-db";
      flake = false;
    };
  };

  outputs = { self, nixpkgs, crane, flake-utils, advisory-db, rust-overlay, ... } @inputs:
    flake-utils.lib.eachDefaultSystem
      (system:
        let
          pkgs = import nixpkgs {
            inherit system;
            overlays = [ (import rust-overlay) ];
          };

          rustTarget = pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default.override {
            extensions = [ "rust-src" "rust-analyzer" ];
            targets = [ "x86_64-unknown-linux-gnu" "wasm32-unknown-unknown" ];
          });

          # NB: we don't need to overlay our custom toolchain for the *entire*
          # pkgs (which would require rebuidling anything else which uses rust).
          # Instead, we just want to update the scope that crane will use by appendings
          inherit (pkgs) lib;
          # our specific toolchain there.
          craneLib = (crane.mkLib pkgs).overrideToolchain rustTarget;
          #craneLib = crane.lib.${system};
          # Only keeps markdown files
          protoFilter = path: _type: builtins.match ".*proto$" path != null;
          sqlxFilter = path: _type: builtins.match ".*json$" path != null;
          sqlFilter = path: _type: builtins.match ".*sql$" path != null;
          cssFilter = path: _type: builtins.match ".*css$" path != null;
          ttfFilter = path: _type: builtins.match ".*ttf$" path != null;
          woff2Filter = path: _type: builtins.match ".*woff2$" path != null;
          webpFilter = path: _type: builtins.match ".*webp$" path != null;
          jpegFilter = path: _type: builtins.match ".*jpeg$" path != null;
          pngFilter = path: _type: builtins.match ".*png$" path != null;
          icoFilter = path: _type: builtins.match ".*ico$" path != null;
          protoOrCargo = path: type:
            (protoFilter path type) || (craneLib.filterCargoSources path type) || (sqlxFilter path type) || (sqlFilter path type) || (cssFilter path type) || (woff2Filter path type) || (ttfFilter path type) || (webpFilter path type) || (icoFilter path type) || (jpegFilter path type) || (pngFilter path type);
          # other attributes omitted

          # Include more types of files in our bundle
          src = lib.cleanSourceWith {
            src = ./.; # The original, unfiltered source
            filter = protoOrCargo;
          };
          #    src = craneLib.cleanCargoSource ./.;

          # Common arguments can be set here
          commonArgs = {
            inherit src;
          buildInputs = [
            # Add additional build inputs here
            cargo-leptos
            pkgs.pkg-config
            pkgs.openssl
            pkgs.protobuf
            pkgs.binaryen
            pkgs.cargo-generate
          ] ++ lib.optionals pkgs.stdenv.isDarwin [
            # Additional darwin specific inputs can be set here
            pkgs.libiconv
          ];
        };

          # Build *just* the cargo dependencies, so we can reuse
          # all of that work (e.g. via cachix) when running in CI
          cargoArtifacts = craneLib.buildDepsOnly (commonArgs // {
            cargoCheckExtraArgs = "--target=x86_64-unknown-linux-gnu";
            RUST_SRC_PATH = "${rustTarget}/lib/rustlib/src/rust/library";
          });

          # Build the actual crate itself, reusing the dependency
          # artifacts from above.
          benwis_leptos = craneLib.buildPackage (commonArgs // {
            pname = "benwis_leptos";
            buildPhaseCargoCommand = "LEPTOS_BIN_TARGET_TRIPLE=x86_64-unknown-linux-gnu cargo leptos build --release";
            installPhaseCommand = ''
            mkdir -p $out/bin
            cp target/server/release/benwis_leptos $out/bin/
            cp -r target/site $out/bin/
            '';
            # Prevnt cargo test and nextest from duplicating tests
            doCheck = false;
            inherit cargoArtifacts;
            # ALL CAPITAL derivations will get forwarded to mkDerivation and will set the env var during build
            SQLX_OFFLINE = "true";
            #LEPTOS_BIN_TARGET_TRIPLE = "x86_64-unknown-linux-gnu"; # Adding this allows -Zbuild-std to work and shave 100kb off the WASM
            APP_ENVIRONMENT = "production";
          });
          cargo-leptos = pkgs.rustPlatform.buildRustPackage rec {
            pname = "cargo-leptos";
            #version = "0.1.7";
            version = "0.1.8.1";
            buildFeatures = ["no_downloads"]; # cargo-leptos will try to download Ruby and other things without this feature

            src = inputs.cargo-leptos; 

            cargoSha256 = "sha256-FBtbVli9qJQYsd6aLiizy9qup8E0VOVxkmYX6K09aO0=";

            nativeBuildInputs = [pkgs.pkg-config pkgs.openssl];

            buildInputs = with pkgs;
              [openssl pkg-config]
              ++ lib.optionals stdenv.isDarwin [
              Security
            ];

            doCheck = false; # integration tests depend on changing cargo config

            meta = with lib; {
            description = "A build tool for the Leptos web framework";
            homepage = "https://github.com/leptos-rs/cargo-leptos";
            changelog = "https://github.com/leptos-rs/cargo-leptos/blob/v${version}/CHANGELOG.md";
            license = with licenses; [mit];
            maintainers = with maintainers; [benwis];
          };
      };
          flyConfig = ./fly.toml;

          # Deploy the image to Fly with our own bash script
          flyDeploy = pkgs.writeShellScriptBin "flyDeploy" ''
            OUT_PATH=$(nix build --print-out-paths .#container)
            HASH=$(echo $OUT_PATH | grep -Po "(?<=store\/)(.*?)(?=-)")
            ${pkgs.skopeo}/bin/skopeo --insecure-policy --debug copy docker-archive:"$OUT_PATH" docker://registry.fly.io/$FLY_PROJECT_NAME:$HASH --dest-creds x:"$FLY_AUTH_TOKEN" --format v2s2
            ${pkgs.flyctl}/bin/flyctl deploy -i registry.fly.io/$FLY_PROJECT_NAME:$HASH -c ${flyConfig} --remote-only
          '';
        in
        {
          checks = {
            # Build the crate as part of `nix flake check` for convenience
            inherit benwis_leptos;

            # Run clippy (and deny all warnings) on the crate source,
            # again, resuing the dependency artifacts from above.
            #
            # Note that this is done as a separate derivation so that
            # we can block the CI if there are issues here, but not
            # prevent downstream consumers from building our crate by itself.
            #benwis_leptos-clippy = craneLib.cargoClippy (commonArgs // {
            #  inherit cargoArtifacts;
            #  cargoClippyExtraArgs = "--all-targets -- --deny warnings";
            #});

            benwis_leptos-doc = craneLib.cargoDoc (commonArgs //{
              inherit cargoArtifacts;
            });

            # Check formatting
            benwis_leptos-fmt = craneLib.cargoFmt {
              inherit src;
            };

            # Audit dependencies
            benwis_leptos-audit = craneLib.cargoAudit {
              inherit src advisory-db;
            };

            # Run tests with cargo-nextest
            # Consider setting `doCheck = false` on `benwis_leptos` if you do not want
            # the tests to run twice
            # benwis_leptos-nextest = craneLib.cargoNextest {
            #  inherit cargoArtifacts src buildInputs;
            #  partitions = 1;
            #  partitionType = "count";
            #};
          } // lib.optionalAttrs (system == "x86_64-linux") {
            # NB: cargo-tarpaulin only supports x86_64 systems
            # Check code coverage (note: this will not upload coverage anywhere)
            #benwis_leptos-coverage = craneLib.cargoTarpaulin {
            #  inherit cargoArtifacts src;
            #};

          };

          packages.default = benwis_leptos;

          apps.default = flake-utils.lib.mkApp {
            drv = benwis_leptos;
          };

          # Create an option to build a docker image from this package 
          packages.container = pkgs.dockerTools.buildImage {
            name = "benwis_leptos";
            #tag = "latest";
            created = "now";
            copyToRoot = pkgs.buildEnv {
              name = "image-root";
              paths = [ pkgs.cacert ./.  ];
              pathsToLink = [ "/bin" "/db" "/migrations" ];
            };
            config = {
              Env = [ "PATH=${benwis_leptos}/bin" "APP_ENVIRONMENT=production" "LEPTOS_OUTPUT_NAME=benwis_leptos" "LEPTOS_SITE_ADDR=0.0.0.0:3000" "LEPTOS_SITE_ROOT=${benwis_leptos}/bin/site" ];

              ExposedPorts = {
                "3000/tcp" = { };
              };

              Cmd = [ "${benwis_leptos}/bin/benwis_leptos" ];
            };

          };

          apps.flyDeploy = flake-utils.lib.mkApp {
            drv = flyDeploy;
          };
          devShells.default = pkgs.mkShell {
            inputsFrom = builtins.attrValues self.checks;

            # Extra inputs can be added here
            nativeBuildInputs = with pkgs; [
              rustTarget
              openssl
              mysql80
              dive
              sqlx-cli
              wasm-pack
              pkg-config
              binaryen
              nodejs
              nodePackages.tailwindcss
              cargo-leptos
              protobuf
              skopeo
              flyctl
            ];
            RUST_SRC_PATH = "${rustTarget}/lib/rustlib/src/rust/library";
          };
        });
}

I have this in my cargo config, but as I said it works fine in the dev shell

[unstable]
build-std = ["std", "panic_abort", "core", "alloc"]
build-std-features = ["panic_immediate_abort"]

[ipetkov]

Hi @benwis thanks for the detailed report and config!

The limitation here is (soon to be was!) that the standard library has its own Cargo.lock file which specifies the crates it consumes, but the crates cannot be downloaded while running in the builder sandbox unless they are vendored ahead of time.

I've opened https://github.com/ipetkov/crane/pull/287 which should address this! It adds vendorMultipleCargoDeps which allows specifying any number of Cargo.lock files (or their read/parsed equivalents), including the Cargo.lock file associated with the toolchain. Check out the added example for how to get things working once the PR lands!

[benwis]

Awesome!

Makes me feel better knowing I wasn't doing something dumb. Thanks for working on this and the example, looking forward to the PR!

------- Original Message ------- On Sunday, April 2nd, 2023 at 10:22 AM, Ivan Petkov @.***> wrote:

Hi @.***(https://github.com/benwis) thanks for the detailed report and config!

The limitation here is (soon to be was!) that the standard library has its own Cargo.lock file which specifies the crates it consumes, but the crates cannot be downloaded while running in the builder sandbox unless they are vendored ahead of time.

I've opened #287 which should address this! It adds vendorMultipleCargoDeps which allows specifying any number of Cargo.lock files (or their read/parsed equivalents), including the Cargo.lock file associated with the toolchain. Check out the added example for how to get things working once the PR lands!

— Reply to this email directly, view it on GitHub, or unsubscribe. You are receiving this because you were mentioned.Message ID: @.***>

[benwis]

Closer, now I have a new error to chase down

nix build .
error: builder for '/nix/store/s688mf6mrkqyg6h1ycfmcaz8l444cc9y-vendor-registry.drv' failed with exit code 1;
       last 1 log lines:
       > ln: failed to create symbolic link '/nix/store/qkgigprai39h404v4qch5p549aawjafj-vendor-registry/adler-1.0.2/jgzk2hdvph1ljkpyrb1kvpzz35999s7j-cargo-package-adler-1.0.2': Permission denied
       For full logs, run 'nix log /nix/store/s688mf6mrkqyg6h1ycfmcaz8l444cc9y-vendor-registry.drv'.
error: 1 dependencies of derivation '/nix/store/i169a57m8v30b1nj5gp3h0ak2ashzg1k-vendor-cargo-deps.drv' failed to build
error: 1 dependencies of derivation '/nix/store/p71n0sfnz38q4f65x7qwk9alm1z82ivp-benwis_leptos-0.1.0.drv' failed to build

[benwis]

@ipetkov It looks like adding any dependency, including to the example provided, causes it to fail with that error for me.

[ipetkov]

@benwis do you have a (ideally minimal) repro I can check out locally? Maybe there is something wrong with the package deduplication logic

[benwis]

@ipetkov Sure, here's the build-std example with the reqwest dep added to Cargo.toml. No other changes https://github.com/benwis/build-std-example-with-dep

[ipetkov]

Perfect, thanks for the repo!

As I suspected the dedup logic isn't actually deduplicating certain crates :thinking: Taking a closer look

nix show-derivation /nix/store/x5pc04h681al3s1b4nqwmx0s4d59fndl-vendor-registry | jq 'to_entries[0].value.env.buildCommand' -r | sort | uniq -c | rg -v '^\s+1'
      2 ln -s '/nix/store/6xsxdzkaglm9jm909ks2z06yma7iclvx-cargo-package-indexmap-1.9.3' $out/'indexmap-1.9.3'
      2 ln -s '/nix/store/88h5fhi5qxrc04xkw6h1wcjk70pdri96-cargo-package-wasi-0.11.0+wasi-snapshot-preview1' $out/'wasi-0.11.0+wasi-snapshot-preview1'
      2 ln -s '/nix/store/gd8v69x2c4s6lnlhz0rny304rxz4cvkm-cargo-package-libc-0.2.140' $out/'libc-0.2.140'
      2 ln -s '/nix/store/hfawbfg4wjaiflg3xksnaf6dcda1kjkz-cargo-package-hashbrown-0.12.3' $out/'hashbrown-0.12.3'
      2 ln -s '/nix/store/hm24b0gcslpr9m8ni7m299zd0xmh26yy-cargo-package-cfg-if-1.0.0' $out/'cfg-if-1.0.0'

[ipetkov]

Thanks again for reporting the breakage, fix should be in #289!