Missing toRustTarget in rust-overlay

[FintanH]

Hey hey, long time listener first-time caller :wave: I was greeted with an error in the overlay this morning and I was wondering if I could get some help on figuring out what went wrong :)

I have the usual line in my shell.nix:

moz_overlay = import (builtins.fetchTarball https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz);

And it errored with:

error: attribute 'toRustTarget' missing, at /nix/store/jazsvnvfwlzd1mry86fgi9jbwp6hqh1p-source/rust-overlay.nix:122:28

The file in question -- rust-overlay.nix -- looks like below. It's giving out about hostTargets in getComponents

# This file provide a Rust overlay, which provides pre-packaged bleeding edge versions of rustc
# and cargo.
self: super:

let
  fromTOML =
    # nix 2.1 added the fromTOML builtin
    if builtins ? fromTOML
    then builtins.fromTOML
    else (import ./lib/parseTOML.nix).fromTOML;

  parseRustToolchain = file: with builtins;
    if file == null then
      {}
    else
    let
      # matches toolchain descriptions of type "nightly" or "nightly-2020-01-01"
      channel_by_name = match "([a-z]+)(-([0-9]{4}-[0-9]{2}-[0-9]{2}))?.*" (readFile file);
      # matches toolchain descriptions of type "1.34.0" or "1.34.0-2019-04-10"
      channel_by_version = match "([0-9]+\\.[0-9]+\\.[0-9]+)(-([0-9]{4}-[0-9]{2}-[0-9]{2}))?.*" (readFile file);
    in
      (x: { channel = head x; date = (head (tail (tail x))); }) (
        if channel_by_name != null then
          channel_by_name
        else
          channel_by_version
        );

  # See https://github.com/rust-lang-nursery/rustup.rs/blob/master/src/dist/src/dist.rs
  defaultDistRoot = "https://static.rust-lang.org";
  manifest_v1_url = {
    dist_root ? defaultDistRoot + "/dist",
    date ? null,
    staging ? false,
    # A channel can be "nightly", "beta", "stable", or "\d{1}\.\d{1,3}\.\d{1,2}".
    channel ? "nightly",
    # A path that points to a rust-toolchain file, typically ./rust-toolchain.
    rustToolchain ? null,
    ...
  }:
    let args = { inherit channel date; } // parseRustToolchain rustToolchain; in
    let inherit (args) date channel; in
    if date == null && staging == false
    then "${dist_root}/channel-rust-${channel}"
    else if date != null && staging == false
    then "${dist_root}/${date}/channel-rust-${channel}"
    else if date == null && staging == true
    then "${dist_root}/staging/channel-rust-${channel}"
    else throw "not a real-world case";

  manifest_v2_url = args: (manifest_v1_url args) + ".toml";

  getComponentsWithFixedPlatform = pkgs: pkgname: stdenv:
    let
      pkg = pkgs.${pkgname};
      srcInfo = pkg.target.${super.rust.toRustTarget stdenv.targetPlatform} or pkg.target."*";
      components = srcInfo.components or [];
      componentNamesList =
        builtins.map (pkg: pkg.pkg) (builtins.filter (pkg: (pkg.target != "*")) components);
    in
      componentNamesList;

  getExtensions = pkgs: pkgname: stdenv:
    let
      inherit (super.lib) unique;
      pkg = pkgs.${pkgname};
      srcInfo = pkg.target.${super.rust.toRustTarget stdenv.targetPlatform} or pkg.target."*";
      extensions = srcInfo.extensions or [];
      extensionNamesList = unique (builtins.map (pkg: pkg.pkg) extensions);
    in
      extensionNamesList;

  hasTarget = pkgs: pkgname: target:
    pkgs ? ${pkgname}.target.${target};

  getTuples = pkgs: name: targets:
    builtins.map (target: { inherit name target; }) (builtins.filter (target: hasTarget pkgs name target) targets);

  # In the manifest, a package might have different components which are bundled with it, as opposed as the extensions which can be added.
  # By default, a package will include the components for the same architecture, and offers them as extensions for other architectures.
  #
  # This functions returns a list of { name, target } attribute sets, which includes the current system package, and all its components for the selected targets.
  # The list contains the package for the pkgTargets as well as the packages for components for all compTargets
  getTargetPkgTuples = pkgs: pkgname: pkgTargets: compTargets: stdenv:
    let
      inherit (builtins) elem;
      inherit (super.lib) intersectLists;
      components = getComponentsWithFixedPlatform pkgs pkgname stdenv;
      extensions = getExtensions pkgs pkgname stdenv;
      compExtIntersect = intersectLists components extensions;
      tuples = (getTuples pkgs pkgname pkgTargets) ++ (builtins.map (name: getTuples pkgs name compTargets) compExtIntersect);
    in
      tuples;

  getFetchUrl = pkgs: pkgname: target: stdenv: fetchurl:
    let
      pkg = pkgs.${pkgname};
      srcInfo = pkg.target.${target};
    in
      (super.fetchurl { url = srcInfo.xz_url; sha256 = srcInfo.xz_hash; });

  checkMissingExtensions = pkgs: pkgname: stdenv: extensions:
    let
      inherit (builtins) head;
      inherit (super.lib) concatStringsSep subtractLists;
      availableExtensions = getExtensions pkgs pkgname stdenv;
      missingExtensions = subtractLists availableExtensions extensions;
      extensionsToInstall =
        if missingExtensions == [] then extensions else throw ''
          While compiling ${pkgname}: the extension ${head missingExtensions} is not available.
          Select extensions from the following list:
          ${concatStringsSep "\n" availableExtensions}'';
    in
      extensionsToInstall;

  getComponents = pkgs: pkgname: targets: extensions: targetExtensions: stdenv: fetchurl:
    let
      inherit (builtins) head map;
      inherit (super.lib) flatten remove subtractLists unique;
      targetExtensionsToInstall = checkMissingExtensions pkgs pkgname stdenv targetExtensions;
      extensionsToInstall = checkMissingExtensions pkgs pkgname stdenv extensions;
      hostTargets = [ "*" (super.rust.toRustTarget stdenv.hostPlatform) (super.rust.toRustTarget stdenv.targetPlatform) ];
      pkgTuples = flatten (getTargetPkgTuples pkgs pkgname hostTargets targets stdenv);
      extensionTuples = flatten (map (name: getTargetPkgTuples pkgs name hostTargets targets stdenv) extensionsToInstall);
      targetExtensionTuples = flatten (map (name: getTargetPkgTuples pkgs name targets targets stdenv) targetExtensionsToInstall);
      pkgsTuples = pkgTuples ++ extensionTuples ++ targetExtensionTuples;
      missingTargets = subtractLists (map (tuple: tuple.target) pkgsTuples) (remove "*" targets);
      pkgsTuplesToInstall =
        if missingTargets == [] then pkgsTuples else throw ''
          While compiling ${pkgname}: the target ${head missingTargets} is not available for any package.'';
    in
      map (tuple: { name = tuple.name; src = (getFetchUrl pkgs tuple.name tuple.target stdenv fetchurl); }) pkgsTuplesToInstall;

  installComponents = stdenv: namesAndSrcs:
    let
      inherit (builtins) map;
      installComponent = name: src:
        stdenv.mkDerivation {
          inherit name;
          inherit src;

          # No point copying src to a build server, then copying back the
          # entire unpacked contents after just a little twiddling.
          preferLocalBuild = true;

          # (@nbp) TODO: Check on Windows and Mac.
          # This code is inspired by patchelf/setup-hook.sh to iterate over all binaries.
          installPhase = ''
            patchShebangs install.sh
            CFG_DISABLE_LDCONFIG=1 ./install.sh --prefix=$out --verbose

            setInterpreter() {
              local dir="$1"
              [ -e "$dir" ] || return 0

              header "Patching interpreter of ELF executables and libraries in $dir"
              local i
              while IFS= read -r -d ''$'\0' i; do
                if [[ "$i" =~ .build-id ]]; then continue; fi
                if ! isELF "$i"; then continue; fi
                echo "setting interpreter of $i"

                if [[ -x "$i" ]]; then
                  # Handle executables
                  patchelf \
                    --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \
                    --set-rpath "${super.lib.makeLibraryPath [ self.zlib ]}:$out/lib" \
                    "$i" || true
                else
                  # Handle libraries
                  patchelf \
                    --set-rpath "${super.lib.makeLibraryPath [ self.zlib ]}:$out/lib" \
                    "$i" || true
                fi
              done < <(find "$dir" -type f -print0)
            }

            setInterpreter $out
          '';

          postFixup = ''
            # Function moves well-known files from etc/
            handleEtc() {
              local oldIFS="$IFS"

              # Directories we are aware of, given as substitution lists
              for paths in \
                "etc/bash_completion.d","share/bash_completion/completions","etc/bash_completions.d","share/bash_completions/completions";
                do
                # Some directoties may be missing in some versions. If so we just skip them.
                # See https://github.com/mozilla/nixpkgs-mozilla/issues/48 for more infomation.
                if [ ! -e $paths ]; then continue; fi

                IFS=","
                set -- $paths
                IFS="$oldIFS"

                local orig_path="$1"
                local wanted_path="$2"

                # Rename the files
                if [ -d ./"$orig_path" ]; then
                  mkdir -p "$(dirname ./"$wanted_path")"
                fi
                mv -v ./"$orig_path" ./"$wanted_path"

                # Fail explicitly if etc is not empty so we can add it to the list and/or report it upstream
                rmdir ./etc || {
                  echo Installer tries to install to /etc:
                  find ./etc
                  exit 1
                }
              done
            }

            if [ -d "$out"/etc ]; then
              pushd "$out"
              handleEtc
              popd
            fi
          '';

          dontStrip = true;
        };
    in
      map (nameAndSrc: (installComponent nameAndSrc.name nameAndSrc.src)) namesAndSrcs;

  # Manifest files are organized as follow:
  # { date = "2017-03-03";
  #   pkg.cargo.version= "0.18.0-nightly (5db6d64 2017-03-03)";
  #   pkg.cargo.target.x86_64-unknown-linux-gnu = {
  #     available = true;
  #     hash = "abce..."; # sha256
  #     url = "https://static.rust-lang.org/dist/....tar.gz";
  #     xz_hash = "abce..."; # sha256
  #     xz_url = "https://static.rust-lang.org/dist/....tar.xz";
  #   };
  # }
  #
  # The packages available usually are:
  #   cargo, rust-analysis, rust-docs, rust-src, rust-std, rustc, and
  #   rust, which aggregates them in one package.
  #
  # For each package the following options are available:
  #   extensions        - The extensions that should be installed for the package.
  #                       For example, install the package rust and add the extension rust-src.
  #   targets           - The package will always be installed for the host system, but with this option
  #                       extra targets can be specified, e.g. "mips-unknown-linux-musl". The target
  #                       will only apply to components of the package that support being installed for
  #                       a different architecture. For example, the rust package will install rust-std
  #                       for the host system and the targets.
  #   targetExtensions  - If you want to force extensions to be installed for the given targets, this is your option.
  #                       All extensions in this list will be installed for the target architectures.
  #                       *Attention* If you want to install an extension like rust-src, that has no fixed architecture (arch *),
  #                       you will need to specify this extension in the extensions options or it will not be installed!
  fromManifestFile = manifest: { stdenv, fetchurl, patchelf }:
    let
      inherit (builtins) elemAt;
      inherit (super) makeOverridable;
      inherit (super.lib) flip mapAttrs;
      pkgs = fromTOML (builtins.readFile manifest);
    in
    flip mapAttrs pkgs.pkg (name: pkg:
      makeOverridable ({extensions, targets, targetExtensions}:
        let
          version' = builtins.match "([^ ]*) [(]([^ ]*) ([^ ]*)[)]" pkg.version;
          version = "${elemAt version' 0}-${elemAt version' 2}-${elemAt version' 1}";
          namesAndSrcs = getComponents pkgs.pkg name targets extensions targetExtensions stdenv fetchurl;
          components = installComponents stdenv namesAndSrcs;
          componentsOuts = builtins.map (comp: (super.lib.strings.escapeNixString (super.lib.getOutput "out" comp))) components;
        in
          super.pkgs.symlinkJoin {
            name = name + "-" + version;
            paths = components;
            postBuild = ''
              # If rustc or rustdoc is in the derivation, we need to copy their
              # executable into the final derivation. This is required
              # for making them find the correct SYSROOT.
              for target in $out/bin/{rustc,rustdoc}; do
                if [ -e $target ]; then
                  cp --remove-destination "$(realpath -e $target)" $target
                fi
              done
            '';

            # Add the compiler as part of the propagated build inputs in order
            # to run:
            #
            #    $ nix-shell -p rustChannels.stable.rust
            #
            # And get a fully working Rust compiler, with the stdenv linker.
            propagatedBuildInputs = [ stdenv.cc ];

            meta.platforms = stdenv.lib.platforms.all;
          }
      ) { extensions = []; targets = []; targetExtensions = []; }
    );

  fromManifest = sha256: manifest: { stdenv, fetchurl, patchelf }:
    let manifestFile = if sha256 == null then builtins.fetchurl manifest else fetchurl { url = manifest; inherit sha256; };
    in fromManifestFile manifestFile { inherit stdenv fetchurl patchelf; };

in

rec {
  lib = super.lib // {
    inherit fromTOML;
    rustLib = {
      inherit fromManifest fromManifestFile manifest_v2_url;
    };
  };

  rustChannelOf = { sha256 ? null, ... } @ manifest_args: fromManifest
    sha256 (manifest_v2_url manifest_args)
    { inherit (self) stdenv fetchurl patchelf; }
    ;

  # Set of packages which are automagically updated. Do not rely on these for
  # reproducible builds.
  latest = (super.latest or {}) // {
    rustChannels = {
      nightly = rustChannelOf { channel = "nightly"; };
      beta    = rustChannelOf { channel = "beta"; };
      stable  = rustChannelOf { channel = "stable"; };
    };
  };

  # Helper builder
  rustChannelOfTargets = channel: date: targets:
    (rustChannelOf { inherit channel date; })
      .rust.override { inherit targets; };

  # For backward compatibility
  rustChannels = latest.rustChannels;

  # For each channel:
  #   latest.rustChannels.nightly.cargo
  #   latest.rustChannels.nightly.rust   # Aggregate all others. (recommended)
  #   latest.rustChannels.nightly.rustc
  #   latest.rustChannels.nightly.rust-analysis
  #   latest.rustChannels.nightly.rust-docs
  #   latest.rustChannels.nightly.rust-src
  #   latest.rustChannels.nightly.rust-std

  # For a specific date:
  #   (rustChannelOf { date = "2017-06-06"; channel = "beta"; }).rust
}

[FintanH]

@matthewbauer: do you think it could be to do with this commit? https://github.com/mozilla/nixpkgs-mozilla/commit/efda5b357451dbb0431f983cca679ae3cd9b9829 :eyes:

Also, hey again :grin:

[FintanH]

I'm able to work around it for now by pointing to https://github.com/mozilla/nixpkgs-mozilla/commit/4521bc61c2332f41e18664812a808294c8c78580 instead of master.

[matthewbauer]

Hey! Sorry about that. It looks like toRustTarget was added only in 20.03, so many users won't have it.

It's easiest to just provide a backup in this case, so done in:

https://github.com/mozilla/nixpkgs-mozilla/pull/234

[FintanH]

Hmmm weird... I'm on 20.03 as far as I'm aware (still getting used to the nix ecosystem). But glad you have a fix for pre-20.03 too :raised_hands:

[mayl]

It also seems like the combination of efda5b3 and this fix the problems I was having with cargo doc in #223.

[dysinger]

I'm running master of nixpkgs and I still hit this error. I have to roll back to July 14.

tim@sirius ~/s/nixpkgs> jq '."nixpkgs"|{ branch, sha256}' <nix/sources.json 
{
  "branch": "master",
  "sha256": "0lb5hh2fscvkdb30mbslqlwwxqvgc6a6bmk2aj6aj0sp49fky18n"
}
tim@sirius ~/s/nixpkgs> jq '."nixpkgs-mozilla"|{ branch, sha256}' <nix/sources.json 
{
  "branch": "master",
  "sha256": "11wqrg86g3qva67vnk81ynvqyfj0zxk83cbrf0p9hsvxiwxs8469"
}
tim@sirius ~/s/nixpkgs> nix-env -f ./. -iA desktop --show-trace
replacing old 'desktop'
installing 'desktop'
error: while evaluating the attribute 'passAsFile' of the derivation 'desktop' at /nix/store/ymajliyf22m3fcvmqdmlpij6grjrjl7s-nixpkgs-src/pkgs/build-support/trivial-builders.nix:7:7:
while evaluating the attribute 'buildInputs' of the derivation 'crate-cargo2nix-0.8.3' at /nix/store/j5x5vsdmsi1r63xzqlb7mf42rxfb966r-cargo2nix-src/overlay/mkcrate.nix:99:5:
while evaluating the attribute 'buildInputs' of the derivation 'crate-cargo-0.41.0' at /nix/store/j5x5vsdmsi1r63xzqlb7mf42rxfb966r-cargo2nix-src/overlay/mkcrate.nix:99:5:
while evaluating the attribute 'buildInputs' of the derivation 'crate-atty-0.2.13' at /nix/store/j5x5vsdmsi1r63xzqlb7mf42rxfb966r-cargo2nix-src/overlay/mkcrate.nix:99:5:
while evaluating the attribute 'nativeBuildInputs' of the derivation 'crate-libc-0.2.65' at /nix/store/j5x5vsdmsi1r63xzqlb7mf42rxfb966r-cargo2nix-src/overlay/mkcrate.nix:99:5:
while evaluating the attribute 'paths' of the derivation 'cargo-0.38.0-2019-08-02-9edd08916' at /nix/store/zdrn0r93cyds8spjfx9his52ksqj6rv7-source/pkgs/build-support/trivial-builders.nix:7:14:
while evaluating 'installComponents' at /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:134:31, called from /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:269:24:
while evaluating 'getComponents' at /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:116:81, called from /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:268:26:
while evaluating 'flatten' at /nix/store/zdrn0r93cyds8spjfx9his52ksqj6rv7-source/lib/lists.nix:124:13, called from /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:123:19:
while evaluating 'getTargetPkgTuples' at /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:84:64, called from /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:123:28:
while evaluating 'getTuples' at /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:76:27, called from /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:91:17:
while evaluating anonymous function at /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:77:71, called from undefined position:
while evaluating 'hasTarget' at /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:73:30, called from /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:77:79:
while evaluating the attribute 'rust.toRustTarget' at /nix/store/zdrn0r93cyds8spjfx9his52ksqj6rv7-source/pkgs/top-level/all-packages.nix:8184:3:
attribute 'toRustTarget' missing, at /nix/store/xcwy5w0b8b1kvnccjw5472lggr3kglq5-nixpkgs-mozilla-src/rust-overlay.nix:122:28
tim@sirius ~/s/nixpkgs> 

[dysinger]

Rolling back to July 14 worked great. I will hold here until the problem is fixed.

[nrolland]

same here, I had to niv add mozilla/nixpkgs-mozilla -a rev=18cd4300e9bf61c7b8b372f07af827f6ddc835bb

[ericbmerritt]

I am running 20.03, it definitely has rust.toRustTarget, yet I am still getting this error. Interestingly enough, going back to before July 14 didn't work for me as it worked for @dysinger (hey Tim). I am stuck at this point.

Ok. It seems to come up when you mix packages from the overlay and from not the overlay. For example, if I do this

rustc = super.latest.rustChannels.stable.rust;
inherit (super.latest.rustChannels.stable) cargo rust rust-fmt rust-std clippy;

Any included in the shell that is not explicitly in that list causes problems. However, if I change it to

rustc = super.latest.rustChannels.stable.rust;
inherit (super.latest.rustChannels.stable);

things work just fine. That solves my problem (well enough at least).

[Ericson2314]

I am a bit confused what is going on, but I made https://github.com/NixOS/nixpkgs/pull/144193/files in part so we could make toRustTarget and friends less tied to specific rust packages. I hope that might help going forward.