Could not read the output of 'nix-prefetch-url'

[rscorer]

Hi, learning nix etc using nix.dev and ran into this issue when executing

nix-shell -p niv --run "niv init --nixpkgs nixos/nixpkgs --nixpkgs-branch nixos-23.11"

Initializing
  Creating nix/sources.nix
  Creating nix/sources.json
  Importing 'nixpkgs' ...
  Adding package nixpkgs
    FATAL: Could not read the output of 'nix-prefetch-url'. This is a bug. Please create a
    ticket:

      https://github.com/nmattia/niv/issues/new

    Thanks! I'll buy you a beer.
    command:
    nix-prefetch-url--unpack https://github.com/nixos/nixpkgs/archive/20f65b86b6485decb43c5498780c223571dd56ef.tar.gz --name 20f65b86b6485decb43c5498780c223571dd56ef.tar.gz
    stdout:

    stderr:
    error: failed to extract archive (Write failed)

    FATAL: One or more packages failed to update:
    nixpkgs: ExitFailure 1
  ERROR: ExitFailure 1
ERROR: ExitFailure 1

[rscorer]

$ niv --version
0.2.22

Ubuntu 23.10
Linux nixie 6.5.0-17-generic #17-Ubuntu SMP PREEMPT_DYNAMIC Thu Jan 11 14:01:59 UTC 2024 x86_64 GNU/Linux

On DigitalOcean

Nix installed with The Determinate Nix Installer

[nmattia]

Hi @rscorer, thanks for the report. Does this happen consistently? What happens if you run the failing command?

$ nix-prefetch-url --unpack https://github.com/nixos/nixpkgs/archive/20f65b86b6485decb43c5498780c223571dd56ef.tar.gz --name 20f65b86b6485decb43c5498780c223571dd56ef.tar.gz

[rscorer]

When I remember to delete the local ./nix folder, it happens each time - and I tried various nixOS versions too, all with the same result...

# nix-prefetch-url --unpack https://github.com/nixos/nixpkgs/archive/20f65b86b6485decb43c5498780c223571dd56ef.tar.gz --name 20f65b86b6485decb43c5498780c223571dd56ef.tar.gz
path is '/nix/store/brkw8r1qx8m53zgl585qcrm79chbcb3b-20f65b86b6485decb43c5498780c223571dd56ef.tar.gz'
04n3qgkgj90qzxm8wsgr2pb8q3qvxjrygcchg8sr34p29cb2mv33

Contents of the ./nix folder - maybe this shows how far your tool got?

# tree -a nix
nix
├── sources.json
└── sources.nix

1 directory, 2 files

cat nix/sources.json

```{}```

cat nix/sources.nix

``` # This file has been generated by Niv. let # # The fetchers. fetch_ fetches specs of type . # fetch_file = pkgs: name: spec: let name' = sanitizeName name + "-src"; in if spec.builtin or true then builtins_fetchurl { inherit (spec) url sha256; name = name'; } else pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; fetch_tarball = pkgs: name: spec: let name' = sanitizeName name + "-src"; in if spec.builtin or true then builtins_fetchTarball { name = name'; inherit (spec) url sha256; } else pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; fetch_git = name: spec: let ref = spec.ref or ( if spec ? branch then "refs/heads/${spec.branch}" else if spec ? tag then "refs/tags/${spec.tag}" else abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!" ); submodules = spec.submodules or false; submoduleArg = let nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; emptyArgWithWarning = if submodules then builtins.trace ( "The niv input \"${name}\" uses submodules " + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + "does not support them" ) { } else { }; in if nixSupportsSubmodules then { inherit submodules; } else emptyArgWithWarning; in builtins.fetchGit ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); fetch_local = spec: spec.path; fetch_builtin-tarball = name: throw ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. $ niv modify ${name} -a type=tarball -a builtin=true''; fetch_builtin-url = name: throw ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. $ niv modify ${name} -a type=file -a builtin=true''; # # Various helpers # # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 sanitizeName = name: ( concatMapStrings (s: if builtins.isList s then "-" else s) ( builtins.split "[^[:alnum:]+._?=-]+" ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) ) ); # The set of packages used when specs are fetched using non-builtins. mkPkgs = sources: system: let sourcesNixpkgs = import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; hasThisAsNixpkgsPath = == ./.; in if builtins.hasAttr "nixpkgs" sources then sourcesNixpkgs else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then import { } else abort '' Please specify either (through -I or NIX_PATH=nixpkgs=...) or add a package called "nixpkgs" to your sources.json. ''; # The actual fetching function. fetch = pkgs: name: spec: if ! builtins.hasAttr "type" spec then abort "ERROR: niv spec ${name} does not have a 'type' attribute" else if spec.type == "file" then fetch_file pkgs name spec else if spec.type == "tarball" then fetch_tarball pkgs name spec else if spec.type == "git" then fetch_git name spec else if spec.type == "local" then fetch_local spec else if spec.type == "builtin-tarball" then fetch_builtin-tarball name else if spec.type == "builtin-url" then fetch_builtin-url name else abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; # If the environment variable NIV_OVERRIDE_${name} is set, then use # the path directly as opposed to the fetched source. replace = name: drv: let saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; in if ersatz == "" then drv else # this turns the string into an actual Nix path (for both absolute and # relative paths) if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; # Ports of functions for older nix versions # a Nix version of mapAttrs if the built-in doesn't exist mapAttrs = builtins.mapAttrs or ( f: set: with builtins; listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) ); # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); concatMapStrings = f: list: concatStrings (map f list); concatStrings = builtins.concatStringsSep ""; # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 optionalAttrs = cond: as: if cond then as else { }; # fetchTarball version that is compatible between all the versions of Nix builtins_fetchTarball = { url, name ? null, sha256 }@attrs: let inherit (builtins) lessThan nixVersion fetchTarball; in if lessThan nixVersion "1.12" then fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) else fetchTarball attrs; # fetchurl version that is compatible between all the versions of Nix builtins_fetchurl = { url, name ? null, sha256 }@attrs: let inherit (builtins) lessThan nixVersion fetchurl; in if lessThan nixVersion "1.12" then fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) else fetchurl attrs; # Create the final "sources" from the config mkSources = config: mapAttrs ( name: spec: if builtins.hasAttr "outPath" spec then abort "The values in sources.json should not have an 'outPath' attribute" else spec // { outPath = replace name (fetch config.pkgs name spec); } ) config.sources; # The "config" used by the fetchers mkConfig = { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile) , system ? builtins.currentSystem , pkgs ? mkPkgs sources system }: rec { # The sources, i.e. the attribute set of spec name to spec inherit sources; # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers inherit pkgs; }; in mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } ```

HTH

[nmattia]

@rscorer thanks, I think it might be an issue with the environment set up by nix-shell. Can you try to run the niv command outside of the nix-shell (after having wiped the nix/ folder of course, good point)?

$ $(nix-shell -p niv --run "which niv") init --nixpkgs nixos/nixpkgs --nixpkgs-branch nixos-23.11

[rscorer]

@nmattia cool, that seems to have worked 👍

$(nix-shell -p niv --run "which niv") init --nixpkgs nixos/nixpkgs --nixpkgs-branch nixos-23.11
Initializing
  Creating nix/sources.nix
  Creating nix/sources.json
  Importing 'nixpkgs' ...
  Adding package nixpkgs
    Writing new sources file
  Done: Adding package nixpkgs
Done: Initializing

If it would help - I can add your public ssh key to my server so you can poke around and, once you stopped laughing at my mistakes, figure out what I've broken!. It's a VM running on a Digital Ocean trial, I can send you details etc. I've set it up to learn nix, and have rebuilt it a few times already :grinning:

cat nix/sources.json

```
{
    "nixpkgs": {
        "branch": "nixos-23.11",
        "description": "Nix Packages collection & NixOS",
        "homepage": "",
        "owner": "nixos",
        "repo": "nixpkgs",
        "rev": "809cca784b9f72a5ad4b991e0e7bcf8890f9c3a6",
        "sha256": "174d7cbgjyhlin2lm71chgdqs2cprqphnka9w33nlc1qh226ks3l",
        "type": "tarball",
        "url": "https://github.com/nixos/nixpkgs/archive/809cca784b9f72a5ad4b991e0e7bcf8890f9c3a6.tar.gz",
        "url_template": "https://github.com///archive/.tar.gz"
    }
}
```

cat nix/sources.nix

``` # This file has been generated by Niv. let # # The fetchers. fetch_ fetches specs of type . # fetch_file = pkgs: name: spec: let name' = sanitizeName name + "-src"; in if spec.builtin or true then builtins_fetchurl { inherit (spec) url sha256; name = name'; } else pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; fetch_tarball = pkgs: name: spec: let name' = sanitizeName name + "-src"; in if spec.builtin or true then builtins_fetchTarball { name = name'; inherit (spec) url sha256; } else pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; fetch_git = name: spec: let ref = spec.ref or ( if spec ? branch then "refs/heads/${spec.branch}" else if spec ? tag then "refs/tags/${spec.tag}" else abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!" ); submodules = spec.submodules or false; submoduleArg = let nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; emptyArgWithWarning = if submodules then builtins.trace ( "The niv input \"${name}\" uses submodules " + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + "does not support them" ) { } else { }; in if nixSupportsSubmodules then { inherit submodules; } else emptyArgWithWarning; in builtins.fetchGit ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); fetch_local = spec: spec.path; fetch_builtin-tarball = name: throw ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. $ niv modify ${name} -a type=tarball -a builtin=true''; fetch_builtin-url = name: throw ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. $ niv modify ${name} -a type=file -a builtin=true''; # # Various helpers # # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 sanitizeName = name: ( concatMapStrings (s: if builtins.isList s then "-" else s) ( builtins.split "[^[:alnum:]+._?=-]+" ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) ) ); # The set of packages used when specs are fetched using non-builtins. mkPkgs = sources: system: let sourcesNixpkgs = import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; hasThisAsNixpkgsPath = == ./.; in if builtins.hasAttr "nixpkgs" sources then sourcesNixpkgs else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then import { } else abort '' Please specify either (through -I or NIX_PATH=nixpkgs=...) or add a package called "nixpkgs" to your sources.json. ''; # The actual fetching function. fetch = pkgs: name: spec: if ! builtins.hasAttr "type" spec then abort "ERROR: niv spec ${name} does not have a 'type' attribute" else if spec.type == "file" then fetch_file pkgs name spec else if spec.type == "tarball" then fetch_tarball pkgs name spec else if spec.type == "git" then fetch_git name spec else if spec.type == "local" then fetch_local spec else if spec.type == "builtin-tarball" then fetch_builtin-tarball name else if spec.type == "builtin-url" then fetch_builtin-url name else abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; # If the environment variable NIV_OVERRIDE_${name} is set, then use # the path directly as opposed to the fetched source. replace = name: drv: let saneName = stringAsChars (c: if (builtins.match "[a-zA-Z0-9]" c) == null then "_" else c) name; ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; in if ersatz == "" then drv else # this turns the string into an actual Nix path (for both absolute and # relative paths) if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; # Ports of functions for older nix versions # a Nix version of mapAttrs if the built-in doesn't exist mapAttrs = builtins.mapAttrs or ( f: set: with builtins; listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) ); # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 range = first: last: if first > last then [ ] else builtins.genList (n: first + n) (last - first + 1); # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); concatMapStrings = f: list: concatStrings (map f list); concatStrings = builtins.concatStringsSep ""; # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 optionalAttrs = cond: as: if cond then as else { }; # fetchTarball version that is compatible between all the versions of Nix builtins_fetchTarball = { url, name ? null, sha256 }@attrs: let inherit (builtins) lessThan nixVersion fetchTarball; in if lessThan nixVersion "1.12" then fetchTarball ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) else fetchTarball attrs; # fetchurl version that is compatible between all the versions of Nix builtins_fetchurl = { url, name ? null, sha256 }@attrs: let inherit (builtins) lessThan nixVersion fetchurl; in if lessThan nixVersion "1.12" then fetchurl ({ inherit url; } // (optionalAttrs (name != null) { inherit name; })) else fetchurl attrs; # Create the final "sources" from the config mkSources = config: mapAttrs ( name: spec: if builtins.hasAttr "outPath" spec then abort "The values in sources.json should not have an 'outPath' attribute" else spec // { outPath = replace name (fetch config.pkgs name spec); } ) config.sources; # The "config" used by the fetchers mkConfig = { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null , sources ? if sourcesFile == null then { } else builtins.fromJSON (builtins.readFile sourcesFile) , system ? builtins.currentSystem , pkgs ? mkPkgs sources system }: rec { # The sources, i.e. the attribute set of spec name to spec inherit sources; # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers inherit pkgs; }; in mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } ```

Proof I'm learning :wink: - also, would this shell.nix be interfering with niv?

cat shell.nix

let
  nixpkgs = fetchTarball "https://github.com/NixOS/nixpkgs/tarball/nixos-23.11";
  pkgs = import nixpkgs { config = {}; overlays = []; };
in

pkgs.mkShell {
  packages = with pkgs; [
    cowsay
    lolcat
    niv
  ];

  GREETING = "Hello, Nix!";

shellHook = ''
    echo $GREETING | cowsay | lolcat
  '';
}