Invalid path name with slash-2.0.0

[mweinelt]

I'm updating evcc (https://github.com/NixOS/nixpkgs/pull/131475) from 0.61 to 0.66 and after regenerating with node2nix I run into the following error:

❯ nom-build -A "evcc" --
error: The path name 'slash-2.0.0.tgz?cache=0&sync_timestamp=1618384600356&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fslash%2Fdownload%2Fslash-2.0.0.tgz' is invalid: the '&' character is invalid. Path names are alphanumeric and can include the symbols +-._?= and must not begin with a period. Note: If 'slash-2.0.0.tgz?cache=0&sync_timestamp=1618384600356&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fslash%2Fdownload%2Fslash-2.0.0.tgz' is a source file and you cannot rename it on disk, builtins.path { name = ... } can be used to give it an alternative name.

The offending package seems to be this one:

# This file has been generated by node2nix 1.9.0. Do not edit!

{nodeEnv, fetchurl, fetchgit, nix-gitignore, stdenv, lib, globalBuildInputs ? []}:

let
  sources = {
    [...]
    "slash-2.0.0" = {
      name = "slash";
      packageName = "slash";
      version = "2.0.0";
      src = fetchurl {
        url = "https://registry.npm.taobao.org/slash/download/slash-2.0.0.tgz?cache=0&sync_timestamp=1618384600356&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fslash%2Fdownload%2Fslash-2.0.0.tgz";
        sha1 = "de552851a1759df3a8f206535442f5ec4ddeab44";
      };
    };
    [...]

Parts of the url seem to leak into the path name.

[mweinelt]

Dropping the GET parameter from all urls unbreaks this.

:s/tgz?cache.*/tgz";/

Substituted around 50 occassions of these urls.

[svanderburg]

I tried deploying the dependencies with the following pkgs.json:

[
  {"evcc": "andig/evcc"}
]

and generating expressions with:

node2nix -i pkgs.json

and then deploying with:

nix-build -A '"evcc-andig/evcc"'

the only problem I ran into is that the package.json file does not contain a version attribute, but I implemented a workaround for that in https://github.com/svanderburg/node2nix/commit/5ebb644c37fb3b6ecdd2a21692381d2fbd6ccd7a

Can you give the development version a try?

[mweinelt]

Using node2nix on eb26451 building the package errors out with:

error: The path name 'read-pkg-3.0.0.tgz?cache=0&sync_timestamp=1616915181709&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fread-pkg%2Fdownload%2Fread-pkg-3.0.0.tgz' is invalid: the '&' character is invalid. Path names are alphanumeric and can include the symbols +-._?= and must not begin with a period. Note: If 'read-pkg-3.0.0.tgz?cache=0&sync_timestamp=1616915181709&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fread-pkg%2Fdownload%2Fread-pkg-3.0.0.tgz' is a source file and you cannot rename it on disk, builtins.path { name = ... } can be used to give it an alternative name.

Again there are a bunch of these URLs that we can't seem to handle:

❯ rg "cache=0" node-packages.nix
3180:        url = "https://registry.nlark.com/ansi-styles/download/ansi-styles-2.2.1.tgz?cache=0&sync_timestamp=1618995588464&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fansi-styles%2Fdownload%2Fansi-styles-2.2.1.tgz";
4395:        url = "https://registry.npm.taobao.org/camelcase/download/camelcase-4.1.0.tgz?cache=0&sync_timestamp=1603921779318&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fcamelcase%2Fdownload%2Fcamelcase-4.1.0.tgz";
4485:        url = "https://registry.nlark.com/chalk/download/chalk-1.1.3.tgz?cache=0&sync_timestamp=1618995297666&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fchalk%2Fdownload%2Fchalk-1.1.3.tgz";
9696:        url = "https://registry.npm.taobao.org/js-beautify/download/js-beautify-1.13.13.tgz?cache=0&sync_timestamp=1618610187862&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fjs-beautify%2Fdownload%2Fjs-beautify-1.13.13.tgz";
11676:        url = "https://registry.npm.taobao.org/p-reduce/download/p-reduce-1.0.0.tgz?cache=0&sync_timestamp=1617907054299&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fp-reduce%2Fdownload%2Fp-reduce-1.0.0.tgz";
13260:        url = "https://registry.npm.taobao.org/read-pkg/download/read-pkg-3.0.0.tgz?cache=0&sync_timestamp=1616915181709&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fread-pkg%2Fdownload%2Fread-pkg-3.0.0.tgz";
14817:        url = "https://registry.npm.taobao.org/strip-json-comments/download/strip-json-comments-2.0.1.tgz?cache=0&sync_timestamp=1594567789043&other_urls=https%3A%2F%2Fregistry.npm.taobao.org%2Fstrip-json-comments%2Fdownload%2Fstrip-json-comments-2.0.1.tgz";
14862:        url = "https://registry.nlark.com/supports-color/download/supports-color-2.0.0.tgz?cache=0&sync_timestamp=1622293670728&other_urls=https%3A%2F%2Fregistry.nlark.com%2Fsupports-color%2Fdownload%2Fsupports-color-2.0.0.tgz";

They only appear on the nlark and taobao mirrors. Also not for all packages from there.

This is the updateScript I'm using:

#!/nix/store/b45zavallnsvqwjs9wg9xw167jcs0935-bash-4.4-p23/bin/bash
export PATH=/nix/store/nvlnl2swxmk50x25mzn71rhc0gzpylfr-coreutils-8.32/bin:/nix/store/fli1q9xmdnyk15cy48hmsz1ij8g74v8b-curl-7.76.1-bin/bin:/nix/store/mg538v5gv7zm0gbcnzakzficxa9lzp8i-git-2.32.0/bin:/nix/store/kd9m465ik64606s2ipal22bdpnnks803-jq-1.6-bin/bin:/nix/store/cc37n826sii4qmhh5i4vnq0j9b23rcjw-nix-prefetch-git/bin:/nix/store/57m9l26lhb8wy009258j60kizvx046hi-nix-update-0.4.0/bin:/nix/store/fwy1bvxzw0kcqc9sha6aa7fr2b03xs14-node2nix-1.9.0/bin

set -x

cd pkgs/servers/home-automation/evcc

latest=$(curl -L https://api.github.com/repos/andig/evcc/releases/latest | jq -r '.name')

nix-prefetch-git --rev "$latest" --url "https://github.com/andig/evcc" | \
  jq > /home/hexa/git/nixos/mfc9140cdn/pkgs/servers/home-automation/evcc/src.json \
  --arg version "$latest" \
  '.version |= $version'

srcpath=$(jq '.path' -r src.json)

tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' exit

# add missing version attribute
jq < "$srcpath"/package.json > "$tmp"/package.json \
  --arg version "$latest" \
  '.version |= $version'

~/git/nixos/node2nix/result-2/bin/node2nix \
 --input "$tmp/package.json" \
 --lock "$srcpath/package-lock.json" \
 --no-copy-node-env \
 --development \
 --composition /dev/null \
 --output ./node-packages.nix

And indeed I can't reproduce with your instructions, so I wonder what is different with mine? Maybe the lock? Indeed the URLs come from the lockfile.

https://github.com/evcc-io/evcc/blob/110408005754ebdd07e36a44cb7af7f312097ad4/package-lock.json#L2085-L2095

Without the lock file I get requirements for a package version not on npmjs.

http request GET https://registry.npmjs.org/vue-loader-v16
http 200 https://registry.npmjs.org/vue-loader-v16
Cannot resolve version: vue-loader-v16@undefined

[mweinelt]

Workaround:

sed 's/tgz?[^"]\+/tgz/' "$srcpath/package-lock.json" > "$tmp/package-lock.json"