Integrity override mechanism for resolution=tarball dependencies

[steinuil]

Hi! I'm trying to package misskey using this project. Misskey uses some dependencies that are not published on NPM and are resolved using resolution=tarball instead: https://github.com/misskey-dev/misskey/blob/develop/pnpm-lock.yaml#L21139

pnpm doesn't provide an integrity hash for this kind of dependencies, so they can't be fetched in a pure build. There's an issue for it but it hasn't seen any activity since 2018. https://github.com/pnpm/pnpm/issues/1035

I think an easy way to fix this would be adding an argument to mkPnpmPackage accepting an attrset that you override dependencies manually, maybe like this:

{
  "github.com/misskey-dev/browser-image-resizer/0227e860621e55cbed0aabe6dc601096a7748c4a" = fetchFromGitHub {
    owner = "misskey-dev";
    repo = "browser-image-resizer";
    rev = "0227e860621e55cbed0aabe6dc601096a7748c4a";
    sha256 = "...";
  };
}

And then in lockfile.nix check if the package name exists in that attrset and use that derivation instead of generating one.

It's kind of a hack but I can't think of a better way until pnpm provides an integrity hash for that kind of dependencies in the lockfile. I'll probably fork this repo to add this tomorrow so I can get the misskey package done, would you accept a PR that adds this mechanism? Or do you have another idea for how to solve this?

[nzbr]

I added a patch that automatically detects git dependencies and downloads them through fetchGit (the url and the revision are in the lockfile, so it's possible to download the repo in restricted evaluation mode). It patches the lockfile to point to a tarball built from the repo. I also tried adding a fake git to the build script so that it'd catch the git clone and put the dependency there instead, but I think patching the lockfile is the cleaner solution

Let me know if it works for you

[CHN-beta]

I have tried to use the master version of this repository to package rsshub, and find it failed to parse dependencies. It complains "attribute 'id' missing" at here.

I package it like this:

{ lib, mkPnpmPackage, fetchFromGitHub }:
mkPnpmPackage
{
    src = fetchFromGitHub
    {
        owner = "DIYgod";
        repo = "RSSHub";
        rev = "ecab0c0882a17b9b70431aca0c155a2da9d2c4fa";
        hash = "sha256-VAIUQCQcKYaav4Ch73Cn7poO0/VCGtWkWJkPJ3Qp31A=";
    };
}

Output error is:

error:
… while calling the 'derivationStrict' builtin

at //builtin/derivation.nix:9:12: (source not available)

… while evaluating derivation 'rsshub'
whose name attribute is located at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/pkgs/stdenv/generic/make-derivation.nix:300:7

… while evaluating attribute 'configurePhase' of derivation 'rsshub'

at «none»:0: (source not available)

… while calling anonymous lambda

at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/lib/attrsets.nix:816:24:

815|     let f = attrPath:
816|       zipAttrsWith (n: values:
|                        ^
817|         let here = attrPath ++ [n]; in

… while evaluating derivation 'rsshub-node-modules'
whose name attribute is located at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/pkgs/stdenv/generic/make-derivation.nix:300:7

… while evaluating attribute 'buildPhase' of derivation 'rsshub-node-modules'

at /nix/store/5p4hshhzfwn3ksm5qfjlxy2q7z1hdq3g-source/derivation.nix:125:15:

124|
125|               buildPhase = ''
|               ^
126|                 export HOME=$NIX_BUILD_TOP # Some packages need a writable HOME

… while evaluating derivation 'pnpm-lock.yaml'
whose name attribute is located at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/pkgs/stdenv/generic/make-derivation.nix:300:7

… while evaluating attribute 'text' of derivation 'pnpm-lock.yaml'

at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/pkgs/build-support/trivial-builders/default.nix:148:16:

147|     runCommand name
148|       { inherit text executable checkPhase allowSubstitutes preferLocalBuild;
|                ^
149|         passAsFile = [ "text" ];

… while evaluating derivation 'difflib.js.tgz'
whose name attribute is located at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/pkgs/stdenv/generic/make-derivation.nix:300:7

… while evaluating attribute 'buildCommand' of derivation 'difflib.js.tgz'

at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/pkgs/build-support/trivial-builders/default.nix:87:14:

86|       enableParallelBuilding = true;
87|       inherit buildCommand name;
|              ^
88|       passAsFile = [ "buildCommand" ]

… from call site

at /nix/store/5p4hshhzfwn3ksm5qfjlxy2q7z1hdq3g-source/lockfile.nix:29:52:

28|           fetchGit {
29|             url = "https://${concatStringsSep "/" (init split)}.git";
|                                                    ^
30|             rev = (last split);

… while calling 'init'

at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/lib/lists.nix:750:10:

749|   */
750|   init = list:
|          ^
751|     assert lib.assertMsg (list != []) "lists.init: list must not be empty!";

… from call site

at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/lib/lists.nix:751:12:

750|   init = list:
751|     assert lib.assertMsg (list != []) "lists.init: list must not be empty!";
|            ^
752|     take (length list - 1) list;

… while calling 'assertMsg'

at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/lib/asserts.nix:23:5:

22|     # Message to throw in case `pred` fails
23|     msg:
|     ^
24|     pred || builtins.throw msg;

… from call site

at /nix/store/5p4hshhzfwn3ksm5qfjlxy2q7z1hdq3g-source/lockfile.nix:26:21:

25|           let
26|             split = splitString "/" v.id;
|                     ^
27|           in

… while calling 'splitString'

at /nix/store/s4jqyj35hii03rs7j5n6vn7gpgp6ja81-source/lib/strings.nix:600:22:

599|   */
600|   splitString = sep: s:
|                      ^
601|     let

error: attribute 'id' missing

at /nix/store/5p4hshhzfwn3ksm5qfjlxy2q7z1hdq3g-source/lockfile.nix:26:37:

25|           let
26|             split = splitString "/" v.id;
|                                     ^
27|           in

[nzbr]

@CHN-beta sorry for the (very) late answer. I have now reworked the dependency download logic and am able to build rsshub with the following derivation:

mkPnpmPackage {
  src = pkgs.fetchFromGitHub
    {
      owner = "DIYgod";
      repo = "RSSHub";
      rev = "ecab0c0882a17b9b70431aca0c155a2da9d2c4fa";
      hash = "sha256-VAIUQCQcKYaav4Ch73Cn7poO0/VCGtWkWJkPJ3Qp31A=";
    };

  installEnv = {
    PUPPETEER_EXECUTABLE_PATH = "${pkgs.coreutils}/bin/true"; # Prevent puppeteer from trying to connect to the internet
  };

  script = "build:all";

  distDir = "."; # Copy everything from the build directory to the output
}

[CHN-beta]

@nzbr Thank you for your great job. I tried to package rsshub with the following code and it works well:

{
  lib, mkPnpmPackage, nodejs, writeShellScriptBin, src,
  chromium, bash
}:
let
  unwrapped = mkPnpmPackage
  {
    name = "rsshub-unwrapped";
    inherit src nodejs;
    installEnv.PUPPETEER_SKIP_DOWNLOAD = "1";
    script = "build:all";
    distDir = ".";
  };
in writeShellScriptBin "rsshub"
''
  cd ${unwrapped}
  export PATH=${lib.makeBinPath [ bash nodejs nodejs.pkgs.pnpm chromium ]}:$PATH
  export CHROMIUM_EXECUTABLE_PATH=chromium
  pnpm start
''

I also tried to package misskey, but it still does not works, with another error:

{
  lib, mkPnpmPackage, nodejs, writeShellScriptBin, # use nodejs 21
  bash, cypress, vips, pkg-config, src
}:
let
  unwrapped = mkPnpmPackage
  {
    inherit src nodejs;
    name = "misskey-unwrapped";
    installEnv = { CYPRESS_RUN_BINARY = "${cypress}/bin/Cypress"; NODE_ENV = "production"; };
    copyPnpmStore = true;
    distDir = ".";
    noDevDependencies = true;
  };
in writeShellScriptBin "misskey"
''
  cd ${unwrapped}
  export PATH=${lib.makeBinPath [ bash nodejs nodejs.pkgs.pnpm nodejs.pkgs.gulp cypress ]}:$PATH
  export CYPRESS_RUN_BINARY="${cypress}/bin/Cypress"
  export NODE_ENV=production
  pnpm run migrateandstart
''

The error is:

misskey-unwrapped-node-modules> building '/nix/store/bm03kinnizfrwqnrprx2byrdwncpr7pi-misskey-unwrapped-node-modules.drv'
misskey-unwrapped-node-modules> Running phase: unpackPhase
misskey-unwrapped-node-modules> '/nix/store/gm1ba72jn5af1pvkjy4rcyc0lragypy5-source/package.json' -> 'package.json'
misskey-unwrapped-node-modules> '/nix/store/nxpyw32cy1zi152nj3l6lkpfq9yj3jr8-pnpm-lock.yaml' -> 'pnpm-lock.yaml'
misskey-unwrapped-node-modules> Running phase: patchPhase
misskey-unwrapped-node-modules> Running phase: updateAutotoolsGnuConfigScriptsPhase
misskey-unwrapped-node-modules> Running phase: configurePhase
misskey-unwrapped-node-modules> no configure script, doing nothing
misskey-unwrapped-node-modules> Running phase: buildPhase
misskey-unwrapped-node-modules> cp: target '/build/.local/share/pnpm/store/v3': No such file or directory

Could you please help me?