`allow-prerelease` source fails to resolve `torch`

[ahirner]

Describe the issue

Adding torch as prerelease dependency fails with nix {develop,run}:

error: Missing suitable source/wheel file entry for torch
(use '--show-trace' to show detailed location information)

I expected that nix develop drops into a shell with python torch installed.

poetry lock resolves this dependency. allow-prerelease might be ignored by poetry2nix.

Additional context

Repo with relevant PR: https://github.com/ahirner/y-torch/pull/1

• flake.nix

  {
  description = "Application packaged using poetry2nix";
  
  inputs.flake-utils.url = "github:numtide/flake-utils";
  inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
  inputs.poetry2nix = {
  url = "github:nix-community/poetry2nix";
  inputs.nixpkgs.follows = "nixpkgs";
  inputs.flake-utils.follows = "flake-utils";
  };
  
  outputs = {
  self,
  nixpkgs,
  flake-utils,
  poetry2nix,
  }:
  {
    # Nixpkgs overlay providing the application
    overlay = nixpkgs.lib.composeManyExtensions [
      poetry2nix.overlay
      (final: prev: {
        y-torch = prev.poetry2nix.mkPoetryApplication {
          projectDir = ./.;
          preferWheels = true;
        };
      })
    ];
  }
  // (flake-utils.lib.eachDefaultSystem (system: let
    pkgs = import nixpkgs {
      inherit system;
      overlays = [self.overlay];
      config.allowUnfree = true;
    };
  
    customOverrides = self: super: {};
    env =
      pkgs.poetry2nix.mkPoetryEnv
      {
        projectDir = ./.;
        preferWheels = true;
        overrides = pkgs.poetry2nix.overrides.withDefaults customOverrides;
      };
  in {
    apps.default = {
      program = "${pkgs.y-torch}/bin/y-torch";
      type = "app";
    };
  
    packages.default = pkgs.poetry;
  
    devShell = pkgs.mkShell {
      buildInputs = with pkgs; [
        poetry
        env
      ];
    };
  }));
  }

• pyproject.toml

  
  # <snip>
  [tool.poetry.dependencies]
  python = "^3.10"
  numpy = "*"
  pydantic = "*"

Root cause of failure:

[tool.poetry.dependencies.torch] version = "2.0.0.dev20230101+cpu" allow-prereleases = true source = "torch"

[tool.poetry.scripts] y-torch = 'y_torch:main'

[tool.poetry.dev-dependencies] python-lsp-server = "" python-lsp-ruff = ""

[tool.ruff.per-file-ignores] "init.py" = ["E402"]

[build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api"

[[tool.poetry.source]] name = "torch" url = "https://download.pytorch.org/whl/nightly/cpu" default = false secondary = true

- [poetry.lock](https://github.com/ahirner/y-torch/blob/24a56afc7fc48f829a485dab4c2aebe5b5a6fb63/poetry.lock)

- `--show-trace` tail:

   at /nix/store/5ip8szfhmy5ify1pbbpra3h6y9bz6rii-source/pkgs/development/interpreters/python/with-packages.nix:3:19:

        2|
        3| f: let packages = f pythonPackages; in buildEnv.override { extraLibs = packages; }
         |                   ^

   … while calling 'requiredPythonModules'

   at /nix/store/5ip8szfhmy5ify1pbbpra3h6y9bz6rii-source/pkgs/development/interpreters/python/python-packages-base.nix:50:27:

       49|   # Get list of required Python modules given a list of derivations.
       50|   requiredPythonModules = drvs: let
         |                           ^
       51|     modules = lib.filter hasPythonModule drvs;

   … from call site

   at /nix/store/5ip8szfhmy5ify1pbbpra3h6y9bz6rii-source/pkgs/development/interpreters/python/wrapper.nix:20:13:

       19|   env = let
       20|     paths = requiredPythonModules (extraLibs ++ [ python ] ) ;
         |             ^
       21|     pythonPath = "${placeholder "out"}/${python.sitePackages}";

   … while evaluating the attribute 'passAsFile'

   at /nix/store/5ip8szfhmy5ify1pbbpra3h6y9bz6rii-source/pkgs/build-support/buildenv/default.nix:77:5:

       76|     # XXX: The size is somewhat arbitrary
       77|     passAsFile = if builtins.stringLength pkgs >= 128*1024 then [ "pkgs" ] else [ ];
         |     ^
       78|   }

   … while evaluating the attribute 'passAsFile' of the derivation 'python3-3.10.9-env'

   at /nix/store/5ip8szfhmy5ify1pbbpra3h6y9bz6rii-source/pkgs/stdenv/generic/make-derivation.nix:270:7:

      269|     // (lib.optionalAttrs (attrs ? name || (attrs ? pname && attrs ? version)) {
      270|       name =
         |       ^
      271|         let

   … while evaluating the attribute 'buildInputs' of the derivation 'nix-shell'

   at /nix/store/5ip8szfhmy5ify1pbbpra3h6y9bz6rii-source/pkgs/stdenv/generic/make-derivation.nix:270:7:

      269|     // (lib.optionalAttrs (attrs ? name || (attrs ? pname && attrs ? version)) {
      270|       name =
         |       ^
      271|         let

Edit: changed torch wheel due to https://pytorch.org/blog/compromised-nightly-dependency/

[kelszo]

Can confirm this is an issue, even without allow-prerelease/nightly build.

error: Missing suitable source/wheel file entry for torch

• pyproject.toml

  
  [tool.poetry.dependencies]
  python = ">=3.8,<3.10"
  catboost = "^1.1.1"
  imbalanced-learn = "^0.10.1"
  joblib = "^1.2.0"
  lightgbm = "^3.3.4"
  numpy = "^1.24.1"
  optuna = "^2"
  pandas = "^1.5.2"
  scikit-learn = "^1.2.0"
  tqdm = "^4.64.1"
  xgboost = "^1.7.3"
  sdv = "^0.17.2"

[tool.poetry.group.dev.dependencies] black = "^22.12.0" python-lsp-server = "^1.7.0" pyls-isort = "^0.2.2" jupyter-core = "^5.1.3" nbconvert = "^7.2.7" pyls-black = "^0.4.7" jedi = "^0.18.2"

[build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api"

[tool.poetry.dependencies.torch] version = "1.13.1+cpu" source = "torch"

[[tool.poetry.source]] name = "torch" url = "https://download.pytorch.org/whl/cpu" default = false secondary = true

- `shell.nix`
```nix
{ pkgs ? import <nixpkgs> { } }:
let
  myAppEnv = pkgs.poetry2nix.mkPoetryEnv {
    projectDir = ./.;
    python = pkgs.python39;
    editablePackageSources = { my-app = ./src; };
    preferWheels = true;
  };
in myAppEnv.env

[andronat]

I'm also affected by this.