search

astral-sh / uv

An extremely fast Python package and project manager, written in Rust.
https://docs.astral.sh/uv
Apache License 2.0
22.21k stars 648 forks source link

cannot install package if module is used in top level of setup.py #3742

Closed brian316 closed 4 months ago

brian316 commented 4 months ago

Issue

I cannot install a library if a module not in standard library is used during build process... in this case torch

Setup

This code comes from the following open-source project https://github.com/idiap/fast-transformers/

Code:

#!/usr/bin/env python
#
# Copyright (c) 2020 Idiap Research Institute, http://www.idiap.ch/
# Written by Angelos Katharopoulos <angelos.katharopoulos@idiap.ch>,
# Apoorv Vyas <avyas@idiap.ch>
#

"""Setup fast transformers"""

from functools import lru_cache
from itertools import dropwhile
import os
from os import path
from setuptools import find_packages, setup
from subprocess import DEVNULL, call
import sys

try:
    import torch
    from torch.utils.cpp_extension import BuildExtension, CppExtension
except ImportError as e:
    raise ImportError(
        ("PyTorch is required to install pytorch-fast-transformers. Please "
         "install your favorite version of PyTorch, we support 1.3.1, 1.5.0 "
         "and >=1.6"),
        name=e.name,
        path=e.path
    ) from e

@lru_cache(None)
def cuda_toolkit_available():
    try:
        call(["nvcc"], stdout=DEVNULL, stderr=DEVNULL)
        return True
    except FileNotFoundError:
        return False

def collect_docstring(lines):
    """Return document docstring if it exists"""
    lines = dropwhile(lambda x: not x.startswith('"""'), lines)
    doc = ""
    for line in lines:
        doc += line
        if doc.endswith('"""\n'):
            break

    return doc[3:-4].replace("\r", "").replace("\n", " ")

def collect_metadata():
    meta = {}
    with open(path.join("fast_transformers", "__init__.py")) as f:
        lines = iter(f)
        meta["description"] = collect_docstring(lines)
        for line in lines:
            if line.startswith("__"):
                key, value = map(lambda x: x.strip(), line.split("="))
                meta[key[2:-2]] = value[1:-1]

    return meta

@lru_cache()
def _get_cpu_extra_compile_args():
    base_args = ["-fopenmp", "-ffast-math"]

    if sys.platform == "darwin":
        return ["-Xpreprocessor"] + base_args
    else:
        return base_args

@lru_cache()
def _get_gpu_extra_compile_args():
    if torch.cuda.is_available():
        return []
    else:
        return ["-arch=compute_60"]

def get_extensions():
    extensions = [
        CppExtension(
            "fast_transformers.hashing.hash_cpu",
            sources=[
                "fast_transformers/hashing/hash_cpu.cpp"
            ],
            extra_compile_args=_get_cpu_extra_compile_args()
        ),
        CppExtension(
            "fast_transformers.aggregate.aggregate_cpu",
            sources=[
               "fast_transformers/aggregate/aggregate_cpu.cpp"
            ],
            extra_compile_args=_get_cpu_extra_compile_args()
        ),
        CppExtension(
            "fast_transformers.clustering.hamming.cluster_cpu",
            sources=[
               "fast_transformers/clustering/hamming/cluster_cpu.cpp"
            ],
            extra_compile_args=_get_cpu_extra_compile_args()
        ),
        CppExtension(
            "fast_transformers.sparse_product.sparse_product_cpu",
            sources=[
                "fast_transformers/sparse_product/sparse_product_cpu.cpp"
            ],
            extra_compile_args=_get_cpu_extra_compile_args()
        ),
        CppExtension(
            "fast_transformers.sparse_product.clustered_sparse_product_cpu",
            sources=[
                "fast_transformers/sparse_product/clustered_sparse_product_cpu.cpp"
            ],
            extra_compile_args=_get_cpu_extra_compile_args()
        ),
        CppExtension(
            "fast_transformers.causal_product.causal_product_cpu",
            sources=[
                "fast_transformers/causal_product/causal_product_cpu.cpp"
            ],
            extra_compile_args=_get_cpu_extra_compile_args()
        ),
        CppExtension(
            "fast_transformers.local_product.local_product_cpu",
            sources=[
                "fast_transformers/local_product/local_product_cpu.cpp"
            ],
            extra_compile_args=_get_cpu_extra_compile_args()
        )
    ]
    if cuda_toolkit_available():
        from torch.utils.cpp_extension import CUDAExtension
        extensions += [
            CUDAExtension(
                "fast_transformers.hashing.hash_cuda",
                sources=[
                    "fast_transformers/hashing/hash_cuda.cu",
                ],
                extra_compile_args=_get_gpu_extra_compile_args()
            ),
            CUDAExtension(
                "fast_transformers.aggregate.aggregate_cuda",
                sources=[
                    "fast_transformers/aggregate/aggregate_cuda.cu"
                ],
                extra_compile_args=_get_gpu_extra_compile_args()
            ),
            CUDAExtension(
                "fast_transformers.aggregate.clustered_aggregate_cuda",
                sources=[
                    "fast_transformers/aggregate/clustered_aggregate_cuda.cu"
                ],
                extra_compile_args=_get_gpu_extra_compile_args()
            ),
            CUDAExtension(
                "fast_transformers.clustering.hamming.cluster_cuda",
                sources=[
                    "fast_transformers/clustering/hamming/cluster_cuda.cu"
                ],
                extra_compile_args=_get_gpu_extra_compile_args()
            ),
            CUDAExtension(
                "fast_transformers.sparse_product.sparse_product_cuda",
                sources=[
                    "fast_transformers/sparse_product/sparse_product_cuda.cu"
                ],
                extra_compile_args=_get_gpu_extra_compile_args()
            ),
            CUDAExtension(
                "fast_transformers.sparse_product.clustered_sparse_product_cuda",
                sources=[
                    "fast_transformers/sparse_product/clustered_sparse_product_cuda.cu"
                ],
                extra_compile_args=_get_gpu_extra_compile_args()
            ),
            CUDAExtension(
                "fast_transformers.causal_product.causal_product_cuda",
                sources=[
                    "fast_transformers/causal_product/causal_product_cuda.cu"
                ],
                extra_compile_args=_get_gpu_extra_compile_args()
            ),
            CUDAExtension(
                "fast_transformers.local_product.local_product_cuda",
                sources=[
                    "fast_transformers/local_product/local_product_cuda.cu"
                ],
                extra_compile_args=_get_gpu_extra_compile_args()
            )
        ]
    return extensions

def setup_package():
    with open("README.rst") as f:
        long_description = f.read()
    meta = collect_metadata()
    version_suffix = os.getenv("FAST_TRANSFORMERS_VERSION_SUFFIX", "")
    setup(
        name="pytorch-fast-transformers",
        version=meta["version"] + version_suffix,
        description=meta["description"],
        long_description=long_description,
        long_description_content_type="text/x-rst",
        maintainer=meta["maintainer"],
        maintainer_email=meta["email"],
        url=meta["url"],
        license=meta["license"],
        classifiers=[
            "Intended Audience :: Science/Research",
            "Intended Audience :: Developers",
            "License :: OSI Approved :: MIT License",
            "Topic :: Scientific/Engineering",
            "Programming Language :: Python",
            "Programming Language :: Python :: 3",
            "Programming Language :: Python :: 3.6",
        ],
        packages=find_packages(exclude=["docs", "tests", "scripts", "examples"]),
        ext_modules=get_extensions(),
        cmdclass={"build_ext": BuildExtension},
        install_requires=["torch"]
    )

if __name__ == "__main__":
    setup_package()

Stacktrace

❯ uv pip install -e . -vvv
 uv_requirements::specification::from_source source=-e .
    0.000922s  INFO uv_interpreter::virtualenv Found a virtualenv through VIRTUAL_ENV at: /home/dev/work/open-source/fast-transformers/.venv
    0.001004s DEBUG uv_interpreter::interpreter Cached interpreter info for Python 3.10.12, skipping probing: .venv/bin/python
    0.001013s DEBUG uv::commands::pip::install Using Python 3.10.12 environment at .venv/bin/python
    0.001044s DEBUG uv_fs Trying to lock if free: .venv/.lock
    0.001114s DEBUG uv::commands::pip::install At least one requirement is not satisfied: file:///home/dev/work/open-source/fast-transformers
 uv_client::linehaul::linehaul 
    0.001354s DEBUG uv_client::base_client Using registry request timeout of 30s
 uv_resolver::flat_index::from_entries 
 uv_installer::downloader::build_editables 
      0.001934s   0ms DEBUG uv_distribution::source Building (editable) file:///home/dev/work/open-source/fast-transformers
   uv_dispatch::setup_build version_id="file:///home/dev/work/open-source/fast-transformers", subdirectory=None
 uv_resolver::resolver::solve 
   uv_resolver::resolver::solve_tracked 
        0.002494s   0ms DEBUG uv_resolver::resolver Solving with target Python version 3.10.12
     uv_resolver::resolver::choose_version package=root
     uv_resolver::resolver::get_dependencies package=root, version=0a0.dev0
          0.002537s   0ms DEBUG uv_resolver::resolver Adding direct dependency: setuptools>=40.8.0
     uv_resolver::resolver::choose_version package=setuptools
     uv_resolver::resolver::process_request request=Versions setuptools
       uv_client::registry_client::simple_api package=setuptools
         uv_client::cached_client::get_cacheable 
           uv_client::cached_client::read_and_parse_cache file=/home/dev/.cache/uv/simple-v7/pypi/setuptools.rkyv
     uv_resolver::resolver::process_request request=Prefetch setuptools >=40.8.0
 uv_client::cached_client::from_path_sync path="/home/dev/.cache/uv/simple-v7/pypi/setuptools.rkyv"
              0.003245s   0ms DEBUG uv_client::cached_client Found fresh response for: https://pypi.org/simple/setuptools/
       uv_resolver::version_map::from_metadata 
          0.003627s   1ms DEBUG uv_resolver::resolver Searching for a compatible version of setuptools (>=40.8.0)
          0.003638s   1ms DEBUG uv_resolver::resolver Selecting: setuptools==70.0.0 (setuptools-70.0.0-py3-none-any.whl)
       uv_distribution::distribution_database::get_or_build_wheel_metadata dist=setuptools==70.0.0
     uv_resolver::resolver::get_dependencies package=setuptools, version=70.0.0
         uv_client::registry_client::wheel_metadata built_dist=setuptools==70.0.0
           uv_client::cached_client::get_serde 
             uv_client::cached_client::get_cacheable 
               uv_client::cached_client::read_and_parse_cache file=/home/dev/.cache/uv/wheels-v1/pypi/setuptools/setuptools-70.0.0-py3-none-any.msgpack
 uv_client::cached_client::from_path_sync path="/home/dev/.cache/uv/wheels-v1/pypi/setuptools/setuptools-70.0.0-py3-none-any.msgpack"
                  0.003790s   0ms DEBUG uv_client::cached_client Found fresh response for: https://files.pythonhosted.org/packages/de/88/70c5767a0e43eb4451c2200f07d042a4bcd7639276003a9c54a68cfcc1f8/setuptools-70.0.0-py3-none-any.whl.metadata
        0.003915s   1ms DEBUG uv_resolver::resolver::batch_prefetch Tried 2 versions: root 1, setuptools 1
     uv_dispatch::install resolution="setuptools==70.0.0", venv="/home/dev/.cache/uv/.tmpEGmPw6/.venv"
          0.003992s   0ms DEBUG uv_dispatch Installing in setuptools==70.0.0 in /home/dev/.cache/uv/.tmpEGmPw6/.venv
          0.004053s   0ms DEBUG uv_installer::plan Requirement already cached: setuptools==70.0.0
          0.004065s   0ms DEBUG uv_dispatch Installing build requirement: setuptools==70.0.0
       uv_installer::installer::install num_wheels=1
        0.006135s   4ms DEBUG uv_build Calling `setuptools.build_meta:__legacy__.get_requires_for_build_editable()`
     uv_build::run_python_script script="get_requires_for_build_editable", python_version=3.10.12
error: Failed to build editables
  Caused by: Failed to build editable: `file:///home/dev/work/open-source/fast-transformers`
  Caused by: Build backend failed to determine extra requires with `build_editable()` with exit status: 1
--- stdout:

--- stderr:
Traceback (most recent call last):
  File "<string>", line 20, in <module>
ModuleNotFoundError: No module named 'torch'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "<string>", line 14, in <module>
  File "/home/dev/.cache/uv/.tmpEGmPw6/.venv/lib/python3.10/site-packages/setuptools/build_meta.py", line 448, in get_requires_for_build_editable
    return self.get_requires_for_build_wheel(config_settings)
  File "/home/dev/.cache/uv/.tmpEGmPw6/.venv/lib/python3.10/site-packages/setuptools/build_meta.py", line 325, in get_requires_for_build_wheel
    return self._get_build_requires(config_settings, requirements=['wheel'])
  File "/home/dev/.cache/uv/.tmpEGmPw6/.venv/lib/python3.10/site-packages/setuptools/build_meta.py", line 295, in _get_build_requires
    self.run_setup()
  File "/home/dev/.cache/uv/.tmpEGmPw6/.venv/lib/python3.10/site-packages/setuptools/build_meta.py", line 487, in run_setup
    super().run_setup(setup_script=setup_script)
  File "/home/dev/.cache/uv/.tmpEGmPw6/.venv/lib/python3.10/site-packages/setuptools/build_meta.py", line 311, in run_setup
    exec(code, locals())
  File "<string>", line 23, in <module>
ImportError: PyTorch is required to install pytorch-fast-transformers. Please install your favorite version of PyTorch, we support 1.3.1, 1.5.0 and >=1.6
charliermarsh commented 4 months ago

This is all correct behavior. You can try running with --no-build-isolation if you want to discover packages in your environment.

brian316 commented 4 months ago

appreciate you!

charliermarsh commented 4 months ago

No prob. The downside is you have to make sure you have all of the build dependencies installed upfront which can be a little tedious and error-prone, but some packages require it (especially if they depend on Torch at build-time).

brian316 commented 4 months ago

is there any workaround for that? like having torch and fast-transformers in 1 requirements.txt file?