search

thoth-station / micropipenv

A lightweight wrapper for pip to support requirements.txt, Pipenv and Poetry lock files or converting them to pip-tools compatible output. Designed for containerized Python applications but not limited to them.
https://pypi.org/project/micropipenv/
GNU Lesser General Public License v3.0
231 stars 25 forks source link

[Poetry] Installing a package in main deps and its extras in group deps breaks requirements string #283

Open matt-carr opened 1 year ago

matt-carr commented 1 year ago

Describe the bug When specifying package as a core dependency and package[extras] as a dev dependency in pyproject.toml, micropipenv requirements --no-dev renders package[extras].

To Reproduce

[tool.poetry]
name = "micropiptest"
version = "0.1.0"
description = "test project for micropipenv"
authors = ["Matt Carr <mcarr@kinaxis.com>"]
readme = "README.md"

[tool.poetry.dependencies]
python = "^3.10"
fastapi = "^0.96.0"

[tool.poetry.group.dev.dependencies]
fastapi = {extras = ["all"], version = "^0.96.0"}

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

poetry lock

micropipenv requirements --no-dev

#
# Default dependencies
#
anyio==3.7.0 \
    --hash=sha256:eddca883c4175f14df8aedce21054bfca3adb70ffe76a9f607aef9d7fa2ea7f0 \
    --hash=sha256:275d9973793619a5374e1c89a4f4ad3f4b0a5510a2b5b939444bee8f4c4d37ce
exceptiongroup==1.1.1; (python_version < "3.11") \
    --hash=sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e \
    --hash=sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785
fastapi[all]==0.96.0 \
    --hash=sha256:b8e11fe81e81eab4e1504209917338e0b80f783878a42c2b99467e5e1019a1e9 \
    --hash=sha256:71232d47c2787446991c81c41c249f8a16238d52d779c0e6b43927d3773dbe3c
idna==3.4 \
    --hash=sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2 \
    --hash=sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4
pydantic==1.10.9 \
    --hash=sha256:e692dec4a40bfb40ca530e07805b1208c1de071a18d26af4a2a0d79015b352ca \
    --hash=sha256:3c52eb595db83e189419bf337b59154bdcca642ee4b2a09e5d7797e41ace783f \
    --hash=sha256:939328fd539b8d0edf244327398a667b6b140afd3bf7e347cf9813c736211896 \
    --hash=sha256:b48d3d634bca23b172f47f2335c617d3fcb4b3ba18481c96b7943a4c634f5c8d \
    --hash=sha256:f0b7628fb8efe60fe66fd4adadd7ad2304014770cdc1f4934db41fe46cc8825f \
    --hash=sha256:e1aa5c2410769ca28aa9a7841b80d9d9a1c5f223928ca8bec7e7c9a34d26b1d4 \
    --hash=sha256:eec39224b2b2e861259d6f3c8b6290d4e0fbdce147adb797484a42278a1a486f \
    --hash=sha256:d111a21bbbfd85c17248130deac02bbd9b5e20b303338e0dbe0faa78330e37e0 \
    --hash=sha256:2e9aec8627a1a6823fc62fb96480abe3eb10168fd0d859ee3d3b395105ae19a7 \
    --hash=sha256:07293ab08e7b4d3c9d7de4949a0ea571f11e4557d19ea24dd3ae0c524c0c334d \
    --hash=sha256:7ee829b86ce984261d99ff2fd6e88f2230068d96c2a582f29583ed602ef3fc2c \
    --hash=sha256:4b466a23009ff5cdd7076eb56aca537c745ca491293cc38e72bf1e0e00de5b91 \
    --hash=sha256:7847ca62e581e6088d9000f3c497267868ca2fa89432714e21a4fb33a04d52e8 \
    --hash=sha256:7845b31959468bc5b78d7b95ec52fe5be32b55d0d09983a877cca6aedc51068f \
    --hash=sha256:517a681919bf880ce1dac7e5bc0c3af1e58ba118fd774da2ffcd93c5f96eaece \
    --hash=sha256:67195274fd27780f15c4c372f4ba9a5c02dad6d50647b917b6a92bf00b3d301a \
    --hash=sha256:2196c06484da2b3fded1ab6dbe182bdabeb09f6318b7fdc412609ee2b564c49a \
    --hash=sha256:6257bb45ad78abacda13f15bde5886efd6bf549dd71085e64b8dcf9919c38b60 \
    --hash=sha256:3283b574b01e8dbc982080d8287c968489d25329a463b29a90d4157de4f2baaf \
    --hash=sha256:5f8bbaf4013b9a50e8100333cc4e3fa2f81214033e05ac5aa44fa24a98670a29 \
    --hash=sha256:b9cd67fb763248cbe38f0593cd8611bfe4b8ad82acb3bdf2b0898c23415a1f82 \
    --hash=sha256:f50e1764ce9353be67267e7fd0da08349397c7db17a562ad036aa7c8f4adfdb6 \
    --hash=sha256:73ef93e5e1d3c8e83f1ff2e7fdd026d9e063c7e089394869a6e2985696693766 \
    --hash=sha256:128d9453d92e6e81e881dd7e2484e08d8b164da5507f62d06ceecf84bf2e21d3 \
    --hash=sha256:ad428e92ab68798d9326bb3e5515bc927444a3d71a93b4a2ca02a8a5d795c572 \
    --hash=sha256:fab81a92f42d6d525dd47ced310b0c3e10c416bbfae5d59523e63ea22f82b31e \
    --hash=sha256:963671eda0b6ba6926d8fc759e3e10335e1dc1b71ff2a43ed2efd6996634dafb \
    --hash=sha256:970b1bdc6243ef663ba5c7e36ac9ab1f2bfecb8ad297c9824b542d41a750b298 \
    --hash=sha256:7e1d5290044f620f80cf1c969c542a5468f3656de47b41aa78100c5baa2b8276 \
    --hash=sha256:83fcff3c7df7adff880622a98022626f4f6dbce6639a88a15a3ce0f96466cb60 \
    --hash=sha256:0da48717dc9495d3a8f215e0d012599db6b8092db02acac5e0d58a65248ec5bc \
    --hash=sha256:0a2aabdc73c2a5960e87c3ffebca6ccde88665616d1fd6d3db3178ef427b267a \
    --hash=sha256:9863b9420d99dfa9c064042304868e8ba08e89081428a1c471858aa2af6f57c4 \
    --hash=sha256:e7c9900b43ac14110efa977be3da28931ffc74c27e96ee89fbcaaf0b0fe338e1 \
    --hash=sha256:6cafde02f6699ce4ff643417d1a9223716ec25e228ddc3b436fe7e2d25a1f305 \
    --hash=sha256:95c70da2cd3b6ddf3b9645ecaa8d98f3d80c606624b6d245558d202cd23ea3be
sniffio==1.3.0 \
    --hash=sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384 \
    --hash=sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101
starlette==0.27.0 \
    --hash=sha256:918416370e846586541235ccd38a474c08b80443ed31c578a418e2209b3eef91 \
    --hash=sha256:6a6b0d042acb8d469a01eba54e9cda6cbd24ac602c4cd016723117d6a7e73b75
typing-extensions==4.6.3 \
    --hash=sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26 \
    --hash=sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5

Expected behavior The package, without extras, should be output

Additional context Related to assumptions around the extras resolver

https://github.com/thoth-station/micropipenv/blob/master/micropipenv.py#L838-L849

Because all the extras are present and the root dependency is flagged as a main dependency, the resolver adds the extras.

Especially with 1.5 and on I'm not sure how you'd solve this issue besides checking the pyproject.toml file. Pre-1.5 you could theoretically check to see if all the extra dependencies had a different category than the parent but that might be a little heavy and especially considering https://github.com/thoth-station/micropipenv/issues/249 this whole discussion could be moot anyways

goern commented 1 year ago

sounds reasonable. @frenzymadness wdyt? please re-label if needed

/kind bug

frenzymadness commented 10 months ago

I have an implementation that should fix this but it causes another 14 tests to fail which might be okay but I'd like to verify my understanding of the problem first.

Let's use httpcore as an example. httpcore requires h2 for extras named http2.

If I have something like this in my pyproject.toml:

[tool.poetry.dependencies]
python = "^3.10"
httpcore = "^0.17.3"

[tool.poetry.group.dev.dependencies]
httpcore = {extras = ["http2"], version = "^0.17.3"}

The expected result of micropipenv requirements --method poetry --no-dev is:

#
# Default dependencies
#
httpcore==0.17.3 \
    --hash=sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87 \
    --hash=sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888

because I want only dependencies from the main category and h2 is not there, there is no reason to add [http2].

But when I manually add h2 into the main dependencies so then I have:

[tool.poetry.dependencies]
python = "^3.10"
httpcore = "^0.17.3"
h2 = "*"

[tool.poetry.group.dev.dependencies]
httpcore = {extras = ["http2"], version = "^0.17.3"}

then the result of the same command is:

#
# Default dependencies
#
h2==4.1.0 \
    --hash=sha256:03a46bcf682256c95b5fd9e9a99c1323584c3eec6440d379b9903d709476bc6d \
    --hash=sha256:a83aca08fbe7aacb79fec788c9c0bac936343560ed9ec18b82a13a12c28d2abb
httpcore[http2]==0.17.3 \
    --hash=sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87 \
    --hash=sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888

The reason for this is that h2 and httpcore are now in the same category so the detection of extras thinks that h2 is there because httpcore has been requested with [http2] - this is not true in this case but causes no harm because httpcore and httpcore[http2] represent the same set of packages to be installed.

But there is another problem with this implementation - recursion. Let's say I have this part in poetry.lock:

[[package]]
name = "boto3"
version = "1.26.42"
description = "The AWS SDK for Python"
category = "main"
optional = false
python-versions = ">= 3.7"
files = [
    {file = "boto3-1.26.42-py3-none-any.whl", hash = "sha256:75c995a04723f23e35e16ea491ed91a1345e2fa6492678a216488512308dada1"},
    {file = "boto3-1.26.42.tar.gz", hash = "sha256:4cfd7e05e4033dbca2cc59bcfdafbdaef9d83dc3c0448917569b301d85766d9d"},
]

[package.dependencies]
botocore = ">=1.29.42,<1.30.0"
jmespath = ">=0.7.1,<2.0.0"
s3transfer = ">=0.6.0,<0.7.0"

[package.extras]
crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]

[[package]]
name = "botocore"
version = "1.29.42"
description = "Low-level, data-driven core of boto 3."
category = "main"
optional = false
python-versions = ">= 3.7"
files = [
    {file = "botocore-1.29.42-py3-none-any.whl", hash = "sha256:f52f9dbd7ad42b3528c1052086c1a7b6122a018f919afdb604f2889caefe8092"},
    {file = "botocore-1.29.42.tar.gz", hash = "sha256:d05c62f64e76194c40f598f5f7c804ec50d9820e9f03f6e0198558e4ace167c4"},
]

[package.dependencies]
jmespath = ">=0.7.1,<2.0.0"
python-dateutil = ">=2.1,<3.0.0"
urllib3 = ">=1.25.4,<1.27"

[package.extras]
crt = ["awscrt (==0.15.3)"]

The question to answer is: Should we install boto3 or boto3[crt]? To answer this, we have to check whether all the dependencies of boto3[crt] are present but that means checking presence of botocore[crt] and that's another level. botocore is there but awscrt (dependency of crt extra of botocore) is not. What should happen in those cases?

Also, thanks to this reproducer, I've realized that we have another problem: dependency can only belong to one category and the main/default one has a higher priority for us which means that when we use this example and run micropipenv with --no-default there is no fastapi in devel dependencies :disappointed: