Support `fetchFromGitLab` with `forceFetchGit`

[pbsds]

I've been trying to debug why nix-update does not work for spade in nixpkgs. The issue is that its use of fetchFromGitLab relies on fetchSubmodules, which in turn makes fetchFromGitLab use fetchgit with a .git url.

I've been playing around trying to solve this problem, and I now see three different solutions:

(1) implement a gitlab version fetcher for .git urls (2) passthru the gitlab api url in fetchFromGitLab which is used instead of .url (3) implement a generic version fetcher for .git urls (git ls-remote --tags --sort)

Possible nix-update patch for (1)

```patch diff --git a/nix_update/version/__init__.py b/nix_update/version/__init__.py index 11dcb71..d567b12 100644 --- a/nix_update/version/__init__.py +++ b/nix_update/version/__init__.py @@ -9,7 +9,7 @@ from .bitbucket import fetch_bitbucket_snapshots, fetch_bitbucket_versions from .crate import fetch_crate_versions from .gitea import fetch_gitea_snapshots, fetch_gitea_versions from .github import fetch_github_snapshots, fetch_github_versions -from .gitlab import fetch_gitlab_snapshots, fetch_gitlab_versions +from .gitlab import fetch_gitlab_snapshots, fetch_gitlab_versions, fetch_gitlab_dotgit_versions from .npm import fetch_npm_versions from .pypi import fetch_pypi_versions from .rubygems import fetch_rubygem_versions @@ -42,6 +42,7 @@ fetchers: list[Callable[[ParseResult], list[Version]]] = [ fetch_sourcehut_versions, fetch_bitbucket_versions, # all entries below perform requests to check if the target url is of that type + fetch_gitlab_dotgit_versions, fetch_gitea_versions, ] diff --git a/nix_update/version/gitlab.py b/nix_update/version/gitlab.py index 2bc0703..b97d4b3 100644 --- a/nix_update/version/gitlab.py +++ b/nix_update/version/gitlab.py @@ -8,9 +8,27 @@ from ..errors import VersionError from ..utils import info from .version import Version +# found when fetchFromGitLab internally uses fetchzip GITLAB_API = re.compile( r"http(s)?://(?P[^/]+)/api/v4/projects/(?P[^/]*)/repository/archive.tar.gz\?sha=(?P.+)" ) +# found when fetchFromGitLab internally uses fetchgit +GITLAB_GIT = re.compile( + r"http(s)?://(?P[^/]+)/((?P[^/]*)/)?(?P[^/]*)/(?P[^/]*).git" +) + +KNOWN_GITLAB_HOSTS = [ + "code.videolan.org" + "framagit.org" + "gitlab.com" + "gitlab.freedesktop.org" + "gitlab.gnome.org" + "gitlab.inria.fr" + "gitlab.linphone.org" + "gitlab.torproject.org" + "invent.kde.org" + "salsa.debian.org" +] def fetch_gitlab_versions(url: ParseResult) -> list[Version]: @@ -19,6 +37,34 @@ def fetch_gitlab_versions(url: ParseResult) -> list[Version]: return [] domain = match.group("domain") project_id = match.group("project_id") + return _fetch_gitlab_versions(domain, project_id) + +def fetch_gitlab_dotgit_versions(url: ParseResult) -> list[Version]: + match = GITLAB_GIT.match(url.geturl()) + if not match: + return [] + + domain = match.group("domain") + group = match.group("group") + owner = match.group("owner") + repo = match.group("repo") + if group is None: + project_id = quote_plus(f"{owner}/{repo}") + else: + project_id = quote_plus(f"{group}/{owner}/{repo}") + + if not domain in KNOWN_GITLAB_HOSTS: + endpoint = f"https://{domain}/api/v4/projects/{project_id}" + try: + resp = urllib.request.urlopen(endpoint) + except URLError: + return [] + if resp.status != 200: + return [] + + return _fetch_gitlab_versions(domain, project_id) + +def _fetch_gitlab_versions(domain: str, project_id: str) -> list[Version]: gitlab_url = f"https://{domain}/api/v4/projects/{project_id}/repository/tags" info(f"fetch {gitlab_url}") resp = urllib.request.urlopen(gitlab_url) diff --git a/tests/test_gitlab.py b/tests/test_gitlab.py index 6c5733c..fa1a2d2 100644 --- a/tests/test_gitlab.py +++ b/tests/test_gitlab.py @@ -1,13 +1,15 @@ import subprocess import conftest +import pytest from nix_update import main -def test_main(helpers: conftest.Helpers) -> None: +@pytest.mark.parametrize("attrpath", ["gitlab", "gitlab-git"]) +def test_main(helpers: conftest.Helpers, attrpath: str) -> None: with helpers.testpkgs(init_git=True) as path: - main(["--file", str(path), "--commit", "gitlab"]) + main(["--file", str(path), "--commit", attrpath]) version = subprocess.run( [ "nix", @@ -17,7 +19,7 @@ def test_main(helpers: conftest.Helpers) -> None: "nix-command", "-f", path, - "gitlab.version", + f"{attrpath}.version", ], check=True, text=True, @@ -32,7 +34,8 @@ def test_main(helpers: conftest.Helpers) -> None: ).stdout.strip() print(commit) assert version in commit - assert "gitlab" in commit - assert ( - "https://gitlab.gnome.org/world/phosh/phosh/-/compare/v0.20.0...v" in commit - ) + assert attrpath in commit + if attrpath == "gitlab": + assert ( + "https://gitlab.gnome.org/world/phosh/phosh/-/compare/v0.20.0...v" in commit + ) diff --git a/tests/testpkgs/default.nix b/tests/testpkgs/default.nix index 006bfc0..8d47553 100644 --- a/tests/testpkgs/default.nix +++ b/tests/testpkgs/default.nix @@ -13,6 +13,7 @@ github = pkgs.callPackage ./github.nix { }; github-no-release = pkgs.callPackage ./github-no-release.nix { }; gitlab = pkgs.callPackage ./gitlab.nix { }; + gitlab-git = pkgs.callPackage ./gitlab-git.nix { }; pypi = pkgs.python3.pkgs.callPackage ./pypi.nix { }; sourcehut = pkgs.python3.pkgs.callPackage ./sourcehut.nix { }; savanna = pkgs.python3.pkgs.callPackage ./savanna.nix { }; diff --git a/tests/testpkgs/gitlab-git.nix b/tests/testpkgs/gitlab-git.nix new file mode 100644 index 0000000..9df9e98 --- /dev/null +++ b/tests/testpkgs/gitlab-git.nix @@ -0,0 +1,16 @@ +{ stdenv, fetchFromGitLab }: + +stdenv.mkDerivation rec { + pname = "phosh"; + version = "0.20.0"; + + src = fetchFromGitLab { + domain = "gitlab.gnome.org"; + group = "world"; + owner = "phosh"; + repo = pname; + rev = "v${version}"; + sha256 = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + forceFetchGit = true; + }; +} ```

Possible nixpkgs patch for (2)

```patch diff --git a/pkgs/build-support/fetchgit/default.nix b/pkgs/build-support/fetchgit/default.nix index 1b000fb49a99..00d59b8a61bd 100644 --- a/pkgs/build-support/fetchgit/default.nix +++ b/pkgs/build-support/fetchgit/default.nix @@ -28,6 +28,7 @@ lib.makeOverridable (lib.fetchers.withNormalizedHash { } ( , # Impure env vars (https://nixos.org/nix/manual/#sec-advanced-attributes) # needed for netrcPhase netrcImpureEnvVars ? [] +, passthru ? {} , meta ? {} , allowedRequisites ? null }: @@ -96,6 +97,6 @@ stdenvNoCC.mkDerivation { passthru = { gitRepoUrl = url; - }; + } // passthru; } )) diff --git a/pkgs/build-support/fetchgitlab/default.nix b/pkgs/build-support/fetchgitlab/default.nix index 749883f2365e..bad0fabc7e69 100644 --- a/pkgs/build-support/fetchgitlab/default.nix +++ b/pkgs/build-support/fetchgitlab/default.nix @@ -6,6 +6,7 @@ lib.makeOverridable ( , fetchSubmodules ? false, leaveDotGit ? false , deepClone ? false, forceFetchGit ? false , sparseCheckout ? [] +, passthru ? {} , ... # For hash agility } @ args: @@ -13,7 +14,9 @@ let slug = lib.concatStringsSep "/" ((lib.optional (group != null) group) ++ [ owner repo ]); escapedSlug = lib.replaceStrings [ "." "/" ] [ "%2E" "%2F" ] slug; escapedRev = lib.replaceStrings [ "+" "%" "/" ] [ "%2B" "%25" "%2F" ] rev; - passthruAttrs = removeAttrs args [ "protocol" "domain" "owner" "group" "repo" "rev" "fetchSubmodules" "forceFetchGit" "leaveDotGit" "deepClone" ]; + passthruAttrs = removeAttrs args [ "protocol" "domain" "owner" "group" "repo" "rev" "fetchSubmodules" "forceFetchGit" "leaveDotGit" "deepClone" "passthru" ]; + + gitlabApiUrl = "${protocol}://${domain}/api/v4/projects/${escapedSlug}"; useFetchGit = fetchSubmodules || leaveDotGit || deepClone || forceFetchGit || (sparseCheckout != []); fetcher = if useFetchGit then fetchgit else fetchzip; @@ -23,12 +26,16 @@ let fetcherArgs = (if useFetchGit then { inherit rev deepClone fetchSubmodules sparseCheckout leaveDotGit; url = gitRepoUrl; + + passthru = { + inherit gitlabApiUrl; + } // passthru; } else { - url = "${protocol}://${domain}/api/v4/projects/${escapedSlug}/repository/archive.tar.gz?sha=${escapedRev}"; + url = "${gitlabApiUrl}/repository/archive.tar.gz?sha=${escapedRev}"; passthru = { - inherit gitRepoUrl; - }; + inherit gitRepoUrl gitlabApiUrl; + } // passthru; }) // passthruAttrs // { inherit name; }; in ```

Any preferences?

[Mic92]

I think the nixpkgs patch looks more elegant and also helps other nix tooling beyond nix-update.