v1.8.0: credentials not found in native keychain

[ekhabarov]

Encountered the error after upgrading rules_oci v1.7.6 => 1.8.0 on Mac M2.

repro: https://github.com/ekhabarov/rules_oci_1_8_0

WORKSPACE:

load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

http_archive(
    name = "rules_oci",
    # v1.8.0 Failed with
    # ERROR: credential helper failed:
    # STDOUT:
    # credentials not found in native keychain
    sha256 = "46ce9edcff4d3d7b3a550774b82396c0fa619cc9ce9da00c1b09a08b45ea5a14",
    strip_prefix = "rules_oci-1.8.0",
    url = "https://github.com/bazel-contrib/rules_oci/releases/download/v1.8.0/rules_oci-v1.8.0.tar.gz",

    # Works with v1.7.6
    # sha256 = "647f4c6fd092dc7a86a7f79892d4b1b7f1de288bdb4829ca38f74fd430fcd2fe",
    # strip_prefix = "rules_oci-1.7.6",
    # url = "https://github.com/bazel-contrib/rules_oci/releases/download/v1.7.6/rules_oci-v1.7.6.tar.gz",
)

load("@rules_oci//oci:dependencies.bzl", "rules_oci_dependencies")

rules_oci_dependencies()

load("@rules_oci//oci:repositories.bzl", "LATEST_CRANE_VERSION", "oci_register_toolchains")

oci_register_toolchains(
    name = "oci",
    crane_version = LATEST_CRANE_VERSION,
    # Uncommenting the zot toolchain will cause it to be used instead of crane for some tasks.
    # Note that it does not support docker-format images.
    # zot_version = LATEST_ZOT_VERSION,
)

# You can pull your base images using oci_pull like this:
load("@rules_oci//oci:pull.bzl", "oci_pull")

oci_pull(
    name = "distroless_base",
    digest = "sha256:ccaef5ee2f1850270d453fdf700a5392534f8d1a8ca2acda391fbb6a06b81c86",
    image = "gcr.io/distroless/base",
    platforms = [
        "linux/amd64",
        "linux/arm64",
    ],
)

.bazelversion

6.5.0

% bazel build @distroless_base
INFO: Repository distroless_base instantiated at:
  /Users/ekhabarov/develop/rules_oci_1_8_0/WORKSPACE:36:9: in <toplevel>
  /private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/pull.bzl:216:14: in oci_pull
Repository rule oci_alias defined at:
  /private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl:414:28: in <toplevel>
ERROR: An error occurred during the fetch of repository 'distroless_base':
   Traceback (most recent call last):
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl", line 347, column 55, in _oci_alias_impl
                manifest, _, digest = downloader.download_manifest(rctx.attr.identifier, "mf.json")
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl", line 163, column 74, in lambda
                download_manifest = lambda identifier, output: _download_manifest(rctx, authn, identifier, output),
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl", line 123, column 23, in _download_manifest
                result = _download(rctx, authn, identifier, output, "manifests", allow_fail = True)
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl", line 89, column 27, in _download
                auth = authn.get_token(rctx.attr.registry, rctx.attr.repository)
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/authn.bzl", line 259, column 49, in lambda
                get_token = lambda reg, repo: _get_token(rctx, state, reg, repo),
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/authn.bzl", line 194, column 24, in _get_token
                pattern = _get_auth(rctx, state, registry)
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/authn.bzl", line 186, column 47, in _get_auth
                pattern = _fetch_auth_via_creds_helper(rctx, registry, config["credsStore"])
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/authn.bzl", line 123, column 13, in _fetch_auth_via_creds_helper
                fail("credential helper failed: \nSTDOUT:\n{}\nSTDERR:\n{}".format(result.stdout, result.stderr))
Error in fail: credential helper failed:
STDOUT:
credentials not found in native keychain

STDERR:
ERROR: /Users/ekhabarov/develop/rules_oci_1_8_0/WORKSPACE:36:9: fetching oci_alias rule //external:distroless_base: Traceback (most recent call last):
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl", line 347, column 55, in _oci_alias_impl
                manifest, _, digest = downloader.download_manifest(rctx.attr.identifier, "mf.json")
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl", line 163, column 74, in lambda
                download_manifest = lambda identifier, output: _download_manifest(rctx, authn, identifier, output),
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl", line 123, column 23, in _download_manifest
                result = _download(rctx, authn, identifier, output, "manifests", allow_fail = True)
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/pull.bzl", line 89, column 27, in _download
                auth = authn.get_token(rctx.attr.registry, rctx.attr.repository)
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/authn.bzl", line 259, column 49, in lambda
                get_token = lambda reg, repo: _get_token(rctx, state, reg, repo),
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/authn.bzl", line 194, column 24, in _get_token
                pattern = _get_auth(rctx, state, registry)
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/authn.bzl", line 186, column 47, in _get_auth
                pattern = _fetch_auth_via_creds_helper(rctx, registry, config["credsStore"])
        File "/private/var/tmp/_bazel_ekhabarov/b9418b43feba4dce05aac254447716d0/external/rules_oci/oci/private/authn.bzl", line 123, column 13, in _fetch_auth_via_creds_helper
                fail("credential helper failed: \nSTDOUT:\n{}\nSTDERR:\n{}".format(result.stdout, result.stderr))
Error in fail: credential helper failed:
STDOUT:
credentials not found in native keychain

STDERR:
ERROR: credential helper failed:
STDOUT:
credentials not found in native keychain

STDERR:
INFO: Elapsed time: 0.733s
INFO: 0 processes.
FAILED: Build did NOT complete successfully (0 packages loaded)

[ekhabarov]

It looks like introduced in #602 cc @msiebuhr

[Zemnmez]

I'm getting this same issue on wsl2 under Windows

[JohnAmican]

+1 same issue. I'm working around it for now locally by passing in dummy creds for the registry it's failing to find creds for:

echo '{"ServerURL":"gcr.io","Username":"none"}' | docker-credential-osxkeychain store

[msiebuhr]

I'm guessing your .docker/config.json looks something like the following, right?

{
    "credsStore": "osxkeychain"
}

My guess is that the built-in client tries to use any authentication method given to it, regardless of the upstream actually requiring authentication or not. (That used to be safe, as it only looked at explicitly configured registry names).

Now that my patch also makes it return the generic credentialsStore, the client will use that regardless of the registry actually requires authentication or not.

Off the bat, I can think of two fixes:

1. Always "taste" the upstream to see if it requires authentication or not.
2. Have the code I wrote check ask the credentials-helper for credentials to the specific registry and only return it if the call is successful.

[ekhabarov]

@msiebuhr my ~/.docker/config.json looks like this

{
        "auths": {
          ...
        },
        "credsStore": "desktop",
}

while gcr.io is a public registry I'm trying to pull an image from is not in auths list.

[lbcjbb]

Same for me. The credentials helper should not be called when the registry is not defined in the auths section. No problem with a revert of https://github.com/bazel-contrib/rules_oci/pull/602

[apesternikov]

+1 confirming same issue

[msiebuhr]

Same for me. The credentials helper should not be called when the registry is not defined in the auths section. No problem with a revert of #602

If we need to copy the behavior of Docker, we should copy it if the upstream registry returns HTTP 401 when trying to fetch from it.

[msiebuhr]

On the down-side, the Bazel downloader won't let us look at the WWW-Authenticate-header (https://github.com/bazel-contrib/rules_oci/blob/main/oci/private/authn.bzl#L7-L8):

# Unfortunately bazel downloader doesn't let us sniff the WWW-Authenticate header, therefore we need to
# keep a map of known registries that require us to acquire a temporary token for authentication.

On the flip-side, it looks like @thesayyn de-facto fixed the issue in 9f0079bd by adding an allow_fail = True for this case:

+def _fetch_auth_via_creds_helper(rctx, raw_host, helper_name, allow_fail = False):
     if rctx.os.name.startswith("windows"):
         executable = "{}.bat".format(helper_name)
         rctx.file(
@@ -120,7 +119,10 @@ exec "docker-credential-{}" get <<< "$1" """.format(helper_name),
         )
     result = rctx.execute([rctx.path(executable), raw_host])
     if result.return_code:
-        fail("credential helper failed: \nSTDOUT:\n{}\nSTDERR:\n{}".format(result.stdout, result.stderr))
+        if not allow_fail:
+            fail("credential helper failed: \nSTDOUT:\n{}\nSTDERR:\n{}".format(result.stdout, result.stderr))
+        else:
+            return {}

     response = json.decode(result.stdout)

@@ -183,7 +185,7 @@ def _get_auth(rctx, state, registry):

     # look for generic credentials-store all lookups for host-specific auth fails
     if "credsStore" in config and len(pattern.keys()) == 0:
-        pattern = _fetch_auth_via_creds_helper(rctx, registry, config["credsStore"])
+        pattern = _fetch_auth_via_creds_helper(rctx, registry, config["credsStore"], allow_fail = True)

     # cache the result so that we don't do this again unnecessarily.
     state["auth"][registry] = pattern

[thesayyn]

fixed by https://github.com/bazel-contrib/rules_oci/pull/664, thank you @msiebuhr

[lbcjbb]

Mmmm, a credential helper which should not be called is different from a credential helper which can failed. This issue is not really fixed but just mitigated IMHO.

[thesayyn]

Mmmm, a credential helper which should not be called is different from a credential helper which can failed. This issue is not really fixed but just mitigated IMHO.

Calling the global cred helper is a fix for https://github.com/bazel-contrib/rules_oci/issues/388, we can't fix both without doing something like this.

[lbcjbb]

I thought I'd tested with a custom credential helper and a ~/.docker/config.json without the auths section, but I've just retested and this custom credential helper is well called. Sorry for the noice.