com.azure.identity.CredentialUnavailableException: ManagedIdentityCredential authentication unavailable. Connection to IMDS endpoint cannot be established.
at com.azure.identity.implementation.IdentityClient.lambda$authenticateToIMDSEndpoint$66(IdentityClient.java:1223)
at reactor.core.publisher.MonoCallable.call(MonoCallable.java:92)
at reactor.core.publisher.FluxFlatMap.trySubscribeScalarMap(FluxFlatMap.java:174)
at reactor.core.publisher.MonoFlatMap.subscribeOrReturn(MonoFlatMap.java:53)
at reactor.core.publisher.Mono.subscribe(Mono.java:4476)
at reactor.core.publisher.Mono.subscribeWith(Mono.java:4606)
at reactor.core.publisher.Mono.toFuture(Mono.java:5011)
at com.azure.identity.implementation.IdentityClientBase.lambda$getManagedIdentityConfidentialClient$3(IdentityClientBase.java:426)
at com.microsoft.aad.msal4j.AcquireTokenByAppProviderSupplier.fetchTokenUsingAppTokenProvider(AcquireTokenByAppProviderSupplier.java:75)
Caused: java.util.concurrent.ExecutionException
at java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:395)
at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1999)
at com.microsoft.aad.msal4j.AcquireTokenByAppProviderSupplier.fetchTokenUsingAppTokenProvider(AcquireTokenByAppProviderSupplier.java:76)
Caused: com.microsoft.aad.msal4j.MsalAzureSDKException
at com.microsoft.aad.msal4j.AcquireTokenByAppProviderSupplier.fetchTokenUsingAppTokenProvider(AcquireTokenByAppProviderSupplier.java:79)
at com.microsoft.aad.msal4j.AcquireTokenByAppProviderSupplier.execute(AcquireTokenByAppProviderSupplier.java:56)
at com.microsoft.aad.msal4j.AcquireTokenByClientCredentialSupplier.acquireTokenByClientCredential(AcquireTokenByClientCredentialSupplier.java:78)
at com.microsoft.aad.msal4j.AcquireTokenByClientCredentialSupplier.execute(AcquireTokenByClientCredentialSupplier.java:49)
at com.microsoft.aad.msal4j.AuthenticationResultSupplier.get(AuthenticationResultSupplier.java:69)
at com.microsoft.aad.msal4j.AuthenticationResultSupplier.get(AuthenticationResultSupplier.java:18)
at java.base/java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1700)
Also: java.lang.Exception: #block terminated with an error
at reactor.core.publisher.BlockingSingleSubscriber.blockingGet(BlockingSingleSubscriber.java:100)
at reactor.core.publisher.Mono.block(Mono.java:1742)
at com.azure.core.credential.TokenCredential.getTokenSync(TokenCredential.java:110)
at com.azure.core.implementation.AccessTokenCache.lambda$new$2(AccessTokenCache.java:63)
at com.azure.core.implementation.AccessTokenCache.lambda$retrieveTokenSync$11(AccessTokenCache.java:228)
at com.azure.core.implementation.AccessTokenCache.getTokenSync(AccessTokenCache.java:91)
at com.azure.core.http.policy.BearerTokenAuthenticationPolicy.setAuthorizationHeaderHelperSync(BearerTokenAuthenticationPolicy.java:194)
at com.azure.core.http.policy.BearerTokenAuthenticationPolicy.setAuthorizationHeaderSync(BearerTokenAuthenticationPolicy.java:181)
at com.azure.security.keyvault.secrets.implementation.KeyVaultCredentialPolicy.authorizeRequestSync(KeyVaultCredentialPolicy.java:227)
at com.azure.core.http.policy.BearerTokenAuthenticationPolicy.processSync(BearerTokenAuthenticationPolicy.java:148)
at com.azure.core.http.HttpPipelineNextSyncPolicy.processSync(HttpPipelineNextSyncPolicy.java:53)
at com.azure.core.http.policy.RetryPolicy.attemptSync(RetryPolicy.java:211)
at com.azure.core.http.policy.RetryPolicy.attemptSync(RetryPolicy.java:224)
at com.azure.core.http.policy.RetryPolicy.attemptSync(RetryPolicy.java:224)
at com.azure.core.http.policy.RetryPolicy.attemptSync(RetryPolicy.java:224)
at com.azure.core.http.policy.RetryPolicy.processSync(RetryPolicy.java:161)
at com.azure.core.http.HttpPipelineNextSyncPolicy.processSync(HttpPipelineNextSyncPolicy.java:53)
at com.azure.core.http.policy.AddHeadersPolicy.processSync(AddHeadersPolicy.java:66)
at com.azure.core.http.HttpPipelineNextSyncPolicy.processSync(HttpPipelineNextSyncPolicy.java:53)
at com.azure.core.http.policy.HttpPipelineSyncPolicy.processSync(HttpPipelineSyncPolicy.java:51)
at com.azure.core.http.policy.UserAgentPolicy.processSync(UserAgentPolicy.java:174)
at com.azure.core.http.HttpPipelineNextSyncPolicy.processSync(HttpPipelineNextSyncPolicy.java:53)
at com.azure.core.http.HttpPipeline.sendSync(HttpPipeline.java:138)
at com.azure.core.implementation.http.rest.SyncRestProxy.send(SyncRestProxy.java:62)
at com.azure.core.implementation.http.rest.SyncRestProxy.invoke(SyncRestProxy.java:83)
at com.azure.core.implementation.http.rest.RestProxyBase.invoke(RestProxyBase.java:124)
at com.azure.core.http.rest.RestProxy.invoke(RestProxy.java:95)
at com.sun.proxy.$Proxy212.getSecretSync(Unknown Source)
at com.azure.security.keyvault.secrets.implementation.SecretClientImpl.getSecretWithResponse(SecretClientImpl.java:1133)
at com.azure.security.keyvault.secrets.SecretClient.lambda$getSecretWithResponse$1(SecretClient.java:360)
at com.azure.security.keyvault.secrets.SecretClient.callWithMappedException(SecretClient.java:1025)
at com.azure.security.keyvault.secrets.SecretClient.getSecretWithResponse(SecretClient.java:359)
at com.azure.security.keyvault.secrets.SecretClient.getSecret(SecretClient.java:296)
at org.jenkinsci.plugins.azurekeyvaultplugin.AzureKeyVaultCredentialRetriever.getSecretBundle(AzureKeyVaultCredentialRetriever.java:58)
at org.jenkinsci.plugins.azurekeyvaultplugin.AzureKeyVaultStep$ExecutionImpl.getSecret(AzureKeyVaultStep.java:184)
at org.jenkinsci.plugins.azurekeyvaultplugin.AzureKeyVaultStep$ExecutionImpl.getSecretsMap(AzureKeyVaultStep.java:197)
at org.jenkinsci.plugins.azurekeyvaultplugin.AzureKeyVaultStep$ExecutionImpl.start(AzureKeyVaultStep.java:170)
at org.jenkinsci.plugins.workflow.cps.DSL.invokeStep(DSL.java:323)
at org.jenkinsci.plugins.workflow.cps.DSL.invokeMethod(DSL.java:196)
at org.jenkinsci.plugins.workflow.cps.CpsScript.invokeMethod(CpsScript.java:124)
at jdk.internal.reflect.GeneratedMethodAccessor128.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.codehaus.groovy.reflection.CachedMethod.invoke(CachedMethod.java:98)
at groovy.lang.MetaMethod.doMethodInvoke(MetaMethod.java:325)
at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1225)
at groovy.lang.MetaClassImpl.invokeMethod(MetaClassImpl.java:1034)
at org.codehaus.groovy.runtime.callsite.PogoMetaClassSite.call(PogoMetaClassSite.java:41)
at org.codehaus.groovy.runtime.callsite.CallSiteArray.defaultCall(CallSiteArray.java:47)
at org.codehaus.groovy.runtime.callsite.AbstractCallSite.call(AbstractCallSite.java:116)
at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:180)
at org.kohsuke.groovy.sandbox.GroovyInterceptor.onMethodCall(GroovyInterceptor.java:23)
at org.jenkinsci.plugins.scriptsecurity.sandbox.groovy.SandboxInterceptor.onMethodCall(SandboxInterceptor.java:163)
at org.kohsuke.groovy.sandbox.impl.Checker$1.call(Checker.java:178)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:182)
at org.kohsuke.groovy.sandbox.impl.Checker.checkedCall(Checker.java:152)
at com.cloudbees.groovy.cps.sandbox.SandboxInvoker.methodCall(SandboxInvoker.java:17)
at org.jenkinsci.plugins.workflow.cps.LoggingInvoker.methodCall(LoggingInvoker.java:105)
at com.cloudbees.groovy.cps.impl.ContinuationGroup.methodCall(ContinuationGroup.java:90)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.dispatchOrArg(FunctionCallBlock.java:116)
at com.cloudbees.groovy.cps.impl.FunctionCallBlock$ContinuationImpl.fixArg(FunctionCallBlock.java:85)
at jdk.internal.reflect.GeneratedMethodAccessor62.invoke(Unknown Source)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at com.cloudbees.groovy.cps.impl.ContinuationPtr$ContinuationImpl.receive(ContinuationPtr.java:72)
at com.cloudbees.groovy.cps.impl.ClosureBlock.eval(ClosureBlock.java:46)
at com.cloudbees.groovy.cps.Next.step(Next.java:83)
at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:152)
at com.cloudbees.groovy.cps.Continuable$1.call(Continuable.java:146)
at org.codehaus.groovy.runtime.GroovyCategorySupport$ThreadCategoryInfo.use(GroovyCategorySupport.java:136)
at org.codehaus.groovy.runtime.GroovyCategorySupport.use(GroovyCategorySupport.java:275)
at com.cloudbees.groovy.cps.Continuable.run0(Continuable.java:146)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.access$001(SandboxContinuable.java:18)
at org.jenkinsci.plugins.workflow.cps.SandboxContinuable.run0(SandboxContinuable.java:51)
at org.jenkinsci.plugins.workflow.cps.CpsThread.runNextChunk(CpsThread.java:187)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup.run(CpsThreadGroup.java:423)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:331)
at org.jenkinsci.plugins.workflow.cps.CpsThreadGroup$2.call(CpsThreadGroup.java:295)
at org.jenkinsci.plugins.workflow.cps.CpsVmExecutorService$2.call(CpsVmExecutorService.java:97)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at hudson.remoting.SingleLaneExecutorService$1.run(SingleLaneExecutorService.java:139)
at jenkins.util.ContextResettingExecutorService$1.run(ContextResettingExecutorService.java:28)
at jenkins.security.ImpersonatingExecutorService$1.run(ImpersonatingExecutorService.java:68)
at jenkins.util.ErrorLoggingExecutorService.lambda$wrap$0(ErrorLoggingExecutorService.java:51)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
Anything else?
sh """curl -H 'Metadata: true' 'http://169.254.169.254/metadata/identity/oauth2/token?api-version=2019-11-01&resource=https://vault.azure.net&client_id=xxxx'"""
Above works just fine in the same job.
Please help me to understand whether it is a bug or configuration issue.
Thank in advance!
Jenkins and plugins versions report
Environment
```text Jenkins: 2.387.3 OS: Linux - 5.15.0-1064-azure Java: 11.0.4 - Alpine (OpenJDK 64-Bit Server VM) --- ace-editor:1.1 ansicolor:1.0.4 antisamy-markup-formatter:162.v0e6ec0fcfcf6 apache-httpcomponents-client-4-api:4.5.14-208.v438351942757 apache-httpcomponents-client-5-api:5.3.1-110.v77252fb_d4da_5 authentication-tokens:1.113.v81215a_241826 azure-credentials:312.v0f3973cd1e59 azure-keyvault:251.vcfe31c013dc7 azure-sdk:174.va_89c1df897d2 blueocean:1.27.5.1 blueocean-autofavorite:1.2.5 blueocean-bitbucket-pipeline:1.27.5.1 blueocean-commons:1.27.5.1 blueocean-config:1.27.5.1 blueocean-core-js:1.27.5.1 blueocean-dashboard:1.27.5.1 blueocean-display-url:2.4.2 blueocean-events:1.27.5.1 blueocean-git-pipeline:1.27.5.1 blueocean-github-pipeline:1.27.5.1 blueocean-i18n:1.27.5.1 blueocean-jwt:1.27.5.1 blueocean-personalization:1.27.5.1 blueocean-pipeline-api-impl:1.27.5.1 blueocean-pipeline-editor:1.27.5.1 blueocean-pipeline-scm-api:1.27.5.1 blueocean-rest:1.27.5.1 blueocean-rest-impl:1.27.5.1 blueocean-web:1.27.5.1 bootstrap5-api:5.3.2-3 bouncycastle-api:2.30.1.78.1-233.vfdcdeb_0a_08a_a_ branch-api:2.1128.v717130d4f816 caffeine-api:3.1.8-133.v17b_1ff2e0599 checks-api:2.0.2 cloud-stats:336.v788e4055508b_ cloudbees-bitbucket-branch-source:856.v04c46c86f911 cloudbees-disk-usage-simple:203.v3f46a_7462b_1a_ cloudbees-folder:6.858.v898218f3609d command-launcher:107.v773860566e2e commons-lang3-api:3.14.0-76.vda_5591261cfe commons-text-api:1.11.0-94.v3e1f4a_926e49 copyartifact:722.v0662a_9b_e22a_c credentials:1319.v7eb_51b_3a_c97b_ credentials-binding:642.v737c34dea_6c2 data-tables-api:1.13.6-5 display-url-api:2.204.vf6fddd8a_8b_e9 docker-commons:439.va_3cb_0a_6a_fb_29 docker-java-api:3.3.6-90.ve7c5c7535ddd docker-plugin:1.5 docker-workflow:580.vc0c340686b_54 durable-task:555.v6802fe0f0b_82 echarts-api:5.4.0-7 extended-choice-parameter:382.v5697b_32134e8 favorite:2.4.3 font-awesome-api:6.5.1-2 git:5.2.1 git-client:4.6.0 github:1.37.3.1 github-api:1.318-461.v7a_c09c9fa_d63 github-branch-source:1771.v59b_6a_fa_1b_89e handy-uri-templates-2-api:2.1.8-22.v77d5b_75e6953 htmlpublisher:1.35 http_request:1.18 instance-identity:185.v303dc7c645f9 ionicons-api:74.v93d5eb_813d5f jackson2-api:2.15.3-363.v82c51b_de9f60 jakarta-activation-api:2.1.3-1 jakarta-mail-api:2.1.3-1 javax-activation-api:1.2.0-7 javax-mail-api:1.6.2-10 jaxb:2.3.9-1 jdk-tool:73.vddf737284550 jenkins-design-language:1.27.5.1 jira-steps:2.0.165.v8846cf59f3db jjwt-api:0.11.5-112.ve82dfb_224b_a_d job-dsl:1.87 jquery3-api:3.7.1-1 jsch:0.2.16-86.v42e010d9484b_ junit:1265.v65b_14fa_f12f0 leastload:3.0.0 locale:314.v22ce953dfe9e lockable-resources:1245.vb_05f8a_4e28db_ mailer:470.vc91f60c5d8e2 matrix-project:818.v7eb_e657db_924 mattermost:3.1.3 metrics:4.2.21-451.vd51df8df52ec mina-sshd-api-common:2.12.1-101.v85b_e08b_780dd mina-sshd-api-core:2.12.1-101.v85b_e08b_780dd nested-view:1.33 okhttp-api:4.11.0-172.vda_da_1feeb_c6e pipeline-build-step:540.vb_e8849e1a_b_d8 pipeline-graph-analysis:216.vfd8b_ece330ca_ pipeline-groovy-lib:689.veec561a_dee13 pipeline-input-step:477.v339683a_8d55e pipeline-milestone-step:119.vdfdc43fc3b_9a_ pipeline-model-api:2.2150.v4cfd8916915c pipeline-model-definition:2.2150.v4cfd8916915c pipeline-model-extensions:2.2150.v4cfd8916915c pipeline-rest-api:2.34 pipeline-stage-step:312.v8cd10304c27a_ pipeline-stage-tags-metadata:2.2150.v4cfd8916915c pipeline-stage-view:2.34 pipeline-utility-steps:2.16.0 plain-credentials:182.v468b_97b_9dcb_8 plugin-util-api:3.8.0 popper2-api:2.11.6-4 prometheus:773.v3b_62d8178eec pubsub-light:1.18 pyenv-pipeline:2.1.2 rebuild:332.va_1ee476d8f6d resource-disposer:0.23 role-strategy:689.v731678c3e0eb_ saml:4.429.v9a_781a_61f1da_ scm-api:676.v886669a_199a_a_ script-security:1341.va_2819b_414686 snakeyaml-api:2.2-111.vc6598e30cc65 sse-gateway:1.27 ssh-agent:367.vf9076cd4ee21 ssh-credentials:337.v395d2403ccd4 ssh-slaves:2.948.vb_8050d697fec sshd:3.322.v159e91f6a_550 structs:325.vcb_307d2a_2782 timestamper:1.27 token-macro:384.vf35b_f26814ec trilead-api:2.84.86.vf9c960e9b_458 variant:60.v7290fc0eb_b_cd workflow-api:1291.v51fd2a_625da_7 workflow-basic-steps:1058.vcb_fc1e3a_21a_9 workflow-cps:3837.v305192405b_c0 workflow-durable-task-step:1331.vc8c2fed35334 workflow-job:1326.ve643e00e9220 workflow-multibranch:770.v1a_d0708dd1f6 workflow-scm-step:427.v4ca_6512e7df1 workflow-step-api:639.v6eca_cd8c04a_a_ workflow-support:907.v6713a_ed8a_573 ws-cleanup:0.46 ```What Operating System are you using (both controller, and any agents involved in the problem)?
Jenkins is running in AKS K8s cluster. User managed identity attached to Jenkins's VMSS.
Reproduction steps
Expected Results
Secret retrieved
Actual Results
Anything else?
Above works just fine in the same job.
Please help me to understand whether it is a bug or configuration issue. Thank in advance!
Are you interested in contributing a fix?
No response