search

spring-cloud / spring-cloud-release

Spring Cloud Release Train - dependency management across a wide range of Spring Cloud projects.
http://projects.spring.io/spring-cloud
Apache License 2.0
874 stars 179 forks source link

Connection pool shut down issue when actuator restart executed #243

Closed viccsjain closed 2 years ago

viccsjain commented 2 years ago

Describe the bug Hi Team,

I am using spring cloud dependencies to read data from AWS secrets manager at the application startup. When i start the application for the first time, it works fine. It is able to read data from secrets and have it in environment properties. Recently we have a requirement to enable actuator restart endpoint. After enabling, if we execute actuator restart, application does not start. It throws the following exception:

Starting Application service :: 1.87.8-SNAPSHOT

2022-04-18 17:52:58,965 ERROR ApiService [                 Thread-4] AwsSecretsManagerPropertySourceLocator   --- [] Message: Fail fast is set and there was an error reading configuration from AWS Secrets Manager:
Connection pool shut down
2022-04-18 17:52:58,978 ERROR ApiService [                 Thread-4] SpringApplication                        --- [] Message: Application run failed
java.lang.IllegalStateException: Connection pool shut down
    at org.apache.http.util.Asserts.check(Asserts.java:34) ~[httpcore-4.4.15.jar:4.4.15]
    at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.requestConnection(PoolingHttpClientConnectionManager.java:269) ~[httpclient-4.5.13.jar:4.5.13]
    at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?]
    at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:?]
    at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?]
    at java.lang.reflect.Method.invoke(Method.java:566) ~[?:?]
    at com.amazonaws.http.conn.ClientConnectionManagerFactory$Handler.invoke(ClientConnectionManagerFactory.java:76) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.conn.$Proxy48.requestConnection(Unknown Source) ~[?:?]
    at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:176) ~[httpclient-4.5.13.jar:4.5.13]
    at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186) ~[httpclient-4.5.13.jar:4.5.13]
    at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185) ~[httpclient-4.5.13.jar:4.5.13]
    at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83) ~[httpclient-4.5.13.jar:4.5.13]
    at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56) ~[httpclient-4.5.13.jar:4.5.13]
    at com.amazonaws.http.apache.client.impl.SdkHttpClient.execute(SdkHttpClient.java:72) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1343) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1154) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:811) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:779) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:753) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:713) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:695) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:559) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:539) ~[aws-java-sdk-core-1.12.129.jar:?]
    at com.amazonaws.services.secretsmanager.AWSSecretsManagerClient.doInvoke(AWSSecretsManagerClient.java:2634) ~[aws-java-sdk-secretsmanager-1.11.792.jar:?]
    at com.amazonaws.services.secretsmanager.AWSSecretsManagerClient.invoke(AWSSecretsManagerClient.java:2601) ~[aws-java-sdk-secretsmanager-1.11.792.jar:?]
    at com.amazonaws.services.secretsmanager.AWSSecretsManagerClient.invoke(AWSSecretsManagerClient.java:2590) ~[aws-java-sdk-secretsmanager-1.11.792.jar:?]
    at com.amazonaws.services.secretsmanager.AWSSecretsManagerClient.executeGetSecretValue(AWSSecretsManagerClient.java:1213) ~[aws-java-sdk-secretsmanager-1.11.792.jar:?]
    at com.amazonaws.services.secretsmanager.AWSSecretsManagerClient.getSecretValue(AWSSecretsManagerClient.java:1184) ~[aws-java-sdk-secretsmanager-1.11.792.jar:?]
    at org.springframework.cloud.aws.secretsmanager.AwsSecretsManagerPropertySource.readSecretValue(AwsSecretsManagerPropertySource.java:74) ~[spring-cloud-aws-secrets-manager-config-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.cloud.aws.secretsmanager.AwsSecretsManagerPropertySource.init(AwsSecretsManagerPropertySource.java:57) ~[spring-cloud-aws-secrets-manager-config-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.cloud.aws.secretsmanager.AwsSecretsManagerPropertySourceLocator.create(AwsSecretsManagerPropertySourceLocator.java:127) ~[spring-cloud-aws-secrets-manager-config-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.cloud.aws.secretsmanager.AwsSecretsManagerPropertySourceLocator.locate(AwsSecretsManagerPropertySourceLocator.java:105) ~[spring-cloud-aws-secrets-manager-config-2.2.6.RELEASE.jar:2.2.6.RELEASE]
    at org.springframework.cloud.bootstrap.config.PropertySourceLocator.locateCollection(PropertySourceLocator.java:51) ~[spring-cloud-context-3.1.0.jar:3.1.0]
    at org.springframework.cloud.bootstrap.config.PropertySourceLocator.locateCollection(PropertySourceLocator.java:47) ~[spring-cloud-context-3.1.0.jar:3.1.0]
    at org.springframework.cloud.bootstrap.config.PropertySourceBootstrapConfiguration.initialize(PropertySourceBootstrapConfiguration.java:95) ~[spring-cloud-context-3.1.0.jar:3.1.0]
    at org.springframework.boot.SpringApplication.applyInitializers(SpringApplication.java:613) ~[spring-boot-2.6.6.jar:2.6.6]
    at org.springframework.boot.SpringApplication.prepareContext(SpringApplication.java:381) ~[spring-boot-2.6.6.jar:2.6.6]
    at org.springframework.boot.SpringApplication.run(SpringApplication.java:302) ~[spring-boot-2.6.6.jar:2.6.6]
    at org.springframework.cloud.context.restart.RestartEndpoint.doRestart(RestartEndpoint.java:144) ~[spring-cloud-context-3.1.0.jar:3.1.0]
    at org.springframework.cloud.context.restart.RestartEndpoint.safeRestart(RestartEndpoint.java:110) ~[spring-cloud-context-3.1.0.jar:3.1.0]
    at java.lang.Thread.run(Thread.java:829) [?:?]
2022-04-18 17:52:58,980 INFO  ApiService [                 Thread-4] RestartEndpoint                          --- [] Message: Could not doRestart: Connection pool shut down

Process finished with exit code 0

Note: Both these properties are set to true: aws.secretsmanager.failFast= true aws.secretsmanager.enabled= true

Sample Unfortunately i dont have a sample code, but this is issue is frequently reproducible

viccsjain commented 2 years ago

I guess this issue does not belong to spring-cloud-release library. I will close the issue