search

aws / aws-sdk-java

The official AWS SDK for Java 1.x. The AWS SDK for Java 2.x is available here: https://github.com/aws/aws-sdk-java-v2/
https://aws.amazon.com/sdkforjava
Apache License 2.0
4.12k stars 2.83k forks source link

"AmazonS3Exception: We encountered an internal error. Please try again" on set retention date after large amount of uploaded files to S3 #2932

Closed GeksterOne closed 1 year ago

GeksterOne commented 1 year ago

Describe the bug

Our script perfroms the following:

  1. Uploads large amount of zip files to S3.
  2. Set retention for these files. If we deal with large amount of files in a raw (few gigabytes) we receive the following error on a first attempt to extend retention: `2023-02-14 15:20:09.804 ERROR 1 --- [nio-8445-exec-3] c.s.a.aws.services.impl.AwsService : BucketName = 00-sizing-main-archive-2f5ed23587d8, ArchivedFile = dr_smtpemail_2023.01.01_999/1215D4DA74E53E5981A1CD3304B046D0.json.zip. We encountered an internal error. Please try again. (Service: Amazon S3; Status Code: 500; Error Code: InternalError; Request ID: ZF4SE29Z7AT7V5GB; S3 Extended Request ID: zDX1oZrUxNzKRtvt5p2YmTe4E/TZlgxO7fDtKCUvIQpeogF35zpPa/zA7/u6h58yvVIWyOnJuk0=; Proxy: null)

com.amazonaws.services.s3.model.AmazonS3Exception: We encountered an internal error. Please try again. (Service: Amazon S3; Status Code: 500; Error Code: InternalError; Request ID: ZF4SE29Z7AT7V5GB; S3 Extended Request ID: zDX1oZrUxNzKRtvt5p2YmTe4E/TZlgxO7fDtKCUvIQpeogF35zpPa/zA7/u6h58yvVIWyOnJuk0=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1879) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1418) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1387) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1157) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:814) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:781) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:755) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:715) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:697) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:561) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:541) ~[aws-java-sdk-core-1.12.272.jar!/:na] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5456) ~[aws-java-sdk-s3-1.12.281.jar!/:na] at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5403) ~[aws-java-sdk-s3-1.12.281.jar!/:na] at com.amazonaws.services.s3.AmazonS3Client.setObjectRetention(AmazonS3Client.java:3347) ~[aws-java-sdk-s3-1.12.281.jar!/:na] at com.shield.archive.aws.services.impl.AwsService.lambda$extendRetentionDto$2(AwsService.java:302) ~[aws-archive-common-100-SNAPSHOT.jar!/:100-SNAPSHOT] at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183) ~[na:na] at java.base/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1707) ~[na:na] at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509) ~[na:na] at java.base/java.util.stream.ForEachOps$ForEachTask.compute(ForEachOps.java:290) ~[na:na] at java.base/java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:754) ~[na:na] at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:373) ~[na:na] at java.base/java.util.concurrent.ForkJoinTask.invoke(ForkJoinTask.java:686) ~[na:na] at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateParallel(ForEachOps.java:159) ~[na:na] at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateParallel(ForEachOps.java:173) ~[na:na] at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:233) ~[na:na] at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:596) ~[na:na] at java.base/java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:765) ~[na:na] at com.shield.archive.aws.services.impl.AwsService.extendRetentionDto(AwsService.java:287) ~[aws-archive-common-100-SNAPSHOT.jar!/:100-SNAPSHOT] at com.shield.archive.aws.controller.AwsArchiveController.extendRetention(AwsArchiveController.java:53) ~[aws-archive-common-100-SNAPSHOT.jar!/:100-SNAPSHOT] at jdk.internal.reflect.GeneratedMethodAccessor130.invoke(Unknown Source) ~[na:na] at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:na] at java.base/java.lang.reflect.Method.invoke(Method.java:568) ~[na:na] at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) ~[spring-web-5.3.21.jar!/:5.3.21] at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150) ~[spring-web-5.3.21.jar!/:5.3.21] at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1067) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:963) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:909) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at javax.servlet.http.HttpServlet.service(HttpServlet.java:681) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883) ~[spring-webmvc-5.3.21.jar!/:5.3.21] at javax.servlet.http.HttpServlet.service(HttpServlet.java:764) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:227) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53) ~[tomcat-embed-websocket-9.0.64.jar!/:na] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100) ~[spring-web-5.3.21.jar!/:5.3.21] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) ~[spring-web-5.3.21.jar!/:5.3.21] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93) ~[spring-web-5.3.21.jar!/:5.3.21] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) ~[spring-web-5.3.21.jar!/:5.3.21] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.springframework.boot.actuate.metrics.web.servlet.WebMvcMetricsFilter.doFilterInternal(WebMvcMetricsFilter.java:96) ~[spring-boot-actuator-2.6.9.jar!/:2.6.9] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) ~[spring-web-5.3.21.jar!/:5.3.21] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201) ~[spring-web-5.3.21.jar!/:5.3.21] at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117) ~[spring-web-5.3.21.jar!/:5.3.21] at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:197) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.valves.RemoteIpValve.invoke(RemoteIpValve.java:769) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:360) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:399) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:890) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1787) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659) ~[tomcat-embed-core-9.0.64.jar!/:na] at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) ~[tomcat-embed-core-9.0.64.jar!/:na] at java.base/java.lang.Thread.run(Thread.java:833) ~[na:na]`

If we deal with small amount of files everything works well.

Expected Behavior

Retention should be set and code 200 should be returned.

Current Behavior

Error 500 returned with the following message: com.amazonaws.services.s3.model.AmazonS3Exception: We encountered an internal error. Please try again. (Service: Amazon S3; Status Code: 500; Error Code: InternalError; Request ID: ZF4SE29Z7AT7V5GB; S3 Extended Request ID: zDX1oZrUxNzKRtvt5p2YmTe4E/TZlgxO7fDtKCUvIQpeogF35zpPa/zA7/u6h58yvVIWyOnJuk0=; Proxy: null)

Reproduction Steps

In one script takedn about 10 Gb zipped json files. For each filescript performs the following:

  1. Upload file
  2. Extend retention for it Error 500 happens after few hours when script after attempt to extend retention of random file (every time is different, not dependant on a content)

Possible Solution

No response

Additional Information/Context

No response

AWS Java SDK version used

1.12.281

JDK version used

17.0.1

Operating System and version

openjdk:17.0.1-jdk-slim docker image

debora-ito commented 1 year ago

@GeksterOne some questions to help in the investigation:

debora-ito commented 1 year ago

Moving this to Java SDK 1.x repo.

GeksterOne commented 1 year ago

Debora, It occurs randomly on different files after hours of flow running (we deal with 3 gb of files 2-10 KB each) I checked the content of files which caused an issue and haven't noticed anything strange. Also we perfrom additional encryption and zip compresion before uploading these files. I created a code with the same flow we perfrom but it there a possibility to send it directly, not in forum?

debora-ito commented 1 year ago

I created a code with the same flow we perfrom but it there a possibility to send it directly, not in forum?

I don't think so, you could open a Premium Support case if you have access to an AWS support plan.

500 errors usually indicate an issue on the server side, so if you can't identify a pattern and they occur at random I would just put retry strategies in the SDK, if the request is not being retried already. If you want to investigate it further, please provide a new fresh requestID when the error occur so I can pass to the S3 team.

github-actions[bot] commented 1 year ago

It looks like this issue has not been active for more than five days. In the absence of more information, we will be closing this issue soon. If you find that this is still a problem, please add a comment to prevent automatic closure, or if the issue is already closed please feel free to reopen it.

yushihui commented 1 year ago

I got a similar error during I uploads half million files to s3. here is the error

com.amazonaws.services.s3.model.AmazonS3Exception: We encountered an internal error. Please try again. (Service: Amazon S3; Status Code: 500; Error Code: InternalError; Request ID: ZW6TPSAFYR0HWT7M; S3 Extended Request ID: 2KC+LxVFWlqyGb0wdzkL+ZHdriMDKSt6uLEsUg6CQSDRd+a8BexHiTddjJNf7hRWSXJBzSqGuW8=; Proxy: null) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleErrorResponse(AmazonHttpClient.java:1879) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.handleServiceErrorResponse(AmazonHttpClient.java:1418) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeOneRequest(AmazonHttpClient.java:1387) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeHelper(AmazonHttpClient.java:1157) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.doExecute(AmazonHttpClient.java:814) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.executeWithTimer(AmazonHttpClient.java:781) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.execute(AmazonHttpClient.java:755) at com.amazonaws.http.AmazonHttpClient$RequestExecutor.access$500(AmazonHttpClient.java:715) at com.amazonaws.http.AmazonHttpClient$RequestExecutionBuilderImpl.execute(AmazonHttpClient.java:697) at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:561) at com.amazonaws.http.AmazonHttpClient.execute(AmazonHttpClient.java:541) at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5456) at com.amazonaws.services.s3.AmazonS3Client.invoke(AmazonS3Client.java:5403) at com.amazonaws.services.s3.AmazonS3Client.access$300(AmazonS3Client.java:421) at com.amazonaws.services.s3.AmazonS3Client$PutObjectStrategy.invokeServiceCall(AmazonS3Client.java:6532) at com.amazonaws.services.s3.AmazonS3Client.uploadObject(AmazonS3Client.java:1861) at com.amazonaws.services.s3.AmazonS3Client.putObject(AmazonS3Client.java:1821) at com.veeva.ctv.core.service.S3Service.uploadFile(S3Service.java:154) at com.veeva.ctv.ctgservice.util.CtgS3Util.lambda$uploadByPath$1(CtgS3Util.java:254) at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:179) at java.base/java.util.stream.ReferencePipeline$3$1.accept(ReferencePipeline.java:197) at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:992) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:509) at java.base/java.util.stream.ForEachOps$ForEachTask.compute(ForEachOps.java:290) at java.base/java.util.concurrent.CountedCompleter.exec(CountedCompleter.java:754) at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:373) at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1182) at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1655) at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1622) at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)