search

linagora / james-project

Mirror of Apache James Project
Apache License 2.0
70 stars 63 forks source link

Weird S3 error #5259

Closed chibenwa closed 3 days ago

chibenwa commented 2 weeks ago 
org.apache.james.blob.api.ObjectStoreIOException: Exception occurred while saving bytearray
    at org.apache.james.blob.aes.AESBlobStoreDAO.lambda$save$8(AESBlobStoreDAO.java:144)
    at reactor.core.publisher.Mono.lambda$onErrorMap$28(Mono.java:3854)
    at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:94)
    at reactor.core.publisher.MonoSubscribeOn$SubscribeOnSubscriber.onError(MonoSubscribeOn.java:152)
    at reactor.core.publisher.MonoUsingWhen$MonoUsingWhenSubscriber.deferredError(MonoUsingWhen.java:277)
    at reactor.core.publisher.FluxUsingWhen$RollbackInner.onComplete(FluxUsingWhen.java:480)
    at reactor.core.publisher.FluxSubscribeOnCallable$CallableSubscribeOnSubscription.run(FluxSubscribeOnCallable.java:238)
    at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:68)
    at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:28)
    at java.base/java.util.concurrent.FutureTask.run(Unknown Source)
    at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
    at java.base/java.lang.Thread.run(Unknown Source)
    Suppressed: java.lang.Exception: #block terminated with an error
        at reactor.core.publisher.BlockingSingleSubscriber.blockingGet(BlockingSingleSubscriber.java:104)
        at reactor.core.publisher.Mono.block(Mono.java:1779)
        at org.apache.james.queue.rabbitmq.RabbitMQMailQueue.enQueue(RabbitMQMailQueue.java:95)
        at org.apache.james.smtpserver.SendMailHandler.lambda$onMessage$7(SendMailHandler.java:116)
        at com.github.fge.lambdas.runnable.RunnableChainer.doRun(RunnableChainer.java:18)
        at com.github.fge.lambdas.runnable.ThrowingRunnable.run(ThrowingRunnable.java:16)
        at java.base/java.util.Optional.ifPresentOrElse(Unknown Source)
        at org.apache.james.smtpserver.SendMailHandler.onMessage(SendMailHandler.java:89)
        at org.apache.james.smtpserver.DataLineJamesMessageHookHandler.executeJamesMessageHooks(DataLineJamesMessageHookHandler.java:215)
        at org.apache.james.smtpserver.DataLineJamesMessageHookHandler.processExtensions(DataLineJamesMessageHookHandler.java:197)
        at org.apache.james.smtpserver.DataLineJamesMessageHookHandler.onLine(DataLineJamesMessageHookHandler.java:114)
        at org.apache.james.protocols.smtp.core.DataCmdHandler$DataLineFilterWrapper.onLine(DataCmdHandler.java:87)
        at org.apache.james.protocols.smtp.core.DataCmdHandler$DataLineFilterWrapper.onLine(DataCmdHandler.java:75)
        at org.apache.james.protocols.smtp.core.SeparatingDataLineFilter.onBodyLine(SeparatingDataLineFilter.java:100)
        at org.apache.james.protocols.smtp.core.SeparatingDataLineFilter.onLine(SeparatingDataLineFilter.java:58)
        at org.apache.james.protocols.smtp.core.DataCmdHandler$DataLineFilterWrapper.onLine(DataCmdHandler.java:87)
        at org.apache.james.protocols.smtp.core.DataCmdHandler$DataLineFilterWrapper.onLine(DataCmdHandler.java:75)
        at org.apache.james.protocols.smtp.core.esmtp.MailSizeEsmtpExtension.onLine(MailSizeEsmtpExtension.java:150)
        at org.apache.james.protocols.smtp.core.DataCmdHandler$DataLineFilterWrapper.onLine(DataCmdHandler.java:87)
        at org.apache.james.protocols.smtp.core.DataCmdHandler$DataLineFilterWrapper.onLine(DataCmdHandler.java:75)
        at org.apache.james.protocols.netty.LineHandlerUpstreamHandler.channelRead(LineHandlerUpstreamHandler.java:52)
        at org.apache.james.protocols.netty.BasicChannelInboundHandler.channelRead(BasicChannelInboundHandler.java:177)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420)
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412)
        at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346)
        at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:333)
        at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:455)
        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:290)
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444)
        at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:61)
        at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:425)
        at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)
        at io.netty.util.concurrent.DefaultEventExecutor.run(DefaultEventExecutor.java:66)
        at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)
        at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
        ... 1 common frames omitted
Caused by: reactor.core.Exceptions$RetryExhaustedException: Retries exhausted: 5/5
    at reactor.core.Exceptions.retryExhausted(Exceptions.java:308)
    at reactor.util.retry.RetryBackoffSpec.lambda$static$0(RetryBackoffSpec.java:68)
    at reactor.util.retry.RetryBackoffSpec.lambda$null$4(RetryBackoffSpec.java:560)
    at reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.onNext(FluxConcatMapNoPrefetch.java:183)
    at reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onNext(FluxContextWrite.java:107)
    at reactor.core.publisher.SinkManyEmitterProcessor.drain(SinkManyEmitterProcessor.java:476)
    at reactor.core.publisher.SinkManyEmitterProcessor.tryEmitNext(SinkManyEmitterProcessor.java:273)
    at reactor.core.publisher.SinkManySerialized.tryEmitNext(SinkManySerialized.java:100)
    at reactor.core.publisher.InternalManySink.emitNext(InternalManySink.java:27)
    at reactor.core.publisher.FluxRetryWhen$RetryWhenMainSubscriber.onError(FluxRetryWhen.java:194)
    at reactor.core.publisher.MonoCompletionStage$MonoCompletionStageSubscription.apply(MonoCompletionStage.java:115)
    at reactor.core.publisher.MonoCompletionStage$MonoCompletionStageSubscription.apply(MonoCompletionStage.java:67)
    at java.base/java.util.concurrent.CompletableFuture.uniHandle(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallMetricCollectionStage.lambda$execute$0(AsyncApiCallMetricCollectionStage.java:56)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallTimeoutTrackingStage.lambda$execute$2(AsyncApiCallTimeoutTrackingStage.java:67)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.maybeAttemptExecute(AsyncRetryableStage2.java:135)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.maybeRetryExecute(AsyncRetryableStage2.java:152)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.lambda$attemptExecute$1(AsyncRetryableStage2.java:113)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$execute$0(MakeAsyncHttpRequestStage.java:108)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.completeResponseFuture(MakeAsyncHttpRequestStage.java:255)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$executeHttpRequest$3(MakeAsyncHttpRequestStage.java:167)
    at java.base/java.util.concurrent.CompletableFuture.uniHandle(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source)
    ... 3 common frames omitted
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Request content was only 0 bytes, but the specified content-length was 7526 bytes.
    at software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
    at software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper2.setLastException(RetryableStageHelper2.java:226)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper2.setLastException(RetryableStageHelper2.java:221)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.maybeRetryExecute(AsyncRetryableStage2.java:151)
    ... 23 common frames omitted
Caused by: java.lang.IllegalStateException: Request content was only 0 bytes, but the specified content-length was 7526 bytes.
    at software.amazon.awssdk.http.nio.netty.internal.NettyRequestExecutor$StreamedRequest$1.onComplete(NettyRequestExecutor.java:477)
    at software.amazon.awssdk.services.s3.internal.checksums.ChecksumCalculatingAsyncRequestBody$ChecksumCalculatingSubscriber.onComplete(ChecksumCalculatingAsyncRequestBody.java:116)
    at reactor.core.publisher.StrictSubscriber.onComplete(StrictSubscriber.java:123)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.onComplete(FluxSubscribeOn.java:166)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.onComplete(FluxSubscribeOn.java:166)
    at reactor.core.publisher.Operators$BaseFluxToMonoOperator.completePossiblyEmpty(Operators.java:2098)
    at reactor.core.publisher.FluxDefaultIfEmpty$DefaultIfEmptySubscriber.onComplete(FluxDefaultIfEmpty.java:134)
    at reactor.core.publisher.FluxGenerate$GenerateSubscription.complete(FluxGenerate.java:205)
    at org.apache.james.util.ReactorUtils.lambda$toChunks$9(ReactorUtils.java:122)
    at reactor.core.publisher.FluxGenerate.lambda$new$1(FluxGenerate.java:58)
    at reactor.core.publisher.FluxGenerate$GenerateSubscription.slowPath(FluxGenerate.java:274)
    at reactor.core.publisher.FluxGenerate$GenerateSubscription.request(FluxGenerate.java:216)
    at reactor.core.publisher.FluxDefaultIfEmpty$DefaultIfEmptySubscriber.request(FluxDefaultIfEmpty.java:98)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.requestUpstream(FluxSubscribeOn.java:131)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.onSubscribe(FluxSubscribeOn.java:124)
    at reactor.core.publisher.Operators$BaseFluxToMonoOperator.onSubscribe(Operators.java:2051)
    at reactor.core.publisher.FluxGenerate.subscribe(FluxGenerate.java:88)
    at reactor.core.publisher.InternalFluxOperator.subscribe(InternalFluxOperator.java:68)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.run(FluxSubscribeOn.java:194)
    at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84)
    at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37)
    at java.base/java.util.concurrent.FutureTask.run(Unknown Source)
    at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)
    ... 3 common frames omitted

Was not happening before the 0.10.2 release.

chibenwa commented 2 weeks ago

Failure seems sporadic and object storage eventually succeed.

CF 

Exception calling RemoteDelivery: Exception occurred while saving bytearray
org.apache.james.blob.api.ObjectStoreIOException: Exception occurred while saving bytearray
    at org.apache.james.blob.aes.AESBlobStoreDAO.lambda$save$8(AESBlobStoreDAO.java:144)
    at reactor.core.publisher.Mono.lambda$onErrorMap$28(Mono.java:3854)
    at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:94)
    at reactor.core.publisher.MonoSubscribeOn$SubscribeOnSubscriber.onError(MonoSubscribeOn.java:152)
    at reactor.core.publisher.MonoUsingWhen$MonoUsingWhenSubscriber.deferredError(MonoUsingWhen.java:277)
    at reactor.core.publisher.FluxUsingWhen$RollbackInner.onComplete(FluxUsingWhen.java:480)
    at reactor.core.publisher.FluxSubscribeOnCallable$CallableSubscribeOnSubscription.run(FluxSubscribeOnCallable.java:238)
    at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:68)
    at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:28)
    at java.base/java.util.concurrent.FutureTask.run(Unknown Source)
    at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
    at java.base/java.lang.Thread.run(Unknown Source)
    Suppressed: java.lang.Exception: #block terminated with an error
        at reactor.core.publisher.BlockingSingleSubscriber.blockingGet(BlockingSingleSubscriber.java:104)
        at reactor.core.publisher.Mono.block(Mono.java:1779)
        at org.apache.james.queue.rabbitmq.RabbitMQMailQueue.enQueue(RabbitMQMailQueue.java:95)
        at org.apache.james.transport.mailets.RemoteDelivery.serviceWithGateway(RemoteDelivery.java:252)
        at org.apache.james.transport.mailets.RemoteDelivery.service(RemoteDelivery.java:228)
        at org.apache.james.mailetcontainer.impl.ProcessorImpl.process(ProcessorImpl.java:81)
        at com.github.fge.lambdas.consumers.ConsumerChainer.lambda$sneakyThrow$9(ConsumerChainer.java:73)
        at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)
        at java.base/java.util.Collections$2.tryAdvance(Unknown Source)
        at java.base/java.util.Collections$2.forEachRemaining(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)
        at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline.forEach(Unknown Source)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.executeProcessingStep(MailetProcessorImpl.java:162)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$service$0(MailetProcessorImpl.java:130)
        at java.base/java.util.stream.ReduceOps$1ReducingSink.accept(Unknown Source)
        at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)
        at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline.reduce(Unknown Source)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.service(MailetProcessorImpl.java:128)
        at org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.handleWithProcessor(AbstractStateCompositeProcessor.java:98)
        at org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.service(AbstractStateCompositeProcessor.java:80)
        at org.apache.james.mailetcontainer.lib.AbstractStateMailetProcessor.toProcessor(AbstractStateMailetProcessor.java:151)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$executeProcessingStep$7(MailetProcessorImpl.java:167)
        at com.github.fge.lambdas.consumers.ConsumerChainer.lambda$sneakyThrow$9(ConsumerChainer.java:73)
        at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)
        at java.base/java.util.Collections$2.tryAdvance(Unknown Source)
        at java.base/java.util.Collections$2.forEachRemaining(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)
        at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline.forEach(Unknown Source)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.executeProcessingStep(MailetProcessorImpl.java:167)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$service$0(MailetProcessorImpl.java:130)
        at java.base/java.util.stream.ReduceOps$1ReducingSink.accept(Unknown Source)
        at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)
        at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline.reduce(Unknown Source)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.service(MailetProcessorImpl.java:128)
        at org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.handleWithProcessor(AbstractStateCompositeProcessor.java:98)
        at org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.service(AbstractStateCompositeProcessor.java:80)
        at org.apache.james.mailetcontainer.lib.AbstractStateMailetProcessor.toProcessor(AbstractStateMailetProcessor.java:151)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$executeProcessingStep$7(MailetProcessorImpl.java:167)
        at com.github.fge.lambdas.consumers.ConsumerChainer.lambda$sneakyThrow$9(ConsumerChainer.java:73)
        at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)
        at java.base/java.util.Collections$2.tryAdvance(Unknown Source)
        at java.base/java.util.Collections$2.forEachRemaining(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)
        at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline.forEach(Unknown Source)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.executeProcessingStep(MailetProcessorImpl.java:167)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$service$0(MailetProcessorImpl.java:130)
        at java.base/java.util.stream.ReduceOps$1ReducingSink.accept(Unknown Source)
        at java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)
        at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(Unknown Source)
        at java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)
        at java.base/java.util.stream.ReferencePipeline.reduce(Unknown Source)
        at org.apache.james.mailetcontainer.impl.MailetProcessorImpl.service(MailetProcessorImpl.java:128)
        at org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.handleWithProcessor(AbstractStateCompositeProcessor.java:98)
        at org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.service(AbstractStateCompositeProcessor.java:80)
        at org.apache.james.mailetcontainer.impl.JamesMailSpooler$Runner.performProcessMail(JamesMailSpooler.java:135)
        at org.apache.james.mailetcontainer.impl.JamesMailSpooler$Runner.lambda$processMail$4(JamesMailSpooler.java:127)
        at reactor.core.publisher.MonoRunnable.subscribe(MonoRunnable.java:49)
        at reactor.core.publisher.MonoUsing.subscribe(MonoUsing.java:109)
        at reactor.core.publisher.Mono.subscribe(Mono.java:4568)
        at reactor.core.publisher.FluxFlatMap.trySubscribeScalarMap(FluxFlatMap.java:202)
        at reactor.core.publisher.MonoFlatMap.subscribeOrReturn(MonoFlatMap.java:53)
        at reactor.core.publisher.Mono.subscribe(Mono.java:4552)
        at reactor.core.publisher.MonoSubscribeOn$SubscribeOnSubscriber.run(MonoSubscribeOn.java:126)
        at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84)
        at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37)
        ... 5 more
Caused by: reactor.core.Exceptions$RetryExhaustedException: Retries exhausted: 5/5
    at reactor.core.Exceptions.retryExhausted(Exceptions.java:308)
    at reactor.util.retry.RetryBackoffSpec.lambda$static$0(RetryBackoffSpec.java:68)
    at reactor.util.retry.RetryBackoffSpec.lambda$null$4(RetryBackoffSpec.java:560)
    at reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.onNext(FluxConcatMapNoPrefetch.java:183)
    at reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onNext(FluxContextWrite.java:107)
    at reactor.core.publisher.SinkManyEmitterProcessor.drain(SinkManyEmitterProcessor.java:476)
    at reactor.core.publisher.SinkManyEmitterProcessor.tryEmitNext(SinkManyEmitterProcessor.java:273)
    at reactor.core.publisher.SinkManySerialized.tryEmitNext(SinkManySerialized.java:100)
    at reactor.core.publisher.InternalManySink.emitNext(InternalManySink.java:27)
    at reactor.core.publisher.FluxRetryWhen$RetryWhenMainSubscriber.onError(FluxRetryWhen.java:194)
    at reactor.core.publisher.MonoCompletionStage$MonoCompletionStageSubscription.apply(MonoCompletionStage.java:115)
    at reactor.core.publisher.MonoCompletionStage$MonoCompletionStageSubscription.apply(MonoCompletionStage.java:67)
    at java.base/java.util.concurrent.CompletableFuture.uniHandle(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallMetricCollectionStage.lambda$execute$0(AsyncApiCallMetricCollectionStage.java:56)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallTimeoutTrackingStage.lambda$execute$2(AsyncApiCallTimeoutTrackingStage.java:67)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.maybeAttemptExecute(AsyncRetryableStage2.java:135)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.maybeRetryExecute(AsyncRetryableStage2.java:152)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.lambda$attemptExecute$1(AsyncRetryableStage2.java:113)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$execute$0(MakeAsyncHttpRequestStage.java:108)
    at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.completeResponseFuture(MakeAsyncHttpRequestStage.java:255)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$executeHttpRequest$3(MakeAsyncHttpRequestStage.java:167)
    at java.base/java.util.concurrent.CompletableFuture.uniHandle(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source)
    at java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source)
    ... 3 more
Caused by: software.amazon.awssdk.core.exception.SdkClientException: Unable to execute HTTP request: Request content was only 0 bytes, but the specified content-length was 1366 bytes.
    at software.amazon.awssdk.core.exception.SdkClientException$BuilderImpl.build(SdkClientException.java:111)
    at software.amazon.awssdk.core.exception.SdkClientException.create(SdkClientException.java:47)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper2.setLastException(RetryableStageHelper2.java:226)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.utils.RetryableStageHelper2.setLastException(RetryableStageHelper2.java:221)
    at software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.maybeRetryExecute(AsyncRetryableStage2.java:151)
    ... 23 more
Caused by: java.lang.IllegalStateException: Request content was only 0 bytes, but the specified content-length was 1366 bytes.
    at software.amazon.awssdk.http.nio.netty.internal.NettyRequestExecutor$StreamedRequest$1.onComplete(NettyRequestExecutor.java:477)
    at software.amazon.awssdk.services.s3.internal.checksums.ChecksumCalculatingAsyncRequestBody$ChecksumCalculatingSubscriber.onComplete(ChecksumCalculatingAsyncRequestBody.java:116)
    at reactor.core.publisher.StrictSubscriber.onComplete(StrictSubscriber.java:123)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.onComplete(FluxSubscribeOn.java:166)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.onComplete(FluxSubscribeOn.java:166)
    at reactor.core.publisher.Operators$BaseFluxToMonoOperator.completePossiblyEmpty(Operators.java:2098)
    at reactor.core.publisher.FluxDefaultIfEmpty$DefaultIfEmptySubscriber.onComplete(FluxDefaultIfEmpty.java:134)
    at reactor.core.publisher.FluxGenerate$GenerateSubscription.complete(FluxGenerate.java:205)
    at org.apache.james.util.ReactorUtils.lambda$toChunks$9(ReactorUtils.java:122)
    at reactor.core.publisher.FluxGenerate.lambda$new$1(FluxGenerate.java:58)
    at reactor.core.publisher.FluxGenerate$GenerateSubscription.slowPath(FluxGenerate.java:274)
    at reactor.core.publisher.FluxGenerate$GenerateSubscription.request(FluxGenerate.java:216)
    at reactor.core.publisher.FluxDefaultIfEmpty$DefaultIfEmptySubscriber.request(FluxDefaultIfEmpty.java:98)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.requestUpstream(FluxSubscribeOn.java:131)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.onSubscribe(FluxSubscribeOn.java:124)
    at reactor.core.publisher.Operators$BaseFluxToMonoOperator.onSubscribe(Operators.java:2051)
    at reactor.core.publisher.FluxGenerate.subscribe(FluxGenerate.java:88)
    at reactor.core.publisher.InternalFluxOperator.subscribe(InternalFluxOperator.java:68)
    at reactor.core.publisher.FluxSubscribeOn$SubscribeOnSubscriber.run(FluxSubscribeOn.java:194)
    at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84)
    at reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37)
    at java.base/java.util.concurrent.FutureTask.run(Unknown Source)
    at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)
    ... 3 more
Arsnael commented 2 weeks ago 
Caused by: java.lang.IllegalStateException: Request content was only 0 bytes, but the specified content-length was 1366 bytes.

Makes me think of this that it could maybe go together as part of a bigger problem? https://github.com/linagora/james-project/issues/5252#issue-2487133343

Would need to investigate deeply I agree

CNB correct? OVH S3 object storage?

chibenwa commented 2 weeks ago

Makes me think of this that it could maybe go together as part of a bigger problem? https://github.com/linagora/james-project/issues/5252#issue-2487133343

Unrelated (call eventually succeed, and this issue did not happen before while it was the case for #5252 ) here we have the differential between the computed size and the actual payload. Looks like the payload is dropped / truncated...

chibenwa commented 2 weeks ago

CNB correct? OVH S3 object storage?

Correct

chibenwa commented 2 weeks ago

https://github.com/chibenwa/james-project/tree/cnb could contain related fixes and might deserve an cautious review.

Arsnael commented 2 weeks ago

Timebox: 5 man day

chibenwa commented 2 weeks ago

image

Proof not happening before the release.

quantranhong1999 commented 2 weeks ago

Audit CNB branch

Can you share the branch?

Arsnael commented 2 weeks ago

Can you share the branch?

I believe it's this: https://github.com/chibenwa/james-project/tree/cnb

quantranhong1999 commented 2 weeks ago

Similar error: https://github.com/aws/aws-sdk-java-v2/issues/4900

But not too much context/response got from that issue.

Only 2 things similar to us:

He had the issue in the 2.20.101 driver version, while we were fine at 2.21.26 (before 0.10.2). So maybe the version is not the issue.

quantranhong1999 commented 2 weeks ago

I believe it's this: https://github.com/chibenwa/james-project/tree/cnb

Thanks. I missed the one above.

quantranhong1999 commented 2 weeks ago

FYI not related to this issue but I checked the OVH s3 change log. It now supports the Server-Side Encryption. That means for future customers we may not need to use AESBlobStore anymore and remove the computation burden out of James.

chibenwa commented 2 weeks ago

Server-Side Encryption is transparently handled by the S3 API if I am correct.

While cool, our AES implemention is more equivalent to SSE-C (Client-Provided Keys), which is supported by OVH

https://help.ovhcloud.com/csm/en-public-cloud-storage-s3-encrypt-objects-sse-c?id=kb_article_view&sysparm_article=KB0047314

Will create a dedicated ticket as, YES, this is a nice enhancement.

To be noted this won't be applicable for CNB... Migrating from James-side AES to SSE-C is not compatible and we likely do not want to undertake the migration.

chibenwa commented 2 weeks ago

(Opened issue for SSE-C)

quantranhong1999 commented 1 week ago

Some actions are taken:

quantranhong1999 commented 1 week ago

Diagnostic ideas are welcome.

chibenwa commented 1 week ago

https://github.com/aws/aws-sdk-java-v2/issues/4900 is IMO not related: the guy customized heavily the driver. Error message is not the same either: we consistantly have 0 whereas he gets an unrelated value...

chibenwa commented 1 week ago

This bug did not occur again on the last 24h hours.

Please review the CNB diff, then we can IMO put it on hold and see if this happens again by the end of the week.

quantranhong1999 commented 1 week ago

Please review the CNB diff

AESBlobStore changes: James master vs CNB fix: https://www.diffchecker.com/cTlkt9JL/

image

Which version is being used on CNB? My guess is that the CNB fix one has been used before but we migrated to the James master one recently?

Maybe something is wrong with calculating the size of the OutputStream in the James master one (which is the specified content-length in later code): https://github.com/apache/james-project/blob/master/server/blob/blob-aes/src/main/java/org/apache/james/blob/aes/AESBlobStoreDAO.java#L75?

I don't dig deeply into the code yet. But either calculating the size is wrong or somehow the OutputStream is wrongly empty?

My feeling is that AESBlobStore change could be related as the performance test and Linagora does not use AESBlobStore therefore not exercising that change.

chibenwa commented 1 week ago

That's minor syntax changes and BTW I no longer see those logs onto CNB

Maybe we got unlucky on OVH side.

quantranhong1999 commented 1 week ago

image

Still happened a few ones yesterday but yes it is way less.

chibenwa commented 1 week ago

What's the exact error message?

chibenwa commented 1 week ago

image

quantranhong1999 commented 1 week ago

Yes, it was not exactly this issue.

Same error Exception occurred while saving bytearray but other root cause.

Example:

{
    "timestamp": "2024-09-04T00:41:39.696Z",
    "level": "ERROR",
    "thread": "spooler-6616",
    "logger": "org.apache.james.mailetcontainer.impl.ProcessorImpl",
    "message": "Exception calling RemoteDelivery: Exception occurred while saving bytearray\norg.apache.james.blob.api.ObjectStoreIOException: Exception occurred while saving bytearray\n\tat org.apache.james.blob.aes.AESBlobStoreDAO.lambda$save$8(AESBlobStoreDAO.java:144)\n\tat reactor.core.publisher.Mono.lambda$onErrorMap$28(Mono.java:3854)\n\tat reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:94)\n\tat reactor.core.publisher.MonoSubscribeOn$SubscribeOnSubscriber.onError(MonoSubscribeOn.java:152)\n\tat reactor.core.publisher.MonoUsingWhen$MonoUsingWhenSubscriber.deferredError(MonoUsingWhen.java:277)\n\tat reactor.core.publisher.FluxUsingWhen$RollbackInner.onComplete(FluxUsingWhen.java:480)\n\tat reactor.core.publisher.FluxSubscribeOnCallable$CallableSubscribeOnSubscription.run(FluxSubscribeOnCallable.java:238)\n\tat reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:68)\n\tat reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:28)\n\tat java.base/java.util.concurrent.FutureTask.run(Unknown Source)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)\n\tat java.base/java.lang.Thread.run(Unknown Source)\n\tSuppressed: java.lang.Exception: #block terminated with an error\n\t\tat reactor.core.publisher.BlockingSingleSubscriber.blockingGet(BlockingSingleSubscriber.java:104)\n\t\tat reactor.core.publisher.Mono.block(Mono.java:1779)\n\t\tat org.apache.james.queue.rabbitmq.RabbitMQMailQueue.enQueue(RabbitMQMailQueue.java:95)\n\t\tat org.apache.james.transport.mailets.RemoteDelivery.serviceWithGateway(RemoteDelivery.java:252)\n\t\tat org.apache.james.transport.mailets.RemoteDelivery.service(RemoteDelivery.java:228)\n\t\tat org.apache.james.mailetcontainer.impl.ProcessorImpl.process(ProcessorImpl.java:81)\n\t\tat com.github.fge.lambdas.consumers.ConsumerChainer.lambda$sneakyThrow$9(ConsumerChainer.java:73)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)\n\t\tat java.base/java.util.Collections$2.tryAdvance(Unknown Source)\n\t\tat java.base/java.util.Collections$2.forEachRemaining(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline.forEach(Unknown Source)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.executeProcessingStep(MailetProcessorImpl.java:162)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$service$0(MailetProcessorImpl.java:130)\n\t\tat java.base/java.util.stream.ReduceOps$1ReducingSink.accept(Unknown Source)\n\t\tat java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)\n\t\tat java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline.reduce(Unknown Source)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.service(MailetProcessorImpl.java:128)\n\t\tat org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.handleWithProcessor(AbstractStateCompositeProcessor.java:98)\n\t\tat org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.service(AbstractStateCompositeProcessor.java:80)\n\t\tat org.apache.james.mailetcontainer.lib.AbstractStateMailetProcessor.toProcessor(AbstractStateMailetProcessor.java:151)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$executeProcessingStep$7(MailetProcessorImpl.java:167)\n\t\tat com.github.fge.lambdas.consumers.ConsumerChainer.lambda$sneakyThrow$9(ConsumerChainer.java:73)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)\n\t\tat java.base/java.util.Collections$2.tryAdvance(Unknown Source)\n\t\tat java.base/java.util.Collections$2.forEachRemaining(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline.forEach(Unknown Source)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.executeProcessingStep(MailetProcessorImpl.java:167)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$service$0(MailetProcessorImpl.java:130)\n\t\tat java.base/java.util.stream.ReduceOps$1ReducingSink.accept(Unknown Source)\n\t\tat java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)\n\t\tat java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline.reduce(Unknown Source)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.service(MailetProcessorImpl.java:128)\n\t\tat org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.handleWithProcessor(AbstractStateCompositeProcessor.java:98)\n\t\tat org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.service(AbstractStateCompositeProcessor.java:80)\n\t\tat org.apache.james.mailetcontainer.lib.AbstractStateMailetProcessor.toProcessor(AbstractStateMailetProcessor.java:151)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$executeProcessingStep$7(MailetProcessorImpl.java:167)\n\t\tat com.github.fge.lambdas.consumers.ConsumerChainer.lambda$sneakyThrow$9(ConsumerChainer.java:73)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline$2$1.accept(Unknown Source)\n\t\tat java.base/java.util.Collections$2.tryAdvance(Unknown Source)\n\t\tat java.base/java.util.Collections$2.forEachRemaining(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline.forEach(Unknown Source)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.executeProcessingStep(MailetProcessorImpl.java:167)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.lambda$service$0(MailetProcessorImpl.java:130)\n\t\tat java.base/java.util.stream.ReduceOps$1ReducingSink.accept(Unknown Source)\n\t\tat java.base/java.util.Spliterators$ArraySpliterator.forEachRemaining(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.copyInto(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(Unknown Source)\n\t\tat java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(Unknown Source)\n\t\tat java.base/java.util.stream.AbstractPipeline.evaluate(Unknown Source)\n\t\tat java.base/java.util.stream.ReferencePipeline.reduce(Unknown Source)\n\t\tat org.apache.james.mailetcontainer.impl.MailetProcessorImpl.service(MailetProcessorImpl.java:128)\n\t\tat org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.handleWithProcessor(AbstractStateCompositeProcessor.java:98)\n\t\tat org.apache.james.mailetcontainer.lib.AbstractStateCompositeProcessor.service(AbstractStateCompositeProcessor.java:80)\n\t\tat org.apache.james.mailetcontainer.impl.JamesMailSpooler$Runner.performProcessMail(JamesMailSpooler.java:135)\n\t\tat org.apache.james.mailetcontainer.impl.JamesMailSpooler$Runner.lambda$processMail$4(JamesMailSpooler.java:127)\n\t\tat reactor.core.publisher.MonoRunnable.subscribe(MonoRunnable.java:49)\n\t\tat reactor.core.publisher.MonoUsing.subscribe(MonoUsing.java:109)\n\t\tat reactor.core.publisher.Mono.subscribe(Mono.java:4568)\n\t\tat reactor.core.publisher.FluxFlatMap.trySubscribeScalarMap(FluxFlatMap.java:202)\n\t\tat reactor.core.publisher.MonoFlatMap.subscribeOrReturn(MonoFlatMap.java:53)\n\t\tat reactor.core.publisher.Mono.subscribe(Mono.java:4552)\n\t\tat reactor.core.publisher.MonoSubscribeOn$SubscribeOnSubscriber.run(MonoSubscribeOn.java:126)\n\t\tat reactor.core.scheduler.WorkerTask.call(WorkerTask.java:84)\n\t\tat reactor.core.scheduler.WorkerTask.call(WorkerTask.java:37)\n\t\t... 5 more\nCaused by: reactor.core.Exceptions$RetryExhaustedException: Retries exhausted: 5/5\n\tat reactor.core.Exceptions.retryExhausted(Exceptions.java:308)\n\tat reactor.util.retry.RetryBackoffSpec.lambda$static$0(RetryBackoffSpec.java:68)\n\tat reactor.util.retry.RetryBackoffSpec.lambda$null$4(RetryBackoffSpec.java:560)\n\tat reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.onNext(FluxConcatMapNoPrefetch.java:183)\n\tat reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onNext(FluxContextWrite.java:107)\n\tat reactor.core.publisher.SinkManyEmitterProcessor.drain(SinkManyEmitterProcessor.java:476)\n\tat reactor.core.publisher.SinkManyEmitterProcessor.tryEmitNext(SinkManyEmitterProcessor.java:273)\n\tat reactor.core.publisher.SinkManySerialized.tryEmitNext(SinkManySerialized.java:100)\n\tat reactor.core.publisher.InternalManySink.emitNext(InternalManySink.java:27)\n\tat reactor.core.publisher.FluxRetryWhen$RetryWhenMainSubscriber.onError(FluxRetryWhen.java:194)\n\tat reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:106)\n\tat reactor.core.publisher.Operators.error(Operators.java:198)\n\tat reactor.core.publisher.MonoError.subscribe(MonoError.java:53)\n\tat reactor.core.publisher.Mono.subscribe(Mono.java:4568)\n\tat reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onError(FluxOnErrorResume.java:103)\n\tat reactor.core.publisher.SerializedSubscriber.onError(SerializedSubscriber.java:124)\n\tat reactor.core.publisher.FluxRetryWhen$RetryWhenMainSubscriber.whenError(FluxRetryWhen.java:229)\n\tat reactor.core.publisher.FluxRetryWhen$RetryWhenOtherSubscriber.onError(FluxRetryWhen.java:279)\n\tat reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onError(FluxContextWrite.java:121)\n\tat reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.maybeOnError(FluxConcatMapNoPrefetch.java:327)\n\tat reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.innerError(FluxConcatMapNoPrefetch.java:298)\n\tat reactor.core.publisher.FluxConcatMap$ConcatMapInner.onError(FluxConcatMap.java:877)\n\tat reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onError(FluxContextWrite.java:121)\n\tat reactor.core.publisher.MonoFlatMap$FlatMapMain.onError(MonoFlatMap.java:180)\n\tat reactor.core.publisher.MonoIgnoreThen$ThenIgnoreMain.onError(MonoIgnoreThen.java:280)\n\tat reactor.core.publisher.MonoIgnoreThen$ThenIgnoreMain.subscribeNext(MonoIgnoreThen.java:256)\n\tat reactor.core.publisher.MonoIgnoreThen.subscribe(MonoIgnoreThen.java:51)\n\tat reactor.core.publisher.Mono.subscribe(Mono.java:4568)\n\tat reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.onNext(FluxConcatMapNoPrefetch.java:207)\n\tat reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.onNext(FluxContextWrite.java:107)\n\tat reactor.core.publisher.SinkManyEmitterProcessor.drain(SinkManyEmitterProcessor.java:476)\n\tat reactor.core.publisher.SinkManyEmitterProcessor$EmitterInner.drainParent(SinkManyEmitterProcessor.java:620)\n\tat reactor.core.publisher.FluxPublish$PubSubInner.request(FluxPublish.java:874)\n\tat reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.request(FluxContextWrite.java:136)\n\tat reactor.core.publisher.FluxConcatMapNoPrefetch$FluxConcatMapNoPrefetchSubscriber.request(FluxConcatMapNoPrefetch.java:337)\n\tat reactor.core.publisher.FluxContextWrite$ContextWriteSubscriber.request(FluxContextWrite.java:136)\n\tat reactor.core.publisher.Operators$DeferredSubscription.request(Operators.java:1743)\n\tat reactor.core.publisher.FluxRetryWhen$RetryWhenMainSubscriber.onError(FluxRetryWhen.java:196)\n\tat reactor.core.publisher.MonoFlatMap$FlatMapMain.secondError(MonoFlatMap.java:241)\n\tat reactor.core.publisher.MonoFlatMap$FlatMapInner.onError(MonoFlatMap.java:315)\n\tat reactor.core.publisher.MonoUsingWhen$MonoUsingWhenSubscriber.deferredError(MonoUsingWhen.java:277)\n\tat reactor.core.publisher.FluxUsingWhen$RollbackInner.onComplete(FluxUsingWhen.java:480)\n\tat reactor.core.publisher.MonoRunnable.subscribe(MonoRunnable.java:50)\n\tat reactor.core.publisher.Mono.subscribe(Mono.java:4568)\n\tat reactor.core.publisher.FluxUsingWhen$UsingWhenSubscriber.onError(FluxUsingWhen.java:368)\n\tat reactor.core.publisher.MonoCompletionStage$MonoCompletionStageSubscription.apply(MonoCompletionStage.java:115)\n\tat reactor.core.publisher.MonoCompletionStage$MonoCompletionStageSubscription.apply(MonoCompletionStage.java:67)\n\tat java.base/java.util.concurrent.CompletableFuture.uniHandle(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)\n\tat software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)\n\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallMetricCollectionStage.lambda$execute$0(AsyncApiCallMetricCollectionStage.java:56)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)\n\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncApiCallTimeoutTrackingStage.lambda$execute$2(AsyncApiCallTimeoutTrackingStage.java:67)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)\n\tat software.amazon.awssdk.utils.CompletableFutureUtils.lambda$forwardExceptionTo$0(CompletableFutureUtils.java:79)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source)\n\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.maybeAttemptExecute(AsyncRetryableStage2.java:135)\n\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.maybeRetryExecute(AsyncRetryableStage2.java:152)\n\tat software.amazon.awssdk.core.internal.http.pipeline.stages.AsyncRetryableStage2$RetryingExecutor.lambda$attemptExecute$1(AsyncRetryableStage2.java:123)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(Unknown Source)\n\tat software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$execute$0(MakeAsyncHttpRequestStage.java:110)\n\tat java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(Unknown Source)\n\tat software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.completeResponseFuture(MakeAsyncHttpRequestStage.java:253)\n\tat software.amazon.awssdk.core.internal.http.pipeline.stages.MakeAsyncHttpRequestStage.lambda$executeHttpRequest$3(MakeAsyncHttpRequestStage.java:167)\n\tat java.base/java.util.concurrent.CompletableFuture.uniHandle(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture$Completion.run(Unknown Source)\n\t... 3 more\nCaused by: software.amazon.awssdk.services.s3.model.S3Exception: Missing X-Amz-Server-Side-Encryption-Customer-Key header (Service: S3, Status Code: 400, Request ID: tx565ee3bd4c914c7bb4b52-0066d7aca7, Extended Request ID: tx565ee3bd4c914c7bb4b52-0066d7aca7)\n\tat software.amazon.awssdk.protocols.xml.internal.unmarshall.AwsXmlPredicatedResponseHandler.handleErrorResponse(AwsXmlPredicatedResponseHandler.java:156)\n\tat software.amazon.awssdk.protocols.xml.internal.unmarshall.AwsXmlPredicatedResponseHandler.handleResponse(AwsXmlPredicatedResponseHandler.java:108)\n\tat software.amazon.awssdk.protocols.xml.internal.unmarshall.AwsXmlPredicatedResponseHandler.handle(AwsXmlPredicatedResponseHandler.java:85)\n\tat software.amazon.awssdk.protocols.xml.internal.unmarshall.AwsXmlPredicatedResponseHandler.handle(AwsXmlPredicatedResponseHandler.java:43)\n\tat software.amazon.awssdk.core.internal.handler.BaseClientHandler.lambda$successTransformationResponseHandler$7(BaseClientHandler.java:279)\n\tat software.amazon.awssdk.core.internal.http.async.AsyncResponseHandler.lambda$prepare$0(AsyncResponseHandler.java:92)\n\tat java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.postComplete(Unknown Source)\n\tat java.base/java.util.concurrent.CompletableFuture.complete(Unknown Source)\n\tat software.amazon.awssdk.core.internal.http.async.AsyncResponseHandler$BaosSubscriber.onComplete(AsyncResponseHandler.java:135)\n\tat software.amazon.awssdk.core.internal.metrics.BytesReadTrackingPublisher$BytesReadTracker.onComplete(BytesReadTrackingPublisher.java:74)\n\tat software.amazon.awssdk.http.nio.netty.internal.ResponseHandler$DataCountingPublisher$1.onComplete(ResponseHandler.java:519)\n\tat software.amazon.awssdk.http.nio.netty.internal.ResponseHandler.runAndLogError(ResponseHandler.java:254)\n\tat software.amazon.awssdk.http.nio.netty.internal.ResponseHandler.access$600(ResponseHandler.java:77)\n\tat software.amazon.awssdk.http.nio.netty.internal.ResponseHandler$PublisherAdapter$1.onComplete(ResponseHandler.java:375)\n\tat software.amazon.awssdk.http.nio.netty.internal.nrs.HandlerPublisher.publishMessage(HandlerPublisher.java:402)\n\tat software.amazon.awssdk.http.nio.netty.internal.nrs.HandlerPublisher.flushBuffer(HandlerPublisher.java:338)\n\tat software.amazon.awssdk.http.nio.netty.internal.nrs.HandlerPublisher.receivedDemand(HandlerPublisher.java:291)\n\tat software.amazon.awssdk.http.nio.netty.internal.nrs.HandlerPublisher.access$200(HandlerPublisher.java:61)\n\tat software.amazon.awssdk.http.nio.netty.internal.nrs.HandlerPublisher$ChannelSubscription$1.run(HandlerPublisher.java:495)\n\tat io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173)\n\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469)\n\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:566)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\t... 1 more\n",
    "context": "default"
}

Caused by: software.amazon.awssdk.services.s3.model.S3Exception: Missing X-Amz-Server-Side-Encryption-Customer-Key header (Service: S3, Status Code: 400, Request ID: tx565ee3bd4c914c7bb4b52-0066d7aca7, Extended Request ID: tx565ee3bd4c914c7bb4b52-0066d7aca7).

=> Experimenting SSE-C on CNB bucket?

chibenwa commented 1 week ago

=> Experimenting SSE-C on CNB bucket?

Nope not that I am aware of but there was a S3 outage on that time period.

quantranhong1999 commented 6 days ago

No similar error so far from 04/09.

Updated: upto 12/09.