zy857189690
commented
2 years ago zy-topic 50 partition , double replica
Open zy857189690 opened 2 years ago
zy857189690
commented
2 years ago zy-topic 50 partition , double replica
github-actions[bot]
commented
2 years ago The issue had no activity for 30 days, mark with Stale label.
github-actions[bot]
commented
2 years ago The issue had no activity for 30 days, mark with Stale label.
Use version 2.7.4 Use pressure test command ： pulsar-perf produce persistent://zy-tenant/zy-namespace/zy-topic -threads 10 -n 10 -r 50000 I use three machines for pressure measurement, and each machine produces 50000 messages per second
After the pressure measurement command runs for a few minutes, it begins to appear abnormal, And broker and bookie are down ，as follows:
15:34:49.465 [pulsar-client-io-7-1] WARN org.apache.pulsar.client.impl.ProducerImpl - [persistent://zy-tenant/zy-namespace/zy-topic-partition-27] [pulsar-cluster-20-18863] error while create opSendMsg by batch message container java.lang.RuntimeException: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 28621930496, max: 28631367680) at org.apache.pulsar.client.impl.BatchMessageContainerImpl.getCompressedBatchMetadataAndPayload(BatchMessageContainerImpl.java:121) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.BatchMessageContainerImpl.createOpSendMsg(BatchMessageContainerImpl.java:183) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.ProducerImpl.batchMessageAndSend(ProducerImpl.java:1669) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.ProducerImpl.lambda$null$15(ProducerImpl.java:1343) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:176) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:387) [io.netty-netty-transport-native-epoll-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_201] Caused by: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 28621930496, max: 28631367680) at io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:802) ~[io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:731) ~[io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:632) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:607) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:202) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.tcacheAllocateNormal(PoolArena.java:186) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.allocate(PoolArena.java:136) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.reallocate(PoolArena.java:286) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PooledByteBuf.capacity(PooledByteBuf.java:118) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable0(AbstractByteBuf.java:305) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:280) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1103) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1096) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1087) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at org.apache.pulsar.common.protocol.Commands.serializeSingleMessageInBatchWithPayload(Commands.java:2033) ~[org.apache.pulsar-pulsar-common-2.7.4.jar:2.7.4] at org.apache.pulsar.common.protocol.Commands.serializeSingleMessageInBatchWithPayload(Commands.java:2071) ~[org.apache.pulsar-pulsar-common-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.BatchMessageContainerImpl.getCompressedBatchMetadataAndPayload(BatchMessageContainerImpl.java:110) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] ... 12 more 15:34:49.465 [pulsar-client-io-5-1] WARN org.apache.pulsar.client.impl.ProducerImpl - [persistent://zy-tenant/zy-namespace/zy-topic-partition-38] [pulsar-cluster-18-14633] error while create opSendMsg by batch message container java.lang.RuntimeException: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 28621930496, max: 28631367680) at org.apache.pulsar.client.impl.BatchMessageContainerImpl.getCompressedBatchMetadataAndPayload(BatchMessageContainerImpl.java:121) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.BatchMessageContainerImpl.createOpSendMsg(BatchMessageContainerImpl.java:183) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.ProducerImpl.batchMessageAndSend(ProducerImpl.java:1669) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.ProducerImpl.lambda$null$15(ProducerImpl.java:1343) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:176) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:387) [io.netty-netty-transport-native-epoll-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_201] Caused by: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 28621930496, max: 28631367680) at io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:802) ~[io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:731) ~[io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:632) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:607) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:202) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.tcacheAllocateNormal(PoolArena.java:186) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.allocate(PoolArena.java:136) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.reallocate(PoolArena.java:286) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PooledByteBuf.capacity(PooledByteBuf.java:118) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable0(AbstractByteBuf.java:305) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:280) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1103) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1096) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1087) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at org.apache.pulsar.common.protocol.Commands.serializeSingleMessageInBatchWithPayload(Commands.java:2033) ~[org.apache.pulsar-pulsar-common-2.7.4.jar:2.7.4] at org.apache.pulsar.common.protocol.Commands.serializeSingleMessageInBatchWithPayload(Commands.java:2071) ~[org.apache.pulsar-pulsar-common-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.BatchMessageContainerImpl.getCompressedBatchMetadataAndPayload(BatchMessageContainerImpl.java:110) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] ... 12 more 15:34:49.465 [pulsar-client-io-10-1] WARN org.apache.pulsar.client.impl.ProducerImpl - [persistent://zy-tenant/zy-namespace/zy-topic-partition-49] [pulsar-cluster-18-12396] error while create opSendMsg by batch message container java.lang.RuntimeException: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 28621930496, max: 28631367680) at org.apache.pulsar.client.impl.BatchMessageContainerImpl.getCompressedBatchMetadataAndPayload(BatchMessageContainerImpl.java:121) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.BatchMessageContainerImpl.createOpSendMsg(BatchMessageContainerImpl.java:183) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.ProducerImpl.batchMessageAndSend(ProducerImpl.java:1669) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.ProducerImpl.lambda$null$15(ProducerImpl.java:1343) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:176) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:387) [io.netty-netty-transport-native-epoll-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_201] Caused by: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 28621930496, max: 28631367680) at io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:802) ~[io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:731) ~[io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:632) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:607) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:202) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.tcacheAllocateNormal(PoolArena.java:186) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.allocate(PoolArena.java:136) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.reallocate(PoolArena.java:286) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PooledByteBuf.capacity(PooledByteBuf.java:118) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable0(AbstractByteBuf.java:305) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:280) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1103) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1096) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1087) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at org.apache.pulsar.common.protocol.Commands.serializeSingleMessageInBatchWithPayload(Commands.java:2033) ~[org.apache.pulsar-pulsar-common-2.7.4.jar:2.7.4] at org.apache.pulsar.common.protocol.Commands.serializeSingleMessageInBatchWithPayload(Commands.java:2071) ~[org.apache.pulsar-pulsar-common-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.BatchMessageContainerImpl.getCompressedBatchMetadataAndPayload(BatchMessageContainerImpl.java:110) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] ... 12 more 15:34:49.466 [pulsar-client-io-7-1] WARN org.apache.pulsar.client.impl.ProducerImpl - [persistent://zy-tenant/zy-namespace/zy-topic-partition-20] [pulsar-cluster-18-13242] error while create opSendMsg by batch message container java.lang.RuntimeException: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 28621930496, max: 28631367680) at org.apache.pulsar.client.impl.BatchMessageContainerImpl.getCompressedBatchMetadataAndPayload(BatchMessageContainerImpl.java:121) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.BatchMessageContainerImpl.createOpSendMsg(BatchMessageContainerImpl.java:183) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.ProducerImpl.batchMessageAndSend(ProducerImpl.java:1669) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.ProducerImpl.lambda$null$15(ProducerImpl.java:1343) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] at io.netty.util.concurrent.PromiseTask.runTask(PromiseTask.java:98) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.ScheduledFutureTask.run(ScheduledFutureTask.java:176) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:469) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:387) [io.netty-netty-transport-native-epoll-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:986) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at java.lang.Thread.run(Thread.java:748) [?:1.8.0_201] Caused by: io.netty.util.internal.OutOfDirectMemoryError: failed to allocate 16777216 byte(s) of direct memory (used: 28621930496, max: 28631367680) at io.netty.util.internal.PlatformDependent.incrementMemoryCounter(PlatformDependent.java:802) ~[io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.util.internal.PlatformDependent.allocateDirectNoCleaner(PlatformDependent.java:731) ~[io.netty-netty-common-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena$DirectArena.allocateDirect(PoolArena.java:632) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena$DirectArena.newChunk(PoolArena.java:607) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.allocateNormal(PoolArena.java:202) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.tcacheAllocateNormal(PoolArena.java:186) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.allocate(PoolArena.java:136) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PoolArena.reallocate(PoolArena.java:286) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.PooledByteBuf.capacity(PooledByteBuf.java:118) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable0(AbstractByteBuf.java:305) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.ensureWritable(AbstractByteBuf.java:280) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1103) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1096) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1087) ~[io.netty-netty-buffer-4.1.68.Final.jar:4.1.68.Final] at org.apache.pulsar.common.protocol.Commands.serializeSingleMessageInBatchWithPayload(Commands.java:2033) ~[org.apache.pulsar-pulsar-common-2.7.4.jar:2.7.4] at org.apache.pulsar.common.protocol.Commands.serializeSingleMessageInBatchWithPayload(Commands.java:2071) ~[org.apache.pulsar-pulsar-common-2.7.4.jar:2.7.4] at org.apache.pulsar.client.impl.BatchMessageContainerImpl.getCompressedBatchMetadataAndPayload(BatchMessageContainerImpl.java:110) ~[org.apache.pulsar-pulsar-client-original-2.7.4.jar:2.7.4] ... 12 more