search

AutoMQ / automq

AutoMQ is a cloud-first alternative to Kafka by decoupling durability to S3 and EBS. 10x Cost-Effective. No Cross-AZ Traffic Cost. Autoscale in seconds. Single-digit ms latency.
https://www.automq.com/docs
Other
3.85k stars 214 forks source link

[BUG] ERROR log "Channel xxxx removed from selector before processing completed receive" #2094

Open Chillax-0v0 opened 4 weeks ago

Chillax-0v0 commented 4 weeks ago

server.log:

...
[2024-10-19 16:39:20,238] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:37376-2493 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:37376-2493 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,238] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:42602-2499 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:42602-2499 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,239] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:60488-429 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:60488-429 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,239] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:43634-2506 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:43634-2506 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,239] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:45400-2430 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:45400-2430 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,239] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:48040-2448 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:48040-2448 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,239] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:35958-2483 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:35958-2483 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,239] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:47602-2445 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:47602-2445 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,239] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:57526-2415 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:57526-2415 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,240] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:34516-2526 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:34516-2526 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,240] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:45736-2432 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:45736-2432 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,240] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:45192-2516 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:45192-2516 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,240] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:57972-2418 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:57972-2418 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,240] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:35554-2480 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:35554-2480 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,240] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:45544-2431 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:45544-2431 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,241] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:39350-2549 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:39350-2549 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,250] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:47680-2462 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:47680-2462 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,250] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:36996-2507 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:36996-2507 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,250] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:47386-2460 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:47386-2460 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,250] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:56244-2423 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:56244-2423 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,251] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:35578-2549 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:35578-2549 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,251] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:46488-2454 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:46488-2454 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,252] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:42008-2584 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:42008-2584 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,252] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:52352-2590 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:52352-2590 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,252] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:57438-2431 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:57438-2431 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,252] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:46688-2468 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:46688-2468 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,252] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:35284-2547 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:35284-2547 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,252] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:48192-2478 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:48192-2478 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
[2024-10-19 16:39:20,252] ERROR Exception while processing request from 10.0.0.94:9092-10.0.0.96:46346-2453 (kafka.network.Processor)
java.lang.IllegalStateException: Channel 10.0.0.94:9092-10.0.0.96:46346-2453 removed from selector before processing completed receive
        at kafka.network.Processor.$anonfun$processCompletedReceives$1(SocketServer.scala:1195)
        at java.base/java.util.LinkedHashMap$LinkedValues.forEach(LinkedHashMap.java:647)
        at kafka.network.Processor.processCompletedReceives(SocketServer.scala:1141)
        at kafka.network.Processor.run(SocketServer.scala:1015)
        at java.base/java.lang.Thread.run(Thread.java:840)
...
lifepuzzlefun commented 2 days ago

we also found this. any fix handle this ? : - )

Chillax-0v0 commented 1 day ago

we also found this. any fix handle this ? : - )

Sorry, we haven't found the reason yet.