search

streamnative / pulsar-archived

Apache Pulsar - distributed pub-sub messaging system
https://pulsar.apache.org
Apache License 2.0
72 stars 25 forks source link

ISSUE-14575: broker restart failed #3871

Open sijie opened 2 years ago

sijie commented 2 years ago

Original Issue: apache/pulsar#14575 

ERROR org.apache.pulsar.functions.worker.FunctionMetaDataManager - Failed to initialize meta data store
org.apache.pulsar.client.api.PulsarClientException$BrokerMetadataException: The subscription c-pulsar-cluster-zk-fw-192.168.228.59-8080-function-metadata-tailer-reader-55d9f6694a of the topic persistent://public/functions/metadata gets the last message id was failed
{"errorMsg":"Failed to read last entry of the compacted Ledger Error while reading ledger","reqId":240264000667504132, "remote":"192.168.228.107/192.168.228.107:6650", "local":"/192.168.228.59:46798"}
    at org.apache.pulsar.client.api.PulsarClientException.unwrap(PulsarClientException.java:1034) ~[org.apache.pulsar-pulsar-client-api-2.9.1.jar:2.9.1]
    at org.apache.pulsar.client.impl.ConsumerImpl.hasMessageAvailable(ConsumerImpl.java:1992) ~[org.apache.pulsar-pulsar-client-original-2.9.1.jar:2.9.1]
    at org.apache.pulsar.client.impl.ReaderImpl.hasMessageAvailable(ReaderImpl.java:181) ~[org.apache.pulsar-pulsar-client-original-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.FunctionMetaDataManager.initialize(FunctionMetaDataManager.java:109) [org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:504) [org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.startWorkerService(PulsarService.java:1538) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.start(PulsarService.java:750) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.PulsarBrokerStarter$BrokerStarter.start(PulsarBrokerStarter.java:273) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.PulsarBrokerStarter.main(PulsarBrokerStarter.java:350) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
2022-03-07T15:47:11,871+0800 [main] ERROR org.apache.pulsar.functions.worker.PulsarWorkerService - Error Starting up in worker
java.lang.RuntimeException: Failed to initialize Metadata Manager
    at org.apache.pulsar.functions.worker.FunctionMetaDataManager.initialize(FunctionMetaDataManager.java:115) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:504) [org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.startWorkerService(PulsarService.java:1538) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.start(PulsarService.java:750) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.PulsarBrokerStarter$BrokerStarter.start(PulsarBrokerStarter.java:273) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.PulsarBrokerStarter.main(PulsarBrokerStarter.java:350) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
Caused by: org.apache.pulsar.client.api.PulsarClientException$BrokerMetadataException: The subscription c-pulsar-cluster-zk-fw-192.168.228.59-8080-function-metadata-tailer-reader-55d9f6694a of the topic persistent://public/functions/metadata gets the last message id was failed
{"errorMsg":"Failed to read last entry of the compacted Ledger Error while reading ledger","reqId":240264000667504132, "remote":"192.168.228.107/192.168.228.107:6650", "local":"/192.168.228.59:46798"}
    at org.apache.pulsar.client.api.PulsarClientException.unwrap(PulsarClientException.java:1034) ~[org.apache.pulsar-pulsar-client-api-2.9.1.jar:2.9.1]
    at org.apache.pulsar.client.impl.ConsumerImpl.hasMessageAvailable(ConsumerImpl.java:1992) ~[org.apache.pulsar-pulsar-client-original-2.9.1.jar:2.9.1]
    at org.apache.pulsar.client.impl.ReaderImpl.hasMessageAvailable(ReaderImpl.java:181) ~[org.apache.pulsar-pulsar-client-original-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.FunctionMetaDataManager.initialize(FunctionMetaDataManager.java:109) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    ... 5 more
2022-03-07T15:47:11,872+0800 [main] ERROR org.apache.pulsar.broker.PulsarService - Failed to start Pulsar service: java.lang.RuntimeException: Failed to initialize Metadata Manager
java.lang.RuntimeException: java.lang.RuntimeException: Failed to initialize Metadata Manager
    at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:587) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.startWorkerService(PulsarService.java:1538) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.start(PulsarService.java:750) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.PulsarBrokerStarter$BrokerStarter.start(PulsarBrokerStarter.java:273) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.PulsarBrokerStarter.main(PulsarBrokerStarter.java:350) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
Caused by: java.lang.RuntimeException: Failed to initialize Metadata Manager
    at org.apache.pulsar.functions.worker.FunctionMetaDataManager.initialize(FunctionMetaDataManager.java:115) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:504) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    ... 4 more
Caused by: org.apache.pulsar.client.api.PulsarClientException$BrokerMetadataException: The subscription c-pulsar-cluster-zk-fw-192.168.228.59-8080-function-metadata-tailer-reader-55d9f6694a of the topic persistent://public/functions/metadata gets the last message id was failed
{"errorMsg":"Failed to read last entry of the compacted Ledger Error while reading ledger","reqId":240264000667504132, "remote":"192.168.228.107/192.168.228.107:6650", "local":"/192.168.228.59:46798"}
    at org.apache.pulsar.client.api.PulsarClientException.unwrap(PulsarClientException.java:1034) ~[org.apache.pulsar-pulsar-client-api-2.9.1.jar:2.9.1]
    at org.apache.pulsar.client.impl.ConsumerImpl.hasMessageAvailable(ConsumerImpl.java:1992) ~[org.apache.pulsar-pulsar-client-original-2.9.1.jar:2.9.1]
    at org.apache.pulsar.client.impl.ReaderImpl.hasMessageAvailable(ReaderImpl.java:181) ~[org.apache.pulsar-pulsar-client-original-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.FunctionMetaDataManager.initialize(FunctionMetaDataManager.java:109) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:504) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    ... 4 more
2022-03-07T15:47:11,873+0800 [main] ERROR org.apache.pulsar.PulsarBrokerStarter - Failed to start pulsar service.
org.apache.pulsar.broker.PulsarServerException: java.lang.RuntimeException: java.lang.RuntimeException: Failed to initialize Metadata Manager
    at org.apache.pulsar.broker.PulsarService.start(PulsarService.java:782) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.PulsarBrokerStarter$BrokerStarter.start(PulsarBrokerStarter.java:273) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.PulsarBrokerStarter.main(PulsarBrokerStarter.java:350) [org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
Caused by: java.lang.RuntimeException: java.lang.RuntimeException: Failed to initialize Metadata Manager
    at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:587) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.startWorkerService(PulsarService.java:1538) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.start(PulsarService.java:750) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    ... 2 more
Caused by: java.lang.RuntimeException: Failed to initialize Metadata Manager
    at org.apache.pulsar.functions.worker.FunctionMetaDataManager.initialize(FunctionMetaDataManager.java:115) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:504) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.startWorkerService(PulsarService.java:1538) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.start(PulsarService.java:750) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    ... 2 more
Caused by: org.apache.pulsar.client.api.PulsarClientException$BrokerMetadataException: The subscription c-pulsar-cluster-zk-fw-192.168.228.59-8080-function-metadata-tailer-reader-55d9f6694a of the topic persistent://public/functions/metadata gets the last message id was failed
{"errorMsg":"Failed to read last entry of the compacted Ledger Error while reading ledger","reqId":240264000667504132, "remote":"192.168.228.107/192.168.228.107:6650", "local":"/192.168.228.59:46798"}
    at org.apache.pulsar.client.api.PulsarClientException.unwrap(PulsarClientException.java:1034) ~[org.apache.pulsar-pulsar-client-api-2.9.1.jar:2.9.1]
    at org.apache.pulsar.client.impl.ConsumerImpl.hasMessageAvailable(ConsumerImpl.java:1992) ~[org.apache.pulsar-pulsar-client-original-2.9.1.jar:2.9.1]
    at org.apache.pulsar.client.impl.ReaderImpl.hasMessageAvailable(ReaderImpl.java:181) ~[org.apache.pulsar-pulsar-client-original-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.FunctionMetaDataManager.initialize(FunctionMetaDataManager.java:109) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.functions.worker.PulsarWorkerService.start(PulsarWorkerService.java:504) ~[org.apache.pulsar-pulsar-functions-worker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.startWorkerService(PulsarService.java:1538) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    at org.apache.pulsar.broker.PulsarService.start(PulsarService.java:750) ~[org.apache.pulsar-pulsar-broker-2.9.1.jar:2.9.1]
    ... 2 more
github-actions[bot] commented 2 years ago

The issue had no activity for 30 days, mark with Stale label.