Getting crashloopbackoff error in zookeeper cluster

Here is the more logs of statefulset of zookeeper

Found 2 pods, using pod/tonline-ocr-solrcloud-zookeeper-0

source /conf/env.sh ++ DOMAIN=tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local ++ QUORUM_PORT=2888 ++ LEADER_PORT=3888 ++ CLIENT_HOST=tonline-ocr-solrcloud-zookeeper-client ++ CLIENT_PORT=2181 ++ ADMIN_SERVER_HOST=tonline-ocr-solrcloud-zookeeper-admin-server ++ ADMIN_SERVER_PORT=8080 ++ CLUSTER_NAME=tonline-ocr-solrcloud-zookeeper ++ CLUSTER_SIZE=2
source /usr/local/bin/zookeeperFunctions.sh ++ set -ex ++ hostname -s
HOST=tonline-ocr-solrcloud-zookeeper-0
DATA_DIR=/data
MYID_FILE=/data/myid
LOG4J_CONF=/conf/log4j-quiet.properties
DYNCONFIG=/data/zoo.cfg.dynamic
STATIC_CONFIG=/data/conf/zoo.cfg
[[ tonline-ocr-solrcloud-zookeeper-0 =~ (.*)-([0-9]+)$ ]]
NAME=tonline-ocr-solrcloud-zookeeper
ORD=0
MYID=1
WRITE_CONFIGURATION=true
REGISTER_NODE=true
ONDISK_MYID_CONFIG=false
ONDISK_DYN_CONFIG=false
'[' -f /data/myid ']' ++ cat /data/myid
EXISTING_ID=1
[[ 1 == \1 ]]
[[ -f /data/conf/zoo.cfg ]]
ONDISK_MYID_CONFIG=true
'[' -f /data/zoo.cfg.dynamic ']'
ONDISK_DYN_CONFIG=true
set +e
[[ -n '' ]]
set -e
set +e
getent hosts tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local
[[ 2 -eq 0 ]]
nslookup tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local
grep -q 'server can'\''t find tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local' there is no active ensemble
echo 'there is no active ensemble'
ACTIVE_ENSEMBLE=false
[[ true == true ]]
[[ true == true ]]
WRITE_CONFIGURATION=false
[[ false == false ]]
REGISTER_NODE=false
[[ false == true ]]
[[ false == true ]]
ZOOCFGDIR=/data/conf
export ZOOCFGDIR
echo Copying /conf contents to writable directory, to support Zookeeper dynamic reconfiguration Copying /conf contents to writable directory, to support Zookeeper dynamic reconfiguration
[[ ! -d /data/conf ]] Copying the /conf/zoo.cfg contents except the dynamic config file during restart
echo Copying the /conf/zoo.cfg contents except the dynamic config file during restart ++ head -n -1 /conf/zoo.cfg ++ tail -n 1 /data/conf/zoo.cfg
echo -e '4lw.commands.whitelist=cons, envi, conf, crst, srvr, stat, mntr, ruok dataDir=/data standaloneEnabled=false reconfigEnabled=true skipACL=yes metricsProvider.className=org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider metricsProvider.httpPort=7000 metricsProvider.exportJvmInfo=true initLimit=10 syncLimit=2 tickTime=2000 globalOutstandingLimit=1000 preAllocSize=65536 snapCount=10000 commitLogCount=500 snapSizeLimitInKb=4194304 maxCnxns=0 maxClientCnxns=60 minSessionTimeout=4000 maxSessionTimeout=40000 autopurge.snapRetainCount=3 autopurge.purgeInterval=1 quorumListenOnAllIPs=false admin.serverPort=8080\ndynamicConfigFile=/data/conf/zoo.cfg.dynamic.84000002f4'
cp -f /conf/log4j.properties /data/conf
cp -f /conf/log4j-quiet.properties /data/conf
cp -f /conf/env.sh /data/conf Starting zookeeper service
'[' -f /data/zoo.cfg.dynamic ']'
echo Starting zookeeper service
zkServer.sh --config /data/conf start-foreground ZooKeeper JMX enabled by default Using config: /data/conf/zoo.cfg 2023-06-02 05:41:20,321 [myid:] - INFO [main:QuorumPeerConfig@174] - Reading configuration from: /data/conf/zoo.cfg 2023-06-02 05:41:20,325 [myid:] - INFO [main:QuorumPeerConfig@435] - clientPort is not set 2023-06-02 05:41:20,325 [myid:] - INFO [main:QuorumPeerConfig@448] - secureClientPort is not set 2023-06-02 05:41:20,325 [myid:] - INFO [main:QuorumPeerConfig@464] - observerMasterPort is not set 2023-06-02 05:41:20,326 [myid:] - INFO [main:QuorumPeerConfig@481] - metricsProvider.className is org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider 2023-06-02 05:41:20,338 [myid:] - WARN [main:QuorumPeerConfig@713] - No server failure will be tolerated. You need at least 3 servers. 2023-06-02 05:41:20,340 [myid:1] - INFO [main:DatadirCleanupManager@78] - autopurge.snapRetainCount set to 3 2023-06-02 05:41:20,340 [myid:1] - INFO [main:DatadirCleanupManager@79] - autopurge.purgeInterval set to 1 2023-06-02 05:41:20,341 [myid:1] - INFO [PurgeTask:DatadirCleanupManager$PurgeTask@139] - Purge task started. 2023-06-02 05:41:20,346 [myid:1] - INFO [main:ManagedUtil@44] - Log4j 1.2 jmx support found and enabled. 2023-06-02 05:41:20,349 [myid:1] - INFO [PurgeTask:FileTxnSnapLog@124] - zookeeper.snapshot.trust.empty : false 2023-06-02 05:41:20,356 [myid:1] - INFO [main:QuorumPeerMain@152] - Starting quorum peer, myid=1 2023-06-02 05:41:20,362 [myid:1] - INFO [PurgeTask:SnapStream@61] - zookeeper.snapshot.compression.method = CHECKED 2023-06-02 05:41:20,369 [myid:1] - INFO [PurgeTask:DatadirCleanupManager$PurgeTask@145] - Purge task completed. 2023-06-02 05:41:20,371 [myid:1] - INFO [main:PrometheusMetricsProvider@74] - Initializing metrics, configuration: {exportJvmInfo=true, httpPort=7000} 2023-06-02 05:41:20,371 [myid:1] - INFO [main:PrometheusMetricsProvider@82] - Starting /metrics HTTP endpoint at port 7000 exportJvmInfo: true 2023-06-02 05:41:20,412 [myid:1] - INFO [main:Log@169] - Logging initialized @648ms to org.eclipse.jetty.util.log.Slf4jLog 2023-06-02 05:41:20,487 [myid:1] - INFO [main:Server@375] - jetty-9.4.38.v20210224; built: 2021-02-24T20:25:07.675Z; git: 288f3cc74549e8a913bf363250b0744f2695b8e6; jvm 11.0.15+10 2023-06-02 05:41:20,521 [myid:1] - INFO [main:ContextHandler@916] - Started o.e.j.s.ServletContextHandler@15ca7889{/,null,AVAILABLE} 2023-06-02 05:41:20,536 [myid:1] - INFO [main:AbstractConnector@331] - Started ServerConnector@35cabb2a{HTTP/1.1, (http/1.1)}{0.0.0.0:7000} 2023-06-02 05:41:20,536 [myid:1] - INFO [main:Server@415] - Started @774ms 2023-06-02 05:41:20,547 [myid:1] - INFO [main:ServerMetrics@62] - ServerMetrics initialized with provider org.apache.zookeeper.metrics.prometheus.PrometheusMetricsProvider@74455848 2023-06-02 05:41:20,556 [myid:1] - INFO [main:DigestAuthenticationProvider@47] - ACL digest algorithm is: SHA1 2023-06-02 05:41:20,556 [myid:1] - INFO [main:DigestAuthenticationProvider@61] - zookeeper.DigestAuthenticationProvider.enabled = true 2023-06-02 05:41:20,563 [myid:1] - INFO [main:ServerCnxnFactory@169] - Using org.apache.zookeeper.server.NIOServerCnxnFactory as server connection factory 2023-06-02 05:41:20,565 [myid:1] - WARN [main:ServerCnxnFactory@309] - maxCnxns is not configured, using default value 0. 2023-06-02 05:41:20,566 [myid:1] - INFO [main:NIOServerCnxnFactory@652] - Configuring NIO connection handler with 10s sessionless connection timeout, 1 selector thread(s), 2 worker threads, and 64 kB direct buffers. 2023-06-02 05:41:20,567 [myid:1] - INFO [main:NIOServerCnxnFactory@660] - binding to port /0.0.0.0:2181 2023-06-02 05:41:20,576 [myid:1] - INFO [main:QuorumPeer@796] - zookeeper.quorumCnxnTimeoutMs=-1 2023-06-02 05:41:20,583 [myid:1] - WARN [main:ContextHandler@1660] - o.e.j.s.ServletContextHandler@70e9c95d{/,null,STOPPED} contextPath ends with / 2023-06-02 05:41:20,583 [myid:1] - WARN [main:ContextHandler@1671] - Empty contextPath 2023-06-02 05:41:20,593 [myid:1] - INFO [main:X509Util@77] - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation 2023-06-02 05:41:20,593 [myid:1] - INFO [main:FileTxnSnapLog@124] - zookeeper.snapshot.trust.empty : false 2023-06-02 05:41:20,593 [myid:1] - INFO [main:QuorumPeer@1739] - Local sessions disabled 2023-06-02 05:41:20,593 [myid:1] - INFO [main:QuorumPeer@1750] - Local session upgrading disabled 2023-06-02 05:41:20,593 [myid:1] - INFO [main:QuorumPeer@1717] - tickTime set to 2000 2023-06-02 05:41:20,593 [myid:1] - INFO [main:QuorumPeer@1761] - minSessionTimeout set to 4000 2023-06-02 05:41:20,593 [myid:1] - INFO [main:QuorumPeer@1772] - maxSessionTimeout set to 40000 2023-06-02 05:41:20,593 [myid:1] - INFO [main:QuorumPeer@1797] - initLimit set to 10 2023-06-02 05:41:20,593 [myid:1] - INFO [main:QuorumPeer@1984] - syncLimit set to 2 2023-06-02 05:41:20,593 [myid:1] - INFO [main:QuorumPeer@1999] - connectToLearnerMasterLimit set to 0 2023-06-02 05:41:20,606 [myid:1] - INFO [main:ZookeeperBanner@42] - 2023-06-02 05:41:20,606 [myid:1] - INFO [main:ZookeeperBanner@42] - __ _
2023-06-02 05:41:20,606 [myid:1] - INFO [main:ZookeeperBanner@42] - | / | |
2023-06-02 05:41:20,606 [myid:1] - INFO [main:ZookeeperBanner@42] - / / _ | | _
2023-06-02 05:41:20,606 [myid:1] - INFO [main:ZookeeperBanner@42] - / / / \ / \ | |/ / / \ / \ | ' \ / \ | '| 2023-06-02 05:41:20,606 [myid:1] - INFO [main:ZookeeperBanner@42] - / / | () | | () | | < | / | / | |_) | | / | |
2023-06-02 05:41:20,606 [myid:1] - INFO [main:ZookeeperBanner@42] - /____| \/ _/ ||_\ \| _| | ./ __| || 2023-06-02 05:41:20,607 [myid:1] - INFO [main:ZookeeperBanner@42] - | |
2023-06-02 05:41:20,607 [myid:1] - INFO [main:ZookeeperBanner@42] - |_|
2023-06-02 05:41:20,607 [myid:1] - INFO [main:ZookeeperBanner@42] - 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:zookeeper.version=3.7.0-e3704b390a6697bfdf4b0bef79e3da7a4f6bac4b, built on 2021-03-17 09:46 UTC 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:host.name=tonline-ocr-solrcloud-zookeeper-0.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:java.version=11.0.15 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:java.vendor=Oracle Corporation 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:java.home=/usr/local/openjdk-11 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:java.class.path=/apache-zookeeper-3.7.0-bin/bin/../zookeeper-server/target/classes:/apache-zookeeper-3.7.0-bin/bin/../build/classes:/apache-zookeeper-3.7.0-bin/bin/../zookeeper-server/target/lib/.jar:/apache-zookeeper-3.7.0-bin/bin/../build/lib/.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/zookeeper-prometheus-metrics-3.7.0.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/zookeeper-jute-3.7.0.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/zookeeper-3.7.0.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/snappy-java-1.1.7.7.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/slf4j-log4j12-1.7.30.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/slf4j-api-1.7.30.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/simpleclient_servlet-0.9.0.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/simpleclient_hotspot-0.9.0.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/simpleclient_common-0.9.0.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/simpleclient-0.9.0.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/netty-transport-native-unix-common-4.1.59.Final.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/netty-transport-native-epoll-4.1.59.Final.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/netty-transport-4.1.59.Final.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/netty-resolver-4.1.59.Final.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/netty-handler-4.1.59.Final.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/netty-common-4.1.59.Final.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/netty-codec-4.1.59.Final.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/netty-buffer-4.1.59.Final.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/metrics-core-4.1.12.1.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/log4j-1.2.17.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jline-2.14.6.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jetty-util-ajax-9.4.38.v20210224.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jetty-util-9.4.38.v20210224.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jetty-servlet-9.4.38.v20210224.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jetty-server-9.4.38.v20210224.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jetty-security-9.4.38.v20210224.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jetty-io-9.4.38.v20210224.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jetty-http-9.4.38.v20210224.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/javax.servlet-api-3.1.0.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jackson-databind-2.10.5.1.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jackson-core-2.10.5.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/jackson-annotations-2.10.5.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/commons-cli-1.4.jar:/apache-zookeeper-3.7.0-bin/bin/../lib/audience-annotations-0.12.0.jar:/apache-zookeeper-3.7.0-bin/bin/../zookeeper-.jar:/apache-zookeeper-3.7.0-bin/bin/../zookeeper-server/src/main/resources/lib/.jar:/data/conf: 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:java.io.tmpdir=/tmp 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:java.compiler= 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:os.name=Linux 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:os.arch=amd64 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:os.version=5.15.108 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:user.name=root 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:user.home=/root 2023-06-02 05:41:20,608 [myid:1] - INFO [main:Environment@98] - Server environment:user.dir=/apache-zookeeper-3.7.0-bin 2023-06-02 05:41:20,609 [myid:1] - INFO [main:Environment@98] - Server environment:os.memory.free=8MB 2023-06-02 05:41:20,609 [myid:1] - INFO [main:Environment@98] - Server environment:os.memory.max=966MB 2023-06-02 05:41:20,609 [myid:1] - INFO [main:Environment@98] - Server environment:os.memory.total=15MB 2023-06-02 05:41:20,609 [myid:1] - INFO [main:ZooKeeperServer@138] - zookeeper.enableEagerACLCheck = false 2023-06-02 05:41:20,609 [myid:1] - INFO [main:ZooKeeperServer@142] - zookeeper.skipACL=="yes", ACL checks will be skipped 2023-06-02 05:41:20,609 [myid:1] - INFO [main:ZooKeeperServer@151] - zookeeper.digest.enabled = true 2023-06-02 05:41:20,609 [myid:1] - INFO [main:ZooKeeperServer@155] - zookeeper.closeSessionTxn.enabled = true 2023-06-02 05:41:20,609 [myid:1] - INFO [main:ZooKeeperServer@1499] - zookeeper.flushDelay=0 2023-06-02 05:41:20,609 [myid:1] - INFO [main:ZooKeeperServer@1508] - zookeeper.maxWriteQueuePollTime=0 2023-06-02 05:41:20,609 [myid:1] - INFO [main:ZooKeeperServer@1517] - zookeeper.maxBatchSize=1000 2023-06-02 05:41:20,609 [myid:1] - INFO [main:ZooKeeperServer@260] - zookeeper.intBufferStartingSizeBytes = 1024 2023-06-02 05:41:20,612 [myid:1] - INFO [main:WatchManagerFactory@42] - Using org.apache.zookeeper.server.watch.WatchManager as watch manager 2023-06-02 05:41:20,612 [myid:1] - INFO [main:WatchManagerFactory@42] - Using org.apache.zookeeper.server.watch.WatchManager as watch manager 2023-06-02 05:41:20,614 [myid:1] - INFO [main:ZKDatabase@133] - zookeeper.snapshotSizeFactor = 0.33 2023-06-02 05:41:20,614 [myid:1] - INFO [main:ZKDatabase@153] - zookeeper.commitLogCount=500 2023-06-02 05:41:20,618 [myid:1] - INFO [main:QuorumPeer@2063] - Using insecure (non-TLS) quorum communication 2023-06-02 05:41:20,618 [myid:1] - INFO [main:QuorumPeer@2069] - Port unification disabled 2023-06-02 05:41:20,618 [myid:1] - INFO [main:QuorumPeer@179] - multiAddress.enabled set to false 2023-06-02 05:41:20,618 [myid:1] - INFO [main:QuorumPeer@204] - multiAddress.reachabilityCheckEnabled set to true 2023-06-02 05:41:20,618 [myid:1] - INFO [main:QuorumPeer@191] - multiAddress.reachabilityCheckTimeoutMs set to 1000 2023-06-02 05:41:20,619 [myid:1] - INFO [main:QuorumPeer@2524] - QuorumPeer communication is not secured! (SASL auth disabled) 2023-06-02 05:41:20,619 [myid:1] - INFO [main:QuorumPeer@2549] - quorum.cnxn.threads.size set to 20 2023-06-02 05:41:20,619 [myid:1] - INFO [main:FileSnap@85] - Reading snapshot /data/version-2/snapshot.b400000c34 2023-06-02 05:41:20,628 [myid:1] - INFO [main:DataTree@1709] - The digest in the snapshot has digest version of 2, , with zxid as 0xb400000c34, and digest value as 281242763809 2023-06-02 05:41:20,732 [myid:1] - INFO [main:ZKAuditProvider@42] - ZooKeeper audit is disabled. 2023-06-02 05:41:20,879 [myid:1] - INFO [main:FileTxnSnapLog@372] - 8012 txns loaded in 214 ms 2023-06-02 05:41:20,880 [myid:1] - INFO [main:ZKDatabase@290] - Snapshot loaded in 260 ms, highest zxid is 0x18100000057, digest is 264084859459 2023-06-02 05:41:20,884 [myid:1] - INFO [main:Server@375] - jetty-9.4.38.v20210224; built: 2021-02-24T20:25:07.675Z; git: 288f3cc74549e8a913bf363250b0744f2695b8e6; jvm 11.0.15+10 2023-06-02 05:41:20,895 [myid:1] - INFO [main:DefaultSessionIdManager@334] - DefaultSessionIdManager workerName=node0 2023-06-02 05:41:20,896 [myid:1] - INFO [main:DefaultSessionIdManager@339] - No SessionScavenger set, using defaults 2023-06-02 05:41:20,897 [myid:1] - INFO [main:HouseKeeper@132] - node0 Scavenging every 660000ms 2023-06-02 05:41:20,899 [myid:1] - WARN [main:ConstraintSecurityHandler@759] - ServletContext@o.e.j.s.ServletContextHandler@70e9c95d{/,null,STARTING} has uncovered http methods for path: / 2023-06-02 05:41:20,900 [myid:1] - INFO [main:ContextHandler@916] - Started o.e.j.s.ServletContextHandler@70e9c95d{/,null,AVAILABLE} 2023-06-02 05:41:20,901 [myid:1] - INFO [main:AbstractConnector@331] - Started ServerConnector@d9345cd{HTTP/1.1, (http/1.1)}{0.0.0.0:8080} 2023-06-02 05:41:20,901 [myid:1] - INFO [main:Server@415] - Started @1139ms 2023-06-02 05:41:20,901 [myid:1] - INFO [main:JettyAdminServer@181] - Started AdminServer on address 0.0.0.0, port 8080 and command URL /commands 2023-06-02 05:41:20,902 [myid:1] - INFO [main:QuorumPeer@2566] - Using 4000ms as the quorum cnxn socket timeout 2023-06-02 05:41:20,905 [myid:1] - INFO [main:QuorumCnxManager$Listener@924] - Election port bind maximum retries is 3 2023-06-02 05:41:20,906 [myid:1] - INFO [main:FastLeaderElection@89] - zookeeper.fastleader.minNotificationInterval=200 2023-06-02 05:41:20,906 [myid:1] - INFO [main:FastLeaderElection@91] - zookeeper.fastleader.maxNotificationInterval=60000 2023-06-02 05:41:20,908 [myid:1] - INFO [ListenerHandler-tonline-ocr-solrcloud-zookeeper-0.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local/10.96.146.97:3888:QuorumCnxManager$Listener$ListenerHandler@1066] - 1 is accepting connections now, my election bind port: tonline-ocr-solrcloud-zookeeper-0.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local/10.96.146.97:3888 2023-06-02 05:41:20,912 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):QuorumPeer@1430] - LOOKING 2023-06-02 05:41:20,913 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@945] - New election. My id = 1, proposed zxid=0x18100000057 2023-06-02 05:41:20,915 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection$Messenger$WorkerReceiver@390] - Notification: my state:LOOKING; n.sid:1, n.state:LOOKING, n.leader:1, n.round:0x1, n.peerEpoch:0x181, n.zxid:0x18100000057, message format version:0x2, n.config version:0x84000002f4 2023-06-02 05:41:20,918 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-1:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:21,116 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@980] - Notification time out: 400 2023-06-02 05:41:21,116 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-2:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:21,516 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@980] - Notification time out: 800 2023-06-02 05:41:21,516 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-3:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:22,317 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@980] - Notification time out: 1600 2023-06-02 05:41:22,317 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-3:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:23,917 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@980] - Notification time out: 3200 2023-06-02 05:41:23,917 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-3:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:24,268 [myid:1] - INFO [NIOWorkerThread-1:FourLetterCommands@223] - The list of known four letter word commands is : [{1936881266=srvr, 1937006964=stat, 2003003491=wchc, 1685417328=dump, 1668445044=crst, 1936880500=srst, 1701738089=envi, 1668247142=conf, -720899=telnet close, 1751217000=hash, 2003003507=wchs, 2003003504=wchp, 1684632179=dirs, 1668247155=cons, 1835955314=mntr, 1769173615=isro, 1920298859=ruok, 1735683435=gtmk, 1937010027=stmk}] 2023-06-02 05:41:24,268 [myid:1] - INFO [NIOWorkerThread-1:FourLetterCommands@224] - The list of enabled four letter word commands is : [[stat, envi, conf, ruok, mntr, srvr, crst, cons]] 2023-06-02 05:41:24,268 [myid:1] - INFO [NIOWorkerThread-1:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:59510 2023-06-02 05:41:24,309 [myid:1] - INFO [NIOWorkerThread-2:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:59512 2023-06-02 05:41:27,118 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@980] - Notification time out: 6400 2023-06-02 05:41:27,118 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-3:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:33,523 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@980] - Notification time out: 12800 2023-06-02 05:41:33,524 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-3:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:34,280 [myid:1] - INFO [NIOWorkerThread-1:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:53894 2023-06-02 05:41:34,317 [myid:1] - INFO [NIOWorkerThread-2:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:53896 2023-06-02 05:41:44,276 [myid:1] - INFO [NIOWorkerThread-1:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:35756 2023-06-02 05:41:44,304 [myid:1] - INFO [NIOWorkerThread-2:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:35766 2023-06-02 05:41:46,333 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@980] - Notification time out: 25600 2023-06-02 05:41:46,333 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-3:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:54,257 [myid:1] - INFO [NIOWorkerThread-1:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:44020 2023-06-02 05:41:54,273 [myid:1] - INFO [NIOWorkerThread-2:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:44026 2023-06-02 05:41:55,523 [myid:1] - WARN [NIOWorkerThread-1:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:57,752 [myid:1] - WARN [NIOWorkerThread-2:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:41:59,499 [myid:1] - WARN [NIOWorkerThread-1:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:42:02,521 [myid:1] - WARN [NIOWorkerThread-2:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:42:04,300 [myid:1] - INFO [NIOWorkerThread-1:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:60462 2023-06-02 05:42:04,347 [myid:1] - INFO [NIOWorkerThread-2:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:60476 2023-06-02 05:42:04,410 [myid:1] - WARN [NIOWorkerThread-1:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:42:07,247 [myid:1] - WARN [NIOWorkerThread-2:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:42:10,155 [myid:1] - WARN [NIOWorkerThread-1:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:42:11,940 [myid:1] - INFO [QuorumPeermyid=1(secure=disabled):FastLeaderElection@980] - Notification time out: 51200 2023-06-02 05:42:11,940 [myid:1] - WARN [QuorumConnectionThread-[myid=1]-3:QuorumCnxManager@401] - Cannot open channel to 2 at election address tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local:3888 java.net.UnknownHostException: tonline-ocr-solrcloud-zookeeper-1.tonline-ocr-solrcloud-zookeeper-headless.jx-qat2.svc.cluster.local at java.base/java.net.AbstractPlainSocketImpl.connect(Unknown Source) at java.base/java.net.SocksSocketImpl.connect(Unknown Source) at java.base/java.net.Socket.connect(Unknown Source) at org.apache.zookeeper.server.quorum.QuorumCnxManager.initiateConnection(QuorumCnxManager.java:384) at org.apache.zookeeper.server.quorum.QuorumCnxManager$QuorumConnectionReqThread.run(QuorumCnxManager.java:458) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:42:12,717 [myid:1] - WARN [NIOWorkerThread-2:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source) 2023-06-02 05:42:14,303 [myid:1] - INFO [NIOWorkerThread-1:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:39388 2023-06-02 05:42:14,320 [myid:1] - INFO [NIOWorkerThread-2:NIOServerCnxn@514] - Processing ruok command from /127.0.0.1:39398 2023-06-02 05:42:14,529 [myid:1] - WARN [NIOWorkerThread-1:NIOServerCnxn@380] - Close of session 0x0 java.io.IOException: ZooKeeperServer not running at org.apache.zookeeper.server.NIOServerCnxn.readLength(NIOServerCnxn.java:554) at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:339) at org.apache.zookeeper.server.NIOServerCnxnFactory$IOWorkRequest.doWork(NIOServerCnxnFactory.java:508) at org.apache.zookeeper.server.WorkerService$ScheduledWorkRequest.run(WorkerService.java:154) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) at java.base/java.lang.Thread.run(Unknown Source)

apache / solr-operator

Getting crashloopbackoff error in zookeeper cluster #577