Java.rmi.ConnectException: Connection refused to host: 127.0.0.1

[sureshoao]

kafka-prometheus-jmx-exporter is unable to connect to Kafka broker.

Jul 14, 2019 9:35:42 AM io.prometheus.jmx.JmxCollector collect SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: 127.0.0.1; nested exception is: java.net.ConnectException: Connection refused (Connection refused)] at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369) at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270) at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106) at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134) at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22) at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59) at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79) at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83) at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82) at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675) at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79) at sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)

[solsson]

Please provide background such as what you applied, and please quote terminal output with ```.

[sureshoao]

Yaml file:

apiVersion: v1 kind: ConfigMap metadata: name: kafka-cp-jmx-configmap namespace: kafka labels: app: cp-kafka data: jmx-kafka-prometheus.yaml: |+ jmxUrl: service:jmx:rmi:///jndi/rmi://localhost:5555/jmxrmi lowercaseOutputName: true lowercaseOutputLabelNames: true ssl: false rules:

• pattern : kafka.server<>(Value|OneMinuteRate) name: "cp_kafka_serverreplicamanager$1"
• pattern : kafka.controller<>Value name: "cp_kafka_controllerkafkacontroller$1"
• pattern : kafka.server<>OneMinuteRate name: "cp_kafka_serverbrokertopicmetrics$1"
• pattern : kafka.network<>OneMinuteRate name: "cp_kafka_network_requestmetricsrequestspersec$1"
• pattern : kafka.network<>Value name: "cp_kafka_network_socketserver_networkprocessoravgidlepercent"
• pattern : kafka.server<>Value name: "cp_kafka_server_replicafetchermanagermaxlag$1"
• pattern : kafka.server<>OneMinuteRate name: "cp_kafka_kafkarequesthandlerpool_requesthandleravgidlepercent"
• pattern : kafka.controller<>OneMinuteRate name: "cp_kafka_controllercontrollerstats$1"

• pattern : kafka.server<>OneMinuteRate name: "cp_kafka_serversessionexpirelistener$1"

  apiVersion: apps/v1beta1 kind: StatefulSet metadata: name: kafka-cp namespace: kafka labels: app: cp-kafka spec: serviceName: kafka-cp-headless podManagementPolicy: OrderedReady replicas: 5 updateStrategy: type: RollingUpdate template: metadata: labels: app: cp-kafka annotations: prometheus.io/scrape: "true" prometheus.io/port: "5556" spec: affinity: podAntiAffinity: preferredDuringSchedulingIgnoredDuringExecution:

  • weight: 1 podAffinityTerm: labelSelector: matchExpressions:

    • key: "app" operator: In values:
      • cp-kafka
    • key: "release" operator: In values:
      • kafka topologyKey: "kubernetes.io/hostname" containers:
        • name: prometheus-jmx-exporter image: "solsson/kafka-prometheus-jmx-exporter:latest" command:
        • java
        • -XX:+UnlockExperimentalVMOptions
        • -XX:+UseCGroupMemoryLimitForHeap
        • -XX:MaxRAMFraction=1
        • -XshowSettings:vm
        • -jar
        • jmx_prometheus_httpserver.jar
        • "5556"
        • /etc/jmx-kafka/jmx-kafka-prometheus.yaml ports:
        • containerPort: 5556 resources: null

    volumeMounts:

    • name: jmx-config mountPath: /etc/jmx-kafka
    • name: cp-kafka-broker image: "solsson/kafka:2.2.1" imagePullPolicy: "IfNotPresent" readinessProbe: tcpSocket: port: kafka ports:
    • containerPort: 9092 name: kafka
    • containerPort: 5555 name: jmx resources: requests: memory: 12000Mi cpu: 4000m
      limits: memory: 12000Mi cpu: 4000m
      env:
    • name: POD_IP valueFrom: fieldRef: fieldPath: status.podIP
    • name: HOST_IP valueFrom: fieldRef: fieldPath: status.hostIP
    • name: KAFKA_HEAP_OPTS value: -Xms12G -Xmx12G
    • name: KAFKA_ZOOKEEPER_CONNECT value: "zookeeper-headless.kafka.svc.cluster.local:2181"
    • name: "KAFKA_ADVERTISED_LISTENERS" value: "EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID}))"
    • name: "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP" value: "PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT"
    • name: "KAFKA_LOG_DIRS" value: "/opt/kafka/data/logs"
    • name: "KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE" value: "false"
    • name: "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR" value: "2"
    • name: KAFKA_JMX_PORT value: "5555" command:
    • sh
    • -exc
    • | export KAFKA_BROKER_ID=${HOSTNAME##*-} && \ export KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${POD_IP}:9092,EXTERNAL://${HOST_IP}:$((31090 + ${KAFKA_BROKER_ID})) && \ exec /etc/confluent/docker/run command:
    • sh
    • -c
    • "exec kafka-server-start /etc/kafka/server.properties --override advertised.listeners=PLAINTEXT://clustername-kafka-${HOSTNAME##-}.aws:443 \ --override zookeeper.connect=zookeeper-headless.kafka.svc.cluster.local:2181 \ --override log.dirs=/opt/kafka/data/logs \ --override broker.id=${HOSTNAME##-} \ --override auto.create.topics.enable=true \ --override num.network.threads=3 \ --override num.io.threads=8 \ --override socket.send.buffer.bytes=102400 \ --override socket.receive.buffer.bytes=102400 \ --override socket.request.max.bytes=104857600 \ --override num.partitions=30 \ --override num.recovery.threads.per.data.dir=1 \ --override offsets.topic.replication.factor=2 \ --override transaction.state.log.replication.factor=2 \ --override transaction.state.log.min.isr=1 \ --override log.retention.hours=168 \ --override log.segment.bytes=1073741824 \ --override log.retention.check.interval.ms=300000 \ --override zookeeper.connection.timeout.ms=6000 \ --override group.initial.rebalance.delay.ms=0 \ "
      volumeMounts:
    • name: datadir mountPath: /opt/kafka/data volumes:
    • name: jmx-config configMap: name: kafka-cp-jmx-configmap volumeClaimTemplates:

    • metadata: name: datadir spec: accessModes: [ "ReadWriteOnce" ] resources: requests: storage: "50Gi" storageClassName: infrastructure-storage

logs of prometheus-jmx-exporter:

''' kubectl logs -f kafka-cp-0 -n kafka -c prometheus-jmx-exporter VM settings: Max. Heap Size (Estimated): 13.93G Ergonomics Machine Class: server Using VM: OpenJDK 64-Bit Server VM

Jul 15, 2019 6:10:32 AM io.prometheus.jmx.JmxCollector collect SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is: java.net.ConnectException: Connection refused (Connection refused)] at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369) at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270) at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106) at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134) at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22) at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59) at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79) at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83) at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82) at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675) at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79) at sun.net.httpserver.ServerImpl$Exchange.run(ServerImpl.java:647) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is: java.net.ConnectException: Connection refused (Connection refused)] at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:136) at com.sun.jndi.toolkit.url.GenericURLContext.lookup(GenericURLContext.java:205) at javax.naming.InitialContext.lookup(InitialContext.java:417) at javax.management.remote.rmi.RMIConnector.findRMIServerJNDI(RMIConnector.java:1955) at javax.management.remote.rmi.RMIConnector.findRMIServer(RMIConnector.java:1922) at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:287) ... 17 more Caused by: java.rmi.ConnectException: Connection refused to host: localhost; nested exception is: java.net.ConnectException: Connection refused (Connection refused) at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:619) at sun.rmi.transport.tcp.TCPChannel.createConnection(TCPChannel.java:216) at sun.rmi.transport.tcp.TCPChannel.newConnection(TCPChannel.java:202) at sun.rmi.server.UnicastRef.newCall(UnicastRef.java:338) at sun.rmi.registry.RegistryImpl_Stub.lookup(RegistryImpl_Stub.java:112) at com.sun.jndi.rmi.registry.RegistryContext.lookup(RegistryContext.java:132) ... 22 more Caused by: java.net.ConnectException: Connection refused (Connection refused) at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) at java.net.Socket.connect(Socket.java:589) at java.net.Socket.connect(Socket.java:538) at java.net.Socket.(Socket.java:434) at java.net.Socket.(Socket.java:211) at sun.rmi.transport.proxy.RMIDirectSocketFactory.createSocket(RMIDirectSocketFactory.java:40) at sun.rmi.transport.proxy.RMIMasterSocketFactory.createSocket(RMIMasterSocketFactory.java:148) at sun.rmi.transport.tcp.TCPEndpoint.newSocket(TCPEndpoint.java:613) ... 27 more

Jul 15, 2019 6:10:35 AM io.prometheus.jmx.JmxCollector collect SEVERE: JMX scrape failed: java.io.IOException: Failed to retrieve RMIServer stub: javax.naming.ServiceUnavailableException [Root exception is java.rmi.ConnectException: Connection refused to host: localhost; nested exception is: java.net.ConnectException: Connection refused (Connection refused)] at javax.management.remote.rmi.RMIConnector.connect(RMIConnector.java:369) at javax.management.remote.JMXConnectorFactory.connect(JMXConnectorFactory.java:270) at io.prometheus.jmx.JmxScraper.doScrape(JmxScraper.java:106) at io.prometheus.jmx.JmxCollector.collect(JmxCollector.java:415) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.findNextElement(CollectorRegistry.java:180) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:213) at io.prometheus.client.CollectorRegistry$MetricFamilySamplesEnumeration.nextElement(CollectorRegistry.java:134) at io.prometheus.client.exporter.common.TextFormat.write004(TextFormat.java:22) at io.prometheus.client.exporter.HTTPServer$HTTPMetricHandler.handle(HTTPServer.java:59) at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:79) at sun.net.httpserver.AuthFilter.doFilter(AuthFilter.java:83) at com.sun.net.httpserver.Filter$Chain.doFilter(Filter.java:82) at sun.net.httpserver.ServerImpl$Exchange$LinkHandler.handle(ServerImpl.java:675) '''