search

deadtrickster / prometheus_rabbitmq_exporter

Prometheus.io exporter as a RabbitMQ Managment Plugin plugin
MIT License
291 stars 72 forks source link

prometheus_rabbitmq_exporter crashes while scraping #79

Closed ryanhartkopf closed 5 years ago

ryanhartkopf commented 5 years ago

I'm running RabbitMQ 3.6.14 with these plugins installed: https://github.com/deadtrickster/prometheus_rabbitmq_exporter/releases/tag/rabbitmq-3.6.14.1

I'm getting this error and a 500 when attempting to scrape metrics.

The error isn't providing much for me to go on. So far, I've tried removing and re-installing the plugins and restarting the RabbitMQ process. Any other troubleshooting tips would be very helpful.

=ERROR REPORT==== 29-Apr-2019::17:29:41 ===
Error in process <0.1541.0> on node 'rabbit@server' with exit value:
{[{reason,function_clause},
  {mfa,{prometheus_rabbitmq_exporter_handler,handle,2}},
  {stacktrace,
      [{prometheus_http_impl,reply,
           [{request,true,
                #Fun<prometheus_rabbitmq_exporter_handler.0.118501792>,
                default,false}],
           [{file,"src/prometheus_http_impl.erl"},{line,13}]},
       {prometheus_rabbitmq_exporter_handler,handle,2,
           [{file,"src/prometheus_rabbitmq_exporter_handler.erl"},{line,21}]},
       {cowboy_handler,handler_handle,4,
           [{file,"src/cowboy_handler.erl"},{line,111}]},
       {cowboy_protocol,execute,4,
           [{file,"src/cowboy_protocol.erl"},{line,442}]}]},
  {req,
      [{socket,#Port<0.33716>},
       {transport,ranch_tcp},
       {connection,keepalive},
       {pid,<0.1541.0>},
       {method,<<"GET">>},
       {version,'HTTP/1.1'},
       {peer,{{client},38272}},
       {host,<<"server">>},
       {host_info,undefined},
       {port,15672},
       {path,<<"/api/metrics">>},
       {path_info,undefined},
       {qs,<<>>},
       {qs_vals,[]},
       {bindings,[]},
       {headers,
           [{<<"host">>,<<"server:15672">>},
            {<<"user-agent">>,<<"Prometheus/2.6.0">>},
            {<<"accept">>,
             <<"application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1">>},
            {<<"accept-encoding">>,<<"gzip">>},
            {<<"x-prometheus-scrape-timeout-seconds">>,<<"10.000000">>}]},
       {p_headers,[]},
       {cookies,undefined},
       {meta,[]},
       {body_state,waiting},
       {buffer,<<>>},
       {multipart,undefined},
       {resp_compress,true},
       {resp_state,waiting},
       {resp_headers,[]},
       {resp_body,<<>>},
       {onresponse,#Fun<rabbit_cowboy_middleware.onresponse.4>}]},
  {state,{default}}],
 [{cowboy_handler,handler_handle,4,
      [{file,"src/cowboy_handler.erl"},{line,119}]},
  {cowboy_protocol,execute,4,[{file,"src/cowboy_protocol.erl"},{line,442}]}]}

=ERROR REPORT==== 29-Apr-2019::17:29:41 ===
Ranch listener rabbit_web_dispatch_sup_15672 had connection process started with cowboy_protocol:start_link/4 at <0.1541.0> exit with reason: {[{reason,function_clause},{mfa,{prometheus_rabbitmq_exporter_handler,handle,2}},{stacktrace,[{prometheus_http_impl,reply,[{request,true,#Fun<prometheus_rabbitmq_exporter_handler.0.118501792>,default,false}],[{file,"src/prometheus_http_impl.erl"},{line,13}]},{prometheus_rabbitmq_exporter_handler,handle,2,[{file,"src/prometheus_rabbitmq_exporter_handler.erl"},{line,21}]},{cowboy_handler,handler_handle,4,[{file,"src/cowboy_handler.erl"},{line,111}]},{cowboy_protocol,execute,4,[{file,"src/cowboy_protocol.erl"},{line,442}]}]},{req,[{socket,#Port<0.33716>},{transport,ranch_tcp},{connection,keepalive},{pid,<0.1541.0>},{method,<<"GET">>},{version,'HTTP/1.1'},{peer,{{client},38272}},{host,<<"server">>},{host_info,undefined},{port,15672},{path,<<"/api/metrics">>},{path_info,undefined},{qs,<<>>},{qs_vals,[]},{bindings,[]},{headers,[{<<"host">>,<<"server:15672">>},{<<"user-agent">>,<<"Prometheus/2.6.0">>},{<<"accept">>,<<"application/openmetrics-text; version=0.0.1,text/plain;version=0.0.4;q=0.5,*/*;q=0.1">>},{<<"accept-encoding">>,<<"gzip">>},{<<"x-prometheus-scrape-timeout-seconds">>,<<"10.000000">>}]},{p_headers,[]},{cookies,undefined},{meta,[]},{body_state,waiting},{buffer,<<>>},{multipart,undefined},{resp_compress,true},{resp_state,waiting},{resp_headers,[]},{resp_body,<<>>},{onresponse,#Fun<rabbit_cowboy_middleware.onresponse.4>}]},{state,{default}}],[{cowboy_handler,handler_handle,4,[{file,"src/cowboy_handler.erl"},{line,119}]},{cowboy_protocol,execute,4,[{file,"src/cowboy_protocol.erl"},{line,442}]}]}

Enabled plugins:

$ sudo rabbitmq-plugins list
 Configured: E = explicitly enabled; e = implicitly enabled
 | Status:   * = running on rabbit@server
 |/
[e*] accept                            0.3.3
[e*] amqp_client                       3.6.14
[e*] cowboy                            1.0.4
[e*] cowlib                            1.0.2
[e*] prometheus                        3.4.3
[E*] prometheus_httpd                  2.1.8
[  ] prometheus_process_collector      1.3.0
[E*] prometheus_rabbitmq_exporter      v3.6.14.1
[  ] rabbitmq_amqp1_0                  3.6.14
[  ] rabbitmq_auth_backend_ldap        3.6.14
[  ] rabbitmq_auth_mechanism_ssl       3.6.14
[  ] rabbitmq_consistent_hash_exchange 3.6.14
[  ] rabbitmq_event_exchange           3.6.14
[  ] rabbitmq_federation               3.6.14
[  ] rabbitmq_federation_management    3.6.14
[  ] rabbitmq_jms_topic_exchange       3.6.14
[E*] rabbitmq_management               3.6.14
[e*] rabbitmq_management_agent         3.6.14
[  ] rabbitmq_management_visualiser    3.6.14
[  ] rabbitmq_mqtt                     3.6.14
[  ] rabbitmq_random_exchange          3.6.14
[  ] rabbitmq_recent_history_exchange  3.6.14
[  ] rabbitmq_sharding                 3.6.14
[  ] rabbitmq_shovel                   3.6.14
[  ] rabbitmq_shovel_management        3.6.14
[  ] rabbitmq_stomp                    3.6.14
[  ] rabbitmq_top                      3.6.14
[  ] rabbitmq_tracing                  3.6.14
[  ] rabbitmq_trust_store              3.6.14
[e*] rabbitmq_web_dispatch             3.6.14
[  ] rabbitmq_web_mqtt                 3.6.14
[  ] rabbitmq_web_mqtt_examples        3.6.14
[  ] rabbitmq_web_stomp                3.6.14
[  ] rabbitmq_web_stomp_examples       3.6.14
[  ] sockjs                            0.3.4

RabbitMQ status:

$ sudo rabbitmqctl status
[{pid,4784},
 {running_applications,
     [{prometheus_rabbitmq_exporter,
          "RabbitMQ Prometheus.io metrics exporter","v3.6.14.1"},
      {rabbitmq_management,"RabbitMQ Management Console","3.6.14"},
      {rabbitmq_management_agent,"RabbitMQ Management Agent","3.6.14"},
      {rabbitmq_web_dispatch,"RabbitMQ Web Dispatcher","3.6.14"},
      {cowboy,"Small, fast, modular HTTP server.","1.0.4"},
      {amqp_client,"RabbitMQ AMQP Client","3.6.14"},
      {cowlib,"Support library for manipulating Web protocols.","1.0.2"},
      {prometheus_httpd,"Prometheus.io inets httpd exporter","2.1.8"},
      {prometheus,"Prometheus.io client in Erlang","3.4.3"},
      {accept,"Accept header(s) for Erlang/Elixir","0.3.3"},
      {inets,"INETS  CXC 138 49","6.4.5"},
      {rabbit,"RabbitMQ","3.6.14"},
      {ranch,"Socket acceptor pool for TCP protocols.","1.3.0"},
      {ssl,"Erlang/OTP SSL application","8.2.3"},
      {public_key,"Public key infrastructure","1.5.2"},
      {asn1,"The Erlang ASN1 compiler version 5.0.4","5.0.4"},
      {mnesia,"MNESIA  CXC 138 12","4.15.3"},
      {rabbit_common,
          "Modules shared by rabbitmq-server and rabbitmq-erlang-client",
          "3.6.14"},
      {compiler,"ERTS  CXC 138 10","7.1.4"},
      {os_mon,"CPO  CXC 138 46","2.4.4"},
      {recon,"Diagnostic tools for production use","2.3.2"},
      {crypto,"CRYPTO","4.2"},
      {syntax_tools,"Syntax tools","2.1.4"},
      {xmerl,"XML parser","1.3.16"},
      {sasl,"SASL  CXC 138 11","3.1.1"},
      {stdlib,"ERTS  CXC 138 10","3.4.3"},
      {kernel,"ERTS  CXC 138 10","5.4.1"}]},
 {os,{unix,linux}},
 {erlang_version,
     "Erlang/OTP 20 [erts-9.2] [source] [64-bit] [smp:2:2] [ds:2:2:10] [async-threads:64] [hipe] [kernel-poll:true]\n"},
 {memory,
     [{connection_readers,607584},
      {connection_writers,57280},
      {connection_channels,178280},
      {connection_other,981576},
      {queue_procs,557088},
      {queue_slave_procs,0},
      {plugins,1230424},
      {other_proc,28754896},
      {metrics,238768},
      {mgmt_db,225920},
      {mnesia,237336},
      {other_ets,2232272},
      {binary,3277144},
      {msg_index,113384},
      {code,25886905},
      {atom,1082561},
      {other_system,10533366},
      {allocated_unused,30542880},
      {reserved_unallocated,10768384},
      {total,117506048}]},
 {alarms,[]},
 {listeners,
     [{clustering,25672,"::"},
      {amqp,5672,"::"},
      {'amqp/ssl',5671,"::"},
      {http,15672,"::"}]},
 {vm_memory_calculation_strategy,rss},
 {vm_memory_high_watermark,0.4},
 {vm_memory_limit,6413197312},
 {disk_free_limit,50000000},
 {disk_free,19856879616},
 {file_descriptors,
     [{total_limit,924},{total_used,21},{sockets_limit,829},{sockets_used,9}]},
 {processes,[{limit,1048576},{used,593}]},
 {run_queue,0},
 {uptime,2488},
 {kernel,{net_ticktime,60}}]
gerhard commented 5 years ago

RabbitMQ 3.6.x was released on 2015-12-22, and reached end of life on 2018-05-31 - almost 1 year ago. Please upgrade to latest stable RabbitMQ 3.7.x. It will fix your issue as well as many other issues that have been fixed since RabbitMQ v3.6.14. Watch out for #80.