Closed axpira closed 6 years ago
This is the output of localhost:8001
{
"plugins": {
"enabled_in_cluster": [
"prometheus"
],
"available_on_server": {
"response-transformer": true,
"oauth2": true,
"acl": true,
"correlation-id": true,
"pre-function": true,
"jwt": true,
"cors": true,
"ip-restriction": true,
"basic-auth": true,
"key-auth": true,
"rate-limiting": true,
"request-transformer": true,
"http-log": true,
"file-log": true,
"hmac-auth": true,
"ldap-auth": true,
"datadog": true,
"tcp-log": true,
"zipkin": true,
"post-function": true,
"request-size-limiting": true,
"bot-detection": true,
"syslog": true,
"loggly": true,
"azure-functions": true,
"udp-log": true,
"response-ratelimiting": true,
"aws-lambda": true,
"statsd": true,
"prometheus": true,
"request-termination": true
}
},
"tagline": "Welcome to kong",
"configuration": {
"plugins": [
"bundled"
],
"admin_ssl_enabled": false,
"lua_ssl_verify_depth": 1,
"trusted_ips": {},
"prefix": "/usr/local/kong",
"loaded_plugins": {
"response-transformer": true,
"request-termination": true,
"prometheus": true,
"ip-restriction": true,
"pre-function": true,
"jwt": true,
"cors": true,
"statsd": true,
"basic-auth": true,
"key-auth": true,
"ldap-auth": true,
"aws-lambda": true,
"http-log": true,
"response-ratelimiting": true,
"hmac-auth": true,
"request-size-limiting": true,
"datadog": true,
"tcp-log": true,
"zipkin": true,
"post-function": true,
"bot-detection": true,
"acl": true,
"loggly": true,
"syslog": true,
"azure-functions": true,
"udp-log": true,
"file-log": true,
"request-transformer": true,
"correlation-id": true,
"rate-limiting": true,
"oauth2": true
},
"cassandra_username": "kong",
"admin_ssl_cert_csr_default": "/usr/local/kong/ssl/admin-kong-default.csr",
"dns_resolver": {},
"pg_user": "kong",
"pg_password": "******",
"cassandra_data_centers": [
"dc1:2",
"dc2:3"
],
"nginx_admin_directives": {},
"custom_plugins": {},
"pg_host": "postgres",
"nginx_acc_logs": "/usr/local/kong/logs/access.log",
"proxy_listen": [
"off"
],
"client_ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
"dns_no_sync": false,
"db_update_propagation": 0,
"nginx_err_logs": "/usr/local/kong/logs/error.log",
"cassandra_port": 9042,
"dns_order": [
"LAST",
"SRV",
"A",
"CNAME"
],
"dns_error_ttl": 1,
"headers": [
"server_tokens",
"latency_tokens"
],
"dns_stale_ttl": 4,
"nginx_optimizations": true,
"database": "postgres",
"pg_database": "kong",
"nginx_worker_processes": "auto",
"lua_package_cpath": "",
"admin_acc_logs": "/usr/local/kong/logs/admin_access.log",
"lua_package_path": "./?.lua;./?/init.lua;",
"nginx_pid": "/usr/local/kong/pids/nginx.pid",
"upstream_keepalive": 60,
"admin_access_log": "/dev/stdout",
"client_ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
"proxy_listeners": {},
"proxy_ssl_enabled": false,
"cassandra_contact_points": [
"127.0.0.1"
],
"enabled_headers": {
"latency_tokens": true,
"X-Kong-Proxy-Latency": true,
"Via": true,
"server_tokens": true,
"Server": true,
"X-Kong-Upstream-Latency": true,
"X-Kong-Upstream-Status": false
},
"ssl_ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256",
"cassandra_ssl": false,
"db_resurrect_ttl": 30,
"ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"cassandra_consistency": "ONE",
"db_cache_ttl": 0,
"admin_error_log": "/dev/stderr",
"pg_ssl_verify": false,
"dns_not_found_ttl": 30,
"pg_ssl": false,
"client_max_body_size": "0",
"client_ssl": false,
"cassandra_repl_strategy": "SimpleStrategy",
"ssl_cipher_suite": "modern",
"lua_socket_pool_size": 30,
"db_update_frequency": 5,
"nginx_http_directives": [
{
"value": "prometheus_metrics 5m",
"name": "lua_shared_dict"
}
],
"kong_env": "/usr/local/kong/.kong_env",
"cassandra_schema_consensus_timeout": 10000,
"dns_hostsfile": "/etc/hosts",
"admin_listeners": [
{
"ssl": false,
"ip": "0.0.0.0",
"proxy_protocol": false,
"port": 8001,
"http2": false,
"listener": "0.0.0.0:8001"
}
],
"cassandra_lb_policy": "RoundRobin",
"error_default_type": "text/plain",
"real_ip_header": "X-Real-IP",
"admin_ssl_cert_key_default": "/usr/local/kong/ssl/admin-kong-default.key",
"cassandra_ssl_verify": false,
"proxy_error_log": "logs/error.log",
"nginx_proxy_directives": {},
"real_ip_recursive": "off",
"cassandra_repl_factor": 1,
"client_ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"nginx_daemon": "off",
"anonymous_reports": true,
"cassandra_timeout": 5000,
"proxy_access_log": "logs/access.log",
"pg_port": 5432,
"nginx_kong_conf": "/usr/local/kong/nginx-kong.conf",
"client_body_buffer_size": "8k",
"mem_cache_size": "128m",
"admin_ssl_cert_default": "/usr/local/kong/ssl/admin-kong-default.crt",
"log_level": "notice",
"cassandra_keyspace": "kong",
"ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"nginx_conf": "/usr/local/kong/nginx.conf",
"admin_listen": [
"0.0.0.0:8001"
]
},
"version": "0.14.0",
"node_id": "74ea23ad-dcab-40c9-b8f9-95798a09f26a",
"lua_version": "LuaJIT 2.1.0-beta3",
"prng_seeds": {
"pid: 29": 238661092046,
"pid: 27": 902062212031,
"pid: 30": 254209216119,
"pid: 28": 140134189225
},
"timers": {
"pending": 5,
"running": 0
},
"hostname": "kong-ingress-controller-7d444b4-2258t"
}
Hi @axpira,
Did you throw any traffic on your Kong instance after the plugin was enabled?
Hi @hbagdi ,
Yes, I have traffic.
This is occurring in two different Kong instance, both is in Kubernetes as Ingress Controller
Hi @axpira, Are you using Routes/Service or APIs (which are deprecated) in Kong? Please note that this plugin supports Routes/Services only.
Hi @hbagdi ,
I'm using Routes/Service. Can I do anything to help?
I skipped over the detail that Kong is setup as a k8s ingress controller.
What you see is an expected behavior. Kong can run as an Admin(Control plane) only or a Proxy(Data plane) only mode in addition to running both of those on a node. You can find the documentation here. The ingress controller runs Kong node(s) in control-plane mode (which is what the Ingress controller talk to) and then there is another set of Kong nodes which run in data-plane mode.
You're scrapping metrics from Admin API on the nodes running in control-plane mode, which don't proxy any traffic. You'll need to scrape metrics from the data-plane Kong nodes. The problem here is that you don't have Admin API on those nodes. For problems like these, you'll need to use the last solution mentioned in the README file of using a custom Nginx template to expose metrics on a different port which will then expose all the metrics.
Please note that 0.14.0 is not yet supported by the Ingress Controller. You can either wait for Ingress controller to support 0.14.0 or use a custom template with CE 0.13.1 to add a server block to expose the metrics.
Thank you for opening this issue.
Hi, i have the same problem and i follow the README, but I can't find file kong.conf. i just find nginx.conf and nginx-kong.conf under directory /usr/local/kong. i add prometheus-server.conf to nginx-kong.conf or nginx.conf , and when i execute 'kong reload', it goes to the original .
Hello @WywTed,
Could you elaborate on what you're trying to accomplish?
we deploy kong through ingress controller, and i add a prometheus plugin on a service, well when i request http://192.168.1.162:8001/metrics, it returns data without the information about the service.
# HELP kong_datastore_reachable Datastore reachable from Kong, 0 is unreachable
# TYPE kong_datastore_reachable gauge
kong_datastore_reachable 1
# HELP kong_nginx_http_current_connections Number of HTTP connections
# TYPE kong_nginx_http_current_connections gauge
kong_nginx_http_current_connections{state="accepted"} 782
kong_nginx_http_current_connections{state="active"} 1
kong_nginx_http_current_connections{state="handled"} 782
kong_nginx_http_current_connections{state="reading"} 0
kong_nginx_http_current_connections{state="total"} 782
kong_nginx_http_current_connections{state="waiting"} 0
kong_nginx_http_current_connections{state="writing"} 1
# HELP kong_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE kong_nginx_metric_errors_total counter
kong_nginx_metric_errors_total 0
[root@k8s-master-1 ~]# kubectl get all -nkong
NAME READY STATUS RESTARTS AGE
pod/kong-667bdcccb5-n9p84 1/1 Running 0 16h
pod/kong-667bdcccb5-q6wft 1/1 Running 0 9h
pod/kong-667bdcccb5-wdlq2 1/1 Running 9 11h
pod/kong-ingress-controller-97485d6f8-45b4f 2/2 Running 33 40d
pod/kong-ingress-controller-97485d6f8-jbg4n 2/2 Running 60 40d
pod/kong-ingress-controller-97485d6f8-stgcd 2/2 Running 35 40d
pod/kong-migrations-b5brm 0/1 Completed 0 40d
pod/postgres-0 1/1 Running 0 40d
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kong-ingress-controller LoadBalancer 10.16.30.75 192.168.1.162 8001:30329/TCP 40d
service/kong-proxy LoadBalancer 10.16.89.115 192.168.1.161 80:31852/TCP,443:32087/TCP,9000:31070/TCP 40d
service/postgres LoadBalancer 10.16.162.192 192.168.1.163 5432:31939/TCP 40d
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
deployment.apps/kong 3 3 3 3 40d
deployment.apps/kong-ingress-controller 3 3 3 3 40d
NAME DESIRED CURRENT READY AGE
replicaset.apps/kong-5669944d57 0 0 0 40d
replicaset.apps/kong-5dc9d95955 0 0 0 17h
replicaset.apps/kong-667bdcccb5 3 3 3 16h
replicaset.apps/kong-79698c76c5 0 0 0 39d
replicaset.apps/kong-ingress-controller-97485d6f8 3 3 3 40d
NAME DESIRED CURRENT AGE
statefulset.apps/postgres 1 1 40d
NAME COMPLETIONS DURATION AGE
job.batch/kong-migrations 1/1 18s 40d
so i try to follow what you say adding prometheus-server.conf to kong.conf, i copy /etc/kong/kong.conf.default to /etc/kong/kong.conf and add the config to it. I also mount the directory to hostPath to prevent the starting initializing. But when i exec 'kong restart' , it went wrong.
finally it works. Thanks.
[root@k8s-master-1 ~]# kubectl exec -it kong-667bdcccb5-q6wft -nkong sh
/ # curl localhost:9542
# HELP kong_bandwidth Total bandwidth in bytes consumed per service in Kong
# TYPE kong_bandwidth counter
kong_bandwidth{type="egress",service="default.tracing-demo-a.11001"} 264
kong_bandwidth{type="ingress",service="default.tracing-demo-a.11001"} 222
# HELP kong_datastore_reachable Datastore reachable from Kong, 0 is unreachable
# TYPE kong_datastore_reachable gauge
kong_datastore_reachable 1
# HELP kong_http_status HTTP status codes per service in Kong
# TYPE kong_http_status counter
kong_http_status{code="200",service="default.tracing-demo-a.11001"} 1
# HELP kong_latency Latency added by Kong, total request time and upstream latency for each service in Kong
# TYPE kong_latency histogram
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="00500.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="01000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="02000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="05000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="10000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="30000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="60000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="+Inf"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="01000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="02000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="05000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="10000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="30000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="60000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="+Inf"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00010.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00015.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00020.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00025.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00030.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00040.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00050.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00060.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00070.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00080.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00090.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00100.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00200.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00300.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00400.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00500.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="01000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="02000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="05000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="10000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="30000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="60000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="+Inf"} 1
kong_latency_count{type="kong",service="default.tracing-demo-a.11001"} 1
kong_latency_count{type="request",service="default.tracing-demo-a.11001"} 1
kong_latency_count{type="upstream",service="default.tracing-demo-a.11001"} 1
kong_latency_sum{type="kong",service="default.tracing-demo-a.11001"} 482
kong_latency_sum{type="request",service="default.tracing-demo-a.11001"} 520
kong_latency_sum{type="upstream",service="default.tracing-demo-a.11001"} 9
# HELP kong_nginx_http_current_connections Number of HTTP connections
# TYPE kong_nginx_http_current_connections gauge
kong_nginx_http_current_connections{state="accepted"} 5
kong_nginx_http_current_connections{state="active"} 1
kong_nginx_http_current_connections{state="handled"} 5
kong_nginx_http_current_connections{state="reading"} 0
kong_nginx_http_current_connections{state="total"} 5
kong_nginx_http_current_connections{state="waiting"} 0
kong_nginx_http_current_connections{state="writing"} 1
# HELP kong_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE kong_nginx_metric_errors_total counter
kong_nginx_metric_errors_total 0
/ #
@WywTed Can I take a look at the kong dashboard in your grafana monitor? I have a question. If I configure the 9542 port on every kong Pod, does the dashboard display the monitoring data of each kong or all the kong data?
@dotbalo You cannot point Grafana to Kong pods. You need to setup a Prometheus server which scrapes Kong pods and then point Grafana to the Prometheus server.
@dotbalo we don't use grafana. just expose prometheus server .
This workaround doesn't work for me. I've added Configmap with Nginx configuration with inject directive to expose metrics over port 9542.
[root@kong-rc-6886bb5c6-hwx2b /]# curl -i localhost:9542
HTTP/1.1 200 OK
Server: openresty/1.13.6.2
Date: Thu, 30 May 2019 16:02:28 GMT
Content-Type: text/plain; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive
# HELP kong_datastore_reachable Datastore reachable from Kong, 0 is unreachable
# TYPE kong_datastore_reachable gauge
kong_datastore_reachable 1
# HELP kong_nginx_http_current_connections Number of HTTP connections
# TYPE kong_nginx_http_current_connections gauge
kong_nginx_http_current_connections{state="accepted"} 23
kong_nginx_http_current_connections{state="active"} 3
kong_nginx_http_current_connections{state="handled"} 23
kong_nginx_http_current_connections{state="reading"} 0
kong_nginx_http_current_connections{state="total"} 306
kong_nginx_http_current_connections{state="waiting"} 2
kong_nginx_http_current_connections{state="writing"} 1
# HELP kong_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE kong_nginx_metric_errors_total counter
kong_nginx_metric_errors_total 0
server {
server_name kong_prometheus_exporter;
listen 0.0.0.0:9542; # can be any other port as well
location / {
default_type text/plain;
content_by_lua_block {
local prometheus = require "kong.plugins.prometheus.exporter"
prometheus:collect()
}
}
location /nginx_status {
internal;
access_log off;
stub_status;
}
}
curl http://localhost:8001/plugins
{
"next": null,
"data": [
{
"created_at": 1558420945,
"consumer": null,
"id": "0a332956-c38b-4a0c-be97-2dd0949f2a97",
"service": null,
"name": "cors",
"run_on": "first",
"enabled": true,
"route": null,
"config": {
"methods": [
"PUT",
"POST",
"DELETE",
"GET"
],
"exposed_headers": [
"Origin"
],
"max_age": null,
"credentials": false,
"headers": null,
"origins": [
"*"
],
"preflight_continue": false
}
},
{
"created_at": 1559223092,
"consumer": null,
"id": "49d30d27-b975-436d-abb3-b56de329c0ee",
"service": null,
"name": "prometheus",
"run_on": "first",
"enabled": true,
"route": null,
"config": {}
}
]
}
curl localhost:8001
{
"plugins": {
"enabled_in_cluster": [
"cors",
"prometheus"
],
"available_on_server": {
"response-transformer": true,
"oauth2": true,
"acl": true,
"correlation-id": true,
"pre-function": true,
"jwt": true,
"cors": true,
"ip-restriction": true,
"basic-auth": true,
"key-auth": true,
"rate-limiting": true,
"request-transformer": true,
"http-log": true,
"file-log": true,
"hmac-auth": true,
"ldap-auth": true,
"datadog": true,
"tcp-log": true,
"zipkin": true,
"post-function": true,
"request-size-limiting": true,
"bot-detection": true,
"syslog": true,
"loggly": true,
"azure-functions": true,
"udp-log": true,
"response-ratelimiting": true,
"aws-lambda": true,
"statsd": true,
"prometheus": true,
"request-termination": true
}
},
"tagline": "Welcome to kong",
"configuration": {
"plugins": [
"bundled"
],
"admin_ssl_enabled": true,
"lua_ssl_verify_depth": 1,
"trusted_ips": [
"0.0.0.0/0"
],
"prefix": "/usr/local/kong",
"loaded_plugins": {
"response-transformer": true,
"request-termination": true,
"prometheus": true,
"ip-restriction": true,
"pre-function": true,
"jwt": true,
"cors": true,
"statsd": true,
"basic-auth": true,
"key-auth": true,
"ldap-auth": true,
"aws-lambda": true,
"http-log": true,
"response-ratelimiting": true,
"hmac-auth": true,
"request-size-limiting": true,
"datadog": true,
"tcp-log": true,
"zipkin": true,
"post-function": true,
"bot-detection": true,
"acl": true,
"loggly": true,
"syslog": true,
"azure-functions": true,
"udp-log": true,
"file-log": true,
"request-transformer": true,
"correlation-id": true,
"rate-limiting": true,
"oauth2": true
},
"cassandra_username": "kong",
"ssl_cert_key": "/usr/local/kong/ssl/kong-default.key",
"admin_ssl_cert_key": "/usr/local/kong/ssl/admin-kong-default.key",
"dns_resolver": {},
"pg_user": "kong",
"mem_cache_size": "128m",
"cassandra_data_centers": [
"dc1:2",
"dc2:3"
],
"nginx_admin_directives": {},
"nginx_http_directives": [
{
"value": "/etc/metrics.conf",
"name": "include"
},
{
"value": "prometheus_metrics 5m",
"name": "lua_shared_dict"
}
],
"pg_host": "127.0.0.1",
"nginx_acc_logs": "/usr/local/kong/logs/access.log",
"proxy_listen": [
"0.0.0.0:8000 proxy_protocol",
"0.0.0.0:8443 ssl"
],
"client_ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"dns_no_sync": true,
"db_update_propagation": 2,
"stream_listen": [
"off"
],
"nginx_err_logs": "/usr/local/kong/logs/error.log",
"cassandra_port": 9042,
"dns_order": [
"A"
],
"dns_error_ttl": 1,
"headers": [
"server_tokens",
"latency_tokens"
],
"cassandra_lb_policy": "RequestRoundRobin",
"nginx_optimizations": true,
"pg_timeout": 5000,
"database": "cassandra",
"pg_database": "kong",
"nginx_worker_processes": "auto",
"lua_package_cpath": "",
"admin_acc_logs": "/usr/local/kong/logs/admin_access.log",
"lua_package_path": "./?.lua;./?/init.lua;",
"nginx_pid": "/usr/local/kong/pids/nginx.pid",
"upstream_keepalive": 60,
"admin_listen": [
"0.0.0.0:8001",
"0.0.0.0:8444 ssl"
],
"proxy_access_log": "/dev/stdout",
"admin_access_log": "/dev/stdout",
"nginx_conf": "/usr/local/kong/nginx.conf",
"ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
"proxy_listeners": [
{
"transparent": false,
"ssl": false,
"ip": "0.0.0.0",
"proxy_protocol": true,
"port": 8000,
"http2": false,
"listener": "0.0.0.0:8000 proxy_protocol"
},
{
"transparent": false,
"ssl": true,
"ip": "0.0.0.0",
"proxy_protocol": false,
"port": 8443,
"http2": false,
"listener": "0.0.0.0:8443 ssl"
}
],
"proxy_ssl_enabled": true,
"dns_stale_ttl": 120,
"cassandra_schema_consensus_timeout": 10000,
"enabled_headers": {
"latency_tokens": true,
"X-Kong-Proxy-Latency": true,
"Via": true,
"server_tokens": true,
"Server": true,
"X-Kong-Upstream-Latency": true,
"X-Kong-Upstream-Status": false
},
"admin_ssl_cert_default": "/usr/local/kong/ssl/admin-kong-default.crt",
"origins": {},
"db_resurrect_ttl": 30,
"nginx_daemon": "off",
"cassandra_consistency": "ONE",
"client_max_body_size": "0",
"admin_error_log": "/dev/stderr",
"pg_ssl_verify": false,
"dns_not_found_ttl": 1,
"pg_ssl": false,
"nginx_kong_stream_conf": "/usr/local/kong/nginx-kong-stream.conf",
"error_default_type": "text/plain",
"cassandra_repl_strategy": "SimpleStrategy",
"ssl_ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256",
"log_level": "notice",
"cassandra_contact_points": [
"cassandra-0.cassandra.g-qa.svc.cluster.local"
],
"ssl_cipher_suite": "modern",
"nginx_kong_conf": "/usr/local/kong/nginx-kong.conf",
"real_ip_header": "proxy_protocol",
"dns_hostsfile": "/etc/hosts",
"admin_listeners": [
{
"transparent": false,
"ssl": false,
"ip": "0.0.0.0",
"proxy_protocol": false,
"port": 8001,
"http2": false,
"listener": "0.0.0.0:8001"
},
{
"transparent": false,
"ssl": true,
"ip": "0.0.0.0",
"proxy_protocol": false,
"port": 8444,
"http2": false,
"listener": "0.0.0.0:8444 ssl"
}
],
"proxy_error_log": "/dev/stderr",
"ssl_cert": "/usr/local/kong/ssl/kong-default.crt",
"cassandra_timeout": 5000,
"admin_ssl_cert_key_default": "/usr/local/kong/ssl/admin-kong-default.key",
"cassandra_ssl_verify": false,
"db_cache_ttl": 0,
"db_update_frequency": 5,
"real_ip_recursive": "on",
"cassandra_repl_factor": 2,
"nginx_proxy_directives": {},
"admin_ssl_cert": "/usr/local/kong/ssl/admin-kong-default.crt",
"anonymous_reports": true,
"kong_env": "/usr/local/kong/.kong_env",
"stream_listeners": {},
"pg_port": 5432,
"nginx_http_include": "/etc/metrics.conf",
"client_body_buffer_size": "8k",
"ssl_preread_enabled": true,
"client_ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"lua_socket_pool_size": 30,
"cassandra_keyspace": "kong",
"ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"cassandra_ssl": false,
"client_ssl": false
},
"version": "1.0.2",
"node_id": "bc751165-cd45-4cdc-a65a-b9ffd1669800",
"lua_version": "LuaJIT 2.1.0-beta3",
"prng_seeds": {
"pid: 36": 247225225134,
"pid: 35": 315016017618,
"pid: 40": 118147131211,
"pid: 1": 135211441071,
"pid: 41": 146210822231,
"pid: 37": 239236911497,
"pid: 42": 475683198317,
"pid: 39": 572546425514,
"pid: 38": 160139154206
},
"timers": {
"pending": 4,
"running": 0
},
"hostname": "kong-rc-6886bb5c6-hwx2b"
}
kubernetes deployment manifest
apiVersion: v1
kind: Service
metadata:
name: kong-proxy
spec:
type: NodePort
ports:
- name: kong-proxy
port: 8000
protocol: TCP
selector:
app: kong
---
apiVersion: v1
kind: Service
metadata:
name: kong-proxy-ssl
spec:
type: NodePort
ports:
- name: kong-proxy-ssl
port: 8443
protocol: TCP
selector:
app: kong
---
apiVersion: v1
kind: Service
metadata:
name: kong-admin
spec:
type: NodePort
ports:
- name: kong-admin
port: 8001
protocol: TCP
selector:
app: kong
---
apiVersion: v1
kind: Service
metadata:
name: kong-admin-ssl
spec:
type: NodePort
ports:
- name: kong-admin-ssl
port: 8444
protocol: TCP
selector:
app: kong
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: kong-rc
spec:
replicas: 3
template:
metadata:
labels:
name: kong-rc
app: kong
spec:
volumes:
- name: kong-metrics
configMap:
name: kong-metrics
containers:
- name: kong
volumeMounts:
- name: kong-metrics
mountPath: /etc/metrics.conf
subPath: kong
image: kong:1.0.2-centos
env:
- name: KONG_NGINX_HTTP_INCLUDE
value: /etc/metrics.conf
- name: KONG_TRUSTED_IPS
value: 0.0.0.0/0
- name: KONG_REAL_IP_HEADER
value: proxy_protocol
- name: KONG_REAL_IP_RECURSIVE
value: "on"
- name: KONG_PROXY_LISTEN
value: 0.0.0.0:8000 proxy_protocol, 0.0.0.0:8443 ssl
- name: KONG_ADMIN_LISTEN
value: 0.0.0.0:8001, 0.0.0.0:8444 ssl
- name: KONG_DATABASE
value: cassandra
- name: KONG_CASSANDRA_CONTACT_POINTS
value: cassandra-0.cassandra.g-qa.svc.cluster.local
- name: KONG_CASSANDRA_KEYSPACE
value: kong
- name: KONG_CASSANDRA_REPL_FACTOR
value: "2"
- name: KONG_DB_UPDATE_PROPAGATION
value: "2"
- name: KONG_PROXY_ACCESS_LOG
value: /dev/stdout
- name: KONG_ADMIN_ACCESS_LOG
value: /dev/stdout
- name: KONG_PROXY_ERROR_LOG
value: /dev/stderr
- name: KONG_ADMIN_ERROR_LOG
value: /dev/stderr
- name: KONG_DNS_STALE_TTL
value: "120"
- name: KONG_DNS_ORDER
value: A
- name: KONG_DNS_NOT_FOUND_TTL
value: "1"
- name: KONG_DNS_NO_SYNC
value: "on"
- name: KONG_PROXY_ACCESS_LOG
value: "/dev/stdout"
- name: KONG_ADMIN_ACCESS_LOG
value: "/dev/stdout"
- name: KONG_PROXY_ERROR_LOG
value: "/dev/stderr"
- name: KONG_ADMIN_ERROR_LOG
value: "/dev/stderr"
ports:
- name: admin
containerPort: 8001
protocol: TCP
- name: proxy
containerPort: 8000
protocol: TCP
- name: proxy-ssl
containerPort: 8443
protocol: TCP
- name: admin-ssl
containerPort: 8444
protocol: TCP
Strange, upgraded to 1.0.3 and it works
@WywTed Could you please show us how you solved this issue as we are now facing the same problem ,thanks
@WywTed Could you please show us how you solved this issue as we are now facing the same problem ,thanks
there is a pod named kong-proxy which expose the 80 port, we followed what @hbagdi said to add prometheus.conf into the container /etc/prometheus.conf and use it in the kong.conf . And then expose the 9542 port on the kong-proxy service. Well, at the same time , you need to config the prometheus annotation to the kong-proxy deploy. https://github.com/Kong/kong-plugin-prometheus/issues/54
I add plugin global and the response of metrics is: