Kong / kong-plugin-prometheus

Prometheus plugin for Kong - this plugin has been moved into https://github.com/Kong/kong, please open issues and PRs in that repo
Apache License 2.0
119 stars 57 forks source link

Is not showing all metrics on Kong 0.14.0 #6

Closed axpira closed 6 years ago

axpira commented 6 years ago

I add plugin global and the response of metrics is:

curl -i http://localhost:8001/metrics

HTTP/1.1 200 OK
Server: openresty/1.13.6.2
Date: Fri, 27 Jul 2018 19:03:16 GMT
Content-Type: text/plain; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive
Access-Control-Allow-Origin: *

# HELP kong_datastore_reachable Datastore reachable from Kong, 0 is unreachable
# TYPE kong_datastore_reachable gauge
kong_datastore_reachable 1
# HELP kong_nginx_http_current_connections Number of HTTP connections
# TYPE kong_nginx_http_current_connections gauge
kong_nginx_http_current_connections{state="accepted"} 6298
kong_nginx_http_current_connections{state="active"} 4
kong_nginx_http_current_connections{state="handled"} 6298
kong_nginx_http_current_connections{state="reading"} 0
kong_nginx_http_current_connections{state="total"} 19503
kong_nginx_http_current_connections{state="waiting"} 3
kong_nginx_http_current_connections{state="writing"} 1
# HELP kong_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE kong_nginx_metric_errors_total counter
kong_nginx_metric_errors_total 0
 curl -i http://localhost:8001/plugins

HTTP/1.1 200 OK
Date: Fri, 27 Jul 2018 19:05:11 GMT
Content-Type: application/json; charset=utf-8
Connection: keep-alive
Access-Control-Allow-Origin: *
Server: kong/0.14.0
Content-Length: 141

{"total":1,"data":[{"created_at":1532718301000,"config":{},"id":"18695e96-6f01-4bf1-a937-ebaa77c3c94c","name":"prometheus","enabled":true}]}
axpira commented 6 years ago

This is the output of localhost:8001

{
  "plugins": {
    "enabled_in_cluster": [
      "prometheus"
    ],
    "available_on_server": {
      "response-transformer": true,
      "oauth2": true,
      "acl": true,
      "correlation-id": true,
      "pre-function": true,
      "jwt": true,
      "cors": true,
      "ip-restriction": true,
      "basic-auth": true,
      "key-auth": true,
      "rate-limiting": true,
      "request-transformer": true,
      "http-log": true,
      "file-log": true,
      "hmac-auth": true,
      "ldap-auth": true,
      "datadog": true,
      "tcp-log": true,
      "zipkin": true,
      "post-function": true,
      "request-size-limiting": true,
      "bot-detection": true,
      "syslog": true,
      "loggly": true,
      "azure-functions": true,
      "udp-log": true,
      "response-ratelimiting": true,
      "aws-lambda": true,
      "statsd": true,
      "prometheus": true,
      "request-termination": true
    }
  },
  "tagline": "Welcome to kong",
  "configuration": {
    "plugins": [
      "bundled"
    ],
    "admin_ssl_enabled": false,
    "lua_ssl_verify_depth": 1,
    "trusted_ips": {},
    "prefix": "/usr/local/kong",
    "loaded_plugins": {
      "response-transformer": true,
      "request-termination": true,
      "prometheus": true,
      "ip-restriction": true,
      "pre-function": true,
      "jwt": true,
      "cors": true,
      "statsd": true,
      "basic-auth": true,
      "key-auth": true,
      "ldap-auth": true,
      "aws-lambda": true,
      "http-log": true,
      "response-ratelimiting": true,
      "hmac-auth": true,
      "request-size-limiting": true,
      "datadog": true,
      "tcp-log": true,
      "zipkin": true,
      "post-function": true,
      "bot-detection": true,
      "acl": true,
      "loggly": true,
      "syslog": true,
      "azure-functions": true,
      "udp-log": true,
      "file-log": true,
      "request-transformer": true,
      "correlation-id": true,
      "rate-limiting": true,
      "oauth2": true
    },
    "cassandra_username": "kong",
    "admin_ssl_cert_csr_default": "/usr/local/kong/ssl/admin-kong-default.csr",
    "dns_resolver": {},
    "pg_user": "kong",
    "pg_password": "******",
    "cassandra_data_centers": [
      "dc1:2",
      "dc2:3"
    ],
    "nginx_admin_directives": {},
    "custom_plugins": {},
    "pg_host": "postgres",
    "nginx_acc_logs": "/usr/local/kong/logs/access.log",
    "proxy_listen": [
      "off"
    ],
    "client_ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
    "ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
    "dns_no_sync": false,
    "db_update_propagation": 0,
    "nginx_err_logs": "/usr/local/kong/logs/error.log",
    "cassandra_port": 9042,
    "dns_order": [
      "LAST",
      "SRV",
      "A",
      "CNAME"
    ],
    "dns_error_ttl": 1,
    "headers": [
      "server_tokens",
      "latency_tokens"
    ],
    "dns_stale_ttl": 4,
    "nginx_optimizations": true,
    "database": "postgres",
    "pg_database": "kong",
    "nginx_worker_processes": "auto",
    "lua_package_cpath": "",
    "admin_acc_logs": "/usr/local/kong/logs/admin_access.log",
    "lua_package_path": "./?.lua;./?/init.lua;",
    "nginx_pid": "/usr/local/kong/pids/nginx.pid",
    "upstream_keepalive": 60,
    "admin_access_log": "/dev/stdout",
    "client_ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
    "proxy_listeners": {},
    "proxy_ssl_enabled": false,
    "cassandra_contact_points": [
      "127.0.0.1"
    ],
    "enabled_headers": {
      "latency_tokens": true,
      "X-Kong-Proxy-Latency": true,
      "Via": true,
      "server_tokens": true,
      "Server": true,
      "X-Kong-Upstream-Latency": true,
      "X-Kong-Upstream-Status": false
    },
    "ssl_ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256",
    "cassandra_ssl": false,
    "db_resurrect_ttl": 30,
    "ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
    "cassandra_consistency": "ONE",
    "db_cache_ttl": 0,
    "admin_error_log": "/dev/stderr",
    "pg_ssl_verify": false,
    "dns_not_found_ttl": 30,
    "pg_ssl": false,
    "client_max_body_size": "0",
    "client_ssl": false,
    "cassandra_repl_strategy": "SimpleStrategy",
    "ssl_cipher_suite": "modern",
    "lua_socket_pool_size": 30,
    "db_update_frequency": 5,
    "nginx_http_directives": [
      {
        "value": "prometheus_metrics 5m",
        "name": "lua_shared_dict"
      }
    ],
    "kong_env": "/usr/local/kong/.kong_env",
    "cassandra_schema_consensus_timeout": 10000,
    "dns_hostsfile": "/etc/hosts",
    "admin_listeners": [
      {
        "ssl": false,
        "ip": "0.0.0.0",
        "proxy_protocol": false,
        "port": 8001,
        "http2": false,
        "listener": "0.0.0.0:8001"
      }
    ],
    "cassandra_lb_policy": "RoundRobin",
    "error_default_type": "text/plain",
    "real_ip_header": "X-Real-IP",
    "admin_ssl_cert_key_default": "/usr/local/kong/ssl/admin-kong-default.key",
    "cassandra_ssl_verify": false,
    "proxy_error_log": "logs/error.log",
    "nginx_proxy_directives": {},
    "real_ip_recursive": "off",
    "cassandra_repl_factor": 1,
    "client_ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
    "nginx_daemon": "off",
    "anonymous_reports": true,
    "cassandra_timeout": 5000,
    "proxy_access_log": "logs/access.log",
    "pg_port": 5432,
    "nginx_kong_conf": "/usr/local/kong/nginx-kong.conf",
    "client_body_buffer_size": "8k",
    "mem_cache_size": "128m",
    "admin_ssl_cert_default": "/usr/local/kong/ssl/admin-kong-default.crt",
    "log_level": "notice",
    "cassandra_keyspace": "kong",
    "ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
    "nginx_conf": "/usr/local/kong/nginx.conf",
    "admin_listen": [
      "0.0.0.0:8001"
    ]
  },
  "version": "0.14.0",
  "node_id": "74ea23ad-dcab-40c9-b8f9-95798a09f26a",
  "lua_version": "LuaJIT 2.1.0-beta3",
  "prng_seeds": {
    "pid: 29": 238661092046,
    "pid: 27": 902062212031,
    "pid: 30": 254209216119,
    "pid: 28": 140134189225
  },
  "timers": {
    "pending": 5,
    "running": 0
  },
  "hostname": "kong-ingress-controller-7d444b4-2258t"
}
hbagdi commented 6 years ago

Hi @axpira,

Did you throw any traffic on your Kong instance after the plugin was enabled?

axpira commented 6 years ago

Hi @hbagdi ,

Yes, I have traffic.

This is occurring in two different Kong instance, both is in Kubernetes as Ingress Controller

hbagdi commented 6 years ago

Hi @axpira, Are you using Routes/Service or APIs (which are deprecated) in Kong? Please note that this plugin supports Routes/Services only.

axpira commented 6 years ago

Hi @hbagdi ,

I'm using Routes/Service. Can I do anything to help?

hbagdi commented 6 years ago

I skipped over the detail that Kong is setup as a k8s ingress controller.

What you see is an expected behavior. Kong can run as an Admin(Control plane) only or a Proxy(Data plane) only mode in addition to running both of those on a node. You can find the documentation here. The ingress controller runs Kong node(s) in control-plane mode (which is what the Ingress controller talk to) and then there is another set of Kong nodes which run in data-plane mode.

You're scrapping metrics from Admin API on the nodes running in control-plane mode, which don't proxy any traffic. You'll need to scrape metrics from the data-plane Kong nodes. The problem here is that you don't have Admin API on those nodes. For problems like these, you'll need to use the last solution mentioned in the README file of using a custom Nginx template to expose metrics on a different port which will then expose all the metrics.

Please note that 0.14.0 is not yet supported by the Ingress Controller. You can either wait for Ingress controller to support 0.14.0 or use a custom template with CE 0.13.1 to add a server block to expose the metrics.

Thank you for opening this issue.

huhu-sky commented 5 years ago

Hi, i have the same problem and i follow the README, but I can't find file kong.conf. i just find nginx.conf and nginx-kong.conf under directory /usr/local/kong. i add prometheus-server.conf to nginx-kong.conf or nginx.conf , and when i execute 'kong reload', it goes to the original .

hbagdi commented 5 years ago

Hello @WywTed,

Could you elaborate on what you're trying to accomplish?

huhu-sky commented 5 years ago

we deploy kong through ingress controller, and i add a prometheus plugin on a service, well when i request http://192.168.1.162:8001/metrics, it returns data without the information about the service.

# HELP kong_datastore_reachable Datastore reachable from Kong, 0 is unreachable
# TYPE kong_datastore_reachable gauge
kong_datastore_reachable 1
# HELP kong_nginx_http_current_connections Number of HTTP connections
# TYPE kong_nginx_http_current_connections gauge
kong_nginx_http_current_connections{state="accepted"} 782
kong_nginx_http_current_connections{state="active"} 1
kong_nginx_http_current_connections{state="handled"} 782
kong_nginx_http_current_connections{state="reading"} 0
kong_nginx_http_current_connections{state="total"} 782
kong_nginx_http_current_connections{state="waiting"} 0
kong_nginx_http_current_connections{state="writing"} 1
# HELP kong_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE kong_nginx_metric_errors_total counter
kong_nginx_metric_errors_total 0
[root@k8s-master-1 ~]# kubectl get all -nkong
NAME                                          READY   STATUS      RESTARTS   AGE
pod/kong-667bdcccb5-n9p84                     1/1     Running     0          16h
pod/kong-667bdcccb5-q6wft                     1/1     Running     0          9h
pod/kong-667bdcccb5-wdlq2                     1/1     Running     9          11h
pod/kong-ingress-controller-97485d6f8-45b4f   2/2     Running     33         40d
pod/kong-ingress-controller-97485d6f8-jbg4n   2/2     Running     60         40d
pod/kong-ingress-controller-97485d6f8-stgcd   2/2     Running     35         40d
pod/kong-migrations-b5brm                     0/1     Completed   0          40d
pod/postgres-0                                1/1     Running     0          40d

NAME                              TYPE           CLUSTER-IP      EXTERNAL-IP     PORT(S)                                     AGE
service/kong-ingress-controller   LoadBalancer   10.16.30.75     192.168.1.162   8001:30329/TCP                              40d
service/kong-proxy                LoadBalancer   10.16.89.115    192.168.1.161   80:31852/TCP,443:32087/TCP,9000:31070/TCP   40d
service/postgres                  LoadBalancer   10.16.162.192   192.168.1.163   5432:31939/TCP                              40d

NAME                                      DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/kong                      3         3         3            3           40d
deployment.apps/kong-ingress-controller   3         3         3            3           40d

NAME                                                DESIRED   CURRENT   READY   AGE
replicaset.apps/kong-5669944d57                     0         0         0       40d
replicaset.apps/kong-5dc9d95955                     0         0         0       17h
replicaset.apps/kong-667bdcccb5                     3         3         3       16h
replicaset.apps/kong-79698c76c5                     0         0         0       39d
replicaset.apps/kong-ingress-controller-97485d6f8   3         3         3       40d

NAME                        DESIRED   CURRENT   AGE
statefulset.apps/postgres   1         1         40d

NAME                        COMPLETIONS   DURATION   AGE
job.batch/kong-migrations   1/1           18s        40d

so i try to follow what you say adding prometheus-server.conf to kong.conf, i copy /etc/kong/kong.conf.default to /etc/kong/kong.conf and add the config to it. I also mount the directory to hostPath to prevent the starting initializing. But when i exec 'kong restart' , it went wrong.

huhu-sky commented 5 years ago

finally it works. Thanks.

huhu-sky commented 5 years ago
[root@k8s-master-1 ~]# kubectl exec -it kong-667bdcccb5-q6wft -nkong sh
/ # curl localhost:9542
# HELP kong_bandwidth Total bandwidth in bytes consumed per service in Kong
# TYPE kong_bandwidth counter
kong_bandwidth{type="egress",service="default.tracing-demo-a.11001"} 264
kong_bandwidth{type="ingress",service="default.tracing-demo-a.11001"} 222
# HELP kong_datastore_reachable Datastore reachable from Kong, 0 is unreachable
# TYPE kong_datastore_reachable gauge
kong_datastore_reachable 1
# HELP kong_http_status HTTP status codes per service in Kong
# TYPE kong_http_status counter
kong_http_status{code="200",service="default.tracing-demo-a.11001"} 1
# HELP kong_latency Latency added by Kong, total request time and upstream latency for each service in Kong
# TYPE kong_latency histogram
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="00500.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="01000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="02000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="05000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="10000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="30000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="60000.0"} 1
kong_latency_bucket{type="kong",service="default.tracing-demo-a.11001",le="+Inf"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="01000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="02000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="05000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="10000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="30000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="60000.0"} 1
kong_latency_bucket{type="request",service="default.tracing-demo-a.11001",le="+Inf"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00010.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00015.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00020.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00025.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00030.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00040.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00050.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00060.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00070.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00080.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00090.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00100.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00200.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00300.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00400.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="00500.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="01000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="02000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="05000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="10000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="30000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="60000.0"} 1
kong_latency_bucket{type="upstream",service="default.tracing-demo-a.11001",le="+Inf"} 1
kong_latency_count{type="kong",service="default.tracing-demo-a.11001"} 1
kong_latency_count{type="request",service="default.tracing-demo-a.11001"} 1
kong_latency_count{type="upstream",service="default.tracing-demo-a.11001"} 1
kong_latency_sum{type="kong",service="default.tracing-demo-a.11001"} 482
kong_latency_sum{type="request",service="default.tracing-demo-a.11001"} 520
kong_latency_sum{type="upstream",service="default.tracing-demo-a.11001"} 9
# HELP kong_nginx_http_current_connections Number of HTTP connections
# TYPE kong_nginx_http_current_connections gauge
kong_nginx_http_current_connections{state="accepted"} 5
kong_nginx_http_current_connections{state="active"} 1
kong_nginx_http_current_connections{state="handled"} 5
kong_nginx_http_current_connections{state="reading"} 0
kong_nginx_http_current_connections{state="total"} 5
kong_nginx_http_current_connections{state="waiting"} 0
kong_nginx_http_current_connections{state="writing"} 1
# HELP kong_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE kong_nginx_metric_errors_total counter
kong_nginx_metric_errors_total 0
/ #
dotbalo commented 5 years ago

@WywTed Can I take a look at the kong dashboard in your grafana monitor? I have a question. If I configure the 9542 port on every kong Pod, does the dashboard display the monitoring data of each kong or all the kong data?

hbagdi commented 5 years ago

@dotbalo You cannot point Grafana to Kong pods. You need to setup a Prometheus server which scrapes Kong pods and then point Grafana to the Prometheus server.

huhu-sky commented 5 years ago

@dotbalo we don't use grafana. just expose prometheus server .

paleksic commented 5 years ago

This workaround doesn't work for me. I've added Configmap with Nginx configuration with inject directive to expose metrics over port 9542.

[root@kong-rc-6886bb5c6-hwx2b /]# curl -i localhost:9542
HTTP/1.1 200 OK
Server: openresty/1.13.6.2
Date: Thu, 30 May 2019 16:02:28 GMT
Content-Type: text/plain; charset=UTF-8
Transfer-Encoding: chunked
Connection: keep-alive

# HELP kong_datastore_reachable Datastore reachable from Kong, 0 is unreachable
# TYPE kong_datastore_reachable gauge
kong_datastore_reachable 1
# HELP kong_nginx_http_current_connections Number of HTTP connections
# TYPE kong_nginx_http_current_connections gauge
kong_nginx_http_current_connections{state="accepted"} 23
kong_nginx_http_current_connections{state="active"} 3
kong_nginx_http_current_connections{state="handled"} 23
kong_nginx_http_current_connections{state="reading"} 0
kong_nginx_http_current_connections{state="total"} 306
kong_nginx_http_current_connections{state="waiting"} 2
kong_nginx_http_current_connections{state="writing"} 1
# HELP kong_nginx_metric_errors_total Number of nginx-lua-prometheus errors
# TYPE kong_nginx_metric_errors_total counter
kong_nginx_metric_errors_total 0
server {
    server_name kong_prometheus_exporter;
    listen 0.0.0.0:9542; # can be any other port as well

    location / {
        default_type text/plain;
        content_by_lua_block {
             local prometheus = require "kong.plugins.prometheus.exporter"
             prometheus:collect()
        }
    }

    location /nginx_status {
        internal;
        access_log off;
        stub_status;
    }
}

curl http://localhost:8001/plugins

{
  "next": null,
  "data": [
    {
      "created_at": 1558420945,
      "consumer": null,
      "id": "0a332956-c38b-4a0c-be97-2dd0949f2a97",
      "service": null,
      "name": "cors",
      "run_on": "first",
      "enabled": true,
      "route": null,
      "config": {
        "methods": [
          "PUT",
          "POST",
          "DELETE",
          "GET"
        ],
        "exposed_headers": [
          "Origin"
        ],
        "max_age": null,
        "credentials": false,
        "headers": null,
        "origins": [
          "*"
        ],
        "preflight_continue": false
      }
    },
    {
      "created_at": 1559223092,
      "consumer": null,
      "id": "49d30d27-b975-436d-abb3-b56de329c0ee",
      "service": null,
      "name": "prometheus",
      "run_on": "first",
      "enabled": true,
      "route": null,
      "config": {}
    }
  ]
}

curl localhost:8001

{
  "plugins": {
    "enabled_in_cluster": [
      "cors",
      "prometheus"
    ],
    "available_on_server": {
      "response-transformer": true,
      "oauth2": true,
      "acl": true,
      "correlation-id": true,
      "pre-function": true,
      "jwt": true,
      "cors": true,
      "ip-restriction": true,
      "basic-auth": true,
      "key-auth": true,
      "rate-limiting": true,
      "request-transformer": true,
      "http-log": true,
      "file-log": true,
      "hmac-auth": true,
      "ldap-auth": true,
      "datadog": true,
      "tcp-log": true,
      "zipkin": true,
      "post-function": true,
      "request-size-limiting": true,
      "bot-detection": true,
      "syslog": true,
      "loggly": true,
      "azure-functions": true,
      "udp-log": true,
      "response-ratelimiting": true,
      "aws-lambda": true,
      "statsd": true,
      "prometheus": true,
      "request-termination": true
    }
  },
  "tagline": "Welcome to kong",
  "configuration": {
    "plugins": [
      "bundled"
    ],
    "admin_ssl_enabled": true,
    "lua_ssl_verify_depth": 1,
    "trusted_ips": [
      "0.0.0.0/0"
    ],
    "prefix": "/usr/local/kong",
    "loaded_plugins": {
      "response-transformer": true,
      "request-termination": true,
      "prometheus": true,
      "ip-restriction": true,
      "pre-function": true,
      "jwt": true,
      "cors": true,
      "statsd": true,
      "basic-auth": true,
      "key-auth": true,
      "ldap-auth": true,
      "aws-lambda": true,
      "http-log": true,
      "response-ratelimiting": true,
      "hmac-auth": true,
      "request-size-limiting": true,
      "datadog": true,
      "tcp-log": true,
      "zipkin": true,
      "post-function": true,
      "bot-detection": true,
      "acl": true,
      "loggly": true,
      "syslog": true,
      "azure-functions": true,
      "udp-log": true,
      "file-log": true,
      "request-transformer": true,
      "correlation-id": true,
      "rate-limiting": true,
      "oauth2": true
    },
    "cassandra_username": "kong",
    "ssl_cert_key": "/usr/local/kong/ssl/kong-default.key",
    "admin_ssl_cert_key": "/usr/local/kong/ssl/admin-kong-default.key",
    "dns_resolver": {},
    "pg_user": "kong",
    "mem_cache_size": "128m",
    "cassandra_data_centers": [
      "dc1:2",
      "dc2:3"
    ],
    "nginx_admin_directives": {},
    "nginx_http_directives": [
      {
        "value": "/etc/metrics.conf",
        "name": "include"
      },
      {
        "value": "prometheus_metrics 5m",
        "name": "lua_shared_dict"
      }
    ],
    "pg_host": "127.0.0.1",
    "nginx_acc_logs": "/usr/local/kong/logs/access.log",
    "proxy_listen": [
      "0.0.0.0:8000 proxy_protocol",
      "0.0.0.0:8443 ssl"
    ],
    "client_ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
    "ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
    "dns_no_sync": true,
    "db_update_propagation": 2,
    "stream_listen": [
      "off"
    ],
    "nginx_err_logs": "/usr/local/kong/logs/error.log",
    "cassandra_port": 9042,
    "dns_order": [
      "A"
    ],
    "dns_error_ttl": 1,
    "headers": [
      "server_tokens",
      "latency_tokens"
    ],
    "cassandra_lb_policy": "RequestRoundRobin",
    "nginx_optimizations": true,
    "pg_timeout": 5000,
    "database": "cassandra",
    "pg_database": "kong",
    "nginx_worker_processes": "auto",
    "lua_package_cpath": "",
    "admin_acc_logs": "/usr/local/kong/logs/admin_access.log",
    "lua_package_path": "./?.lua;./?/init.lua;",
    "nginx_pid": "/usr/local/kong/pids/nginx.pid",
    "upstream_keepalive": 60,
    "admin_listen": [
      "0.0.0.0:8001",
      "0.0.0.0:8444 ssl"
    ],
    "proxy_access_log": "/dev/stdout",
    "admin_access_log": "/dev/stdout",
    "nginx_conf": "/usr/local/kong/nginx.conf",
    "ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
    "proxy_listeners": [
      {
        "transparent": false,
        "ssl": false,
        "ip": "0.0.0.0",
        "proxy_protocol": true,
        "port": 8000,
        "http2": false,
        "listener": "0.0.0.0:8000 proxy_protocol"
      },
      {
        "transparent": false,
        "ssl": true,
        "ip": "0.0.0.0",
        "proxy_protocol": false,
        "port": 8443,
        "http2": false,
        "listener": "0.0.0.0:8443 ssl"
      }
    ],
    "proxy_ssl_enabled": true,
    "dns_stale_ttl": 120,
    "cassandra_schema_consensus_timeout": 10000,
    "enabled_headers": {
      "latency_tokens": true,
      "X-Kong-Proxy-Latency": true,
      "Via": true,
      "server_tokens": true,
      "Server": true,
      "X-Kong-Upstream-Latency": true,
      "X-Kong-Upstream-Status": false
    },
    "admin_ssl_cert_default": "/usr/local/kong/ssl/admin-kong-default.crt",
    "origins": {},
    "db_resurrect_ttl": 30,
    "nginx_daemon": "off",
    "cassandra_consistency": "ONE",
    "client_max_body_size": "0",
    "admin_error_log": "/dev/stderr",
    "pg_ssl_verify": false,
    "dns_not_found_ttl": 1,
    "pg_ssl": false,
    "nginx_kong_stream_conf": "/usr/local/kong/nginx-kong-stream.conf",
    "error_default_type": "text/plain",
    "cassandra_repl_strategy": "SimpleStrategy",
    "ssl_ciphers": "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256",
    "log_level": "notice",
    "cassandra_contact_points": [
      "cassandra-0.cassandra.g-qa.svc.cluster.local"
    ],
    "ssl_cipher_suite": "modern",
    "nginx_kong_conf": "/usr/local/kong/nginx-kong.conf",
    "real_ip_header": "proxy_protocol",
    "dns_hostsfile": "/etc/hosts",
    "admin_listeners": [
      {
        "transparent": false,
        "ssl": false,
        "ip": "0.0.0.0",
        "proxy_protocol": false,
        "port": 8001,
        "http2": false,
        "listener": "0.0.0.0:8001"
      },
      {
        "transparent": false,
        "ssl": true,
        "ip": "0.0.0.0",
        "proxy_protocol": false,
        "port": 8444,
        "http2": false,
        "listener": "0.0.0.0:8444 ssl"
      }
    ],
    "proxy_error_log": "/dev/stderr",
    "ssl_cert": "/usr/local/kong/ssl/kong-default.crt",
    "cassandra_timeout": 5000,
    "admin_ssl_cert_key_default": "/usr/local/kong/ssl/admin-kong-default.key",
    "cassandra_ssl_verify": false,
    "db_cache_ttl": 0,
    "db_update_frequency": 5,
    "real_ip_recursive": "on",
    "cassandra_repl_factor": 2,
    "nginx_proxy_directives": {},
    "admin_ssl_cert": "/usr/local/kong/ssl/admin-kong-default.crt",
    "anonymous_reports": true,
    "kong_env": "/usr/local/kong/.kong_env",
    "stream_listeners": {},
    "pg_port": 5432,
    "nginx_http_include": "/etc/metrics.conf",
    "client_body_buffer_size": "8k",
    "ssl_preread_enabled": true,
    "client_ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
    "lua_socket_pool_size": 30,
    "cassandra_keyspace": "kong",
    "ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
    "cassandra_ssl": false,
    "client_ssl": false
  },
  "version": "1.0.2",
  "node_id": "bc751165-cd45-4cdc-a65a-b9ffd1669800",
  "lua_version": "LuaJIT 2.1.0-beta3",
  "prng_seeds": {
    "pid: 36": 247225225134,
    "pid: 35": 315016017618,
    "pid: 40": 118147131211,
    "pid: 1": 135211441071,
    "pid: 41": 146210822231,
    "pid: 37": 239236911497,
    "pid: 42": 475683198317,
    "pid: 39": 572546425514,
    "pid: 38": 160139154206
  },
  "timers": {
    "pending": 4,
    "running": 0
  },
  "hostname": "kong-rc-6886bb5c6-hwx2b"
}

kubernetes deployment manifest

apiVersion: v1
kind: Service
metadata:
  name: kong-proxy
spec:
  type: NodePort
  ports:
  - name: kong-proxy
    port: 8000
    protocol: TCP
  selector:
    app: kong

---
apiVersion: v1
kind: Service
metadata:
  name: kong-proxy-ssl
spec:
  type: NodePort
  ports:
  - name: kong-proxy-ssl
    port: 8443
    protocol: TCP
  selector:
    app: kong

---
apiVersion: v1
kind: Service
metadata:
  name: kong-admin
spec:
  type: NodePort
  ports:
  - name: kong-admin
    port: 8001
    protocol: TCP
  selector:
    app: kong

---
apiVersion: v1
kind: Service
metadata:
  name: kong-admin-ssl
spec:
  type: NodePort
  ports:
  - name: kong-admin-ssl
    port: 8444
    protocol: TCP
  selector:
    app: kong

---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: kong-rc
spec:
  replicas: 3
  template:
    metadata:
      labels:
        name: kong-rc
        app: kong
    spec:
      volumes:
      - name: kong-metrics
        configMap:
          name: kong-metrics
      containers:
      - name: kong
        volumeMounts:
        - name: kong-metrics
          mountPath: /etc/metrics.conf
          subPath: kong
        image: kong:1.0.2-centos
        env:
          - name: KONG_NGINX_HTTP_INCLUDE
            value: /etc/metrics.conf
          - name: KONG_TRUSTED_IPS
            value: 0.0.0.0/0
          - name: KONG_REAL_IP_HEADER
            value: proxy_protocol
          - name: KONG_REAL_IP_RECURSIVE
            value: "on"
          - name: KONG_PROXY_LISTEN
            value: 0.0.0.0:8000 proxy_protocol, 0.0.0.0:8443 ssl
          - name: KONG_ADMIN_LISTEN
            value: 0.0.0.0:8001, 0.0.0.0:8444 ssl
          - name: KONG_DATABASE
            value: cassandra
          - name: KONG_CASSANDRA_CONTACT_POINTS
            value: cassandra-0.cassandra.g-qa.svc.cluster.local
          - name: KONG_CASSANDRA_KEYSPACE
            value: kong
          - name: KONG_CASSANDRA_REPL_FACTOR
            value: "2"
          - name: KONG_DB_UPDATE_PROPAGATION
            value: "2"
          - name: KONG_PROXY_ACCESS_LOG
            value: /dev/stdout
          - name: KONG_ADMIN_ACCESS_LOG
            value: /dev/stdout
          - name: KONG_PROXY_ERROR_LOG
            value: /dev/stderr
          - name: KONG_ADMIN_ERROR_LOG
            value: /dev/stderr
          - name: KONG_DNS_STALE_TTL
            value: "120"
          - name: KONG_DNS_ORDER
            value: A
          - name: KONG_DNS_NOT_FOUND_TTL
            value: "1"
          - name: KONG_DNS_NO_SYNC
            value: "on"
          - name: KONG_PROXY_ACCESS_LOG
            value: "/dev/stdout"
          - name: KONG_ADMIN_ACCESS_LOG
            value: "/dev/stdout"
          - name: KONG_PROXY_ERROR_LOG
            value: "/dev/stderr"
          - name: KONG_ADMIN_ERROR_LOG
            value: "/dev/stderr"
        ports:
        - name: admin
          containerPort: 8001
          protocol: TCP
        - name: proxy
          containerPort: 8000
          protocol: TCP
        - name: proxy-ssl
          containerPort: 8443
          protocol: TCP
        - name: admin-ssl
          containerPort: 8444
          protocol: TCP
paleksic commented 5 years ago

Strange, upgraded to 1.0.3 and it works

Eslamanwar commented 5 years ago

@WywTed Could you please show us how you solved this issue as we are now facing the same problem ,thanks

huhu-sky commented 5 years ago

@WywTed Could you please show us how you solved this issue as we are now facing the same problem ,thanks

there is a pod named kong-proxy which expose the 80 port, we followed what @hbagdi said to add prometheus.conf into the container /etc/prometheus.conf and use it in the kong.conf . And then expose the 9542 port on the kong-proxy service. Well, at the same time , you need to config the prometheus annotation to the kong-proxy deploy. https://github.com/Kong/kong-plugin-prometheus/issues/54