Kong / kong

🦍 The Cloud-Native API Gateway and AI Gateway.
https://konghq.com/install/#kong-community
Apache License 2.0
39.28k stars 4.82k forks source link

Aws lambda plugin throwing closed error #5246

Closed ashish2007csGit closed 3 years ago

ashish2007csGit commented 4 years ago

Summary

In Kong 1.3 we have exposed route to call Aws lambda from kong. Intermittent we observed while making the http post request is to connect aws lamda getting closed error.

IN kong error log can see ony response getting closed .

handler.lua:188 [aws-lambda] closed, client: 10.24.1.213, server: kong, request: "GET /v1/apis

below clock is throwing the error

local res, err = client:request { method = "POST", path = request.url, body = request.body, headers = request.headers }

SUMMARY_GOES_HERE getting

Steps To Reproduce

  1. create a route and apply lambda plugin
  2. hit 150 RPS
  3. getting closed error in log and getting 500 error from plugin

Additional Details & Logs

// 20191122140614 // http://cg-proxy-lb-poola-1965500442.us-east-1.elb.amazonaws.com:8000/

{ "plugins": { "enabled_in_cluster": [ "custom-aws-lambda", "rate-limiting", "correlation-id", "request-size-limiting", "kong-plugin-hmac-authentication", "custom-kong-websockets-metering", "custom-rate-limiting", "response-transformer" ], "available_on_server": { "correlation-id": true, "pre-function": true, "cors": true, "ldap-auth": true, "loggly": true, "hmac-auth": true, "custom-aws-lambda": true, "zipkin": true, "request-size-limiting": true, "azure-functions": true, "custom-rate-limiting": true, "request-transformer": true, "kong-plugin-jwt-claims-validate": true, "oauth2": true, "response-transformer": true, "ip-restriction": true, "statsd": true, "jwt": true, "proxy-cache": true, "kong-plugin-hmac-authentication": true, "basic-auth": true, "key-auth": true, "kong-plugin-hmac-auth": true, "stdout-log": true, "http-log": true, "custom-prometheus": true, "custom-kong-websockets-metering": true, "kong-plugin-template-transformer": true, "datadog": true, "tcp-log": true, "rate-limiting": true, "post-function": true, "prometheus": true, "acl": true, "kubernetes-sidecar-injector": true, "syslog": true, "file-log": true, "udp-log": true, "response-ratelimiting": true, "aws-lambda": true, "session": true, "bot-detection": true, "request-termination": true } }, "tagline": "Welcome to kong", "configuration": { "plugins": [ "bundled" ], "admin_ssl_enabled": false, "lua_ssl_verify_depth": 1, "trusted_ips": {

},
"prefix": "/usr/local/kong",
"loaded_plugins": {
  "correlation-id": true,
  "pre-function": true,
  "cors": true,
  "rate-limiting": true,
  "loggly": true,
  "hmac-auth": true,
  "custom-aws-lambda": true,
  "custom-kong-websockets-metering": true,
  "request-size-limiting": true,
  "azure-functions": true,
  "custom-rate-limiting": true,
  "request-transformer": true,
  "kong-plugin-jwt-claims-validate": true,
  "oauth2": true,
  "response-transformer": true,
  "ip-restriction": true,
  "statsd": true,
  "jwt": true,
  "stdout-log": true,
  "ldap-auth": true,
  "basic-auth": true,
  "key-auth": true,
  "kong-plugin-hmac-auth": true,
  "bot-detection": true,
  "http-log": true,
  "session": true,
  "zipkin": true,
  "response-ratelimiting": true,
  "datadog": true,
  "tcp-log": true,
  "proxy-cache": true,
  "post-function": true,
  "kong-plugin-hmac-authentication": true,
  "acl": true,
  "kubernetes-sidecar-injector": true,
  "syslog": true,
  "prometheus": true,
  "udp-log": true,
  "file-log": true,
  "aws-lambda": true,
  "kong-plugin-template-transformer": true,
  "custom-prometheus": true,
  "request-termination": true
},
"cassandra_username": "kong",
"ssl_cert_key": "/usr/local/kong/ssl/kong-default.key",
"dns_resolver": {

},
"pg_user": "user",
"mem_cache_size": "128m",
"cassandra_data_centers": [
  "dc1:2",
  "dc2:3"
],
"nginx_admin_directives": {

},
"nginx_http_upstream_directives": [
  {
    "value": "300",
    "name": "keepalive_timeout"
  },
  {
    "value": "3000000",
    "name": "keepalive_requests"
  },
  {
    "value": "300",
    "name": "keepalive"
  }
],
"nginx_http_directives": [
  {
    "value": "TLSv1.1 TLSv1.2 TLSv1.3",
    "name": "ssl_protocols"
  },
  {
    "value": "3000000",
    "name": "keepalive_requests"
  },
  {
    "value": "300",
    "name": "keepalive_timeout"
  },
  {
    "value": "prometheus_metrics 5m",
    "name": "lua_shared_dict"
  }
],
"pg_host": "cgdatabaseinstancekong.cpmv5lddb0as.us-east-1.rds.amazonaws.com",
"nginx_acc_logs": "/usr/local/kong/logs/access.log",
"pg_semaphore_timeout": 60000,
"proxy_listen": [
  "0.0.0.0:8000",
  "0.0.0.0:8443 ssl"
],
"client_ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"cassandra_ssl": false,
"dns_no_sync": false,
"db_update_propagation": 0,
"stream_listen": [
  "off"
],
"nginx_err_logs": "/usr/local/kong/logs/error.log",
"cassandra_port": 9042,
"dns_order": [
  "LAST",
  "SRV",
  "A",
  "CNAME"
],
"dns_error_ttl": 1,
"headers": [
  "off"
],
"cassandra_lb_policy": "RequestRoundRobin",
"nginx_optimizations": true,
"nginx_http_upstream_keepalive_timeout": "300",
"pg_timeout": 5000,
"nginx_http_upstream_keepalive_requests": "3000000",
"database": "postgres",
"proxy_access_log": "/dev/stdout custom-access-json-log",
"client_ssl": false,
"pg_database": "kong",
"nginx_worker_processes": "auto",
"ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"lua_package_cpath": "",
"nginx_conf": "/usr/local/kong/nginx.conf",
"admin_acc_logs": "/usr/local/kong/logs/admin_access.log",
"dns_stale_ttl": 4,
"client_max_body_size": "10M",
"lua_package_path": "./?.lua;./?/init.lua;",
"nginx_pid": "/usr/local/kong/pids/nginx.pid",
"upstream_keepalive": 60,
"origins": {

},
"nginx_kong_stream_conf": "/usr/local/kong/nginx-kong-stream.conf",
"router_consistency": "strict",
"pg_ssl_verify": false,
"admin_access_log": "logs/admin_access.log",
"error_default_type": "text/plain",
"pg_ssl": false,
"proxy_listeners": [
  {
    "listener": "0.0.0.0:8000",
    "proxy_protocol": false,
    "reuseport": false,
    "transparent": false,
    "ssl": false,
    "ip": "0.0.0.0",
    "deferred": false,
    "http2": false,
    "port": 8000,
    "bind": false
  },
  {
    "listener": "0.0.0.0:8443 ssl",
    "proxy_protocol": false,
    "reuseport": false,
    "transparent": false,
    "ssl": true,
    "ip": "0.0.0.0",
    "deferred": false,
    "http2": false,
    "port": 8443,
    "bind": false
  }
],
"proxy_ssl_enabled": true,
"nginx_http_upstream_keepalive": "300",
"db_cache_warmup_entities": [
  "services",
  "plugins"
],
"lua_socket_pool_size": 30,
"nginx_http_ssl_protocols": "TLSv1.1 TLSv1.2 TLSv1.3",
"nginx_proxy_directives": {

},
"db_resurrect_ttl": 30,
"nginx_stream_directives": {

},
"cassandra_consistency": "ONE",
"db_cache_ttl": 0,
"admin_error_log": "logs/error.log",
"admin_ssl_cert_default": "/usr/local/kong/ssl/admin-kong-default.crt",
"dns_not_found_ttl": 30,
"nginx_http_keepalive_requests": "3000000",
"ssl_cipher_suite": "modern",
"proxy_error_log": "/dev/stderr",
"cassandra_repl_strategy": "SimpleStrategy",
"cassandra_schema_consensus_timeout": 10000,
"pg_max_concurrent_queries": 0,
"nginx_http_keepalive_timeout": "300",
"nginx_kong_conf": "/usr/local/kong/nginx-kong.conf",
"real_ip_header": "X-Real-IP",
"dns_hostsfile": "/etc/hosts",
"admin_listeners": [
  {
    "listener": "0.0.0.0:8001",
    "proxy_protocol": false,
    "reuseport": false,
    "transparent": false,
    "ssl": false,
    "ip": "0.0.0.0",
    "deferred": false,
    "http2": false,
    "port": 8001,
    "bind": false
  }
],
"cassandra_contact_points": [
  "127.0.0.1"
],
"ssl_cert": "/usr/local/kong/ssl/kong-default.crt",
"cassandra_timeout": 5000,
"admin_ssl_cert_key_default": "/usr/local/kong/ssl/admin-kong-default.key",
"cassandra_ssl_verify": false,
"kong_env": "/usr/local/kong/.kong_env",
"log_level": "info",
"real_ip_recursive": "off",
"cassandra_repl_factor": 1,
"client_ssl_cert_key_default": "/usr/local/kong/ssl/kong-default.key",
"nginx_daemon": "off",
"anonymous_reports": true,
"nginx_sproxy_directives": {

},
"stream_listeners": {

},
"pg_port": 8000,
"db_update_frequency": 5,
"client_body_buffer_size": "9M",
"ssl_preread_enabled": true,
"ssl_cert_csr_default": "/usr/local/kong/ssl/kong-default.csr",
"pg_password": "******",
"cassandra_keyspace": "kong",
"ssl_cert_default": "/usr/local/kong/ssl/kong-default.crt",
"enabled_headers": {
  "latency_tokens": false,
  "X-Kong-Proxy-Latency": false,
  "Via": false,
  "server_tokens": false,
  "Server": false,
  "X-Kong-Upstream-Latency": false,
  "X-Kong-Upstream-Status": false
},
"admin_listen": [
  "0.0.0.0:8001"
]

}, "version": "1.3.0", "node_id": "3d525eee-dd15-4c07-b4d6-ffee9f0fa66b", "lua_version": "LuaJIT 2.1.0-beta3", "prng_seeds": { "pid: 26": 183261311391, "pid: 25": 222219522822, "pid: 23": 322241642317, "pid: 24": 217137105237, "pid: 1": 711182254395 }, "timers": { "pending": 145, "running": 1 }, "hostname": "c85549d6fc3c" }

p0pr0ck5 commented 4 years ago

This is likely from AWS closing the tcp connection but Kong not respecting (or realizing) this.

The solution would be to have the Kong AWS client code be more resilient in the face of transient failures like this.

gszr commented 3 years ago

Hey @ashish2007csGit,

Is this issue observed with newer versions of Kong? The plugin received lots of improvements since then.

Closing this for now, but please reopen if the issue persists in newer versions.