Open angrosz opened 4 years ago
another case when KONG_CASSANDRA_CONTACT_POINTS is one IPv6 address
Error:
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:107: [Cassandra error] all hosts tried for query failed. [2001:283:4000:2001::5158]: host seems unhealthy, considering it down (host or service not provided, or not known)
[root@bcmt-sandbox4-3c3w2e-2003-control-01 angrosz]# kubectl logs -f -n csfapiexp-ckng ckng-ckng-init-migrations-q566t
2020/04/01 10:39:16 Provided contact points: ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local
2020/04/01 10:39:16 CONTACT POINT: [ccas-ccas-apache.csfapiexp-ccas.svc.cluster.local]
2020/04/01 10:39:16 CONTACT IPs: [2001:283:4000:2001::5158]
2020/04/01 10:39:16 Resolved KONG_CASSANDRA_CONTACT_POINTS: 2001:283:4000:2001::5158
2020/04/01 10:39:16 Executing kong migrations bootstrap. It could take some time, so please be patient...
2020/04/01 10:39:16 [verbose] Kong: 1.1.1
2020/04/01 10:39:16 [debug] ngx_lua: 10013
2020/04/01 10:39:16 [debug] nginx: 1013006
2020/04/01 10:39:16 [debug] Lua: LuaJIT 2.1.0-beta3
2020/04/01 10:39:16 [verbose] no config file found at /etc/kong/kong.conf
2020/04/01 10:39:16 [verbose] no config file, skip loading
2020/04/01 10:39:16 [debug] reading environment variables
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_USERNAME ENV found with "suadmincass"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_PASSWORD ENV found with "******"
2020/04/01 10:39:16 [debug] KONG_DB_UPDATE_PROPAGATION ENV found with "1"
2020/04/01 10:39:16 [debug] KONG_DATABASE ENV found with "cassandra"
2020/04/01 10:39:16 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_NUMBER ENV found with "4"
2020/04/01 10:39:16 [debug] KONG_CLIENT_HEADER_BUFFER_SIZE ENV found with "1k"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_SCHEMA_CONSENSUS_TIMEOUT ENV found with "180000"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_CONTACT_POINTS ENV found with "2001:283:4000:2001::5158"
2020/04/01 10:39:16 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_SIZE ENV found with "8k"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_REPL_FACTOR ENV found with "1"
2020/04/01 10:39:16 [debug] KONG_NGINX_DAEMON ENV found with "off"
2020/04/01 10:39:16 [debug] KONG_ANONYMOUS_REPORTS ENV found with "off"
2020/04/01 10:39:16 [debug] KONG_CASSANDRA_CONSISTENCY ENV found with "QUORUM"
2020/04/01 10:39:16 [debug] admin_access_log = "logs/admin_access.log"
2020/04/01 10:39:16 [debug] admin_api_spec_listen = {"0.0.0.0:8080","0.0.0.0:9443 ssl"}
2020/04/01 10:39:16 [debug] admin_error_log = "logs/error.log"
2020/04/01 10:39:16 [debug] admin_listen = {"0.0.0.0:8001","0.0.0.0:8444 ssl"}
2020/04/01 10:39:16 [debug] anonymous_reports = false
2020/04/01 10:39:16 [debug] cache_negative_ttl_default = 60
2020/04/01 10:39:16 [debug] cache_negative_ttl_max = 300
2020/04/01 10:39:16 [debug] cache_negative_ttl_min = 1
2020/04/01 10:39:16 [debug] cache_ttl_default = 3600
2020/04/01 10:39:16 [debug] cache_ttl_max = 3600
2020/04/01 10:39:16 [debug] cache_ttl_min = 60
2020/04/01 10:39:16 [debug] cassandra_cluster_refresh_interval = 15
2020/04/01 10:39:16 [debug] cassandra_consistency = "QUORUM"
2020/04/01 10:39:16 [debug] cassandra_contact_points = {"2001:283:4000:2001::5158"}
2020/04/01 10:39:16 [debug] cassandra_data_centers = {"dc1:2","dc2:3"}
2020/04/01 10:39:16 [debug] cassandra_keyspace = "kong"
2020/04/01 10:39:16 [debug] cassandra_lb_policy = "RequestRoundRobin"
2020/04/01 10:39:16 [debug] cassandra_password = "******"
2020/04/01 10:39:16 [debug] cassandra_port = 9042
2020/04/01 10:39:16 [debug] cassandra_repl_factor = 1
2020/04/01 10:39:16 [debug] cassandra_repl_strategy = "SimpleStrategy"
2020/04/01 10:39:16 [debug] cassandra_schema_consensus_timeout = 180000
2020/04/01 10:39:16 [debug] cassandra_ssl = false
2020/04/01 10:39:16 [debug] cassandra_ssl_verify = false
2020/04/01 10:39:16 [debug] cassandra_timeout = 5000
2020/04/01 10:39:16 [debug] cassandra_username = "suadmincass"
2020/04/01 10:39:16 [debug] client_body_buffer_size = "8k"
2020/04/01 10:39:16 [debug] client_header_buffer_size = "1k"
2020/04/01 10:39:16 [debug] client_max_body_size = "0"
2020/04/01 10:39:16 [debug] client_ssl = false
2020/04/01 10:39:16 [debug] database = "cassandra"
2020/04/01 10:39:16 [debug] db_cache_ttl = 0
2020/04/01 10:39:16 [debug] db_resurrect_ttl = 30
2020/04/01 10:39:16 [debug] db_update_frequency = 5
2020/04/01 10:39:16 [debug] db_update_propagation = 1
2020/04/01 10:39:16 [debug] dns_error_ttl = 1
2020/04/01 10:39:16 [debug] dns_hostsfile = "/etc/hosts"
2020/04/01 10:39:16 [debug] dns_no_sync = false
2020/04/01 10:39:16 [debug] dns_not_found_ttl = 30
2020/04/01 10:39:16 [debug] dns_order = {"LAST","SRV","A","CNAME"}
2020/04/01 10:39:16 [debug] dns_resolver = {}
2020/04/01 10:39:16 [debug] dns_stale_ttl = 4
2020/04/01 10:39:16 [debug] error_default_type = "text/plain"
2020/04/01 10:39:16 [debug] headers = {"server_tokens","latency_tokens"}
2020/04/01 10:39:16 [debug] large_client_header_buffers_number = 4
Error:
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:107: [Cassandra error] all hosts tried for query failed. [2001:283:4000:2001::5158]: host seems unhealthy, considering it down (host or service not provided, or not known)
stack traceback:
[C]: in function 'assert'
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:107: in function 'cmd_exec'
/usr/local/share/lua/5.1/kong/cmd/init.lua:90: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:90>
[C]: in function 'xpcall'
/usr/local/share/lua/5.1/kong/cmd/init.lua:90: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:47>
/etc/kong/openresty_temp_script.lua:6: in function 'file_gen'
init_worker_by_lua:49: in function <init_worker_by_lua:47>
[C]: in function 'xpcall'
init_worker_by_lua:56: in function <init_worker_by_lua:54>
2020/04/01 10:39:16 [debug] large_client_header_buffers_size = "8k"
2020/04/01 10:39:16 [debug] log_anonymize = "OFF"
2020/04/01 10:39:16 [debug] log_level = "notice"
2020/04/01 10:39:16 [debug] lru_cache_size = 500000
2020/04/01 10:39:16 [debug] lua_package_cpath = ""
2020/04/01 10:39:16 [debug] lua_package_path = "./?.lua;./?/init.lua;"
2020/04/01 10:39:16 [debug] lua_socket_pool_size = 30
2020/04/01 10:39:16 [debug] lua_ssl_verify_depth = 1
2020/04/01 10:39:16 [debug] maria_database = "kong"
2020/04/01 10:39:16 [debug] maria_host = "127.0.0.1"
2020/04/01 10:39:16 [debug] maria_password = "******"
2020/04/01 10:39:16 [debug] maria_port = 3306
2020/04/01 10:39:16 [debug] maria_socket_keepalive_timeout = 60000
2020/04/01 10:39:16 [debug] maria_socket_pool_size = 20
2020/04/01 10:39:16 [debug] maria_ssl = false
2020/04/01 10:39:16 [debug] maria_ssl_verify = false
2020/04/01 10:39:16 [debug] maria_timeout = 10000
2020/04/01 10:39:16 [debug] maria_user = "kong"
2020/04/01 10:39:16 [debug] mem_cache_size = "128m"
2020/04/01 10:39:16 [debug] nginx_admin_directives = {}
2020/04/01 10:39:16 [debug] nginx_daemon = "off"
2020/04/01 10:39:16 [debug] nginx_http_directives = {}
2020/04/01 10:39:16 [debug] nginx_optimizations = true
2020/04/01 10:39:16 [debug] nginx_proxy_directives = {}
2020/04/01 10:39:16 [debug] nginx_sproxy_directives = {}
2020/04/01 10:39:16 [debug] nginx_stream_directives = {}
2020/04/01 10:39:16 [debug] nginx_user = "nobody nobody"
2020/04/01 10:39:16 [debug] nginx_worker_processes = "auto"
2020/04/01 10:39:16 [debug] origins = {}
2020/04/01 10:39:16 [debug] pg_database = "kong"
2020/04/01 10:39:16 [debug] pg_host = "127.0.0.1"
2020/04/01 10:39:16 [debug] pg_port = 5432
2020/04/01 10:39:16 [debug] pg_ssl = false
2020/04/01 10:39:16 [debug] pg_ssl_verify = false
2020/04/01 10:39:16 [debug] pg_timeout = 5000
2020/04/01 10:39:16 [debug] pg_user = "kong"
2020/04/01 10:39:16 [debug] plugins = {"bundled"}
2020/04/01 10:39:16 [debug] prefix = "/usr/local/kong/"
2020/04/01 10:39:16 [debug] proxy_access_log = "logs/access.log"
2020/04/01 10:39:16 [debug] proxy_error_log = "logs/error.log"
2020/04/01 10:39:16 [debug] proxy_listen = {"0.0.0.0:8000","0.0.0.0:8443 ssl"}
2020/04/01 10:39:16 [debug] real_ip_header = "X-Real-IP"
2020/04/01 10:39:16 [debug] real_ip_recursive = "off"
2020/04/01 10:39:16 [debug] ssl_cipher_suite = "modern"
2020/04/01 10:39:16 [debug] ssl_ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"
2020/04/01 10:39:16 [debug] stream_listen = {"off"}
2020/04/01 10:39:16 [debug] trusted_ips = {}
2020/04/01 10:39:16 [debug] upstream_keepalive = 60
2020/04/01 10:39:16 [debug] validate_certs_refresh_interval = 300
2020/04/01 10:39:16 [debug] validate_certs_warn_before = 7
2020/04/01 10:39:16 [verbose] prefix in use: /usr/local/kong
2020/04/01 10:39:16 [debug] resolved Cassandra contact point '2001:283:4000:2001::5158' to: [2001:283:4000:2001::5158]
2020/04/01 10:39:16 Executing kong pre-upgrade step.
Error: [Cassandra error] all hosts tried for query failed. [2001:283:4000:2001::5158]: host seems unhealthy, considering it down (host or service not provided, or not known) Run with --v (verbose) or --vv (debug) for more details
2020/04/01 10:39:16 Kong pre-upgrade step [kong su_check] failed.
another case when KONG_CASSANDRA_CONTACT_POINTS is set to 3 IPv6 addresses
Error:
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:34: [cassandra error] could not find coordinator: all hosts tried for query failed. 2001:283:4000:2001::5151: host seems unhealthy, considering it down (Address family for hostname not supported). 2001:283:4000:2001::3230: host seems unhealthy, considering it down (Address family for hostname not supported). 2001:283:4000:2001::3077: host seems unhealthy, considering it down (Address family for hostname not supported)
bash-4.2$ kong migrations up --vv
2020/03/31 20:31:40 [verbose] Kong: 0.13.1
2020/03/31 20:31:40 [debug] ngx_lua: 10010
2020/03/31 20:31:40 [debug] nginx: 1011002
2020/03/31 20:31:40 [debug] Lua: LuaJIT 2.1.0-beta3
2020/03/31 20:31:40 [verbose] no config file found at /etc/kong/kong.conf
2020/03/31 20:31:40 [verbose] no config file, skipping loading
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_USERNAME ENV found with "suadmincass"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_PASSWORD ENV found with "******"
2020/03/31 20:31:40 [debug] KONG_DB_UPDATE_PROPAGATION ENV found with "1"
2020/03/31 20:31:40 [debug] KONG_DATABASE ENV found with "cassandra"
2020/03/31 20:31:40 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_NUMBER ENV found with "4"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_SCHEMA_CONSENSUS_TIMEOUT ENV found with "180000"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_CONTACT_POINTS ENV found with "2001:283:4000:2001::5151,2001:283:4000:2001::3230,2001:283:4000:2001::3077"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_REPL_FACTOR ENV found with "3"
2020/03/31 20:31:40 [debug] KONG_CLIENT_HEADER_BUFFER_SIZE ENV found with "1k"
2020/03/31 20:31:40 [debug] KONG_CASSANDRA_CONSISTENCY ENV found with "QUORUM"
2020/03/31 20:31:40 [debug] KONG_NGINX_DAEMON ENV found with "off"
2020/03/31 20:31:40 [debug] KONG_LARGE_CLIENT_HEADER_BUFFERS_SIZE ENV found with "8k"
2020/03/31 20:31:40 [debug] KONG_ANONYMOUS_REPORTS ENV found with "off"
2020/03/31 20:31:40 [debug] admin_access_log = "logs/admin_access.log"
2020/03/31 20:31:40 [debug] admin_api_spec_listen = {"0.0.0.0:8080","0.0.0.0:9443 ssl"}
2020/03/31 20:31:40 [debug] admin_error_log = "logs/error.log"
2020/03/31 20:31:40 [debug] admin_listen = {"0.0.0.0:8001","0.0.0.0:8444 ssl"}
2020/03/31 20:31:40 [debug] anonymous_reports = false
2020/03/31 20:31:40 [debug] cache_negative_ttl_default = 60
2020/03/31 20:31:40 [debug] cache_negative_ttl_max = 300
2020/03/31 20:31:40 [debug] cache_negative_ttl_min = 1
2020/03/31 20:31:40 [debug] cache_ttl_default = 3600
2020/03/31 20:31:40 [debug] cache_ttl_max = 3600
2020/03/31 20:31:40 [debug] cache_ttl_min = 60
2020/03/31 20:31:40 [debug] cassandra_cluster_refresh_interval = 15
2020/03/31 20:31:40 [debug] cassandra_consistency = "QUORUM"
2020/03/31 20:31:40 [debug] cassandra_contact_points = {"2001:283:4000:2001::5151","2001:283:4000:2001::3230","2001:283:4000:2001::3077"}
2020/03/31 20:31:40 [debug] cassandra_data_centers = {"dc1:2","dc2:3"}
2020/03/31 20:31:40 [debug] cassandra_keyspace = "kong"
2020/03/31 20:31:40 [debug] cassandra_lb_policy = "RoundRobin"
2020/03/31 20:31:40 [debug] cassandra_password = "******"
2020/03/31 20:31:40 [debug] cassandra_port = 9042
2020/03/31 20:31:40 [debug] cassandra_repl_factor = 3
2020/03/31 20:31:40 [debug] cassandra_repl_strategy = "SimpleStrategy"
2020/03/31 20:31:40 [debug] cassandra_schema_consensus_timeout = 180000
2020/03/31 20:31:40 [debug] cassandra_ssl = false
2020/03/31 20:31:40 [debug] cassandra_ssl_verify = false
2020/03/31 20:31:40 [debug] cassandra_timeout = 5000
2020/03/31 20:31:40 [debug] cassandra_username = "suadmincass"
2020/03/31 20:31:40 [debug] client_body_buffer_size = "8k"
2020/03/31 20:31:40 [debug] client_header_buffer_size = "1k"
2020/03/31 20:31:40 [debug] client_max_body_size = "0"
2020/03/31 20:31:40 [debug] client_ssl = false
2020/03/31 20:31:40 [debug] custom_plugins = {}
2020/03/31 20:31:40 [debug] database = "cassandra"
2020/03/31 20:31:40 [debug] db_cache_ttl = 3600
2020/03/31 20:31:40 [debug] db_update_frequency = 5
2020/03/31 20:31:40 [debug] db_update_propagation = 1
2020/03/31 20:31:40 [debug] dns_error_ttl = 1
2020/03/31 20:31:40 [debug] dns_hostsfile = "/etc/hosts"
2020/03/31 20:31:40 [debug] dns_no_sync = false
2020/03/31 20:31:40 [debug] dns_not_found_ttl = 30
2020/03/31 20:31:40 [debug] dns_order = {"LAST","SRV","A","CNAME"}
2020/03/31 20:31:40 [debug] dns_resolver = {}
2020/03/31 20:31:40 [debug] dns_stale_ttl = 4
2020/03/31 20:31:40 [debug] error_default_type = "text/plain"
2020/03/31 20:31:40 [debug] large_client_header_buffers_number = 4
Error:
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:34: [cassandra error] could not find coordinator: all hosts tried for query failed. 2001:283:4000:2001::5151: host seems unhealthy, considering it down (Address family for hostname not supported). 2001:283:4000:2001::3230: host seems unhealthy, considering it down (Address family for hostname not supported). 2001:283:4000:2001::3077: host seems unhealthy, considering it down (Address family for hostname not supported)
stack traceback:
[C]: in function 'assert'
/usr/local/share/lua/5.1/kong/cmd/migrations.lua:34: in function 'cmd_exec'
/usr/local/share/lua/5.1/kong/cmd/init.lua:87: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:87>
[C]: in function 'xpcall'
/usr/local/share/lua/5.1/kong/cmd/init.lua:87: in function </usr/local/share/lua/5.1/kong/cmd/init.lua:44>
/usr/local/bin/kong:7: in function 'file_gen'
init_worker_by_lua:46: in function <init_worker_by_lua:44>
[C]: in function 'xpcall'
init_worker_by_lua:53: in function <init_worker_by_lua:51>
2020/03/31 20:31:40 [debug] large_client_header_buffers_size = "8k"
2020/03/31 20:31:40 [debug] latency_tokens = true
2020/03/31 20:31:40 [debug] log_level = "notice"
2020/03/31 20:31:40 [debug] lru_cache_size = 500000
2020/03/31 20:31:40 [debug] lua_package_cpath = ""
2020/03/31 20:31:40 [debug] lua_package_path = "./?.lua;./?/init.lua;"
2020/03/31 20:31:40 [debug] lua_socket_pool_size = 30
2020/03/31 20:31:40 [debug] lua_ssl_verify_depth = 1
2020/03/31 20:31:40 [debug] maria_database = "kong"
2020/03/31 20:31:40 [debug] maria_host = "127.0.0.1"
2020/03/31 20:31:40 [debug] maria_password = "******"
2020/03/31 20:31:40 [debug] maria_port = 3306
2020/03/31 20:31:40 [debug] maria_socket_keepalive_timeout = 60000
2020/03/31 20:31:40 [debug] maria_socket_pool_size = 20
2020/03/31 20:31:40 [debug] maria_ssl = false
2020/03/31 20:31:40 [debug] maria_ssl_verify = false
2020/03/31 20:31:40 [debug] maria_timeout = 3000
2020/03/31 20:31:40 [debug] maria_user = "kong"
2020/03/31 20:31:40 [debug] mem_cache_size = "128m"
2020/03/31 20:31:40 [debug] nginx_daemon = "off"
2020/03/31 20:31:40 [debug] nginx_optimizations = true
2020/03/31 20:31:40 [debug] nginx_user = "nobody nobody"
2020/03/31 20:31:40 [debug] nginx_worker_processes = "auto"
2020/03/31 20:31:40 [debug] pg_database = "kong"
2020/03/31 20:31:40 [debug] pg_host = "127.0.0.1"
2020/03/31 20:31:40 [debug] pg_port = 5432
2020/03/31 20:31:40 [debug] pg_ssl = false
2020/03/31 20:31:40 [debug] pg_ssl_verify = false
2020/03/31 20:31:40 [debug] pg_user = "kong"
2020/03/31 20:31:40 [debug] prefix = "/usr/local/kong/"
2020/03/31 20:31:40 [debug] proxy_access_log = "logs/access.log"
2020/03/31 20:31:40 [debug] proxy_error_log = "logs/error.log"
2020/03/31 20:31:40 [debug] proxy_listen = {"0.0.0.0:8000","0.0.0.0:8443 ssl"}
2020/03/31 20:31:40 [debug] real_ip_header = "X-Real-IP"
2020/03/31 20:31:40 [debug] real_ip_recursive = "off"
2020/03/31 20:31:40 [debug] server_tokens = true
2020/03/31 20:31:40 [debug] ssl_cipher_suite = "modern"
2020/03/31 20:31:40 [debug] ssl_ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256"
2020/03/31 20:31:40 [debug] trusted_ips = {}
2020/03/31 20:31:40 [debug] upstream_keepalive = 60
2020/03/31 20:31:40 [verbose] prefix in use: /usr/local/kong
2020/03/31 20:31:40 [verbose] running datastore migrations
2020/03/31 20:31:40 Kong migrations failed.
@thibaultcha I have Kong 1.1.1 installed in IPv6 only k8s cluster in a namespace ckng. I have also Cassandra installed (cassandra-3.11.6) in a separate namespace ccas. I cannot start kong because I'm receiving the following error:
Debug version:
Please note that getent can find the proper IP for cassandra service name:
I cannot find any Kong documentation that desribes how to configure configure Kong 1.1.x to support IPv6 connections to Cassandra db. Please help.
I appreciate any suggestions, Ania