tokers
commented
3 years ago It seems APISIX doens't use the resolver in /etc/resolv.conf. Could you please paste the nginx.conf in your APISIX container?
Open wgy-home opened 3 years ago
tokers
commented
3 years ago It seems APISIX doens't use the resolver in /etc/resolv.conf. Could you please paste the nginx.conf in your APISIX container?
wgy-home
commented
3 years ago It seems APISIX doens't use the resolver in
/etc/resolv.conf. Could you please paste the nginx.conf in your APISIX container?
This configuration file ?
# cat /usr/local/apisix/conf/nginx.conf
# Configuration File - Nginx Server Configs
# This is a read-only file, do not try to modify it.
master_process on;
worker_processes auto;
worker_cpu_affinity auto;
# main configuration snippet starts
# main configuration snippet ends
error_log /dev/stderr warn;
pid logs/nginx.pid;
worker_rlimit_nofile 20480;
events {
accept_mutex off;
worker_connections 10620;
}
worker_rlimit_core 16G;
worker_shutdown_timeout 240s;
env APISIX_PROFILE;
http {
# put extra_lua_path in front of the builtin path
# so user can override the source code
lua_package_path "$prefix/deps/share/lua/5.1/?.lua;$prefix/deps/share/lua/5.1/?/init.lua;/usr/local/apisix/?.lua;/usr/local/apisix/?/init.lua;;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua;";
lua_package_cpath "$prefix/deps/lib64/lua/5.1/?.so;$prefix/deps/lib/lua/5.1/?.so;;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;";
lua_shared_dict internal_status 10m;
lua_shared_dict plugin-limit-req 10m;
lua_shared_dict plugin-limit-count 10m;
lua_shared_dict prometheus-metrics 10m;
lua_shared_dict plugin-limit-conn 10m;
lua_shared_dict upstream-healthcheck 10m;
lua_shared_dict worker-events 10m;
lua_shared_dict lrucache-lock 10m;
lua_shared_dict skywalking-tracing-buffer 100m;
lua_shared_dict balancer_ewma 10m;
lua_shared_dict balancer_ewma_locks 10m;
lua_shared_dict balancer_ewma_last_touched_at 10m;
lua_shared_dict plugin-limit-count-redis-cluster-slot-lock 1m;
lua_shared_dict tracing_buffer 10m; # plugin: skywalking
lua_shared_dict plugin-api-breaker 10m;
# for openid-connect and authz-keycloak plugin
lua_shared_dict discovery 1m; # cache for discovery metadata documents
# for openid-connect plugin
lua_shared_dict jwks 1m; # cache for JWKs
lua_shared_dict introspection 10m; # cache for JWT verification results
# for authz-keycloak
lua_shared_dict access_tokens 1m; # cache for service account access tokens
# for custom shared dict
# for proxy cache
proxy_cache_path /tmp/disk_cache_one levels=1:2 keys_zone=disk_cache_one:50m inactive=1d max_size=1G use_temp_path=off;
# for proxy cache
map $upstream_cache_zone $upstream_cache_zone_info {
disk_cache_one /tmp/disk_cache_one,1:2;
}
lua_ssl_verify_depth 5;
ssl_session_timeout 86400;
underscores_in_headers on;
lua_socket_log_errors off;
resolver 10.254.0.2 valid=30;
resolver_timeout 5;
lua_http10_buffering off;
lua_regex_match_limit 100000;
lua_regex_cache_max_entries 8192;
log_format main escape=default '$remote_addr - $remote_user [$time_local] $http_host "$request" $status $body_bytes_sent $request_time "$http_referer" "$http_user_agent" $upstream_addr $upstream_status $upstream_response_time "$upstream_scheme://$upstream_host$upstream_uri"';
access_log /dev/stdout main buffer=16384 flush=3;
open_file_cache max=1000 inactive=60;
client_max_body_size 0;
keepalive_timeout 60s;
client_header_timeout 60s;
client_body_timeout 60s;
send_timeout 10s;
server_tokens off;
include mime.types;
charset utf-8;
real_ip_header X-Real-IP;
set_real_ip_from 127.0.0.1;
set_real_ip_from unix:;
# http configuration snippet starts
# http configuration snippet ends
upstream apisix_backend {
server 0.0.0.1;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
keepalive 320;
}
init_by_lua_block {
require "resty.core"
apisix = require("apisix")
local dns_resolver = { "10.254.0.2", }
local args = {
dns_resolver = dns_resolver,
}
apisix.http_init(args)
}
init_worker_by_lua_block {
apisix.http_init_worker()
}
server {
listen 127.0.0.1:9090;
access_log off;
location / {
content_by_lua_block {
apisix.http_control()
}
}
}
server {
listen 9180;
log_not_found off;
# admin configuration snippet starts
# admin configuration snippet ends
set $upstream_scheme 'http';
set $upstream_host $http_host;
set $upstream_uri '';
location /apisix/admin {
allow 0.0.0.0/0;
deny all;
content_by_lua_block {
apisix.http_admin()
}
}
location /apisix/dashboard {
allow 0.0.0.0/0;
deny all;
alias dashboard/;
try_files $uri $uri/index.html /index.html =404;
}
location =/robots.txt {
return 200 'User-agent: *\nDisallow: /';
}
}
server {
listen 9080 reuseport;
# http server configuration snippet starts
# http server configuration snippet ends
set $upstream_scheme 'http';
set $upstream_host $http_host;
set $upstream_uri '';
set $ctx_ref '';
location = /apisix/nginx_status {
allow 127.0.0.0/24;
deny all;
access_log off;
stub_status;
}
location / {
set $upstream_mirror_host '';
set $upstream_upgrade '';
set $upstream_connection '';
access_by_lua_block {
apisix.http_access_phase()
}
proxy_http_version 1.1;
proxy_set_header Host $upstream_host;
proxy_set_header Upgrade $upstream_upgrade;
proxy_set_header Connection $upstream_connection;
proxy_set_header X-Real-IP $remote_addr;
proxy_pass_header Date;
proxy_ssl_name $host;
proxy_ssl_server_name on;
### the following x-forwarded-* headers is to send to upstream server
set $var_x_forwarded_for $remote_addr;
set $var_x_forwarded_proto $scheme;
set $var_x_forwarded_host $host;
set $var_x_forwarded_port $server_port;
if ($http_x_forwarded_for != "") {
set $var_x_forwarded_for "${http_x_forwarded_for}, ${realip_remote_addr}";
}
if ($http_x_forwarded_proto != "") {
set $var_x_forwarded_proto $http_x_forwarded_proto;
}
if ($http_x_forwarded_host != "") {
set $var_x_forwarded_host $http_x_forwarded_host;
}
if ($http_x_forwarded_port != "") {
set $var_x_forwarded_port $http_x_forwarded_port;
}
proxy_set_header X-Forwarded-For $var_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $var_x_forwarded_proto;
proxy_set_header X-Forwarded-Host $var_x_forwarded_host;
proxy_set_header X-Forwarded-Port $var_x_forwarded_port;
### the following configuration is to cache response content from upstream server
set $upstream_cache_zone off;
set $upstream_cache_key '';
set $upstream_cache_bypass '';
set $upstream_no_cache '';
set $upstream_hdr_expires '';
set $upstream_hdr_cache_control '';
proxy_cache $upstream_cache_zone;
proxy_cache_valid any 10s;
proxy_cache_min_uses 1;
proxy_cache_methods GET HEAD;
proxy_cache_lock_timeout 5s;
proxy_cache_use_stale off;
proxy_cache_key $upstream_cache_key;
proxy_no_cache $upstream_no_cache;
proxy_cache_bypass $upstream_cache_bypass;
proxy_hide_header Cache-Control;
proxy_hide_header Expires;
add_header Cache-Control $upstream_hdr_cache_control;
add_header Expires $upstream_hdr_expires;
add_header Apisix-Cache-Status $upstream_cache_status always;
proxy_pass $upstream_scheme://apisix_backend$upstream_uri;
mirror /proxy_mirror;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location @grpc_pass {
access_by_lua_block {
apisix.grpc_access_phase()
}
grpc_set_header Content-Type application/grpc;
grpc_socket_keepalive on;
grpc_pass $upstream_scheme://apisix_backend;
header_filter_by_lua_block {
apisix.http_header_filter_phase()
}
body_filter_by_lua_block {
apisix.http_body_filter_phase()
}
log_by_lua_block {
apisix.http_log_phase()
}
}
location = /proxy_mirror {
internal;
if ($upstream_mirror_host = "") {
return 200;
}
proxy_http_version 1.1;
proxy_set_header Host $upstream_host;
proxy_pass $upstream_mirror_host$request_uri;
}
}
# http end configuration snippet starts
# http end configuration snippet ends
}I tried to modify the DNS configuration, but it still doesn't work
init_by_lua_block {
require "resty.core"
apisix = require("apisix")
local dns_resolver = { "10.254.0.2", }
local args = {
dns_resolver = dns_resolver,
}
apisix.http_init(args)
}I am using Service IP now, it works normally.
# cat conf/config.yaml | grep -A 3 "etcd:" | grep -v '#'
--
etcd:
- "http://10.254.112.146:2379"way to install
git clone https://github.com/apache/apisix-helm-chart.git cd apisix-helm-chart/charts/apisix helm dependency update . helm install apisix . --set gateway.type=NodePort --set allow.ipList="{0.0.0.0/0}" --namespace ingress-apisixapisix image tag : apache/apisix:2.3-alpine
apisix error log
2021/03/29 08:51:52 [error] 49#49: *282942 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/plugin_metadata, context: ngx.timer 2021/03/29 08:51:53 [error] 73#73: *285218 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/consumers, context: ngx.timer 2021/03/29 08:51:53 [error] 51#51: *272261 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/proto, context: ngx.timer 2021/03/29 08:51:53 [error] 43#43: *281732 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/upstreams, context: ngx.timer 2021/03/29 08:51:53 [error] 45#45: *283202 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/services, context: ngx.timer 2021/03/29 08:51:53 [error] 65#65: *277404 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/proto, context: ngx.timer 2021/03/29 08:51:53 [error] 55#55: *284742 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/plugin_metadata, context: ngx.timer 2021/03/29 08:51:53 [error] 59#59: *285624 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/upstreams, context: ngx.timer 2021/03/29 08:51:53 [error] 70#70: *282262 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/global_rules, context: ngx.timer 2021/03/29 08:51:53 [error] 67#67: *279198 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/routes, context: ngx.timerBut it can be connected through ETCDCTL.
I have no name!@apisix-etcd-0:/opt/bitnami/etcd$ etcdctl --endpoints=apisix-etcd.ingress-apisix.svc.cluster.local:2379 put /test apisix OK I have no name!@apisix-etcd-0:/opt/bitnami/etcd$ etcdctl --endpoints=apisix-etcd.ingress-apisix.svc.cluster.local:2379 get /test apisix /test apisixHelp me, Thanks!
Is 10.254.0.2 is the one in /etc/resolv.conf?
wgy-home
commented
3 years ago way to install
git clone https://github.com/apache/apisix-helm-chart.git cd apisix-helm-chart/charts/apisix helm dependency update . helm install apisix . --set gateway.type=NodePort --set allow.ipList="{0.0.0.0/0}" --namespace ingress-apisixapisix image tag : apache/apisix:2.3-alpine
apisix error log
2021/03/29 08:51:52 [error] 49#49: *282942 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/plugin_metadata, context: ngx.timer 2021/03/29 08:51:53 [error] 73#73: *285218 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/consumers, context: ngx.timer 2021/03/29 08:51:53 [error] 51#51: *272261 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/proto, context: ngx.timer 2021/03/29 08:51:53 [error] 43#43: *281732 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/upstreams, context: ngx.timer 2021/03/29 08:51:53 [error] 45#45: *283202 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/services, context: ngx.timer 2021/03/29 08:51:53 [error] 65#65: *277404 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/proto, context: ngx.timer 2021/03/29 08:51:53 [error] 55#55: *284742 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/plugin_metadata, context: ngx.timer 2021/03/29 08:51:53 [error] 59#59: *285624 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/upstreams, context: ngx.timer 2021/03/29 08:51:53 [error] 70#70: *282262 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/global_rules, context: ngx.timer 2021/03/29 08:51:53 [error] 67#67: *279198 [lua] config_etcd.lua:530: failed to fetch data from etcd: apisix-etcd.ingress-apisix.svc.cluster.local could not be resolved (3: Host not found), etcd key: /apisix/routes, context: ngx.timerBut it can be connected through ETCDCTL.
I have no name!@apisix-etcd-0:/opt/bitnami/etcd$ etcdctl --endpoints=apisix-etcd.ingress-apisix.svc.cluster.local:2379 put /test apisix OK I have no name!@apisix-etcd-0:/opt/bitnami/etcd$ etcdctl --endpoints=apisix-etcd.ingress-apisix.svc.cluster.local:2379 get /test apisix /test apisixHelp me, Thanks!
Is
10.254.0.2is the one in/etc/resolv.conf?
yes
# cat /etc/resolv.conf
nameserver 10.254.0.2
search ingress-apisix.svc.cluster.local svc.cluster.local cluster.local
options ndots:5
···Does this problem can be reproduced by executing commands like dig or nslookup in the APISIX container?
wgy-home
commented
3 years ago Does this problem can be reproduced by executing commands like
digornslookupin the APISIX container?
ipv4 can be parsed normally,But I have disabled IPv6 in the cluster.
# nslookup apisix-etcd.ingress-apisix.svc.cluster.local
Server: 10.254.0.2
Address: 10.254.0.2:53
** server can't find apisix-etcd.ingress-apisix.svc.cluster.local: NXDOMAIN
Name: apisix-etcd.ingress-apisix.svc.cluster.local
Address: 10.254.112.146@wgy-home Does this problem exist continuously? In theory, APISIX should parse the etcd ClusterIP normally since it uses the correct nameserver.
wgy-home
commented
3 years ago @wgy-home Does this problem exist continuously? In theory, APISIX should parse the etcd ClusterIP normally since it uses the correct nameserver.
Yes, there have always been problems, but now I'm using Service IP. It can be repeated at any time！
mootezbessifi
commented
1 year ago @wgy-home any fix for this issue? since i am facing the same behavior with apisix v 2.3
anirudhr-sahaj
commented
1 year ago The dashboard is reading the ETCD Host from a configmap
kubectl get configmap
NAME DATA AGE
apache-apisix 1 4m15s
apache-apisix-dashboard 1 4m15s
kube-root-ca.crt 1 92mIf I manage to change the ETCD URL to the proper service URL in apache-apisix-dashboard configmap to http://apache-apisix-etcd.${YOUR_ETCD_SERVICE_NAMESPACE}:2379, I can get the dashboard up.
(Note: Hardcoding the IP is not a great idea - because the SVC type is clusterIP and changes on every deployment).
way to install
apisix image tag : apache/apisix:2.3-alpine
apisix error log
But it can be connected through ETCDCTL.
Help me, Thanks!