bug: Can not ping k8s svc fqdn

[SakaZhang]

Current Behavior

First, logs below:

2023-02-27T16:44:46+08:00   error   apisix/route.go:118 failed to list routes: Get "http://apisix-admin.minum.svc.cluster.local:9180/apisix/admin/routes": dial tcp: lookup apisix-admin.minum.svc.cluster.local: i/o timeout
2023-02-27T16:44:46+08:00   error   apisix/cluster.go:222   failed to list routes in APISIX: Get "http://apisix-admin.minum.svc.cluster.local:9180/apisix/admin/routes": dial tcp: lookup apisix-admin.minum.svc.cluster.local: i/o timeout
2023-02-27T16:44:51+08:00   error   apisix/route.go:118 failed to list routes: Get "http://apisix-admin.minum.svc.cluster.local:9180/apisix/admin/routes": dial tcp: lookup apisix-admin.minum.svc.cluster.local: i/o timeout
2023-02-27T16:44:51+08:00   error   apisix/cluster.go:222   failed to list routes in APISIX: Get "http://apisix-admin.minum.svc.cluster.local:9180/apisix/admin/routes": dial tcp: lookup apisix-admin.minum.svc.cluster.local: i/o timeout

In apisix-ingress-controller :

/ingress-apisix # ping mysql-svc.minum.svc.cluster.local
ping: bad address 'mysql-svc.minum.svc.cluster.local'
/ingress-apisix # ping mysql-svc
PING mysql-svc (10.96.0.167): 56 data bytes
64 bytes from 10.96.0.167: seq=0 ttl=64 time=0.061 ms
64 bytes from 10.96.0.167: seq=1 ttl=64 time=0.054 ms

In another pod:

[root@nacos-0 nacos]# ping mysql-svc
PING mysql-svc.minum.svc.cluster.local (10.96.0.167) 56(84) bytes of data.
64 bytes from mysql-svc.minum.svc.cluster.local (10.96.0.167): icmp_seq=1 ttl=64 time=0.038 ms
^C
--- mysql-svc.minum.svc.cluster.local ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.038/0.038/0.038/0.000 ms
[root@nacos-0 nacos]# ping mysql-svc.minum.svc.cluster.local
PING mysql-svc.minum.svc.cluster.local (10.96.0.167) 56(84) bytes of data.
64 bytes from mysql-svc.minum.svc.cluster.local (10.96.0.167): icmp_seq=1 ttl=64 time=0.044 ms
64 bytes from mysql-svc.minum.svc.cluster.local (10.96.0.167): icmp_seq=2 ttl=64 time=0.041 ms

Expected Behavior

No timeout.

Error Logs

No response

Steps to Reproduce

1.Install apisix with helm. values:

global:
  imagePullSecrets: []
apisix:
  enabled: true
  enableIPv6: true
  enableServerTokens: true
  setIDFromPodUID: false
  customLuaSharedDicts: []
  luaModuleHook:
    enabled: false
    luaPath: ""
    hookPoint: ""
    configMapRef:
      name: ""
      mounts:
        - key: ""
          path: ""
  httpRouter: radixtree_uri
  enableCustomizedConfig: false
  customizedConfig: {}
  image:
    repository: apache/apisix
    pullPolicy: IfNotPresent
    tag: 3.1.0-debian
  kind: Deployment
  replicaCount: 1
  priorityClassName: ""
  podAnnotations: {}
  podSecurityContext: {}
  securityContext: {}
  podDisruptionBudget:
    enabled: false
    minAvailable: 90%
    maxUnavailable: 1
  resources: {}
  hostNetwork: false
  nodeSelector: {}
  tolerations: []
  affinity: {}
  timezone: "Asia/Shanghai"
  extraEnvVars: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
  create: false
  annotations: {}
  name: ""
rbac:
  create: false
deployment:
  mode: traditional
  role: "traditional"
  controlPlane:
    confServerPort: "9280"
    certsSecret: ""
    cert: ""
    certKey: ""
  dataPlane:
    controlPlane:
      host: []
      prefix: "/apisix"
      timeout: 30
  certs:
    certsSecret: ""
    cert: ""
    cert_key: ""
    mTLSCACertSecret: ""
    mTLSCACert: ""
gateway:
  type: NodePort
  externalTrafficPolicy: Cluster
  externalIPs: []
  http:
    enabled: true
    servicePort: 80
    containerPort: 9080
  tls:
    enabled: false
    servicePort: 443
    containerPort: 9443
    existingCASecret: ""
    certCAFilename: ""
    http2:
      enabled: true
    sslProtocols: "TLSv1.2 TLSv1.3"
  stream:
    enabled: false
    only: false
    tcp: []
    udp: []
  ingress:
    enabled: false
    annotations: {}
    hosts:
      - host: apisix.local
        paths: []
    tls: []
  labelsOverride: {}
admin:
  enabled: true
  type: ClusterIP
  externalIPs: []
  port: 9180
  servicePort: 9180
  cors: true
  credentials:
    admin: edd1c9f034335f136f87ad84b625c8f1
    viewer: 4054f7cf07e344346cd3f287985e76a2
  allow:
    ipList:
      - 127.0.0.1/24
  ingress:
    enabled: false
    annotations:
      {}
    hosts:
      - host: apisix-admin.local
        paths:
          - "/apisix"
    tls: []
nginx:
  workerRlimitNofile: "20480"
  workerConnections: "10620"
  workerProcesses: auto
  enableCPUAffinity: true
  envs: []
plugins:
  - api-breaker
  - authz-keycloak
  - basic-auth
  - batch-requests
  - consumer-restriction
  - cors
  - echo
  - fault-injection
  - file-logger
  - grpc-transcode
  - grpc-web
  - hmac-auth
  - http-logger
  - ip-restriction
  - ua-restriction
  - jwt-auth
  - kafka-logger
  - key-auth
  - limit-conn
  - limit-count
  - limit-req
  - node-status
  - openid-connect
  - authz-casbin
  - prometheus
  - proxy-cache
  - proxy-mirror
  - proxy-rewrite
  - redirect
  - referer-restriction
  - request-id
  - request-validation
  - response-rewrite
  - serverless-post-function
  - serverless-pre-function
  - sls-logger
  - syslog
  - tcp-logger
  - udp-logger
  - uri-blocker
  - wolf-rbac
  - zipkin
  - traffic-split
  - gzip
  - real-ip
  - ext-plugin-pre-req
  - ext-plugin-post-req
stream_plugins:
  - mqtt-proxy
  - ip-restriction
  - limit-conn
pluginAttrs: {}
extPlugin:
  enabled: false
  cmd: ["/path/to/apisix-plugin-runner/runner", "run"]
wasmPlugins:
  enabled: false
  plugins: []
customPlugins:
  enabled: false
  luaPath: "/opts/custom_plugins/?.lua"
  plugins:
    - name: ""
      attrs: {}
      configMap:
        name: ""
        mounts:
          - key: ""
            path: ""
          - key: ""
            path: ""
updateStrategy: {}
extraVolumes: []
extraVolumeMounts: []
extraInitContainers: []
discovery:
  enabled: false
  registry: {}
logs:
  enableAccessLog: true
  accessLog: "/dev/stdout"
  accessLogFormat: '$remote_addr - $remote_user [$time_local] $http_host \"$request\" $status $body_bytes_sent $request_time \"$http_referer\" \"$http_user_agent\" $upstream_addr $upstream_status $upstream_response_time \"$upstream_scheme://$upstream_host$upstream_uri\"'
  accessLogFormatEscape: default
  errorLog: "/dev/stderr"
  errorLogLevel: "warn"
dns:
  resolvers:
    - 127.0.0.1
    - 172.20.0.10
    - 114.114.114.114
    - 223.5.5.5
    - 1.1.1.1
    - 8.8.8.8
  validity: 30
  timeout: 5
initContainer:
  image: busybox
  tag: 1.28
autoscaling:
  enabled: false
  minReplicas: 1
  maxReplicas: 100
  targetCPUUtilizationPercentage: 80
  targetMemoryUtilizationPercentage: 80
configurationSnippet:
  main: |
  httpStart: |
  httpEnd: |
  httpSrv: |
  httpAdmin: |
  stream: |
serviceMonitor:
  enabled: false
  namespace: ""
  name: ""
  interval: 15s
  path: /apisix/prometheus/metrics
  metricPrefix: apisix_
  containerPort: 9091
  labels: {}
  annotations: {}
etcd:
  enabled: true
  host:
    - http://etcd.host:2379
  prefix: "/apisix"
  timeout: 30
  auth:
    rbac:
      create: false
      user: ""
      password: ""
    tls:
      enabled: false
      existingSecret: ""
      certFilename: ""
      certKeyFilename: ""
      verify: true
      sni: ""
  service:
    port: 2379
  replicaCount: 3
dashboard:
  enabled: true
ingress-controller:
  enabled: true
  config:
    apisix:
      adminAPIVersion: "v3"
      serviceNamespace: minum
vault:
  enabled: false
  host: ""
  timeout: 10
  token: ""
  prefix: ""

Environment

• Ingress: Version: 1.6.0 Git SHA: no-git-module Go Version: go1.19.4 Building OS/Arch: linux/amd64 Running OS/Arch: linux/amd64

• k8s: Server Version: version.Info{Major:"1", Minor:"25", GitVersion:"v1.25.0", GitCommit:"a866cbe2e5bbaa01cfd5e969aa3e033f3282a8a2", GitTreeState:"clean", BuildDate:"2022-08-23T17:38:15Z", GoVersion:"go1.19", Compiler:"gc", Platform:"linux/amd64"}

[SakaZhang]

see this: https://github.com/coredns/coredns/issues/3982

[SakaZhang]

check /etc/hosts when you are in cloud machine... It's not APISIX problem.