search

CentaurusInfra / mizar

Mizar – Experimental, High Scale and High Performance Cloud Network https://mizar.readthedocs.io
https://mizar.readthedocs.io
GNU General Public License v2.0
112 stars 50 forks source link

CoreDNS pods fail to start in Mizar #504

Closed vinaykul closed 2 years ago

vinaykul commented 3 years ago

Repro steps:

  1. Ubuntu 18.04.05 + persistent net-device naming disabled + 5.6.0-rc2 kernel.
  2. Strart K8s 1.12.1 master via kubeadm
  3. Install Mizar v0.8 using https://github.com/CentaurusInfra/mizar/blob/dev-next/etc/deploy/deploy.mizar.yaml
  4. kubectl get no -owide to check NodeReady
  5. kubectl get po -owide --all-namespaces
root@ip-192-168-1-16:~# kubectl get no -owide
NAME              STATUS   ROLES                  AGE   VERSION   INTERNAL-IP    EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION   CONTAINER-RUNTIME
ip-192-168-1-16   Ready    control-plane,master   17m   v1.21.1   192.168.1.16   <none>        Ubuntu 18.04.5 LTS   5.6.0-rc2        docker://20.10.7
root@ip-192-168-1-16:~# kubectl get po --all-namespaces -owide
NAMESPACE     NAME                                      READY   STATUS    RESTARTS   AGE   IP             NODE              NOMINATED NODE   READINESS GATES
default       mizar-daemon-vjwsk                        1/1     Running   0          16m   192.168.1.16   ip-192-168-1-16   <none>           <none>
default       mizar-operator-79d4846f95-z27pj           1/1     Running   0          16m   192.168.1.16   ip-192-168-1-16   <none>           <none>
kube-system   coredns-558bd4d5db-5plz8                  0/1     Running   6          17m   20.0.0.30      ip-192-168-1-16   <none>           <none>
kube-system   coredns-558bd4d5db-dsjxj                  0/1     Running   6          17m   20.0.0.38      ip-192-168-1-16   <none>           <none>
kube-system   etcd-ip-192-168-1-16                      1/1     Running   0          17m   192.168.1.16   ip-192-168-1-16   <none>           <none>
kube-system   kube-apiserver-ip-192-168-1-16            1/1     Running   0          17m   192.168.1.16   ip-192-168-1-16   <none>           <none>
kube-system   kube-controller-manager-ip-192-168-1-16   1/1     Running   0          17m   192.168.1.16   ip-192-168-1-16   <none>           <none>
kube-system   kube-proxy-nxrq9                          1/1     Running   0          17m   192.168.1.16   ip-192-168-1-16   <none>           <none>
kube-system   kube-scheduler-ip-192-168-1-16            1/1     Running   0          17m   192.168.1.16   ip-192-168-1-16   <none>           <none>
root@ip-192-168-1-16:~#

root@ip-192-168-1-16:~# kubectl logs -n kube-system coredns-558bd4d5db-5plz8
E0610 02:00:00.864954       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:00.864954       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:00.864970       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: Get "https://10.96.0.1:443/api/v1/endpoints?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
.:53
[INFO] plugin/reload: Running configuration MD5 = db32ca3650231d74073ff4cf814959a7
CoreDNS-1.8.0
linux/amd64, go1.15.3, 054c9ae
E0610 02:00:03.936862       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:03.936862       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: Get "https://10.96.0.1:443/api/v1/endpoints?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:03.936944       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:07.008875       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:07.008875       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: Get "https://10.96.0.1:443/api/v1/endpoints?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:07.008875       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
[ERROR] plugin/errors: 2 6919126170054326117.2228872859192345676. HINFO: read udp 20.0.0.30:37769->192.168.0.2:53: i/o timeout
[ERROR] plugin/errors: 2 6919126170054326117.2228872859192345676. HINFO: read udp 20.0.0.30:59473->192.168.0.2:53: i/o timeout
[ERROR] plugin/errors: 2 6919126170054326117.2228872859192345676. HINFO: read udp 20.0.0.30:39627->192.168.0.2:53: i/o timeout
E0610 02:00:13.184862       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:13.184862       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: Get "https://10.96.0.1:443/api/v1/endpoints?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:13.184865       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
[ERROR] plugin/errors: 2 6919126170054326117.2228872859192345676. HINFO: read udp 20.0.0.30:40734->192.168.0.2:53: i/o timeout
[ERROR] plugin/errors: 2 6919126170054326117.2228872859192345676. HINFO: read udp 20.0.0.30:52706->192.168.0.2:53: i/o timeout
[ERROR] plugin/errors: 2 6919126170054326117.2228872859192345676. HINFO: read udp 20.0.0.30:36185->192.168.0.2:53: i/o timeout
[ERROR] plugin/errors: 2 6919126170054326117.2228872859192345676. HINFO: read udp 20.0.0.30:41847->192.168.0.2:53: i/o timeout
E0610 02:00:23.808856       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
E0610 02:00:23.808857       1 reflector.go:127] pkg/mod/k8s.io/client-go@v0.19.2/tools/cache/reflector.go:156: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: Get "https://10.96.0.1:443/api/v1/endpoints?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: connect: no route to host
[ERROR] plugin/errors: 2 6919126170054326117.2228872859192345676. HINFO: read udp 20.0.0.30:53113->192.168.0.2:53: i/o timeout
vinaykul commented 2 years ago

This is fixed.