search

rancher / rke

Rancher Kubernetes Engine (RKE), an extremely simple, lightning fast Kubernetes distribution that runs entirely within containers.
Apache License 2.0
3.22k stars 582 forks source link

x509: cannot validate certificate for x because it doesn't contain any IP SANs seen when using custom certificates #2216

Closed pengmingming closed 4 years ago

pengmingming commented 4 years ago

RKE version:

INFO[0000] Running RKE version: v1.1.4

Docker version: (docker version,docker info preferred)

Client:
 Debug Mode: false

Server:
 Containers: 33
  Running: 21
  Paused: 0
  Stopped: 12
 Images: 73
 Server Version: 19.03.12
 Storage Driver: overlay2
  Backing Filesystem: extfs
  Supports d_type: true
  Native Overlay Diff: true
 Logging Driver: json-file
 Cgroup Driver: cgroupfs
 Plugins:
  Volume: local
  Network: bridge host ipvlan macvlan null overlay
  Log: awslogs fluentd gcplogs gelf journald json-file local logentries splunk syslog
 Swarm: inactive
 Runtimes: runc
 Default Runtime: runc
 Init Binary: docker-init
 containerd version: 7ad184331fa3e55e52b890ea95e65ba581ae3429
 runc version: dc9208a3303feef5b3839f4323d9beb36df0a9dd
 init version: fec3683
 Security Options:
  apparmor
  seccomp
   Profile: default
 Kernel Version: 5.4.0-42-generic
 Operating System: Ubuntu 20.04 LTS
 OSType: linux
 Architecture: x86_64
 CPUs: 8
 Total Memory: 31.15GiB
 Name: deepxi-OptiPlex-7070
 ID: UGAG:VJGK:GDYP:ZEML:G4Q5:2DFA:G2Z7:5U75:J2CX:MKBX:X32M:GM62
 Docker Root Dir: /var/lib/docker
 Debug Mode: false
 Registry: https://index.docker.io/v1/
 Labels:
 Experimental: false
 Insecure Registries:
  10.57.21.5
  10.57.4.17:5000
  127.0.0.0/8
 Registry Mirrors:
  https://7tqpuxme.mirror.aliyuncs.com/
 Live Restore Enabled: false

Operating system and kernel: (cat /etc/os-release, uname -r preferred)

5.4.0-42-generic

Type/provider of hosts: (VirtualBox/Bare-metal/AWS/GCE/DO)

cluster.yml file:

nodes:
- address: 172.16.4.145
  port: "22"
  internal_address: ""
  role:
  - controlplane
  - worker
  - etcd
  hostname_override: ""
  user: deepxi
  docker_socket: /var/run/docker.sock
  ssh_key: ""
  ssh_key_path: /home/deepxi/.ssh/id_rsa
  ssh_cert: ""
  ssh_cert_path: ""
  labels: {}
  taints: []
services:
  etcd:
    image: ""
    extra_args: {}
    extra_binds: []
    extra_env: []
    external_urls: []
    ca_cert: ""
    cert: ""
    key: ""
    path: ""
    uid: 0
    gid: 0
    snapshot: null
    retention: ""
    creation: ""
    backup_config: null
  kube-api:
    image: ""
    extra_args: {} 
    extra_binds: []
    extra_env: []
    service_cluster_ip_range: 10.43.0.0/16
    service_node_port_range: ""
    pod_security_policy: false
    always_pull_images: false
    secrets_encryption_config: null
    audit_log: null
    admission_configuration: null
    event_rate_limit: null
  kube-controller:
    image: ""
    extra_args: 
      cluster-signing-cert-file: /etc/kubernetes/ssl/kube-ca.pem
      cluster-signing-key-file: /etc/kubernetes/ssl/kube-ca-key.pem 
    extra_binds: []
    extra_env: []
    cluster_cidr: 10.42.0.0/16
    service_cluster_ip_range: 10.43.0.0/16
  scheduler:
    image: ""
    extra_args: {}
    extra_binds: []
    extra_env: []
  kubelet:
    image: ""
    extra_args: {}
    extra_binds: []
    extra_env: []
    cluster_domain: cluster.local
    infra_container_image: ""
    cluster_dns_server: 10.43.0.10
    fail_swap_on: false
    generate_serving_certificate: false
  kubeproxy:
    image: ""
    extra_args: {}
    extra_binds: []
    extra_env: []
network:
  plugin: canal
  options: {}
  mtu: 0
  node_selector: {}
  update_strategy: null
authentication:
  strategy: x509
  sans: []
  webhook: null
addons: ""
addons_include: []
system_images:
  etcd: rancher/coreos-etcd:v3.4.3-rancher1
  alpine: rancher/rke-tools:v0.1.59
  nginx_proxy: rancher/rke-tools:v0.1.59
  cert_downloader: rancher/rke-tools:v0.1.59
  kubernetes_services_sidecar: rancher/rke-tools:v0.1.59
  kubedns: rancher/k8s-dns-kube-dns:1.15.2
  dnsmasq: rancher/k8s-dns-dnsmasq-nanny:1.15.2
  kubedns_sidecar: rancher/k8s-dns-sidecar:1.15.2
  kubedns_autoscaler: rancher/cluster-proportional-autoscaler:1.7.1
  coredns: rancher/coredns-coredns:1.6.9
  coredns_autoscaler: rancher/cluster-proportional-autoscaler:1.7.1
  nodelocal: rancher/k8s-dns-node-cache:1.15.7
  kubernetes: rancher/hyperkube:v1.18.6-rancher1
  flannel: rancher/coreos-flannel:v0.12.0
  flannel_cni: rancher/flannel-cni:v0.3.0-rancher6
  calico_node: rancher/calico-node:v3.13.4
  calico_cni: rancher/calico-cni:v3.13.4
  calico_controllers: rancher/calico-kube-controllers:v3.13.4
  calico_ctl: rancher/calico-ctl:v3.13.4
  calico_flexvol: rancher/calico-pod2daemon-flexvol:v3.13.4
  canal_node: rancher/calico-node:v3.13.4
  canal_cni: rancher/calico-cni:v3.13.4
  canal_flannel: rancher/coreos-flannel:v0.12.0
  canal_flexvol: rancher/calico-pod2daemon-flexvol:v3.13.4
  weave_node: weaveworks/weave-kube:2.6.4
  weave_cni: weaveworks/weave-npc:2.6.4
  pod_infra_container: rancher/pause:3.1
  ingress: rancher/nginx-ingress-controller:nginx-0.32.0-rancher1
  ingress_backend: rancher/nginx-ingress-controller-defaultbackend:1.5-rancher1
  metrics_server: rancher/metrics-server:v0.3.6
  windows_pod_infra_container: rancher/kubelet-pause:v0.1.4
ssh_key_path: /home/deepxi/.ssh/id_rsa
ssh_cert_path: ""
ssh_agent_auth: false
authorization:
  mode: rbac
  options: {}
ignore_docker_version: null
kubernetes_version: ""
private_registries: []
ingress:
  provider: ""
  options: {}
  node_selector: {}
  extra_args: {}
  dns_policy: ""
  extra_envs: []
  extra_volumes: []
  extra_volume_mounts: []
  update_strategy: null
cluster_name: ""
cloud_provider:
  name: ""
prefix_path: ""
addon_job_timeout: 0
bastion_host:
  address: ""
  port: ""
  user: ""
  ssh_key: ""
  ssh_key_path: ""
  ssh_cert: ""
  ssh_cert_path: ""
monitoring:
  provider: ""
  options: {}
  node_selector: {}
  update_strategy: null
  replicas: null
restore:
  restore: false
  snapshot_name: ""
dns: null

Steps to Reproduce: 1.rke cert generate-csr

2.openssl genrsa -out kube-ca-key.pem 2048

3.openssl req -x509 -new -nodes -key kube-ca-key.pem -days 10000 -out kube-ca.pem -subj "/CN=kube-ca"

4.openssl req -x509 -nodes -days 10000 -newkey rsa:2048 -keyout ./cluster_certs/kube-service-account-token-key.pem -out ./cluster_certs/kube-service-account-token.pem

5.openssl x509 -req -days 10000 -sha256 -CA ./cluster_certs/kube-ca.pem -CAkey ./cluster_certs/kube-ca-key.pem -CAcreateserial -in ./cluster_certs/kube-apiserver-csr.pem -out ./cluster_certs/kube-apiserver.pem (forearch)

6.tree cluster_certs/

cluster_certs/
├── kube-admin-csr.pem
├── kube-admin-key.pem
├── kube-admin.pem
├── kube-apiserver-csr.pem
├── kube-apiserver-key.pem
├── kube-apiserver.pem
├── kube-apiserver-proxy-client-csr.pem
├── kube-apiserver-proxy-client-key.pem
├── kube-apiserver-proxy-client.pem
├── kube-ca-key.pem
├── kube-ca.pem
├── kube-ca.srl
├── kube-controller-manager-csr.pem
├── kube-controller-manager-key.pem
├── kube-controller-manager.pem
├── kube-etcd-172-16-4-145-csr.pem
├── kube-etcd-172-16-4-145-key.pem
├── kube-etcd-172-16-4-145.pem
├── kube-node-csr.pem
├── kube-node-key.pem
├── kube-node.pem
├── kube-proxy-csr.pem
├── kube-proxy-key.pem
├── kube-proxy.pem
├── kube-scheduler-csr.pem
├── kube-scheduler-key.pem
├── kube-scheduler.pem
├── kube-service-account-token-key.pem
└── kube-service-account-token.pem

0 directories, 29 files

7.rke up --custom-certs

INFO[0000] Running RKE version: v1.1.4                  
INFO[0000] Initiating Kubernetes cluster                
INFO[0000] [dialer] Setup tunnel for host [172.16.4.145] 
INFO[0000] Checking if container [cluster-state-deployer] is running on host [172.16.4.145], try  #1  
WARN[0000] Failed to find RequestHeader CA certificate, using master CA certificate 
INFO[0000] Successfully Deployed state file at [./cluster.rkestate] 
INFO[0000] Building Kubernetes cluster                  
INFO[0000] [dialer] Setup tunnel for host [172.16.4.145] 
INFO[0000] [network] Deploying port listener containers 
INFO[0000] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0000] Starting container [rke-etcd-port-listener] on host [172.16.4.145], try  #1  
INFO[0001] [network] Successfully started [rke-etcd-port-listener] container on host [172.16.4.145] 
INFO[0001] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0001] Starting container [rke-cp-port-listener] on host [172.16.4.145], try  #1  
INFO[0001] [network] Successfully started [rke-cp-port-listener] container on host [172.16.4.145] 
INFO[0001] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0001] Starting container [rke-worker-port-listener] on host [172.16.4.145], try  #1  
INFO[0001] [network] Successfully started [rke-worker-port-listener] container on host [172.16.4.145] 
INFO[0001] [network] Port listener containers deployed successfully 
INFO[0001] [network] Running control plane -> etcd port checks 
INFO[0001] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0001] Starting container [rke-port-checker] on host [172.16.4.145], try  #1  
INFO[0001] [network] Successfully started [rke-port-checker] container on host [172.16.4.145] 
INFO[0002] Removing container [rke-port-checker] on host [172.16.4.145], try  #1  
INFO[0002] [network] Running control plane -> worker port checks 
INFO[0002] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0002] Starting container [rke-port-checker] on host [172.16.4.145], try  #1  
INFO[0002] [network] Successfully started [rke-port-checker] container on host [172.16.4.145] 
INFO[0002] Removing container [rke-port-checker] on host [172.16.4.145], try  #1  
INFO[0002] [network] Running workers -> control plane port checks 
INFO[0002] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0003] Starting container [rke-port-checker] on host [172.16.4.145], try  #1  
INFO[0003] [network] Successfully started [rke-port-checker] container on host [172.16.4.145] 
INFO[0003] Removing container [rke-port-checker] on host [172.16.4.145], try  #1  
INFO[0003] [network] Checking KubeAPI port Control Plane hosts 
INFO[0003] [network] Removing port listener containers  
INFO[0003] Removing container [rke-etcd-port-listener] on host [172.16.4.145], try  #1  
INFO[0003] [remove/rke-etcd-port-listener] Successfully removed container on host [172.16.4.145] 
INFO[0003] Removing container [rke-cp-port-listener] on host [172.16.4.145], try  #1  
INFO[0004] [remove/rke-cp-port-listener] Successfully removed container on host [172.16.4.145] 
INFO[0004] Removing container [rke-worker-port-listener] on host [172.16.4.145], try  #1  
INFO[0004] [remove/rke-worker-port-listener] Successfully removed container on host [172.16.4.145] 
INFO[0004] [network] Port listener containers removed successfully 
INFO[0004] [certificates] Deploying kubernetes certificates to Cluster nodes 
INFO[0004] Checking if container [cert-deployer] is running on host [172.16.4.145], try  #1  
INFO[0004] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0004] Starting container [cert-deployer] on host [172.16.4.145], try  #1  
INFO[0004] Checking if container [cert-deployer] is running on host [172.16.4.145], try  #1  
INFO[0009] Checking if container [cert-deployer] is running on host [172.16.4.145], try  #1  
INFO[0009] Removing container [cert-deployer] on host [172.16.4.145], try  #1  
INFO[0009] [reconcile] Rebuilding and updating local kube config 
INFO[0009] Successfully Deployed local admin kubeconfig at [./kube_config_cluster.yml] 
INFO[0009] [certificates] Successfully deployed kubernetes certificates to Cluster nodes 
INFO[0009] [file-deploy] Deploying file [/etc/kubernetes/audit-policy.yaml] to node [172.16.4.145] 
INFO[0009] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0010] Starting container [file-deployer] on host [172.16.4.145], try  #1  
INFO[0010] Successfully started [file-deployer] container on host [172.16.4.145] 
INFO[0010] Waiting for [file-deployer] container to exit on host [172.16.4.145] 
INFO[0010] Waiting for [file-deployer] container to exit on host [172.16.4.145] 
INFO[0010] Container [file-deployer] is still running on host [172.16.4.145] 
INFO[0011] Waiting for [file-deployer] container to exit on host [172.16.4.145] 
INFO[0011] Removing container [file-deployer] on host [172.16.4.145], try  #1  
INFO[0011] [remove/file-deployer] Successfully removed container on host [172.16.4.145] 
INFO[0011] [/etc/kubernetes/audit-policy.yaml] Successfully deployed audit policy file to Cluster control nodes 
INFO[0011] [reconcile] Reconciling cluster state        
INFO[0011] [reconcile] This is newly generated cluster  
INFO[0011] Pre-pulling kubernetes images                
INFO[0011] Image [rancher/hyperkube:v1.18.6-rancher1] exists on host [172.16.4.145] 
INFO[0011] Kubernetes images pulled successfully        
INFO[0011] [etcd] Building up etcd plane..              
INFO[0011] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0011] Starting container [etcd-fix-perm] on host [172.16.4.145], try  #1  
INFO[0011] Successfully started [etcd-fix-perm] container on host [172.16.4.145] 
INFO[0011] Waiting for [etcd-fix-perm] container to exit on host [172.16.4.145] 
INFO[0011] Waiting for [etcd-fix-perm] container to exit on host [172.16.4.145] 
INFO[0011] Container [etcd-fix-perm] is still running on host [172.16.4.145] 
INFO[0012] Waiting for [etcd-fix-perm] container to exit on host [172.16.4.145] 
INFO[0012] Removing container [etcd-fix-perm] on host [172.16.4.145], try  #1  
INFO[0012] [remove/etcd-fix-perm] Successfully removed container on host [172.16.4.145] 
INFO[0012] Image [rancher/coreos-etcd:v3.4.3-rancher1] exists on host [172.16.4.145] 
INFO[0012] Starting container [etcd] on host [172.16.4.145], try  #1  
INFO[0013] [etcd] Successfully started [etcd] container on host [172.16.4.145] 
INFO[0013] [etcd] Running rolling snapshot container [etcd-snapshot-once] on host [172.16.4.145] 
INFO[0013] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0013] Starting container [etcd-rolling-snapshots] on host [172.16.4.145], try  #1  
INFO[0013] [etcd] Successfully started [etcd-rolling-snapshots] container on host [172.16.4.145] 
INFO[0018] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0018] Starting container [rke-bundle-cert] on host [172.16.4.145], try  #1  
INFO[0018] [certificates] Successfully started [rke-bundle-cert] container on host [172.16.4.145] 
INFO[0018] Waiting for [rke-bundle-cert] container to exit on host [172.16.4.145] 
INFO[0018] Container [rke-bundle-cert] is still running on host [172.16.4.145] 
INFO[0019] Waiting for [rke-bundle-cert] container to exit on host [172.16.4.145] 
INFO[0019] [certificates] successfully saved certificate bundle [/opt/rke/etcd-snapshots//pki.bundle.tar.gz] on host [172.16.4.145] 
INFO[0019] Removing container [rke-bundle-cert] on host [172.16.4.145], try  #1  
INFO[0019] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0019] Starting container [rke-log-linker] on host [172.16.4.145], try  #1  
INFO[0020] [etcd] Successfully started [rke-log-linker] container on host [172.16.4.145] 
INFO[0020] Removing container [rke-log-linker] on host [172.16.4.145], try  #1  
INFO[0020] [remove/rke-log-linker] Successfully removed container on host [172.16.4.145] 
INFO[0020] [etcd] Successfully started etcd plane.. Checking etcd cluster health 
INFO[0020] [controlplane] Building up Controller Plane.. 
INFO[0020] Checking if container [service-sidekick] is running on host [172.16.4.145], try  #1  
INFO[0020] Image [rancher/rke-tools:v0.1.59] exists on host [172.16.4.145] 
INFO[0020] Image [rancher/hyperkube:v1.18.6-rancher1] exists on host [172.16.4.145] 
INFO[0020] Starting container [kube-apiserver] on host [172.16.4.145], try  #1  
INFO[0020] [controlplane] Successfully started [kube-apiserver] container on host [172.16.4.145] 
INFO[0020] [healthcheck] Start Healthcheck on service [kube-apiserver] on host [172.16.4.145] 
FATA[0231] [controlPlane] Failed to bring up Control Plane: [Failed to verify healthcheck: Failed to check https://localhost:6443/healthz for service [kube-apiserver] on host [172.16.4.145]: Get https://localhost:6443/healthz: EOF, log: W0824 03:20:46.585918       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...]

Results: 1.docker ps -a

CONTAINER ID        IMAGE                                 COMMAND                  CREATED             STATUS                  PORTS                                         NAMES
56465ec53a83        rancher/hyperkube:v1.18.6-rancher1    "/opt/rke-tools/entr…"   12 minutes ago      Up 9 seconds                                                          kube-apiserver
5e744d92e9bc        rancher/rke-tools:v0.1.59             "/bin/bash"              12 minutes ago      Created                                                               service-sidekick
addff4e5e2f1        rancher/rke-tools:v0.1.59             "/opt/rke-tools/rke-…"   12 minutes ago      Up 12 minutes                                                         etcd-rolling-snapshots
762ed294bef7        rancher/coreos-etcd:v3.4.3-rancher1   "/usr/local/bin/etcd…"   12 minutes ago      Up 12 minutes                                                         etcd
eeb1f76bbf95        b5af743e5984                          "/server"                3 days ago          Up 3 days                                                             k8s_default-http-backend_default-http-backend-598b7d7dbd-thmk5_ingress-nginx_3d5acd7a-2de8-4f81-838d-80f9fd5de08a_0
21a7c54d398c        rancher/metrics-server                "/metrics-server --k…"   3 days ago          Up 3 days                                                             k8s_metrics-server_metrics-server-697746ff48-kt2b7_kube-system_fe324c08-72e1-48d8-b9b7-f7a7d1b2edea_0
d3ad7965d198        eda78cfd6f9d                          "/usr/bin/dumb-init …"   3 days ago          Up 3 days                                                             k8s_nginx-ingress-controller_nginx-ingress-controller-8nxdv_ingress-nginx_173fa00e-4730-4bb4-a45c-a04c82c32047_0
ba6d3da362e0        rancher/pause:3.1                     "/pause"                 3 days ago          Up 3 days                                                             k8s_POD_nginx-ingress-controller-8nxdv_ingress-nginx_173fa00e-4730-4bb4-a45c-a04c82c32047_0
6db89357f977        rancher/pause:3.1                     "/pause"                 3 days ago          Up 3 days                                                             k8s_POD_default-http-backend-598b7d7dbd-thmk5_ingress-nginx_3d5acd7a-2de8-4f81-838d-80f9fd5de08a_0
db5a91032648        5a1e9f24e782                          "kubectl apply -f /e…"   3 days ago          Exited (0) 3 days ago                                                 k8s_rke-ingress-controller-pod_rke-ingress-controller-deploy-job-d4vk5_kube-system_a903292a-d92f-4657-9ada-41b7b1fa52ca_0
2f16fb749374        rancher/pause:3.1                     "/pause"                 3 days ago          Exited (0) 3 days ago                                                 k8s_POD_rke-ingress-controller-deploy-job-d4vk5_kube-system_a903292a-d92f-4657-9ada-41b7b1fa52ca_0
cea733b4f0e1        rancher/pause:3.1                     "/pause"                 3 days ago          Up 3 days                                                             k8s_POD_metrics-server-697746ff48-kt2b7_kube-system_fe324c08-72e1-48d8-b9b7-f7a7d1b2edea_0
1b66ef9c2ee7        5a1e9f24e782                          "kubectl apply -f /e…"   3 days ago          Exited (0) 3 days ago                                                 k8s_rke-metrics-addon-pod_rke-metrics-addon-deploy-job-7nvd9_kube-system_3817be1d-8499-49f1-bfad-18104a27186f_0
605debff99a6        rancher/pause:3.1                     "/pause"                 3 days ago          Exited (0) 3 days ago                                                 k8s_POD_rke-metrics-addon-deploy-job-7nvd9_kube-system_3817be1d-8499-49f1-bfad-18104a27186f_0
80a2faf88733        14afc47fd5af                          "/cluster-proportion…"   3 days ago          Up 3 days                                                             k8s_autoscaler_coredns-autoscaler-5dcd676cbd-79vz5_kube-system_53e28ff8-81fa-4fd3-a4a7-c103f5e426b1_0
993439a104e5        rancher/pause:3.1                     "/pause"                 3 days ago          Up 3 days                                                             k8s_POD_coredns-autoscaler-5dcd676cbd-79vz5_kube-system_53e28ff8-81fa-4fd3-a4a7-c103f5e426b1_0
a1259226d9fb        4e797b323460                          "/coredns -conf /etc…"   3 days ago          Up 3 days                                                             k8s_coredns_coredns-849545576b-25mks_kube-system_47545378-6101-4b0c-8cac-358458573dd3_0
b880f7102654        rancher/pause:3.1                     "/pause"                 3 days ago          Up 3 days                                                             k8s_POD_coredns-849545576b-25mks_kube-system_47545378-6101-4b0c-8cac-358458573dd3_0
bdee43900c07        5a1e9f24e782                          "kubectl apply -f /e…"   3 days ago          Exited (0) 3 days ago                                                 k8s_rke-coredns-addon-pod_rke-coredns-addon-deploy-job-rw28q_kube-system_3ee050b1-7c6b-421f-8ec9-93a24fb29c53_0
39feb0b49cf5        rancher/pause:3.1                     "/pause"                 3 days ago          Exited (0) 3 days ago                                                 k8s_POD_rke-coredns-addon-deploy-job-rw28q_kube-system_3ee050b1-7c6b-421f-8ec9-93a24fb29c53_0
a881bb89789b        4e9f801d2217                          "/opt/bin/flanneld -…"   3 days ago          Up 3 days                                                             k8s_kube-flannel_canal-qdt4p_kube-system_2a7322ad-e003-4459-a51a-74837d775553_0
e77e0564bdaf        c91d49e6f044                          "start_runit"            3 days ago          Up 3 days                                                             k8s_calico-node_canal-qdt4p_kube-system_2a7322ad-e003-4459-a51a-74837d775553_0
fb4544a2da15        c5dca18c0346                          "/usr/local/bin/flex…"   3 days ago          Exited (0) 3 days ago                                                 k8s_flexvol-driver_canal-qdt4p_kube-system_2a7322ad-e003-4459-a51a-74837d775553_0
8e7f5c161d1e        9e1176a74e85                          "/install-cni.sh"        3 days ago          Exited (0) 3 days ago                                                 k8s_install-cni_canal-qdt4p_kube-system_2a7322ad-e003-4459-a51a-74837d775553_0
6ad41dfd3c97        rancher/pause:3.1                     "/pause"                 3 days ago          Up 3 days                                                             k8s_POD_canal-qdt4p_kube-system_2a7322ad-e003-4459-a51a-74837d775553_0
23026371aa32        5a1e9f24e782                          "kubectl apply -f /e…"   3 days ago          Exited (0) 3 days ago                                                 k8s_rke-network-plugin-pod_rke-network-plugin-deploy-job-zcftf_kube-system_9fb192c9-7474-4a79-9ce5-583fc9e8b24f_0
a89e32b60cbc        rancher/pause:3.1                     "/pause"                 3 days ago          Exited (0) 3 days ago                                                 k8s_POD_rke-network-plugin-deploy-job-zcftf_kube-system_9fb192c9-7474-4a79-9ce5-583fc9e8b24f_0
28642f8641b0        aisuko/rancher:v2.4.51                "entrypoint.sh"          6 days ago          Up 4 days               0.0.0.0:8080->80/tcp, 0.0.0.0:8443->443/tcp   rancher
e7ed865142e3        rancher/rke-tools:v0.1.59             "/bin/bash"              6 days ago          Exited (0) 6 days ago                                                 cluster-state-deployer

2.docker logs kube-apiserver

+ grep -q cloud-provider=azure
+ echo kube-apiserver --cloud-provider= --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-node-port-range=30000-32767 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --requestheader-extra-headers-prefix=X-Remote-Extra- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize --advertise-address=172.16.4.145 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --profiling=false --anonymous-auth=false --service-account-lookup=true --audit-log-maxsize=100 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-prefix=/registry --etcd-servers=https://172.16.4.145:2379 --insecure-port=0 --allow-privileged=true --audit-log-maxbackup=10 --audit-log-format=json --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-group-headers=X-Remote-Group --secure-port=6443 --audit-log-maxage=30 --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --storage-backend=etcd3 --authorization-mode=Node,RBAC
+ '[' kube-apiserver = kubelet ']'
+ exec kube-apiserver --cloud-provider= --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-node-port-range=30000-32767 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --requestheader-extra-headers-prefix=X-Remote-Extra- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize --advertise-address=172.16.4.145 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --profiling=false --anonymous-auth=false --service-account-lookup=true --audit-log-maxsize=100 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-prefix=/registry --etcd-servers=https://172.16.4.145:2379 --insecure-port=0 --allow-privileged=true --audit-log-maxbackup=10 --audit-log-format=json --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-group-headers=X-Remote-Group --secure-port=6443 --audit-log-maxage=30 --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --storage-backend=etcd3 --authorization-mode=Node,RBAC
Flag --insecure-port has been deprecated, This flag will be removed in a future version.
I0824 03:44:26.092307       1 server.go:618] external host was not specified, using 172.16.4.145
I0824 03:44:26.092531       1 server.go:148] Version: v1.18.6
I0824 03:44:26.377339       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0824 03:44:26.377349       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I0824 03:44:26.377890       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0824 03:44:26.377896       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I0824 03:44:26.378531       1 client.go:361] parsed scheme: "endpoint"
I0824 03:44:26.378551       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://172.16.4.145:2379  <nil> 0 <nil>}]
W0824 03:44:26.380457       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
I0824 03:44:27.377396       1 client.go:361] parsed scheme: "endpoint"
I0824 03:44:27.377483       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://172.16.4.145:2379  <nil> 0 <nil>}]
W0824 03:44:27.384598       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:27.386723       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:28.391637       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:28.812683       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:30.124703       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:31.556319       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:32.576726       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:35.022204       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:37.146552       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:41.677594       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:44.060690       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
panic: context deadline exceeded

goroutine 1 [running]:
k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition.NewREST(0xc000756b60, 0x50e7a40, 0xc000169c20, 0xc0001479c8)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go:56 +0x3e7
k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver.completedConfig.New(0xc00091bd20, 0xc00091e888, 0x51a63e0, 0x77457d8, 0x10, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go:145 +0x14ef
k8s.io/kubernetes/cmd/kube-apiserver/app.createAPIExtensionsServer(0xc00091e880, 0x51a63e0, 0x77457d8, 0x0, 0x50e75a0, 0xc00070cd60)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/apiextensions.go:102 +0x59
k8s.io/kubernetes/cmd/kube-apiserver/app.CreateServerChain(0xc000a91080, 0xc0003e0de0, 0x455c0f4, 0xc, 0xc000735c48)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:181 +0x2b8
k8s.io/kubernetes/cmd/kube-apiserver/app.Run(0xc000a91080, 0xc0003e0de0, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:150 +0x101
k8s.io/kubernetes/cmd/kube-apiserver/app.NewAPIServerCommand.func1(0xc000910a00, 0xc000a91340, 0x0, 0x29, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:117 +0x104
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).execute(0xc000910a00, 0xc00004c2d0, 0x29, 0x2b, 0xc000910a00, 0xc00004c2d0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:826 +0x460
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc000910a00, 0x162e176afffa048f, 0x7727600, 0xc000078750)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:914 +0x2fb
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).Execute(...)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:864
main.main()
    _output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/apiserver.go:43 +0xcd
+ grep -q cloud-provider=azure
+ echo kube-apiserver --cloud-provider= --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-node-port-range=30000-32767 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --requestheader-extra-headers-prefix=X-Remote-Extra- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize --advertise-address=172.16.4.145 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --profiling=false --anonymous-auth=false --service-account-lookup=true --audit-log-maxsize=100 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-prefix=/registry --etcd-servers=https://172.16.4.145:2379 --insecure-port=0 --allow-privileged=true --audit-log-maxbackup=10 --audit-log-format=json --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-group-headers=X-Remote-Group --secure-port=6443 --audit-log-maxage=30 --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --storage-backend=etcd3 --authorization-mode=Node,RBAC
+ '[' kube-apiserver = kubelet ']'
+ exec kube-apiserver --cloud-provider= --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-node-port-range=30000-32767 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --requestheader-extra-headers-prefix=X-Remote-Extra- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize --advertise-address=172.16.4.145 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --profiling=false --anonymous-auth=false --service-account-lookup=true --audit-log-maxsize=100 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-prefix=/registry --etcd-servers=https://172.16.4.145:2379 --insecure-port=0 --allow-privileged=true --audit-log-maxbackup=10 --audit-log-format=json --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-group-headers=X-Remote-Group --secure-port=6443 --audit-log-maxage=30 --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --storage-backend=etcd3 --authorization-mode=Node,RBAC
Flag --insecure-port has been deprecated, This flag will be removed in a future version.
I0824 03:44:46.726909       1 server.go:618] external host was not specified, using 172.16.4.145
I0824 03:44:46.727124       1 server.go:148] Version: v1.18.6
I0824 03:44:47.098292       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0824 03:44:47.098301       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I0824 03:44:47.098805       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0824 03:44:47.098812       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I0824 03:44:47.100071       1 client.go:361] parsed scheme: "endpoint"
I0824 03:44:47.100163       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://172.16.4.145:2379  <nil> 0 <nil>}]
W0824 03:44:47.102395       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
I0824 03:44:48.098494       1 client.go:361] parsed scheme: "endpoint"
I0824 03:44:48.098573       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://172.16.4.145:2379  <nil> 0 <nil>}]
W0824 03:44:48.105714       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:48.107262       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:49.113166       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:49.962795       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:50.797107       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:52.836787       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:53.409699       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:56.843843       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:44:57.324548       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:03.227723       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:03.360798       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
panic: context deadline exceeded

goroutine 1 [running]:
k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition.NewREST(0xc0006d10a0, 0x50e7a40, 0xc000171560, 0xc000171788)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go:56 +0x3e7
k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver.completedConfig.New(0xc000ce6c60, 0xc00091dec8, 0x51a63e0, 0x77457d8, 0x10, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go:145 +0x14ef
k8s.io/kubernetes/cmd/kube-apiserver/app.createAPIExtensionsServer(0xc00091dec0, 0x51a63e0, 0x77457d8, 0x0, 0x50e75a0, 0xc000c0ed30)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/apiextensions.go:102 +0x59
k8s.io/kubernetes/cmd/kube-apiserver/app.CreateServerChain(0xc000a37340, 0xc0002de360, 0x455c0f4, 0xc, 0xc000af1c48)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:181 +0x2b8
k8s.io/kubernetes/cmd/kube-apiserver/app.Run(0xc000a37340, 0xc0002de360, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:150 +0x101
k8s.io/kubernetes/cmd/kube-apiserver/app.NewAPIServerCommand.func1(0xc000aa8000, 0xc00035e2c0, 0x0, 0x29, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:117 +0x104
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).execute(0xc000aa8000, 0xc0000de010, 0x29, 0x2b, 0xc000aa8000, 0xc0000de010)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:826 +0x460
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc000aa8000, 0x162e176fcddee333, 0x7727600, 0xc000078750)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:914 +0x2fb
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).Execute(...)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:864
main.main()
    _output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/apiserver.go:43 +0xcd
+ grep -q cloud-provider=azure
+ echo kube-apiserver --cloud-provider= --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-node-port-range=30000-32767 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --requestheader-extra-headers-prefix=X-Remote-Extra- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize --advertise-address=172.16.4.145 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --profiling=false --anonymous-auth=false --service-account-lookup=true --audit-log-maxsize=100 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-prefix=/registry --etcd-servers=https://172.16.4.145:2379 --insecure-port=0 --allow-privileged=true --audit-log-maxbackup=10 --audit-log-format=json --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-group-headers=X-Remote-Group --secure-port=6443 --audit-log-maxage=30 --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --storage-backend=etcd3 --authorization-mode=Node,RBAC
+ '[' kube-apiserver = kubelet ']'
+ exec kube-apiserver --cloud-provider= --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-node-port-range=30000-32767 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --requestheader-extra-headers-prefix=X-Remote-Extra- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize --advertise-address=172.16.4.145 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --profiling=false --anonymous-auth=false --service-account-lookup=true --audit-log-maxsize=100 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-prefix=/registry --etcd-servers=https://172.16.4.145:2379 --insecure-port=0 --allow-privileged=true --audit-log-maxbackup=10 --audit-log-format=json --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-group-headers=X-Remote-Group --secure-port=6443 --audit-log-maxage=30 --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --storage-backend=etcd3 --authorization-mode=Node,RBAC
Flag --insecure-port has been deprecated, This flag will be removed in a future version.
I0824 03:45:07.417225       1 server.go:618] external host was not specified, using 172.16.4.145
I0824 03:45:07.417401       1 server.go:148] Version: v1.18.6
I0824 03:45:07.589387       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0824 03:45:07.589400       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I0824 03:45:07.589960       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0824 03:45:07.589967       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I0824 03:45:07.590656       1 client.go:361] parsed scheme: "endpoint"
I0824 03:45:07.590674       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://172.16.4.145:2379  <nil> 0 <nil>}]
W0824 03:45:07.592556       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
I0824 03:45:08.588661       1 client.go:361] parsed scheme: "endpoint"
I0824 03:45:08.588737       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://172.16.4.145:2379  <nil> 0 <nil>}]
W0824 03:45:08.595718       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:08.597295       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:09.602863       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:10.289836       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:10.916783       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:12.929839       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:13.906769       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:17.298734       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:17.452013       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:24.468824       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:24.833718       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
panic: context deadline exceeded

goroutine 1 [running]:
k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition.NewREST(0xc00076caf0, 0x50e7a40, 0xc00029e900, 0xc000177548)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/registry/customresourcedefinition/etcd.go:56 +0x3e7
k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver.completedConfig.New(0xc000b430e0, 0xc000363488, 0x51a63e0, 0x77457d8, 0x10, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/k8s.io/apiextensions-apiserver/pkg/apiserver/apiserver.go:145 +0x14ef
k8s.io/kubernetes/cmd/kube-apiserver/app.createAPIExtensionsServer(0xc000363480, 0x51a63e0, 0x77457d8, 0x0, 0x50e75a0, 0xc000cb3d80)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/apiextensions.go:102 +0x59
k8s.io/kubernetes/cmd/kube-apiserver/app.CreateServerChain(0xc000b15080, 0xc0000ba3c0, 0x455c0f4, 0xc, 0xc000addc48)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:181 +0x2b8
k8s.io/kubernetes/cmd/kube-apiserver/app.Run(0xc000b15080, 0xc0000ba3c0, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:150 +0x101
k8s.io/kubernetes/cmd/kube-apiserver/app.NewAPIServerCommand.func1(0xc000b26280, 0xc0000e0840, 0x0, 0x29, 0x0, 0x0)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go:117 +0x104
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).execute(0xc000b26280, 0xc0000e0010, 0x29, 0x2b, 0xc000b26280, 0xc0000e0010)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:826 +0x460
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).ExecuteC(0xc000b26280, 0x162e17749f229019, 0x7727600, 0xc000078750)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:914 +0x2fb
k8s.io/kubernetes/vendor/github.com/spf13/cobra.(*Command).Execute(...)
    /workspace/anago-v1.18.6-rc.0.48+a9f7208b601483/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/vendor/github.com/spf13/cobra/command.go:864
main.main()
    _output/dockerized/go/src/k8s.io/kubernetes/cmd/kube-apiserver/apiserver.go:43 +0xcd
+ grep -q cloud-provider=azure
+ echo kube-apiserver --cloud-provider= --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-node-port-range=30000-32767 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --requestheader-extra-headers-prefix=X-Remote-Extra- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize --advertise-address=172.16.4.145 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --profiling=false --anonymous-auth=false --service-account-lookup=true --audit-log-maxsize=100 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-prefix=/registry --etcd-servers=https://172.16.4.145:2379 --insecure-port=0 --allow-privileged=true --audit-log-maxbackup=10 --audit-log-format=json --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-group-headers=X-Remote-Group --secure-port=6443 --audit-log-maxage=30 --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --storage-backend=etcd3 --authorization-mode=Node,RBAC
+ '[' kube-apiserver = kubelet ']'
+ exec kube-apiserver --cloud-provider= --kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem --service-node-port-range=30000-32767 --requestheader-username-headers=X-Remote-User --bind-address=0.0.0.0 --requestheader-extra-headers-prefix=X-Remote-Extra- --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,NodeRestriction,Priority,TaintNodesByCondition,PersistentVolumeClaimResize --advertise-address=172.16.4.145 --audit-policy-file=/etc/kubernetes/audit-policy.yaml --proxy-client-cert-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client.pem --tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem --profiling=false --anonymous-auth=false --service-account-lookup=true --audit-log-maxsize=100 --client-ca-file=/etc/kubernetes/ssl/kube-ca.pem --etcd-prefix=/registry --etcd-servers=https://172.16.4.145:2379 --insecure-port=0 --allow-privileged=true --audit-log-maxbackup=10 --audit-log-format=json --kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem --tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem --requestheader-group-headers=X-Remote-Group --secure-port=6443 --audit-log-maxage=30 --etcd-keyfile=/etc/kubernetes/ssl/kube-node-key.pem --runtime-config=authorization.k8s.io/v1beta1=true --audit-log-path=/var/log/kube-audit/audit-log.json --etcd-cafile=/etc/kubernetes/ssl/kube-ca.pem --etcd-certfile=/etc/kubernetes/ssl/kube-node.pem --requestheader-client-ca-file=/etc/kubernetes/ssl/kube-apiserver-requestheader-ca.pem --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 --requestheader-allowed-names=kube-apiserver-proxy-client --proxy-client-key-file=/etc/kubernetes/ssl/kube-apiserver-proxy-client-key.pem --service-account-key-file=/etc/kubernetes/ssl/kube-service-account-token-key.pem --service-cluster-ip-range=10.43.0.0/16 --storage-backend=etcd3 --authorization-mode=Node,RBAC
Flag --insecure-port has been deprecated, This flag will be removed in a future version.
I0824 03:45:27.955821       1 server.go:618] external host was not specified, using 172.16.4.145
I0824 03:45:27.955971       1 server.go:148] Version: v1.18.6
I0824 03:45:28.272329       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0824 03:45:28.272339       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I0824 03:45:28.272889       1 plugins.go:158] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0824 03:45:28.272895       1 plugins.go:161] Loaded 10 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,CertificateSubjectRestriction,ValidatingAdmissionWebhook,ResourceQuota.
I0824 03:45:28.273530       1 client.go:361] parsed scheme: "endpoint"
I0824 03:45:28.273546       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://172.16.4.145:2379  <nil> 0 <nil>}]
W0824 03:45:28.275493       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
I0824 03:45:29.273146       1 client.go:361] parsed scheme: "endpoint"
I0824 03:45:29.273408       1 endpoint.go:68] ccResolverWrapper: sending new addresses to cc: [{https://172.16.4.145:2379  <nil> 0 <nil>}]
W0824 03:45:29.282858       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...
W0824 03:45:29.283014       1 clientconn.go:1208] grpc: addrConn.createTransport failed to connect to {https://172.16.4.145:2379  <nil> 0 <nil>}. Err :connection error: desc = "transport: authentication handshake failed: x509: cannot validate certificate for 172.16.4.145 because it doesn't contain any IP SANs". Reconnecting...

3.certificate kube-ca.zip

pengmingming commented 4 years ago

@superseb Hi，Please help me！

superseb commented 4 years ago

I think the error is accurate, when signing the certificate, the SANs won't be used by default. You will need to configure it so that the certificate includes those, see https://gist.github.com/croxton/ebfb5f3ac143cd86542788f972434c96 and https://stackoverflow.com/questions/30977264/subject-alternative-name-not-present-in-certificate.

Let me know if that solves it for you.

In general, I think we should add a pre-check for certificates before deploying them in case of custom-certs to make sure the certificates are accurate to the cluster.yml before deploying.

stale[bot] commented 4 years ago

This issue/PR has been automatically marked as stale because it has not had activity (commit/comment/label) for 60 days. It will be closed in 14 days if no further activity occurs. Thank you for your contributions.