neolit123
commented
2 years ago I saw your comments earlier on another issue but i do not think you will get a better response on this new issue. Kubeadm as a client tries to get access the api server trough the lb but it fails. It could be a temp outage or a permanent problem.
To test if the lb works you can deploy a test Go compiled app that serves as a dummy api server and see if curl can reach it trough the lb.
This is not a kubeadm problem per se and the best place is to discuss on support channels.
/kind support
vupv
k8s-ci-robot
LonersBoy
cyford
Error execution phase upload-config/kubelet: Error writing Crisocket information for the control-plane node: timed out waiting for the condition init.log
root@master01:~# cat /etc/os-release NAME="Ubuntu" VERSION="20.04.3 LTS (Focal Fossa)" ID=ubuntu ID_LIKE=debian PRETTY_NAME="Ubuntu 20.04.3 LTS" VERSION_ID="20.04" HOME_URL="https://www.ubuntu.com/" SUPPORT_URL="https://help.ubuntu.com/" BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" VERSION_CODENAME=focal UBUNTU_CODENAME=focal
root@master01:~# kubeadm version kubeadm version: &version.Info{Major:"1", Minor:"22", GitVersion:"v1.22.4", GitCommit:"b695d79d4f967c403a96986f1750a35eb75e75f1", GitTreeState:"clean", BuildDate:"2021-11-17T15:47:19Z", GoVersion:"go1.16.10", Compiler:"gc", Platform:"linux/amd64"}
root@master01:~# etcd version {"level":"info","ts":1637853104.719803,"caller":"etcdmain/etcd.go:72","msg":"Running: ","args":["etcd","version"]} {"level":"warn","ts":1637853104.719844,"caller":"etcdmain/etcd.go:74","msg":"failed to verify flags","error":"'version' is not a valid flag"} root@master01:~#
I plan to setup with below boxes master01 --> 192.168.1.85 master02 --> 192.168.1.86 haproxy01, keepalived --> 192.168.1.87 , VIP --> 192.168.1.88 worker01 --> 192.168.1.90 worker02 --> 192.168.1.91
Install docker, kubelet kubeadm kubectl on all cluster node
HAProxy setup root@haproxy01:~# cat /etc/haproxy/haproxy.cfg global ... defaults ... frontend k8s_frontend bind 192.168.1.88:6443 option tcplog mode tcp default_backend k8s_backend
backend k8s_backend mode tcp balance roundrobin option tcp-check server master01 192.168.1.85:6443 check fall 3 rise 2 server master02 192.168.1.86:6443 check fall 3 rise 2
Keepalived setup
root@haproxy01:~# cat /etc/keepalived/keepalived.conf vrrp_script chk_haproxy { # Requires keepalived-1.1.13 script "killall -0 haproxy" # cheaper than pidof interval 2 # check every 2 seconds weight 2 # add 2 points of prio if OK }
vrrp_instance VI_1 { interface enp0s3 state MASTER virtual_router_id 51 priority 100 # 101 on master, 100 on backup virtual_ipaddress { 192.168.1.88 brd 192.168.1.255 dev enp0s3 label enp0s3:1 } track_script { chk_haproxy } }
Generating the TLS certificates
$ vim ca-config.json { "signing": { "default": { "expiry": "8760h" }, "profiles": { "kubernetes": { "usages": ["signing", "key encipherment", "server auth", "client auth"], "expiry": "8760h" } } } }
$ vim ca-csr.json { "CN": "Kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "IE", "L": "Cork", "O": "Kubernetes", "OU": "CA", "ST": "Cork Co." } ] }
$ cfssl gencert -initca ca-csr.json | cfssljson -bare ca $ vim kubernetes-csr.json { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "IE", "L": "Cork", "O": "Kubernetes", "OU": "Kubernetes", "ST": "Cork Co." } ] }
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -hostname=192.168.1.85,192.168.1.86,192.168.1.87,192.168.1.88,192.168.1.89,127.0.0.1,kubernetes.default -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
scp ca.pem kubernetes.pem kubernetes-key.pem root@192.168.1.85:/etc/etcd/ scp ca.pem kubernetes.pem kubernetes-key.pem root@192.168.1.86:/etc/etcd/
Etcd on two masters setup root@master01:~# cat /etc/systemd/system/etcd.service
[Unit] Description=etcd Documentation=https://github.com/coreos
[Service] ExecStart=/usr/local/bin/etcd --name 192.168.1.85 --cert-file=/etc/etcd/kubernetes.pem --key-file=/etc/etcd/kubernetes-key.pem --peer-cert-file=/etc/etcd/kubernetes.pem --peer-key-file=/etc/etcd/kubernetes-key.pem --trusted-ca-file=/etc/etcd/ca.pem --peer-trusted-ca-file=/etc/etcd/ca.pem --peer-client-cert-auth --client-cert-auth --initial-advertise-peer-urls https://192.168.1.85:2380 --listen-peer-urls https://192.168.1.85:2380 --listen-client-urls https://192.168.1.85:2379,http://127.0.0.1:2379 --advertise-client-urls https://192.168.1.85:2379 --initial-cluster-token etcd-cluster-1 --initial-cluster 192.168.1.85=https://192.168.1.85:2380,192.168.1.86=https://192.168.1.86:2380 --initial-cluster-state new --data-dir=/var/lib/etcd Restart=on-failure RestartSec=5
[Install] WantedBy=multi-user.target
root@master02:~# cat /etc/systemd/system/etcd.service [Unit] Description=etcd Documentation=https://github.com/coreos
[Service] ExecStart=/usr/local/bin/etcd --name 192.168.1.86 --cert-file=/etc/etcd/kubernetes.pem --key-file=/etc/etcd/kubernetes-key.pem --peer-cert-file=/etc/etcd/kubernetes.pem --peer-key-file=/etc/etcd/kubernetes-key.pem --trusted-ca-file=/etc/etcd/ca.pem --peer-trusted-ca-file=/etc/etcd/ca.pem --peer-client-cert-auth --client-cert-auth --initial-advertise-peer-urls https://192.168.1.86:2380 --listen-peer-urls https://192.168.1.86:2380 --listen-client-urls https://192.168.1.86:2379,http://127.0.0.1:2379 --advertise-client-urls https://192.168.1.86:2379 --initial-cluster-token etcd-cluster-0 --initial-cluster 192.168.1.85=https://192.168.1.85:2380,192.168.1.86=https://192.168.1.86:2380 --initial-cluster-state new --data-dir=/var/lib/etcd Restart=on-failure RestartSec=5
[Install] WantedBy=multi-user.target
root@master01:~# ETCDCTL_API=3 etcdctl member list 8a137d8e3cb900a, started, 192.168.1.86, https://192.168.1.86:2380, https://192.168.1.86:2379, false 75b9a29ae5a417ae, started, 192.168.1.85, https://192.168.1.85:2380, https://192.168.1.85:2379, false
Prepare cluster config file root@master01:~# cat cluster.yaml apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration etcd: external: caFile: /etc/etcd/ca.pem certFile: /etc/etcd/kubernetes.pem keyFile: /etc/etcd/kubernetes-key.pem endpoints:
kind: KubeletConfiguration apiVersion: kubelet.config.k8s.io/v1beta1 cgroupDriver: systemd root@master01:~#
root@master01:~# kubeadm init --config=cluster.yaml