seongilp commented 6 years ago

deploy openstack.sh 실행하면 아래와 같이 에러가 나오네요..

[root@localhost taco-scripts]# ./04-deploy-openstack.sh
++ ip route get 8.8.8.8
++ awk '/8.8.8.8/ {print $7}'
+ EXIP=10.211.55.10
++ ip route get 8.8.8.8
++ awk '/8.8.8.8/ {print $3}'
+ EXGW=10.211.55.1
++ ip route get 8.8.8.8
++ awk '/8.8.8.8/ {print $5}'
+ EXNIC=eth0
++ cat /etc/os-release
++ grep PRETTY_NAME
++ sed 's/["]//g'
++ sed s/PRETTY_NAME=//g
++ awk '{print $1}'
+ OS_DISTRO=CentOS
+ '[' CentOS == Red ']'
+ '[' CentOS == CentOS ']'
++ ifconfig eth0
++ awk '/netmask /{ print $4;}'
+ MASK=255.255.255.0
++ ipcalc -n 10.211.55.10 255.255.255.0
++ cut -d= -f2
++ ipcalc -p 10.211.55.10 255.255.255.0
++ cut -d= -f2
+ CIDR=10.211.55.0/24
+ ARMADA_MANIFEST_DIR=/root/apps/armada-manifests
+ '[' -d /root/apps/armada-manifests ']'
+ rm -rf /root/apps/armada-manifests
+ cd /root/apps
+ git clone https://github.com/sktelecom-oslab/armada-manifests.git
Cloning into 'armada-manifests'...
remote: Counting objects: 23, done.
remote: Total 23 (delta 0), reused 0 (delta 0), pack-reused 23
Unpacking objects: 100% (23/23), done.
+ armada apply /root/apps/armada-manifests/taco-aio-manifest.yaml --set chart:ceph:values.network.public=10.211.55.0/24 --set chart:ceph:values.network.cluster=10.211.55.0/24 --set chart:neutron:values.network.interface.tunnel=eth0
2018-02-09 19:48:13.199 9042 CRITICAL armada [-] Unhandled error: FileNotFoundError: [Errno 2] No such file or directory: '/root/.kube/config'
2018-02-09 19:48:13.199 9042 ERROR armada Traceback (most recent call last):
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/armada/handlers/k8s.py", line 43, in __init__
2018-02-09 19:48:13.199 9042 ERROR armada     config.load_incluster_config()
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/kubernetes/config/incluster_config.py", line 93, in load_incluster_config
2018-02-09 19:48:13.199 9042 ERROR armada     cert_filename=SERVICE_CERT_FILENAME).load_and_set()
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/kubernetes/config/incluster_config.py", line 45, in load_and_set
2018-02-09 19:48:13.199 9042 ERROR armada     self._load_config()
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/kubernetes/config/incluster_config.py", line 51, in _load_config
2018-02-09 19:48:13.199 9042 ERROR armada     raise ConfigException("Service host/port is not set.")
2018-02-09 19:48:13.199 9042 ERROR armada kubernetes.config.config_exception.ConfigException: Service host/port is not set.
2018-02-09 19:48:13.199 9042 ERROR armada
2018-02-09 19:48:13.199 9042 ERROR armada During handling of the above exception, another exception occurred:
2018-02-09 19:48:13.199 9042 ERROR armada
2018-02-09 19:48:13.199 9042 ERROR armada Traceback (most recent call last):
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/bin/armada", line 10, in <module>
2018-02-09 19:48:13.199 9042 ERROR armada     sys.exit(main())
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/click/core.py", line 722, in __call__
2018-02-09 19:48:13.199 9042 ERROR armada     return self.main(*args, **kwargs)
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/click/core.py", line 697, in main
2018-02-09 19:48:13.199 9042 ERROR armada     rv = self.invoke(ctx)
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
2018-02-09 19:48:13.199 9042 ERROR armada     return _process_result(sub_ctx.command.invoke(sub_ctx))
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/click/core.py", line 895, in invoke
2018-02-09 19:48:13.199 9042 ERROR armada     return ctx.invoke(self.callback, **ctx.params)
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/click/core.py", line 535, in invoke
2018-02-09 19:48:13.199 9042 ERROR armada     return callback(*args, **kwargs)
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/click/decorators.py", line 17, in new_func
2018-02-09 19:48:13.199 9042 ERROR armada     return f(get_current_context(), *args, **kwargs)
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/armada/cli/apply.py", line 136, in apply_create
2018-02-09 19:48:13.199 9042 ERROR armada     target_manifest).invoke()
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/armada/cli/apply.py", line 216, in invoke
2018-02-09 19:48:13.199 9042 ERROR armada     target_manifest=self.target_manifest)
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/armada/handlers/armada.py", line 91, in __init__
2018-02-09 19:48:13.199 9042 ERROR armada     tiller_namespace=tiller_namespace)
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/armada/handlers/tiller.py", line 66, in __init__
2018-02-09 19:48:13.199 9042 ERROR armada     self.k8s = K8s()
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/armada/handlers/k8s.py", line 45, in __init__
2018-02-09 19:48:13.199 9042 ERROR armada     config.load_kube_config()
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/kubernetes/config/kube_config.py", line 358, in load_kube_config
2018-02-09 19:48:13.199 9042 ERROR armada     config_persister=config_persister)
2018-02-09 19:48:13.199 9042 ERROR armada   File "/usr/lib/python3.6/site-packages/kubernetes/config/kube_config.py", line 315, in _get_kube_config_loader_for_yaml_file
2018-02-09 19:48:13.199 9042 ERROR armada     with open(filename) as f:
2018-02-09 19:48:13.199 9042 ERROR armada FileNotFoundError: [Errno 2] No such file or directory: '/root/.kube/config'
2018-02-09 19:48:13.199 9042 ERROR armada

seungkyua commented 6 years ago

02-install-k8s.sh 를 실행하셨나요? kubernetes 설치가 안된듯 하네요.

seongilp commented 6 years ago

네, 순서대로 설치했습니다..

seungkyua commented 6 years ago

/root/.kube/config 파일은 kubernetes 가 잘 설치되면 자동으로 생성됩니다. 위 로그로 봐서는 kubernetes 설치가 제대로 안된거 같네요. 02-install-k8s.sh 가 제대로 완료되었는지 확인이 필요하고 아니면 02번을 다시 실행해 보시는 것이 좋겠네요.

seongilp commented 6 years ago

네 말씀하신대로 kubenetes 설치가 안된거 같은데 2번 스크립트를 계속 실행해도 특별한 에러는 없네요 스크립트 외에 사전에 설치해야 하거나 환경변수 설정해야 하는게 있지는 않은가요?

Saturday 10 February 2018 09:39:18 +0900 (0:00:00.023) 0:00:21.882 *****

TASK [kubernetes/preinstall : Install epel-release on RedHat/CentOS] *** Saturday 10 February 2018 09:39:19 +0900 (0:00:00.021) 0:00:21.904 * [WARNING]: Consider using yum, dnf or zypper module rather than running rpm

ok: [taco-aio]

TASK [kubernetes/preinstall : Install packages requirements] *** Saturday 10 February 2018 09:39:20 +0900 (0:00:01.155) 0:00:23.059 * changed: [taco-aio] => (item=[u'libselinux-python', u'device-mapper-libs', u'ebtables', u'python-httplib2', u'openssl', u'curl', u'rsync', u'bash-completion', u'socat', u'unzip'])

TASK [kubernetes/preinstall : Confirm selinux deployed] **** Saturday 10 February 2018 09:39:31 +0900 (0:00:11.516) 0:00:34.576 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Set selinux policy] ** Saturday 10 February 2018 09:39:31 +0900 (0:00:00.277) 0:00:34.854 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Disable IPv6 DNS lookup] ***** Saturday 10 February 2018 09:39:32 +0900 (0:00:00.431) 0:00:35.285 *

TASK [kubernetes/preinstall : set default sysctl file path] **** Saturday 10 February 2018 09:39:32 +0900 (0:00:00.017) 0:00:35.303 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Stat sysctl file configuration] ** Saturday 10 February 2018 09:39:32 +0900 (0:00:00.030) 0:00:35.334 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Change sysctl file path to link source if linked] **** Saturday 10 February 2018 09:39:32 +0900 (0:00:00.151) 0:00:35.485 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Enable ip forwarding] **** Saturday 10 February 2018 09:39:32 +0900 (0:00:00.038) 0:00:35.524 ***** changed: [taco-aio]

TASK [kubernetes/preinstall : Write cloud-config] ** Saturday 10 February 2018 09:39:32 +0900 (0:00:00.280) 0:00:35.805 *****

TASK [kubernetes/preinstall : Hosts | populate inventory into hosts file] ** Saturday 10 February 2018 09:39:32 +0900 (0:00:00.024) 0:00:35.829 ***** changed: [taco-aio]

TASK [kubernetes/preinstall : Hosts | populate kubernetes loadbalancer address into hosts file] **** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.333) 0:00:36.163 *****

TASK [kubernetes/preinstall : Hosts | localhost ipv4 in hosts file] **** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.020) 0:00:36.183 ***** changed: [taco-aio]

TASK [kubernetes/preinstall : Hosts | localhost ipv6 in hosts file] **** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.154) 0:00:36.338 ***** changed: [taco-aio]

TASK [kubernetes/preinstall : Configure dhclient to supersede search/domain/nameservers] *** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.142) 0:00:36.480 *

TASK [kubernetes/preinstall : Configue dhclient hooks for resolv.conf (non-RH)] **** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.028) 0:00:36.508 *****

TASK [kubernetes/preinstall : Configue dhclient hooks for resolv.conf (RH-only)] *** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.027) 0:00:36.536 *

TASK [kubernetes/preinstall : Remove kubespray specific config from dhclient config] *** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.030) 0:00:36.567 *

TASK [kubernetes/preinstall : Remove kubespray specific dhclient hook] ***** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.044) 0:00:36.611 * ok: [taco-aio]

TASK [kubernetes/preinstall : Check if we are running inside a Azure VM] *** Saturday 10 February 2018 09:39:33 +0900 (0:00:00.164) 0:00:36.775 * ok: [taco-aio]

TASK [kubernetes/preinstall : install growpart] **** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.133) 0:00:36.909 *****

TASK [kubernetes/preinstall : check if growpart needs to be run] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.022) 0:00:36.931 *

TASK [kubernetes/preinstall : check fs type] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.027) 0:00:36.958 *

TASK [kubernetes/preinstall : run growpart] **** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.026) 0:00:36.985 *****

TASK [kubernetes/preinstall : run xfs_growfs] ** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.033) 0:00:37.019 *****

TASK [docker/docker-storage : docker-storage-setup | install git and make] ***** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.025) 0:00:37.044 *

TASK [docker/docker-storage : docker-storage-setup | docker-storage-setup sysconfig template] ** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.031) 0:00:37.076 *****

TASK [docker/docker-storage : docker-storage-override-directory | docker service storage-setup override dir] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.025) 0:00:37.101 *

TASK [docker/docker-storage : docker-storage-override | docker service storage-setup override file] **** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.023) 0:00:37.125 *****

TASK [docker/docker-storage : docker-storage-setup | install and run container-storage-setup] ** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.029) 0:00:37.154 *****

TASK [docker : gather os specific variables] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.022) 0:00:37.177 * ok: [taco-aio] => (item=/root/apps/kubespray/roles/docker/vars/../vars/redhat.yml)

TASK [docker : set dns server for docker] ** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.057) 0:00:37.235 ***** ok: [taco-aio]

TASK [docker : set base docker dns facts] ** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.047) 0:00:37.283 ***** ok: [taco-aio]

TASK [docker : add upstream dns servers (only when dnsmasq is not used)] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.046) 0:00:37.329 * ok: [taco-aio]

TASK [docker : add global searchdomains] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.043) 0:00:37.373 * ok: [taco-aio]

TASK [docker : check system nameservers] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.043) 0:00:37.417 * ok: [taco-aio]

TASK [docker : check system search domains] **** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.153) 0:00:37.570 ***** ok: [taco-aio]

TASK [docker : add system nameservers to docker options] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.140) 0:00:37.710 * ok: [taco-aio]

TASK [docker : add system search domains to docker options] **** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.038) 0:00:37.749 ***** ok: [taco-aio]

TASK [docker : check number of nameservers] **** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.048) 0:00:37.798 *****

TASK [docker : rtrim number of nameservers to 3] *** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.033) 0:00:37.831 *

TASK [docker : check number of search domains] ***** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.035) 0:00:37.866 *

TASK [docker : check length of search domains] ***** Saturday 10 February 2018 09:39:34 +0900 (0:00:00.029) 0:00:37.896 *

TASK [docker : check for minimum kernel version] *** Saturday 10 February 2018 09:39:35 +0900 (0:00:00.027) 0:00:37.923 *

TASK [docker : ensure docker repository public key is installed] *** Saturday 10 February 2018 09:39:35 +0900 (0:00:00.027) 0:00:37.950 *

TASK [docker : ensure docker repository is enabled] **** Saturday 10 February 2018 09:39:35 +0900 (0:00:00.024) 0:00:37.975 *****

TASK [docker : Configure docker repository on RedHat/CentOS] *** Saturday 10 February 2018 09:39:35 +0900 (0:00:00.020) 0:00:37.995 * changed: [taco-aio]

TASK [docker : ensure docker packages are installed] *** Saturday 10 February 2018 09:39:35 +0900 (0:00:00.433) 0:00:38.429 * ok: [taco-aio] => (item={u'name': u'docker-engine-17.03.0.ce-1.el7.centos'})

TASK [docker : check minimum docker version for docker_dns mode. You need at least docker version >= 1.12 for resolvconf_mode=docker_dns] * Saturday 10 February 2018 09:39:38 +0900 (0:00:02.811) 0:00:41.241 *** ok: [taco-aio]

TASK [docker : Create docker service systemd directory if it doesn't exist] **** Saturday 10 February 2018 09:39:38 +0900 (0:00:00.190) 0:00:41.432 ***** changed: [taco-aio]

TASK [docker : Write docker proxy drop-in] ***** Saturday 10 February 2018 09:39:38 +0900 (0:00:00.138) 0:00:41.570 *

TASK [docker : get systemd version] **** Saturday 10 February 2018 09:39:38 +0900 (0:00:00.023) 0:00:41.593 ***** ok: [taco-aio]

TASK [docker : Write docker.service systemd file] ** Saturday 10 February 2018 09:39:38 +0900 (0:00:00.175) 0:00:41.769 ***** changed: [taco-aio]

TASK [docker : Write docker options systemd drop-in] *** Saturday 10 February 2018 09:39:39 +0900 (0:00:00.307) 0:00:42.077 * changed: [taco-aio]

TASK [docker : Write docker dns systemd drop-in] *** Saturday 10 February 2018 09:39:39 +0900 (0:00:00.302) 0:00:42.379 * changed: [taco-aio]

RUNNING HANDLER [docker : restart docker] ** Saturday 10 February 2018 09:39:39 +0900 (0:00:00.288) 0:00:42.667 ***** changed: [taco-aio]

RUNNING HANDLER [docker : Docker | reload systemd] ***** Saturday 10 February 2018 09:39:39 +0900 (0:00:00.200) 0:00:42.868 * changed: [taco-aio]

RUNNING HANDLER [docker : Docker | reload docker.socket] *** Saturday 10 February 2018 09:39:40 +0900 (0:00:00.201) 0:00:43.070 *

RUNNING HANDLER [docker : Docker | reload docker] ** Saturday 10 February 2018 09:39:40 +0900 (0:00:00.021) 0:00:43.091 ***** fatal: [taco-aio]: FAILED! => {"changed": false, "msg": "Unable to restart service docker: Job for docker.service failed because the control process exited with error code. See \"systemctl status docker.service\" and \"journalctl -xe\" for details.\n"}

RUNNING HANDLER [docker : Docker | pause while Docker restarts] **** Saturday 10 February 2018 09:39:41 +0900 (0:00:01.368) 0:00:44.460 *****

RUNNING HANDLER [docker : Docker | wait for docker] **** Saturday 10 February 2018 09:39:41 +0900 (0:00:00.015) 0:00:44.475 ***** to retry, use: --limit @/root/apps/kubespray/cluster.retry

PLAY RECAP ***** localhost : ok=2 changed=0 unreachable=0 failed=0 taco-aio : ok=69 changed=17 unreachable=0 failed=1

Saturday 10 February 2018 09:39:41 +0900 (0:00:00.001) 0:00:44.477 *****

kubernetes/preinstall : Update package management cache (YUM) ---------------------------------------------------------------------- 13.72s kubernetes/preinstall : Install packages requirements ------------------------------------------------------------------------------ 11.52s docker : ensure docker packages are installed --------------------------------------------------------------------------------------- 2.81s docker : Docker | reload docker ----------------------------------------------------------------------------------------------------- 1.37s kubernetes/preinstall : Install epel-release on RedHat/CentOS ----------------------------------------------------------------------- 1.16s bootstrap-os : Assign inventory name to unconfigured hostnames (non-CoreOS) --------------------------------------------------------- 0.55s bootstrap-os : Gather nodes hostnames ----------------------------------------------------------------------------------------------- 0.55s Gathering Facts --------------------------------------------------------------------------------------------------------------------- 0.52s kubernetes/preinstall : Create kubernetes directories ------------------------------------------------------------------------------- 0.50s adduser : User | Create User -------------------------------------------------------------------------------------------------------- 0.49s docker : Configure docker repository on RedHat/CentOS ------------------------------------------------------------------------------- 0.43s kubernetes/preinstall : Set selinux policy ------------------------------------------------------------------------------------------ 0.43s kubernetes/preinstall : Hosts | populate inventory into hosts file ------------------------------------------------------------------ 0.33s docker : Write docker.service systemd file ------------------------------------------------------------------------------------------ 0.31s docker : Write docker options systemd drop-in --------------------------------------------------------------------------------------- 0.30s docker : Write docker dns systemd drop-in ------------------------------------------------------------------------------------------- 0.29s bootstrap-os : Remove require tty --------------------------------------------------------------------------------------------------- 0.28s kubernetes/preinstall : Enable ip forwarding ---------------------------------------------------------------------------------------- 0.28s download : Download items ----------------------------------------------------------------------------------------------------------- 0.28s kubernetes/preinstall : Create cni directories -------------------------------------------------------------------------------------- 0.28s [root@TACO taco-scripts]# [root@TACO taco-scripts]# [root@TACO taco-scripts]# kubectl bash: kubectl: command not found... [root@TACO taco-scripts]# ./02-install-k8s.sh

export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/bin
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:/usr/local/bin
cd /root/
mkdir -p /root/apps
TACO_KUBESPRAY_DIR=/root/apps/taco-kubespray
'[' -d /root/apps/taco-kubespray ']'
rm -rf /root/apps/taco-kubespray
UPSTREAM_KUBESPRAY_DIR=/root/apps/upstream-kubespray
'[' -d /root/apps/upstream-kubespray ']'
rm -rf /root/apps/upstream-kubespray
KUBESPRAY_DIR=/root/apps/kubespray
'[' -d /root/apps/kubespray ']'
rm -rf /root/apps/kubespray
CACHE_FILE=/tmp/taco-aio
'[' -f /tmp/taco-aio ']'
rm -f /tmp/taco-aio
cd /root/apps
git clone https://github.com/kubernetes-incubator/kubespray.git upstream-kubespray Cloning into 'upstream-kubespray'... remote: Counting objects: 20260, done. remote: Total 20260 (delta 0), reused 0 (delta 0), pack-reused 20259 Receiving objects: 100% (20260/20260), 6.78 MiB | 2.61 MiB/s, done. Resolving deltas: 100% (10998/10998), done.
cd upstream-kubespray
git checkout -b v2.3.0 tags/v2.3.0 Switched to a new branch 'v2.3.0'
pip install -r requirements.txt Requirement already satisfied: pbr>=1.6 in /usr/lib/python2.7/site-packages (from -r requirements.txt (line 1)) Requirement already satisfied: ansible>=2.4.0 in /usr/lib/python2.7/site-packages (from -r requirements.txt (line 2)) Requirement already satisfied: netaddr in /usr/lib/python2.7/site-packages (from -r requirements.txt (line 3)) Requirement already satisfied: jinja2>=2.9.6 in /usr/lib64/python2.7/site-packages (from -r requirements.txt (line 4)) Requirement already satisfied: PyYAML in /usr/lib64/python2.7/site-packages (from ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: paramiko in /usr/lib/python2.7/site-packages (from ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: cryptography in /usr/lib64/python2.7/site-packages (from ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: setuptools in /usr/lib/python2.7/site-packages (from ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: MarkupSafe>=0.23 in /usr/lib64/python2.7/site-packages (from jinja2>=2.9.6->-r requirements.txt (line 4)) Requirement already satisfied: pyasn1>=0.1.7 in /usr/lib/python2.7/site-packages (from paramiko->ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: bcrypt>=3.1.3 in /usr/lib64/python2.7/site-packages (from paramiko->ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: pynacl>=1.0.1 in /usr/lib64/python2.7/site-packages (from paramiko->ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: idna>=2.0 in /usr/lib/python2.7/site-packages (from cryptography->ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: six>=1.4.1 in /usr/lib/python2.7/site-packages (from cryptography->ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: enum34 in /usr/lib/python2.7/site-packages (from cryptography->ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: ipaddress in /usr/lib/python2.7/site-packages (from cryptography->ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: cffi>=1.4.1 in /usr/lib64/python2.7/site-packages (from cryptography->ansible>=2.4.0->-r requirements.txt (line 2)) Requirement already satisfied: pycparser in /usr/lib/python2.7/site-packages (from cffi>=1.4.1->cryptography->ansible>=2.4.0->-r requirements.txt (line 2))
cd /root/apps
git clone https://github.com/sktelecom-oslab/taco-kubespray.git Cloning into 'taco-kubespray'... remote: Counting objects: 133, done. remote: Total 133 (delta 0), reused 0 (delta 0), pack-reused 133 Receiving objects: 100% (133/133), 61.52 KiB | 0 bytes/s, done. Resolving deltas: 100% (18/18), done.
cd taco-kubespray
git checkout -b v2.3.0 tags/v2.3.0 Switched to a new branch 'v2.3.0'
cd /root/apps
cp -R upstream-kubespray kubespray
cp -R taco-kubespray/credentials taco-kubespray/inventory taco-kubespray/monitoring.yml taco-kubespray/README.md taco-kubespray/reset-monitoring.yml taco-kubespray/reset-node.yml taco-kubespray/roles taco-kubespray/scale.yml taco-kubespray/storage.yml kubespray/.
cd kubespray
echo 'taco-aio ansible_connection=local local_release_dir={{ansible_env.HOME}}/releases [kube-master] taco-aio [etcd] taco-aio [kube-node] taco-aio [k8s-cluster:children] kube-node kube-master'
ansible-playbook -u root -b -i /root/apps/kubespray/inventory/taco-aio.cfg /root/apps/kubespray/cluster.yml [DEPRECATION WARNING]: The use of 'include' for tasks has been deprecated. Use 'import_tasks' for static inclusions or 'include_tasks' for dynamic inclusions. This feature will be removed in a future release. Deprecation warnings can be disabled by setting deprecation_warnings=False in ansible.cfg. [DEPRECATION WARNING]: include is kept for backwards compatibility but usage is discouraged. The module documentation details page may explain more about this rationale.. This feature will be removed in a future release. Deprecation warnings can be disabled by setting deprecation_warnings=False in ansible.cfg.

PLAY [localhost] ***

TASK [download : Register docker images info] ** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.075) 0:00:00.075 *****

TASK [download : container_download | Create dest directory for saved/loaded container images] ***** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.021) 0:00:00.097 *

TASK [download : container_download | create local directory for saved/loaded container images] **** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.026) 0:00:00.124 *****

TASK [download : Download items] *** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.019) 0:00:00.144 *

TASK [download : Sync container] *** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.102) 0:00:00.246 *

TASK [kubespray-defaults : Configure defaults] ***** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.093) 0:00:00.340 * ok: [localhost] => { "msg": "Check roles/kubespray-defaults/defaults/main.yml" }

TASK [bastion-ssh-config : set_fact] *** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.023) 0:00:00.364 * ok: [localhost]

TASK [bastion-ssh-config : set_fact] *** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.032) 0:00:00.397 *

TASK [bastion-ssh-config : set_fact] *** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.020) 0:00:00.417 *

TASK [bastion-ssh-config : create ssh bastion conf] **** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.017) 0:00:00.435 ***** [WARNING]: Could not match supplied host pattern, ignoring: calico-rr

PLAY [k8s-cluster:etcd:calico-rr] ** [WARNING]: While constructing a mapping from /root/apps/kubespray/inventory/group_vars/k8s-cluster.yml, line 6, column 1, found a duplicate dict key (kubelet_deployment_type). Using last defined value only.

TASK [download : Register docker images info] ** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.055) 0:00:00.491 *****

TASK [download : container_download | Create dest directory for saved/loaded container images] ***** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.021) 0:00:00.512 *

TASK [download : container_download | create local directory for saved/loaded container images] **** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.021) 0:00:00.534 *****

TASK [download : Download items] *** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.020) 0:00:00.554 *

TASK [download : Sync container] *** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.104) 0:00:00.659 *

TASK [kubespray-defaults : Configure defaults] ***** Saturday 10 February 2018 09:40:02 +0900 (0:00:00.097) 0:00:00.757 * ok: [taco-aio] => { "msg": "Check roles/kubespray-defaults/defaults/main.yml" }

TASK [bootstrap-os : Bootstrap | Check if bootstrap is needed] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.027) 0:00:00.784 *

TASK [bootstrap-os : Bootstrap | Install python 2.x and pip] *** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.045) 0:00:00.829 *

TASK [bootstrap-os : set_fact] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.025) 0:00:00.855 *

TASK [bootstrap-os : Bootstrap | Check if bootstrap is needed] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.019) 0:00:00.874 *

TASK [bootstrap-os : Bootstrap | Install python 2.x, pip, and dbus] **** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.039) 0:00:00.914 *****

TASK [bootstrap-os : set_fact] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.020) 0:00:00.934 *

TASK [bootstrap-os : Bootstrap | Check if bootstrap is needed] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.020) 0:00:00.954 *

TASK [bootstrap-os : Bootstrap | Run bootstrap.sh] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.021) 0:00:00.976 *

TASK [bootstrap-os : set_fact] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.023) 0:00:00.999 *

TASK [bootstrap-os : Bootstrap | Check if we need to install pip] ** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.019) 0:00:01.018 *****

TASK [bootstrap-os : Bootstrap | Copy get-pip.py] ** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.023) 0:00:01.042 *****

TASK [bootstrap-os : Bootstrap | Install pip] ** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.026) 0:00:01.068 *****

TASK [bootstrap-os : Bootstrap | Remove get-pip.py] **** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.027) 0:00:01.096 *****

TASK [bootstrap-os : Bootstrap | Install pip launcher] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.021) 0:00:01.117 *

TASK [bootstrap-os : Install required python modules] ** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.022) 0:00:01.140 *****

TASK [bootstrap-os : Check presence of fastestmirror.conf] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.038) 0:00:01.178 *

TASK [bootstrap-os : Disable fastestmirror plugin] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.025) 0:00:01.203 *

TASK [bootstrap-os : Install packages requirements for bootstrap] ** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.020) 0:00:01.224 *****

TASK [bootstrap-os : Remove require tty] *** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.024) 0:00:01.248 * ok: [taco-aio]

TASK [bootstrap-os : check if atomic host] ***** Saturday 10 February 2018 09:40:03 +0900 (0:00:00.301) 0:00:01.549 * ok: [taco-aio]

TASK [bootstrap-os : set_fact] ***** Saturday 10 February 2018 09:40:04 +0900 (0:00:00.271) 0:00:01.821 * ok: [taco-aio]

TASK [bootstrap-os : Gather nodes hostnames] *** Saturday 10 February 2018 09:40:04 +0900 (0:00:00.028) 0:00:01.850 * ok: [taco-aio]

TASK [bootstrap-os : Assign inventory name to unconfigured hostnames (non-CoreOS)] ***** Saturday 10 February 2018 09:40:04 +0900 (0:00:00.478) 0:00:02.328 * ok: [taco-aio]

TASK [bootstrap-os : Assign inventory name to unconfigured hostnames (CoreOS only)] **** Saturday 10 February 2018 09:40:05 +0900 (0:00:00.500) 0:00:02.829 *****

TASK [bootstrap-os : Update hostname fact (CoreOS only)] *** Saturday 10 February 2018 09:40:05 +0900 (0:00:00.018) 0:00:02.848 *

PLAY [k8s-cluster:etcd:calico-rr] **

TASK [Gathering Facts] ***** Saturday 10 February 2018 09:40:05 +0900 (0:00:00.026) 0:00:02.874 * ok: [taco-aio]

PLAY [k8s-cluster:etcd:calico-rr] **

TASK [download : Register docker images info] ** Saturday 10 February 2018 09:40:05 +0900 (0:00:00.455) 0:00:03.330 *****

TASK [download : container_download | Create dest directory for saved/loaded container images] ***** Saturday 10 February 2018 09:40:05 +0900 (0:00:00.024) 0:00:03.354 *

TASK [download : container_download | create local directory for saved/loaded container images] **** Saturday 10 February 2018 09:40:05 +0900 (0:00:00.027) 0:00:03.382 *****

TASK [download : Download items] *** Saturday 10 February 2018 09:40:05 +0900 (0:00:00.024) 0:00:03.407 *

TASK [download : Sync container] *** Saturday 10 February 2018 09:40:05 +0900 (0:00:00.264) 0:00:03.671 *

TASK [kubespray-defaults : Configure defaults] ***** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.262) 0:00:03.934 * ok: [taco-aio] => { "msg": "Check roles/kubespray-defaults/defaults/main.yml" }

TASK [adduser : User | Create User Group] ** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.033) 0:00:03.968 ***** ok: [taco-aio]

TASK [adduser : User | Create User] **** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.238) 0:00:04.207 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Stop if ansible version is too low] ** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.275) 0:00:04.483 ***** ok: [taco-aio] => { "changed": false, "msg": "All assertions passed" }

TASK [kubernetes/preinstall : Stop if non systemd OS type] ***** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.032) 0:00:04.516 * ok: [taco-aio] => { "changed": false, "msg": "All assertions passed" }

TASK [kubernetes/preinstall : Stop if unknown OS] ** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.041) 0:00:04.557 ***** ok: [taco-aio] => { "changed": false, "msg": "All assertions passed" }

TASK [kubernetes/preinstall : Stop if unknown network plugin] ** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.037) 0:00:04.594 *****

TASK [kubernetes/preinstall : Stop if incompatible network plugin and cloudprovider] *** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.020) 0:00:04.615 *

TASK [kubernetes/preinstall : Stop if known booleans are set as strings (Use JSON format on CLI: -e "{'key': true }")] ***** Saturday 10 February 2018 09:40:06 +0900 (0:00:00.027) 0:00:04.642 * ok: [taco-aio] => (item={u'name': u'kubeadm_enabled', u'value': False}) => { "changed": false, "item": { "name": "kubeadm_enabled", "value": false }, "msg": "All assertions passed" } ok: [taco-aio] => (item={u'name': u'download_run_once', u'value': False}) => { "changed": false, "item": { "name": "download_run_once", "value": false }, "msg": "All assertions passed" } ok: [taco-aio] => (item={u'name': u'deploy_netchecker', u'value': False}) => { "changed": false, "item": { "name": "deploy_netchecker", "value": false }, "msg": "All assertions passed" } ok: [taco-aio] => (item={u'name': u'download_always_pull', u'value': False}) => { "changed": false, "item": { "name": "download_always_pull", "value": false }, "msg": "All assertions passed" } ok: [taco-aio] => (item={u'name': u'efk_enabled', u'value': False}) => { "changed": false, "item": { "name": "efk_enabled", "value": false }, "msg": "All assertions passed" } ok: [taco-aio] => (item={u'name': u'helm_enabled', u'value': True}) => { "changed": false, "item": { "name": "helm_enabled", "value": true }, "msg": "All assertions passed" } ok: [taco-aio] => (item={u'name': u'openstack_lbaas_enabled', u'value': False}) => { "changed": false, "item": { "name": "openstack_lbaas_enabled", "value": false }, "msg": "All assertions passed" } ok: [taco-aio] => (item={u'name': u'rbac_enabled', u'value': True}) => { "changed": false, "item": { "name": "rbac_enabled", "value": true }, "msg": "All assertions passed" }

TASK [kubernetes/preinstall : Stop if even number of etcd hosts] *** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.145) 0:00:04.788 * ok: [taco-aio] => { "changed": false, "msg": "All assertions passed" }

TASK [kubernetes/preinstall : Stop if memory is too small for masters] ***** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.036) 0:00:04.825 * ok: [taco-aio] => { "changed": false, "msg": "All assertions passed" }

TASK [kubernetes/preinstall : Stop if memory is too small for nodes] *** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.040) 0:00:04.866 * ok: [taco-aio] => { "changed": false, "msg": "All assertions passed" }

TASK [kubernetes/preinstall : Stop if ip var does not match local ips] ***** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.034) 0:00:04.900 *

TASK [kubernetes/preinstall : Stop if access_ip is not pingable] *** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.027) 0:00:04.928 *

TASK [kubernetes/preinstall : Stop if swap enabled] **** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.032) 0:00:04.960 ***** ok: [taco-aio] => { "changed": false, "msg": "All assertions passed" }

TASK [kubernetes/preinstall : Force binaries directory for Container Linux by CoreOS] ** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.047) 0:00:05.007 *****

TASK [kubernetes/preinstall : check bin dir exists] **** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.025) 0:00:05.033 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : check if atomic host] **** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.240) 0:00:05.274 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : set_fact] **** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.134) 0:00:05.408 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : set_fact] **** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.032) 0:00:05.441 *****

TASK [kubernetes/preinstall : check resolvconf] **** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.026) 0:00:05.467 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : set_fact] **** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.232) 0:00:05.699 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : set_fact] **** Saturday 10 February 2018 09:40:07 +0900 (0:00:00.041) 0:00:05.741 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : check if kubelet is configured] ** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.058) 0:00:05.800 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : check if early DNS configuration stage] ** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.145) 0:00:05.946 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : target resolv.conf files] **** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.037) 0:00:05.983 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : target temporary resolvconf cloud init file (Container Linux by CoreOS)] ***** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.052) 0:00:06.036 *

TASK [kubernetes/preinstall : check if /etc/dhclient.conf exists] ** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.028) 0:00:06.064 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : target dhclient conf file for /etc/dhclient.conf] **** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.155) 0:00:06.220 *****

TASK [kubernetes/preinstall : check if /etc/dhcp/dhclient.conf exists] ***** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.024) 0:00:06.244 * ok: [taco-aio]

TASK [kubernetes/preinstall : target dhclient conf file for /etc/dhcp/dhclient.conf] *** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.143) 0:00:06.387 *

TASK [kubernetes/preinstall : target dhclient hook file for Red Hat family] **** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.030) 0:00:06.418 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : target dhclient hook file for Debian family] ***** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.045) 0:00:06.464 *

TASK [kubernetes/preinstall : generate search domains to resolvconf] *** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.028) 0:00:06.493 * ok: [taco-aio]

TASK [kubernetes/preinstall : pick dnsmasq cluster IP or default resolver] ***** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.046) 0:00:06.539 * ok: [taco-aio]

TASK [kubernetes/preinstall : generate nameservers to resolvconf] ** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.049) 0:00:06.589 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : gather os specific variables] **** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.041) 0:00:06.630 ***** ok: [taco-aio] => (item=/root/apps/kubespray/roles/kubernetes/preinstall/vars/../vars/centos.yml)

TASK [kubernetes/preinstall : Create kubernetes directories] *** Saturday 10 February 2018 09:40:08 +0900 (0:00:00.051) 0:00:06.682 * ok: [taco-aio] => (item=/etc/kubernetes) ok: [taco-aio] => (item=/etc/kubernetes/ssl) ok: [taco-aio] => (item=/etc/kubernetes/manifests) ok: [taco-aio] => (item=/usr/local/bin/kubernetes-scripts)

TASK [kubernetes/preinstall : check cloud_provider value] ** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.510) 0:00:07.193 *****

TASK [kubernetes/preinstall : include] ***** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.022) 0:00:07.215 *

TASK [kubernetes/preinstall : Create cni directories] ** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.024) 0:00:07.239 ***** ok: [taco-aio] => (item=/etc/cni/net.d) ok: [taco-aio] => (item=/opt/cni/bin)

TASK [kubernetes/preinstall : create temporary resolveconf cloud init file] **** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.273) 0:00:07.512 *****

TASK [kubernetes/preinstall : Add domain/search/nameservers/options to resolv.conf] **** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.024) 0:00:07.536 *****

TASK [kubernetes/preinstall : Remove search/domain/nameserver options before block] **** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.029) 0:00:07.566 *****

TASK [kubernetes/preinstall : Remove search/domain/nameserver options after block] ***** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.076) 0:00:07.642 *

TASK [kubernetes/preinstall : get temporary resolveconf cloud init file content] *** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.065) 0:00:07.708 *

TASK [kubernetes/preinstall : persist resolvconf cloud init file] ** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.029) 0:00:07.737 *****

TASK [kubernetes/preinstall : Update package management cache (YUM)] *** Saturday 10 February 2018 09:40:09 +0900 (0:00:00.029) 0:00:07.767 * ok: [taco-aio]

TASK [kubernetes/preinstall : Install latest version of python-apt for Debian distribs] **** Saturday 10 February 2018 09:40:16 +0900 (0:00:06.123) 0:00:13.890 *****

TASK [kubernetes/preinstall : Install python-dnf for latest RedHat versions] *** Saturday 10 February 2018 09:40:16 +0900 (0:00:00.021) 0:00:13.911 *

TASK [kubernetes/preinstall : Install epel-release on RedHat/CentOS] *** Saturday 10 February 2018 09:40:16 +0900 (0:00:00.023) 0:00:13.935 * [WARNING]: Consider using yum, dnf or zypper module rather than running rpm

ok: [taco-aio]

TASK [kubernetes/preinstall : Install packages requirements] *** Saturday 10 February 2018 09:40:17 +0900 (0:00:01.345) 0:00:15.281 * ok: [taco-aio] => (item=[u'libselinux-python', u'device-mapper-libs', u'ebtables', u'python-httplib2', u'openssl', u'curl', u'rsync', u'bash-completion', u'socat', u'unzip'])

TASK [kubernetes/preinstall : Confirm selinux deployed] **** Saturday 10 February 2018 09:40:27 +0900 (0:00:10.108) 0:00:25.389 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Set selinux policy] ** Saturday 10 February 2018 09:40:27 +0900 (0:00:00.152) 0:00:25.542 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Disable IPv6 DNS lookup] ***** Saturday 10 February 2018 09:40:28 +0900 (0:00:00.456) 0:00:25.998 *

TASK [kubernetes/preinstall : set default sysctl file path] **** Saturday 10 February 2018 09:40:28 +0900 (0:00:00.017) 0:00:26.016 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Stat sysctl file configuration] ** Saturday 10 February 2018 09:40:28 +0900 (0:00:00.033) 0:00:26.049 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Change sysctl file path to link source if linked] **** Saturday 10 February 2018 09:40:28 +0900 (0:00:00.157) 0:00:26.206 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Enable ip forwarding] **** Saturday 10 February 2018 09:40:28 +0900 (0:00:00.036) 0:00:26.243 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Write cloud-config] ** Saturday 10 February 2018 09:40:28 +0900 (0:00:00.278) 0:00:26.521 *****

TASK [kubernetes/preinstall : Hosts | populate inventory into hosts file] ** Saturday 10 February 2018 09:40:28 +0900 (0:00:00.022) 0:00:26.544 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Hosts | populate kubernetes loadbalancer address into hosts file] **** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.318) 0:00:26.862 *****

TASK [kubernetes/preinstall : Hosts | localhost ipv4 in hosts file] **** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.020) 0:00:26.883 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Hosts | localhost ipv6 in hosts file] **** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.146) 0:00:27.029 ***** ok: [taco-aio]

TASK [kubernetes/preinstall : Configure dhclient to supersede search/domain/nameservers] *** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.137) 0:00:27.166 *

TASK [kubernetes/preinstall : Configue dhclient hooks for resolv.conf (non-RH)] **** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.022) 0:00:27.188 *****

TASK [kubernetes/preinstall : Configue dhclient hooks for resolv.conf (RH-only)] *** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.027) 0:00:27.216 *

TASK [kubernetes/preinstall : Remove kubespray specific config from dhclient config] *** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.032) 0:00:27.249 *

TASK [kubernetes/preinstall : Remove kubespray specific dhclient hook] ***** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.044) 0:00:27.294 * ok: [taco-aio]

TASK [kubernetes/preinstall : Check if we are running inside a Azure VM] *** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.176) 0:00:27.470 * ok: [taco-aio]

TASK [kubernetes/preinstall : install growpart] **** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.136) 0:00:27.607 *****

TASK [kubernetes/preinstall : check if growpart needs to be run] *** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.024) 0:00:27.631 *

TASK [kubernetes/preinstall : check fs type] *** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.025) 0:00:27.657 *

TASK [kubernetes/preinstall : run growpart] **** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.025) 0:00:27.682 *****

TASK [kubernetes/preinstall : run xfs_growfs] ** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.025) 0:00:27.708 *****

TASK [docker/docker-storage : docker-storage-setup | install git and make] ***** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.027) 0:00:27.735 *

TASK [docker/docker-storage : docker-storage-setup | docker-storage-setup sysconfig template] ** Saturday 10 February 2018 09:40:29 +0900 (0:00:00.037) 0:00:27.773 *****

TASK [docker/docker-storage : docker-storage-override-directory | docker service storage-setup override dir] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.028) 0:00:27.801 *

TASK [docker/docker-storage : docker-storage-override | docker service storage-setup override file] **** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.026) 0:00:27.828 *****

TASK [docker/docker-storage : docker-storage-setup | install and run container-storage-setup] ** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.025) 0:00:27.853 *****

TASK [docker : gather os specific variables] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.026) 0:00:27.879 * ok: [taco-aio] => (item=/root/apps/kubespray/roles/docker/vars/../vars/redhat.yml)

TASK [docker : set dns server for docker] ** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.059) 0:00:27.939 ***** ok: [taco-aio]

TASK [docker : set base docker dns facts] ** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.051) 0:00:27.990 ***** ok: [taco-aio]

TASK [docker : add upstream dns servers (only when dnsmasq is not used)] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.047) 0:00:28.037 * ok: [taco-aio]

TASK [docker : add global searchdomains] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.046) 0:00:28.084 * ok: [taco-aio]

TASK [docker : check system nameservers] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.044) 0:00:28.129 * ok: [taco-aio]

TASK [docker : check system search domains] **** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.159) 0:00:28.288 ***** ok: [taco-aio]

TASK [docker : add system nameservers to docker options] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.143) 0:00:28.431 * ok: [taco-aio]

TASK [docker : add system search domains to docker options] **** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.040) 0:00:28.472 ***** ok: [taco-aio]

TASK [docker : check number of nameservers] **** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.047) 0:00:28.520 *****

TASK [docker : rtrim number of nameservers to 3] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.028) 0:00:28.548 *

TASK [docker : check number of search domains] ***** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.028) 0:00:28.576 *

TASK [docker : check length of search domains] ***** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.031) 0:00:28.607 *

TASK [docker : check for minimum kernel version] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.029) 0:00:28.637 *

TASK [docker : ensure docker repository public key is installed] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.022) 0:00:28.660 *

TASK [docker : ensure docker repository is enabled] **** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.019) 0:00:28.679 *****

TASK [docker : Configure docker repository on RedHat/CentOS] *** Saturday 10 February 2018 09:40:30 +0900 (0:00:00.022) 0:00:28.702 * ok: [taco-aio]

TASK [docker : ensure docker packages are installed] *** Saturday 10 February 2018 09:40:31 +0900 (0:00:00.275) 0:00:28.978 * ok: [taco-aio] => (item={u'name': u'docker-engine-17.03.0.ce-1.el7.centos'})

TASK [docker : check minimum docker version for docker_dns mode. You need at least docker version >= 1.12 for resolvconf_mode=docker_dns] * Saturday 10 February 2018 09:40:33 +0900 (0:00:02.468) 0:00:31.447 *** ok: [taco-aio]

TASK [docker : Create docker service systemd directory if it doesn't exist] **** Saturday 10 February 2018 09:40:33 +0900 (0:00:00.183) 0:00:31.630 ***** ok: [taco-aio]

TASK [docker : Write docker proxy drop-in] ***** Saturday 10 February 2018 09:40:33 +0900 (0:00:00.134) 0:00:31.765 *

TASK [docker : get systemd version] **** Saturday 10 February 2018 09:40:34 +0900 (0:00:00.023) 0:00:31.789 ***** ok: [taco-aio]

TASK [docker : Write docker.service systemd file] ** Saturday 10 February 2018 09:40:34 +0900 (0:00:00.207) 0:00:31.996 ***** ok: [taco-aio]

TASK [docker : Write docker options systemd drop-in] *** Saturday 10 February 2018 09:40:34 +0900 (0:00:00.312) 0:00:32.309 * ok: [taco-aio]

TASK [docker : Write docker dns systemd drop-in] *** Saturday 10 February 2018 09:40:34 +0900 (0:00:00.312) 0:00:32.622 * ok: [taco-aio]

TASK [docker : Remove docker cert file] **** Saturday 10 February 2018 09:40:35 +0900 (0:00:00.310) 0:00:32.932 ***** ok: [taco-aio]

TASK [docker : Copy docker cert file] ** Saturday 10 February 2018 09:40:35 +0900 (0:00:00.145) 0:00:33.078 ***** changed: [taco-aio]

TASK [docker : Move docker certs directory] **** Saturday 10 February 2018 09:40:36 +0900 (0:00:00.846) 0:00:33.924 ***** changed: [taco-aio]

TASK [docker : ensure docker service is started and enabled] *** Saturday 10 February 2018 09:40:36 +0900 (0:00:00.156) 0:00:34.080 * failed: [taco-aio] (item=docker) => {"changed": false, "item": "docker", "msg": "Unable to start service docker: Job for docker.service failed because the control process exited with error code. See \"systemctl status docker.service\" and \"journalctl -xe\" for details.\n"}

NO MORE HOSTS LEFT ***** to retry, use: --limit @/root/apps/kubespray/cluster.retry

PLAY RECAP ***** localhost : ok=2 changed=0 unreachable=0 failed=0 taco-aio : ok=70 changed=2 unreachable=0 failed=1

Saturday 10 February 2018 09:40:36 +0900 (0:00:00.391) 0:00:34.472 *****

kubernetes/preinstall : Install packages requirements ------------------------------------------------------------------------------ 10.11s kubernetes/preinstall : Update package management cache (YUM) ----------------------------------------------------------------------- 6.12s docker : ensure docker packages are installed --------------------------------------------------------------------------------------- 2.47s kubernetes/preinstall : Install epel-release on RedHat/CentOS ----------------------------------------------------------------------- 1.35s docker : Copy docker cert file ------------------------------------------------------------------------------------------------------ 0.85s kubernetes/preinstall : Create kubernetes directories ------------------------------------------------------------------------------- 0.51s bootstrap-os : Assign inventory name to unconfigured hostnames (non-CoreOS) --------------------------------------------------------- 0.50s bootstrap-os : Gather nodes hostnames ----------------------------------------------------------------------------------------------- 0.48s kubernetes/preinstall : Set selinux policy ------------------------------------------------------------------------------------------ 0.46s Gathering Facts --------------------------------------------------------------------------------------------------------------------- 0.46s docker : ensure docker service is started and enabled ------------------------------------------------------------------------------- 0.39s kubernetes/preinstall : Hosts | populate inventory into hosts file ------------------------------------------------------------------ 0.32s docker : Write docker.service systemd file ------------------------------------------------------------------------------------------ 0.31s docker : Write docker options systemd drop-in --------------------------------------------------------------------------------------- 0.31s docker : Write docker dns systemd drop-in ------------------------------------------------------------------------------------------- 0.31s bootstrap-os : Remove require tty --------------------------------------------------------------------------------------------------- 0.30s kubernetes/preinstall : Enable ip forwarding ---------------------------------------------------------------------------------------- 0.28s adduser : User | Create User -------------------------------------------------------------------------------------------------------- 0.28s docker : Configure docker repository on RedHat/CentOS ------------------------------------------------------------------------------- 0.28s kubernetes/preinstall : Create cni directories -------------------------------------------------------------------------------------- 0.27s [root@TACO taco-scripts]# kubectl bash: kubectl: command not found...

seungkyua commented 6 years ago

이전에 docker 가 따로 설치된 서버 같네요. docker 와 docker option conf 파일을 삭제하고 다시해 보세요.

seongilp commented 6 years ago

위 문제는 해결되었습니다. 맥에서 Parellels라는 가상화 프로그램에서 CentOS 7.4, Ubuntu 17.10 을 이용해서 테스트 했는데 안되었고 VMware Fusion 에서 구동을 시키니깐 잘되네요. 4번 스크립트 실행 시 LC_ALL, LANG이 선언되어 있지 않으면 에러가 나기도 했구요.

그런데 4번 스크립트를 이제 다시 실행하면 아래와 같은 에러가 납니다.

2018-02-10 18:39:24.693 31606 ERROR armada grpc._channel._Rendezvous: <_Rendezvous of RPC that terminated with (StatusCode.UNKNOWN, Get https://10.96.0.1:443/api/v1/namespaces/kube-system/configmaps?labelSelector=OWNER%3DTILLER: dial tcp 10.96.0.1:443: getsockopt: no route to host)> 2018-02-10 18:39:24.693 31606 ERROR armada

prudentcircle commented 6 years ago

위에 @seongilp 님께서 언급을 해 주셨듯이 LC_ALL, LANG이 제대로 설정 되어있지 않으면 아래와 같은 에러메시지가 뜹니다.

Click discovered that you exported a UTF-8 locale
but the locale system could not pick up from it because
it does not exist.  The exported locale is "en_US.utf8" but it
is not supported

혹시나 해답을 찾기 귀찮아 하시는 분들을 위해 정답 아래 적어놓습니다. ^^ 명령어 두 줄 입니다.

[root@taco taco-scripts]# export LC_ALL=en_US.utf-8
[root@taco taco-scripts]# export LANG=en_US.utf-8

seongilp commented 6 years ago

사양을 많이 타네요. 32 core / 32 GB 의 시스템에서는 약 45분만에 에러 없이 모두 수행되었습니다

몇가지 팁을 드리자면

윗분과 마찬가지로 LC_ALL, LANG 설정 필요하고요

#  echo LANG=en_US.utf-8 >> /etc/environment
# echo LC_ALL=en_US.utf-8 >> /etc/environment
# echo export LANG=en_US.utf-8 >> ~/.bashrc
# echo export LC_ALL=en_US.utf-8 >> ~/.bashrc

02번까지 실행 후 weave 접속이 가능합니다. 그리고 00번 쉘 2개를 창 분리 해서 띄워놓으면 어느 단계인지 볼 수 있어서 괜찮았습니다

8 core / 16 GB에서도 느려서 에러가 많이 났고, 32 core 정도까지 올려도 load가 3~4까지 올라 갔으나 특별한 에러 없이 스크립트가 모두 잘 수행되었습니다

sktelecom-oslab / taco-scripts

CentOS 7.3 설치시 에러 - 04-deploy-openstack.sh #13

Saturday 10 February 2018 09:39:41 +0900 (0:00:00.001) 0:00:44.477 *****

Saturday 10 February 2018 09:40:36 +0900 (0:00:00.391) 0:00:34.472 *****