create daemonset failed

yanruogu commented 4 years ago

os: ubuntu 16.04 kbuernetes: v1.11.10

step:

root@k8s-m:/home/www/server/kube-yamls/public# git clone https://github.com/denverdino/lxcfs-admission-webhook.git
Cloning into 'lxcfs-admission-webhook'...
remote: Enumerating objects: 28, done.
remote: Counting objects: 100% (28/28), done.
remote: Compressing objects: 100% (27/27), done.
remote: Total 28 (delta 1), reused 28 (delta 1), pack-reused 0
Unpacking objects: 100% (28/28), done.
Checking connectivity... done.

root@k8s-m:/home/www/server/kube-yamls/public/lxcfs-admission-webhook/deployment# pwd
/home/www/server/kube-yamls/public/lxcfs-admission-webhook/deployment
root@k8s-m:/home/www/server/kube-yamls/public/lxcfs-admission-webhook/deployment# ls
deployment.yaml  install.sh  lxcfs-daemonset.yaml  mutatingwebhook.yaml  service.yaml  uninstall.sh  validatingwebhook.yaml  webhook-create-signed-cert.sh  webhook-patch-ca-bundle.sh  web.yaml

root@k8s-m:/home/www/server/kube-yamls/public/lxcfs-admission-webhook/deployment# kubectl apply -f  lxcfs-daemonset.yaml 
daemonset.apps/lxcfs created

root@k8s-m:/home/www/server/kube-yamls/public/lxcfs-admission-webhook/deployment# kubectl  api-versions |grep admissionregistration.k8s.io/v1beta1
admissionregistration.k8s.io/v1beta1

root@k8s-m:/home/www/server/kube-yamls/public/lxcfs-admission-webhook/deployment# kubectl get pods |grep lxc      
lxcfs-5kv56                                                0/1       CrashLoopBackOff    2          1m
lxcfs-5nbrb                                                0/1       CrashLoopBackOff    1          1m
lxcfs-74txn                                                0/1       CrashLoopBackOff    2          1m
lxcfs-9f5sv                                                0/1       RunContainerError   3          1m
lxcfs-bzhgz                                                0/1       CrashLoopBackOff    2          1m
lxcfs-d7q5k                                                0/1       CrashLoopBackOff    3          1m
lxcfs-dqdw7                                                0/1       CrashLoopBackOff    3          1m
lxcfs-fdsvj                                                0/1       CrashLoopBackOff    3          1m
...

root@k8s-m:/home/www/server/kube-yamls/public/lxcfs-admission-webhook/deployment# kubectl describe po  lxcfs-d7q5k     
Name:               lxcfs-d7q5k
Namespace:          default
Priority:           0
PriorityClassName:  <none>
Node:               ...
Start Time:         Mon, 28 Oct 2019 17:53:42 +0800
Labels:             app=lxcfs
                    controller-revision-hash=1128831819
                    pod-template-generation=1
Annotations:        <none>
Status:             Running
IP:                ...
Controlled By:      DaemonSet/lxcfs
Containers:
  lxcfs:
    Container ID:   docker://d58c2afae24d11d1313f9f7ceb8aa19db4351c3db884f98218e1be655873989e
    Image:          registry.cn-hangzhou.aliyuncs.com/denverdino/lxcfs:3.1.2
    Image ID:       docker-pullable://registry.cn-hangzhou.aliyuncs.com/denverdino/lxcfs@sha256:102ed1896c3bcd5325f293a2758568022c93dd32d8712bc397f48cd38012a441
    Port:           <none>
    Host Port:      <none>
    State:          Waiting
      Reason:       RunContainerError
    Last State:     Terminated
      Reason:       ContainerCannotRun
      Message:      linux mounts: path /var/lib/lxcfs is mounted on /var/lib/lxcfs but it is not a shared mount
      Exit Code:    128
      Started:      Mon, 28 Oct 2019 17:56:49 +0800
      Finished:     Mon, 28 Oct 2019 17:56:49 +0800
    Ready:          False
    Restart Count:  5
    Environment:    <none>
    Mounts:
      /sys/fs/cgroup from cgroup (rw)
      /usr/local from usr-local (rw)
      /var/lib/lxcfs from lxcfs (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-gznmj (ro)
Conditions:
  Type              Status
  Initialized       True 
  Ready             False 
  ContainersReady   False 
  PodScheduled      True 
Volumes:
  cgroup:
    Type:          HostPath (bare host directory volume)
    Path:          /sys/fs/cgroup
    HostPathType:  
  usr-local:
    Type:          HostPath (bare host directory volume)
    Path:          /usr/local
    HostPathType:  
  lxcfs:
    Type:          HostPath (bare host directory volume)
    Path:          /var/lib/lxcfs
    HostPathType:  DirectoryOrCreate
  default-token-gznmj:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  default-token-gznmj
    Optional:    false
QoS Class:       BestEffort
Node-Selectors:  <none>
Tolerations:     node-role.kubernetes.io/master:NoSchedule
                 node.kubernetes.io/disk-pressure:NoSchedule
                 node.kubernetes.io/memory-pressure:NoSchedule
                 node.kubernetes.io/not-ready:NoExecute
                 node.kubernetes.io/unreachable:NoExecute
                 node.kubernetes.io/unschedulable:NoSchedule
Events:
  Type     Reason   Age              From                 Message
  ----     ------   ----             ----                 -------
  Normal   Pulling  1m (x5 over 3m)  kubelet, 10.1.56.56  pulling image "registry.cn-hangzhou.aliyuncs.com/denverdino/lxcfs:3.1.2"
  Normal   Pulled   1m (x5 over 3m)  kubelet, 10.1.56.56  Successfully pulled image "registry.cn-hangzhou.aliyuncs.com/denverdino/lxcfs:3.1.2"
  Normal   Created  1m (x5 over 3m)  kubelet, 10.1.56.56  Created container
  Warning  Failed   1m (x5 over 3m)  kubelet, 10.1.56.56  Error: failed to start container "lxcfs": Error response from daemon: linux mounts: path /var/lib/lxcfs is mounted on /var/lib/lxcfs but it is not a shared mount
  Warning  BackOff  1m (x5 over 2m)  kubelet, 10.1.56.56  Back-off restarting failed container

when i want to create daemonset, then failed.

i dont know what happend.

help.

xigang commented 4 years ago

@yanruogu check if the /var/lib/lxcfs directory already exists on the node? If it exists, delete it, try again.

denverdino commented 4 years ago

Hmm, I don't think it supports Ubuntu/Debian yet. I just build the RPM package.

Can you take a look if you can create one PR for DEB package? Thanks

yanruogu commented 4 years ago

When I manually run the docker container on ubuntu

root@ctnr:~# docker run --privileged -d -v /sys/fs/cgroup:/sys/fs/cgroup -v /var/lib/lxcfs:/var/lib/lxcfs -v /usr/local:/usr/local registry.cn-hangzhou.aliyuncs.com/denverdino/lxcfs:3.1.2
1cf06cd91641741f7960e7409b6fba636765e1dad3a6b47b1c239b167030bb04
root@ctnr:~# docker ps -a 
CONTAINER ID        IMAGE                                                      COMMAND                  CREATED             STATUS                     PORTS               NAMES
1cf06cd91641        registry.cn-hangzhou.aliyuncs.com/denverdino/lxcfs:3.1.2   "/start.sh"              3 seconds ago       Up 1 second                                    nostalgic_volhard

root@ctnr:~# docker logs 1cf06cd91641
mount namespace: 5
hierarchies:
  0: fd:   6: perf_event
  1: fd:   7: pids
  2: fd:   8: hugetlb
  3: fd:   9: blkio
  4: fd:  10: cpuset
  5: fd:  11: memory
  6: fd:  12: freezer
  7: fd:  13: net_cls,net_prio
  8: fd:  14: rdma
  9: fd:  15: cpu,cpuacct
 10: fd:  16: devices
 11: fd:  17: name=systemd

It looks normal

When I deleted the container, then deleted /var/lib/lxcfs, and then re-deployed using kubernetes, the problem remained the same

denverdino / lxcfs-admission-webhook

create daemonset failed #2