The mount pod cannot be created after the pod references juicefs-sc.

1473371932 commented 9 months ago

kubectl describe pods:
Events:
  Type     Reason       Age                   From     Message
  ----     ------       ----                  ----     -------
  Warning  FailedMount  28m (x9 over 96m)     kubelet  Unable to attach or mount volumes: unmounted volumes=[data-path], unattached volumes=[sql-path data-path kube-api-access-zcv8m log-path debug-path mysql-config mysql-config-init-sql]: timed out waiting for the condition
  Warning  FailedMount  17m (x5 over 87m)     kubelet  Unable to attach or mount volumes: unmounted volumes=[data-path], unattached volumes=[debug-path mysql-config mysql-config-init-sql sql-path data-path kube-api-access-zcv8m log-path]: timed out waiting for the condition
  Warning  FailedMount  8m16s (x5 over 89m)   kubelet  Unable to attach or mount volumes: unmounted volumes=[data-path], unattached volumes=[mysql-config-init-sql sql-path data-path kube-api-access-zcv8m log-path debug-path mysql-config]: timed out waiting for the condition
  Warning  FailedMount  2m53s (x55 over 98m)  kubelet  MountVolume.MountDevice failed for volume "pvc-7eb24564-b48a-441d-985a-a1f976c349c9" : kubernetes.io/csi: attacher.MountDevice failed to create newCsiDriverClient: driver name csi.juicefs.com not found in the list of registered CSI drivers
##############################################################
root@k8s-master:test# kubectl describe pvc -n deepflow deepflow-mysql-data 
Name:          deepflow-mysql-data
Namespace:     deepflow
StorageClass:  juicefs-sc
Status:        Bound
Volume:        pvc-7eb24564-b48a-441d-985a-a1f976c349c9
Labels:        app=deepflow-mysql
               app.kubernetes.io/managed-by=Helm
Annotations:   helm.sh/resource-policy: keep
               meta.helm.sh/release-name: mysql
               meta.helm.sh/release-namespace: deepflow
               pv.kubernetes.io/bind-completed: yes
               pv.kubernetes.io/bound-by-controller: yes
               volume.beta.kubernetes.io/storage-provisioner: csi.juicefs.com
               volume.kubernetes.io/storage-provisioner: csi.juicefs.com
Finalizers:    [kubernetes.io/pvc-protection]
Capacity:      100Gi
Access Modes:  RWO
VolumeMode:    Filesystem
Used By:       mysql-deployment-5d5bb9b569-fv4x9
Events:        <none>
##############################################################
root@k8s-master:test# kubectl get pv pvc-7eb24564-b48a-441d-985a-a1f976c349c9
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                          STORAGECLASS   REASON   AGE
pvc-7eb24564-b48a-441d-985a-a1f976c349c9   100Gi      RWO            Delete           Bound    deepflow/deepflow-mysql-data   juicefs-sc              113m
##############################################################
root@k8s-master:test# kubectl describe pv pvc-7eb24564-b48a-441d-985a-a1f976c349c9
Name:            pvc-7eb24564-b48a-441d-985a-a1f976c349c9
Labels:          <none>
Annotations:     pv.kubernetes.io/provisioned-by: csi.juicefs.com
Finalizers:      [kubernetes.io/pv-protection]
StorageClass:    juicefs-sc
Status:          Bound
Claim:           deepflow/deepflow-mysql-data
Reclaim Policy:  Delete
Access Modes:    RWO
VolumeMode:      Filesystem
Capacity:        100Gi
Node Affinity:   <none>
Message:         
Source:
    Type:              CSI (a Container Storage Interface (CSI) volume source)
    Driver:            csi.juicefs.com
    FSType:            juicefs
    VolumeHandle:      pvc-7eb24564-b48a-441d-985a-a1f976c349c9
    ReadOnly:          false
    VolumeAttributes:      capacity=107374182400
                           csi.storage.k8s.io/controller-expand-secret-name=juicefs-sc-secret
                           csi.storage.k8s.io/controller-expand-secret-namespace=kube-system
                           csi.storage.k8s.io/node-publish-secret-name=juicefs-sc-secret
                           csi.storage.k8s.io/node-publish-secret-namespace=kube-system
                           csi.storage.k8s.io/provisioner-secret-name=juicefs-sc-secret
                           csi.storage.k8s.io/provisioner-secret-namespace=kube-system
                           pathPattern=${.PVC.namespace}-${.PVC.name}
                           subPath=deepflow-deepflow-mysql-data
Events:                <none>

What happened:

What you expected to happen:

Successfully mounted after referencing juicefs-sc

How to reproduce it (as minimally and precisely as possible):

  volumeClaimTemplates:
  - metadata:
      name: www
      namespace: default
    spec:
      accessModes:
      - ReadWriteOnce
      storageClassName: "juicefs-sc"
      resources:
        requests:
          storage: 10Gi

root@k8s-master:test# kubectl get node
NAME         STATUS   ROLES           AGE   VERSION
k8s-master   Ready    control-plane   46d   v1.25.13
root@k8s-master:test# 
root@k8s-master:test# kubectl get pods -n kube-system | grep -i 'juicefs'
juicefs-csi-controller-0                                             3/3     Running   0             138m
juicefs-csi-dashboard-85dc4cb47c-gjvs5                               1/1     Running   0             138m
juicefs-csi-node-m66c9                                               3/3     Running   0             138m
juicefs-k8s-master-pvc-35e0b0c1-f948-47b1-acf3-7b4b5cd1a2a3-frubuv   1/1     Running   0             11d
juicefs-k8s-master-pvc-38e48f25-fd8d-452c-96f5-18ad24cd2b66-qjzcwf   1/1     Running   2 (27d ago)   32d
juicefs-k8s-master-pvc-3a1628f7-7632-4cbb-b28a-cde0f518a08f-mxivei   1/1     Running   0             11d
juicefs-k8s-master-pvc-d17fd796-7003-4630-9cab-3a75166c4f05-ffnxja   1/1     Running   1 (27d ago)   27d

root@k8s-master:test# kubectl logs -n kube-system juicefs-csi-node-m66c9 -c juicefs-plugin
I0123 06:19:10.817320       6 main.go:94] Run CSI node
I0123 06:19:10.869238       6 node.go:171] Pod Reconciler Started
I0123 06:19:10.869260       6 driver.go:48] Driver: csi.juicefs.com version v0.23.2 commit 7cf22fd2489f6304ed61ab579b02ba18fa7a4d84 date 2023-12-27T06:32:27Z
I0123 06:19:10.948115       6 driver.go:113] Listening for connection on address: &net.UnixAddr{Name:"/csi/csi.sock", Net:"unix"}

root@k8s-master:test# kubectl get csidrivers csi.juicefs.com
NAME              ATTACHREQUIRED   PODINFOONMOUNT   STORAGECAPACITY   TOKENREQUESTS   REQUIRESREPUBLISH   MODES        AGE
csi.juicefs.com   false            true             false             <unset>         false               Persistent   141m

Snipaste_2024-01-23_16-37-20

Anything else we need to know?

Nothing more is needed

Environment:

JuiceFS version (use juicefs --version) or Hadoop Java SDK version:

root@k8s-master:test# juicefs --version
juicefs version 1.1.1+2023-11-28.437f4e6
root@k8s-master:test# 
root@k8s-master:test# helm list -n kube-system 
NAME                    NAMESPACE       REVISION        UPDATED                                 STATUS          CHART                           APP VERSION
juicefs                 kube-system     1               2024-01-23 14:17:17.962792329 +0800 CST deployed        juicefs-csi-driver-0.19.5       0.23.2

Cloud provider or hardware configuration running JuiceFS:
```
alicloud
8c64g
```

OS (e.g cat /etc/os-release):

root@k8s-master:test# cat /etc/os-release
NAME="Ubuntu"
VERSION="20.04.6 LTS (Focal Fossa)"
ID=ubuntu
ID_LIKE=debian
PRETTY_NAME="Ubuntu 20.04.6 LTS"
VERSION_ID="20.04"
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
VERSION_CODENAME=focal
UBUNTU_CODENAME=focal

Kernel (e.g. uname -a):

root@k8s-master:test# uname -a
Linux k8s-master 5.4.0-164-generic juicedata/juicefs#181-Ubuntu SMP Fri Sep 1 13:41:22 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

Object storage (cloud provider and region, or self maintained):
```
alicloud - OSS
```

Metadata engine info (version, cloud provider managed or self maintained):

root@k8s-master:test# kubectl get pods -n redis redis-76497b8856-fvrfc -o yaml | grep image:
- image: registry.cn-hangzhou.aliyuncs.com/pb-harbor/redis:6.2.6-v6

Network connectivity (JuiceFS to metadata engine, JuiceFS to object storage):
Others:

zxh326 commented 9 months ago

hi @1473371932, Can you provide a log of the following commands：

kubectl -n kube-system logs juicefs-csi-node-m66c9 -c node-driver-registrar

1473371932 commented 9 months ago

hi @1473371932, Can you provide a log of the following commands：
kubectl -n kube-system logs juicefs-csi-node-m66c9 -c node-driver-registrar

root@k8s-master:~# kubectl -n kube-system logs juicefs-csi-node-m66c9 -c node-driver-registrar
I0131 01:59:14.208413       1 main.go:113] Version: v2.1.0-0-g80d42f24
I0131 01:59:14.220774       1 main.go:137] Attempting to open a gRPC connection with: "/csi/csi.sock"
I0131 01:59:14.221037       1 connection.go:153] Connecting to unix:///csi/csi.sock
I0131 01:59:16.319066       1 main.go:144] Calling CSI driver to discover driver name
I0131 01:59:16.319095       1 connection.go:182] GRPC call: /csi.v1.Identity/GetPluginInfo
I0131 01:59:16.319099       1 connection.go:183] GRPC request: {}
I0131 01:59:16.335597       1 connection.go:185] GRPC response: {"name":"csi.juicefs.com","vendor_version":"v0.23.2"}
I0131 01:59:16.335678       1 connection.go:186] GRPC error: <nil>
I0131 01:59:16.335687       1 main.go:154] CSI driver name: "csi.juicefs.com"
I0131 01:59:16.344112       1 node_register.go:52] Starting Registration Server at: /registration/csi.juicefs.com-reg.sock
I0131 01:59:16.345147       1 node_register.go:61] Registration Server started at: /registration/csi.juicefs.com-reg.sock
I0131 01:59:16.345519       1 node_register.go:83] Skipping healthz server because HTTP endpoint is set to: ""
I0131 01:59:17.733272       1 main.go:80] Received GetInfo call: &InfoRequest{}
I0131 01:59:18.341034       1 main.go:90] Received NotifyRegistrationStatus call: &RegistrationStatus{PluginRegistered:true,Error:,}
I0131 01:59:20.477535       1 main.go:80] Received GetInfo call: &InfoRequest{}
I0131 01:59:21.955205       1 main.go:90] Received NotifyRegistrationStatus call: &RegistrationStatus{PluginRegistered:true,Error:,}
E0201 02:05:27.929279       1 connection.go:131] Lost connection to unix:///csi/csi.sock.
E0201 08:02:26.543577       1 connection.go:131] Lost connection to unix:///csi/csi.sock.

juicedata / juicefs-csi-driver

The mount pod cannot be created after the pod references juicefs-sc. #862