search

IBM / ibm-object-csi-driver

Object storage plugin for IBM satellite
Apache License 2.0
3 stars 8 forks source link

Add Delay While fetching Volume Stats from Bucket #125

Open ashimagarg27 opened 5 months ago

ashimagarg27 commented 5 months ago

Describe your changes

Issue ticket number and link

Checklist before requesting a review

ashimagarg27 commented 5 months ago

Test Results

~ ❯ oc get secret | grep csi                                                                                                                                                                        ⎈ ashima-sat/cp0u7tkd0gdarrvi9cbg/admin 14:13:44
cos-s3-csi-pvc-rclone      cos-s3-csi-driver                     6      9d
cos-s3-csi-pvc-s3fs        cos-s3-csi-driver                     6      74m

~ ❯ oc get all -n kube-system | grep csi                                                                                                                                                            ⎈ ashima-sat/cp0u7tkd0gdarrvi9cbg/admin 14:13:49
Warning: apps.openshift.io/v1 DeploymentConfig is deprecated in v4.14+, unavailable in v4.10000+
pod/cos-s3-csi-controller-57966f66c7-s6vvv           3/3     Running   0          30s
pod/cos-s3-csi-driver-2fc6q                          3/3     Running   0          29s
pod/cos-s3-csi-driver-c28vf                          3/3     Running   0          29s
pod/cos-s3-csi-driver-hszkn                          3/3     Running   0          29s
daemonset.apps/cos-s3-csi-driver         3         3         3       3            3           <none>          31s
deployment.apps/cos-s3-csi-controller       1/1     1            1           33s
replicaset.apps/cos-s3-csi-controller-57966f66c7       1         1         1       33s

~ ❯ oc get deployment.apps/cos-s3-csi-controller -o yaml -n kube-system | grep image:                                                                                                            8s ⎈ ashima-sat/cp0u7tkd0gdarrvi9cbg/admin 14:14:15
        image: k8s.gcr.io/sig-storage/csi-provisioner:v3.4.1
        image: docker.io/ashimagarg/stats:09
        image: registry.k8s.io/sig-storage/livenessprobe:v2.9.0

~ ❯ oc get pvc                                                                                                                                                                                      ⎈ ashima-sat/cp0u7tkd0gdarrvi9cbg/admin 14:14:17
NAME                    STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS           AGE
cos-s3-csi-pvc-rclone   Bound    pvc-68c91b3a-773b-4099-aeb7-45aa16558aae   100Gi      RWO            cos-s3-csi-rclone-sc   13s
cos-s3-csi-pvc-s3fs     Bound    pvc-900f3c82-f90e-46a3-a4cd-3cfe1621ccc6   256Mi      RWO            cos-s3-csi-s3fs-sc     12s

~ ❯ oc get pv                                                                                                                                                                                       ⎈ ashima-sat/cp0u7tkd0gdarrvi9cbg/admin 14:14:40
NAME                                       CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS   CLAIM                           STORAGECLASS           REASON   AGE
pvc-68c91b3a-773b-4099-aeb7-45aa16558aae   100Gi      RWO            Retain           Bound    default/cos-s3-csi-pvc-rclone   cos-s3-csi-rclone-sc            16s
pvc-900f3c82-f90e-46a3-a4cd-3cfe1621ccc6   256Mi      RWO            Retain           Bound    default/cos-s3-csi-pvc-s3fs     cos-s3-csi-s3fs-sc              16s

~ ❯ oc get pod                                                                                                                                                                                      ⎈ ashima-sat/cp0u7tkd0gdarrvi9cbg/admin 14:14:44
NAME          READY   STATUS    RESTARTS   AGE
cos-csi-app   1/1     Running   0          12s

~ ❯ oc exec -it cos-csi-app -- /bin/bash                                                                                                                                                            ⎈ ashima-sat/cp0u7tkd0gdarrvi9cbg/admin 14:15:15
root@cos-csi-app:/# mount | grep s3
s3fs on /data1 type fuse.s3fs (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
root@cos-csi-app:/# mount | grep rclone
ibmcos:ashima-bucket02 on /data2 type fuse.rclone (rw,nosuid,nodev,relatime,user_id=0,group_id=0,allow_other)
root@cos-csi-app:/#
root@cos-csi-app:/#
root@cos-csi-app:/# ls /data1
hackathon202404.pptx  hackathon_upd.mp4  pod.yaml  rclonePVC.yaml  rcloneSecret.yaml  s3FSPVC.yaml  s3FSsecert.yaml
root@cos-csi-app:/#
root@cos-csi-app:/#
root@cos-csi-app:/# ls /data2
Medibuddy_2024.pdf  new.txt  test.txt
root@cos-csi-app:/#
root@cos-csi-app:/#
root@cos-csi-app:/# exit
exit

Logs:

I0523 08:45:56.825497       1 nodeserver.go:198] NodeGetVolumeStats: Start getting Stats
I0523 08:45:56.844594       1 driver_utils.go:209] pvc details found. pvc-name: cos-s3-csi-pvc-rclone, pvc-namespace: default
I0523 08:45:56.855475       1 driver_utils.go:220] secret details found. secret-name: cos-s3-csi-pvc-rclone
I0523 08:45:57.082134       1 nodeserver.go:224] NodeGetVolumeStats: Total Capacity of Volume: 107374182400
I0523 08:45:57.082172       1 nodeserver.go:253] NodeGetVolumeStats: Volume Stats usage:<available:107373689615 total:107374182400 used:492785 unit:BYTES > usage:<available:1000000000 total:1000000000 unit:INODES >
I0523 08:45:57.082347       1 server.go:160] GRPC response: usage:<available:107373689615 total:107374182400 used:492785 unit:BYTES > usage:<available:1000000000 total:1000000000 unit:INODES >
I0523 08:45:57.084156       1 server.go:155] GRPC call: /csi.v1.Node/NodeGetCapabilities
I0523 08:45:57.084188       1 nodeserver.go:262] NodeGetCapabilities: Request: {XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}
I0523 08:45:57.084221       1 server.go:160] GRPC response: capabilities:<rpc:<type:GET_VOLUME_STATS > > capabilities:<rpc:<type:VOLUME_CONDITION > > capabilities:<rpc:<type:VOLUME_MOUNT_GROUP > >
I0523 08:45:57.085202       1 server.go:155] GRPC call: /csi.v1.Node/NodeGetVolumeStats
I0523 08:45:57.085222       1 nodeserver.go:186] NodeGetVolumeStats: Request: {VolumeId:pvc-900f3c82-f90e-46a3-a4cd-3cfe1621ccc6 VolumePath:/var/data/kubelet/pods/cb8ba9c3-d413-4a4c-b811-dc5f200b6b4e/volumes/kubernetes.io~csi/pvc-900f3c82-f90e-46a3-a4cd-3cfe1621ccc6/mount StagingTargetPath: XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}
I0523 08:45:57.085240       1 nodeserver.go:198] NodeGetVolumeStats: Start getting Stats
I0523 08:45:57.093570       1 driver_utils.go:209] pvc details found. pvc-name: cos-s3-csi-pvc-s3fs, pvc-namespace: default
I0523 08:45:57.104869       1 driver_utils.go:220] secret details found. secret-name: cos-s3-csi-pvc-s3fs
I0523 08:45:57.248903       1 nodeserver.go:224] NodeGetVolumeStats: Total Capacity of Volume: 268435456
I0523 08:45:57.248941       1 nodeserver.go:253] NodeGetVolumeStats: Volume Stats usage:<available:201738522 total:268435456 used:66696934 unit:BYTES > usage:<unit:INODES >
I0523 08:45:57.249003       1 server.go:160] GRPC response: usage:<available:201738522 total:268435456 used:66696934 unit:BYTES > usage:<unit:INODES >

I0523 09:01:11.750611       1 server.go:155] GRPC call: /csi.v1.Node/NodeGetVolumeStats
I0523 09:01:11.750644       1 nodeserver.go:186] NodeGetVolumeStats: Request: {VolumeId:pvc-68c91b3a-773b-4099-aeb7-45aa16558aae VolumePath:/var/data/kubelet/pods/cb8ba9c3-d413-4a4c-b811-dc5f200b6b4e/volumes/kubernetes.io~csi/pvc-68c91b3a-773b-4099-aeb7-45aa16558aae/mount StagingTargetPath: XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}
I0523 09:01:11.750675       1 nodeserver.go:198] NodeGetVolumeStats: Start getting Stats
I0523 09:01:11.768959       1 driver_utils.go:209] pvc details found. pvc-name: cos-s3-csi-pvc-rclone, pvc-namespace: default
I0523 09:01:11.780424       1 driver_utils.go:220] secret details found. secret-name: cos-s3-csi-pvc-rclone
I0523 09:01:12.203022       1 nodeserver.go:224] NodeGetVolumeStats: Total Capacity of Volume: 107374182400
I0523 09:01:12.203048       1 nodeserver.go:253] NodeGetVolumeStats: Volume Stats usage:<available:107373689615 total:107374182400 used:492785 unit:BYTES > usage:<available:1000000000 total:1000000000 unit:INODES >
I0523 09:01:12.203102       1 server.go:160] GRPC response: usage:<available:107373689615 total:107374182400 used:492785 unit:BYTES > usage:<available:1000000000 total:1000000000 unit:INODES >
I0523 09:01:12.204302       1 server.go:155] GRPC call: /csi.v1.Node/NodeGetCapabilities
I0523 09:01:12.204320       1 nodeserver.go:262] NodeGetCapabilities: Request: {XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}
I0523 09:01:12.204346       1 server.go:160] GRPC response: capabilities:<rpc:<type:GET_VOLUME_STATS > > capabilities:<rpc:<type:VOLUME_CONDITION > > capabilities:<rpc:<type:VOLUME_MOUNT_GROUP > >
I0523 09:01:12.205259       1 server.go:155] GRPC call: /csi.v1.Node/NodeGetVolumeStats
I0523 09:01:12.205282       1 nodeserver.go:186] NodeGetVolumeStats: Request: {VolumeId:pvc-900f3c82-f90e-46a3-a4cd-3cfe1621ccc6 VolumePath:/var/data/kubelet/pods/cb8ba9c3-d413-4a4c-b811-dc5f200b6b4e/volumes/kubernetes.io~csi/pvc-900f3c82-f90e-46a3-a4cd-3cfe1621ccc6/mount StagingTargetPath: XXX_NoUnkeyedLiteral:{} XXX_unrecognized:[] XXX_sizecache:0}
I0523 09:01:12.205295       1 nodeserver.go:198] NodeGetVolumeStats: Start getting Stats
I0523 09:01:12.215802       1 driver_utils.go:209] pvc details found. pvc-name: cos-s3-csi-pvc-s3fs, pvc-namespace: default
I0523 09:01:12.227787       1 driver_utils.go:220] secret details found. secret-name: cos-s3-csi-pvc-s3fs
I0523 09:01:12.536398       1 nodeserver.go:224] NodeGetVolumeStats: Total Capacity of Volume: 268435456
I0523 09:01:12.536433       1 nodeserver.go:253] NodeGetVolumeStats: Volume Stats usage:<available:201738522 total:268435456 used:66696934 unit:BYTES > usage:<unit:INODES >
I0523 09:01:12.536515       1 server.go:160] GRPC response: usage:<available:201738522 total:268435456 used:66696934 unit:BYTES > usage:<unit:INODES >

NodeSummary:

"volume": [
    {
     "time": "2024-05-23T08:47:00Z",
     "availableBytes": 201738522,
     "capacityBytes": 268435456,
     "usedBytes": 66696934,
     "inodesFree": 0,
     "inodes": 0,
     "inodesUsed": 0,
     "name": "cos-csi-volume",
     "pvcRef": {
      "name": "cos-s3-csi-pvc-s3fs",
      "namespace": "default"
     }
    },
    {
     "time": "2024-05-23T08:47:00Z",
     "availableBytes": 107373689615,
     "capacityBytes": 107374182400,
     "usedBytes": 492785,
     "inodesFree": 1000000000,
     "inodes": 1000000000,
     "inodesUsed": 0,
     "name": "cos-csi-rclone",
     "pvcRef": {
      "name": "cos-s3-csi-pvc-rclone",
      "namespace": "default"
     }
    },
    {
     "time": "2024-05-23T08:47:00Z",
     "availableBytes": 61094879232,
     "capacityBytes": 61094903808,
     "usedBytes": 24576,
     "inodesFree": 8176071,
     "inodes": 8176082,
     "inodesUsed": 11,
     "name": "kube-api-access-rk2c5"
    }
   ],

NodeSummary After Some Time:

"volume": [
    {
     "time": "2024-05-23T09:13:49Z",
     "availableBytes": 107373689615,
     "capacityBytes": 107374182400,
     "usedBytes": 492785,
     "inodesFree": 1000000000,
     "inodes": 1000000000,
     "inodesUsed": 0,
     "name": "cos-csi-rclone",
     "pvcRef": {
      "name": "cos-s3-csi-pvc-rclone",
      "namespace": "default"
     }
    },
    {
     "time": "2024-05-23T09:13:49Z",
     "availableBytes": 61094879232,
     "capacityBytes": 61094903808,
     "usedBytes": 24576,
     "inodesFree": 8176071,
     "inodes": 8176082,
     "inodesUsed": 11,
     "name": "kube-api-access-rk2c5"
    },
    {
     "time": "2024-05-23T09:13:49Z",
     "availableBytes": 201738522,
     "capacityBytes": 268435456,
     "usedBytes": 66696934,
     "inodesFree": 0,
     "inodes": 0,
     "inodesUsed": 0,
     "name": "cos-csi-volume",
     "pvcRef": {
      "name": "cos-s3-csi-pvc-s3fs",
      "namespace": "default"
     }
    }
   ],
ashimagarg27 commented 5 months ago

Complete NodeServer Pod Logs:

logs.log