clone of shallow copy volume is not working

Describe the bug

when we try to create clone of shallow copy volume then it gives following error: spectrumscale.csi.ibm.com_ibm-spectrum-scale-csi-provisioner-765d779f6b-gjkhc_b983f2f8-f0f3-4464-a094-f633e7391e29 failed to provision volume with StorageClass "ibm-spectrum-scale-csi": rpc error: code = Internal desc = unable to get details of source fileset link information for [pvc-bfec008e-712a-480a-aa11-44edf3f205ac]. Error : [rpc error: code = Internal desc = remote call failed with response &{[] {400 Invalid value in 'filesetName'} { 0 }}: GET request https://GUI:443/scalemgmt/v2/filesystems/fs0/filesets/pvc-bfec008e-712a-480a-aa11-44edf3f205ac, user: csi-storage-gui-user, param: <nil>, response: &{400 Bad Request 400 HTTP/1.1 1 1 map[Access-Control-Allow-Credentials:[true] Access-Control-Allow-Headers:[Authorization,Content-Type,Accept] Access-Control-Allow-Methods:[GET, POST, PUT, DELETE] Cache-Control:[no-cache="set-cookie, set-cookie2"] Content-Language:[en-US] Content-Length:[89] Content-Type:[application/json] Date:[Sat, 30 Dec 2023 03:26:43 GMT] Expires:[Thu, 01 Dec 1994 16:00:00 GMT] Set-Cookie:[JSESSIONID=0000iwu__xLnn7Uj2ivQxTzYnfW:1eb0a1d7-f9fd-424d-ada5-ecd252ed2642; Path=/; Secure; HttpOnly; SameSite=Strict] Strict-Transport-Security:[max-age=31536000] X-Content-Type-Options:[nosniff]] 0xc0000ba880 89 [] true false map[] 0xc000510500 0xc00036e2c0}]

How to Reproduce?

Install CSI 2.11.0 with driver images from #1067 :

[root@OCP ]# oc get pods
NAME                                                  READY   STATUS    RESTARTS        AGE
csi-scale-fsetdemo-pod-1                              1/1     Running   0               114s
ibm-spectrum-scale-csi-5f9vx                          3/3     Running   0               3d16h
ibm-spectrum-scale-csi-attacher-585ddbd6bb-l4n62      1/1     Running   0               3d16h
ibm-spectrum-scale-csi-attacher-585ddbd6bb-mc5bk      1/1     Running   0               3d16h
ibm-spectrum-scale-csi-f69cw                          3/3     Running   0               3d16h
ibm-spectrum-scale-csi-knlvd                          3/3     Running   0               3d16h
ibm-spectrum-scale-csi-operator-5c6565c84c-q4c7p      1/1     Running   0               3d16h
ibm-spectrum-scale-csi-provisioner-765d779f6b-gjkhc   1/1     Running   0               3d16h
ibm-spectrum-scale-csi-resizer-644c7bd59b-8rnlj       1/1     Running   0               3d16h
ibm-spectrum-scale-csi-snapshotter-7cd97548d5-j8stg   1/1     Running   2 (6h20m ago)   3d16h
[root@OCP ]# oc get cso
NAME                     VERSION   SUCCESS
ibm-spectrum-scale-csi   2.11.0    True
[root@OCP ]# oc describe pod |grep quay
Image:         quay.io/hemalatha_gajendran/driver_shallowcopy_withoutfileset
Image ID:      quay.io/hemalatha_gajendran/driver_shallowcopy_withoutfileset@sha256:8f1d9b7d5c80ca2b7c173584ca97d2587143f8e84d5c7bfa48d3618e43e9119b
Image:         quay.io/hemalatha_gajendran/driver_shallowcopy_withoutfileset
Image ID:      quay.io/hemalatha_gajendran/driver_shallowcopy_withoutfileset@sha256:8f1d9b7d5c80ca2b7c173584ca97d2587143f8e84d5c7bfa48d3618e43e9119b
Image:         quay.io/hemalatha_gajendran/driver_shallowcopy_withoutfileset
Image ID:      quay.io/hemalatha_gajendran/driver_shallowcopy_withoutfileset@sha256:8f1d9b7d5c80ca2b7c173584ca97d2587143f8e84d5c7bfa48d3618e43e9119b
Image:         quay.io/ibm-spectrum-scale-dev/ibm-spectrum-scale-csi-operator@sha256:18264e0c9c112856bc2744f7f971a4b60ecb24de57e46af4d35456dcdf8e3cbf
Image ID:      quay.io/ibm-spectrum-scale-dev/ibm-spectrum-scale-csi-operator@sha256:18264e0c9c112856bc2744f7f971a4b60ecb24de57e46af4d35456dcdf8e3cbf
  CSI_DRIVER_IMAGE:          quay.io/hemalatha_gajendran/driver_shallowcopy_withoutfileset

Create PVC and storage class as following:


[root@OCP ]# cat apply.yaml
apiVersion: v1
kind: Pod
metadata:
name: csi-scale-fsetdemo-pod-1
labels:
app: nginx
spec:
containers:
- name: web-server
 image: docker-na-public.artifactory.swg-devops.com/sys-spectrum-scale-team-test-environment-docker-local/nginx:1.22.0
 volumeMounts:
   - name: mypvc
     mountPath: /usr/share/nginx/html/scale
 ports:
 - containerPort: 80
volumes:
- name: mypvc
 persistentVolumeClaim:
   claimName: scale-pvc-1
   readOnly: false

apiVersion: v1 kind: PersistentVolumeClaim metadata: name: scale-pvc-1 spec: accessModes:

ReadWriteMany resources: requests: storage: 1Gi storageClassName: ibm-spectrum-scale-csi

apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: ibm-spectrum-scale-csi provisioner: spectrumscale.csi.ibm.com parameters: volBackendFs: "fs0" reclaimPolicy: Delete

[root@OCP ]# oc get pvc NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE scale-pvc-1 Bound pvc-1f4cea24-349f-4e5e-bded-77697a4d4273 1Gi RWX ibm-spectrum-scale-csi 68s [root@OCP ]# oc get pods NAME READY STATUS RESTARTS AGE csi-scale-fsetdemo-pod-1 1/1 Running 0 72s

3. Write data inside pod :

root@csi-scale-fsetdemo-pod-1:/usr/share/nginx/html/scale# touch test{1..10} root@csi-scale-fsetdemo-pod-1:/usr/share/nginx/html/scale# ls test1 test10 test2 test3 test4 test5 test6 test7 test8 test9

4. Create snapshot of pvc :

[root@OCP ]# cat snapshot.yaml apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshot metadata: name: ibm-spectrum-scale-snapshot spec: volumeSnapshotClassName: ibm-spectrum-scale-snapshotclass source: persistentVolumeClaimName: scale-pvc-1

apiVersion: snapshot.storage.k8s.io/v1 kind: VolumeSnapshotClass metadata: name: ibm-spectrum-scale-snapshotclass driver: spectrumscale.csi.ibm.com parameters: snapWindow: "30" #Optional : Time in minutes (default=30) deletionPolicy: Delete [root@OCP ]# oc apply -f snapshot.yaml volumesnapshot.snapshot.storage.k8s.io/ibm-spectrum-scale-snapshot created volumesnapshotclass.snapshot.storage.k8s.io/ibm-spectrum-scale-snapshotclass created [root@OCP ]# oc get vs NAME READYTOUSE SOURCEPVC SOURCESNAPSHOTCONTENT RESTORESIZE SNAPSHOTCLASS SNAPSHOTCONTENT CREATIONTIME AGE ibm-spectrum-scale-snapshot true scale-pvc-1 1Gi ibm-spectrum-scale-snapshotclass snapcontent-a7f54b9d-08d3-4087-949c-38c98d9be859 47s 61s

5. Create shallow copy from above snapshot :

[root@OCP ]# cat shallowCopy.yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: scale-shallow_copy-pvc-1 spec: accessModes:

ReadOnlyMany resources: requests: storage: 1Gi storageClassName: ibm-spectrum-scale-csi dataSource: name: ibm-spectrum-scale-snapshot kind: VolumeSnapshot apiGroup: snapshot.storage.k8s.io [root@OCP ]# oc apply -f shallowCopy.yaml persistentvolumeclaim/scale-shallow-copy-pvc-1 created
```
6. Create clone of shallow copy volume:
```
[root@OCP ]# cat clone.yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: name: ibm-spectrum-scale-pvc-clone-from-shallowcopy spec: accessModes:
ReadWriteMany resources: requests: storage: 1Gi storageClassName: ibm-spectrum-scale-csi dataSource: name: scale-advance-pvc-1 kind: PersistentVolumeClaim [root@OCP ]# oc apply -f clone.yaml persistentvolumeclaim/ibm-spectrum-scale-pvc-clone-from-shallowcopy created
```
7. Check the description of clone pvc
```
[root@OCP ]# oc get pvc -w NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE ibm-spectrum-scale-pvc-clone-from-shallowcopy Pending ibm-spectrum-scale-csi 3s scale-pvc-1 Bound pvc-1f4cea24-349f-4e5e-bded-77697a4d4273 1Gi RWX ibm-spectrum-scale-csi 32m scale-shallow-copy-pvc-1 Bound pvc-bfec008e-712a-480a-aa11-44edf3f205ac 1Gi ROX ibm-spectrum-scale-csi 65s [root@OCP ]# ^C [root@OCP ]# oc describe pvc ibm-spectrum-scale-pvc-clone-from-shallowcopy Name: ibm-spectrum-scale-pvc-clone-from-shallowcopy Namespace: ibm-spectrum-scale-csi StorageClass: ibm-spectrum-scale-csi Status: Pending Volume: Labels: Annotations: volume.beta.kubernetes.io/storage-provisioner: spectrumscale.csi.ibm.com volume.kubernetes.io/storage-provisioner: spectrumscale.csi.ibm.com Finalizers: [kubernetes.io/pvc-protection] Capacity: Access Modes: VolumeMode: Filesystem DataSource: Kind: PersistentVolumeClaim Name: scale-shallow-copy-pvc-1 Used By: Events: Type Reason Age From Message

Normal Provisioning 15s spectrumscale.csi.ibm.com_ibm-spectrum-scale-csi-provisioner-765d779f6b-gjkhc_b983f2f8-f0f3-4464-a094-f633e7391e29 External provisioner is provisioning volume for claim "ibm-spectrum-scale-csi/ibm-spectrum-scale-pvc-clone-from-shallowcopy" Normal ExternalProvisioning 12s (x3 over 15s) persistentvolume-controller waiting for a volume to be created, either by external provisioner "spectrumscale.csi.ibm.com" or manually created by system administrator Warning ProvisioningFailed 1s spectrumscale.csi.ibm.com_ibm-spectrum-scale-csi-provisioner-765d779f6b-gjkhc_b983f2f8-f0f3-4464-a094-f633e7391e29 failed to provision volume with StorageClass "ibm-spectrum-scale-csi": rpc error: code = Internal desc = unable to get details of source fileset link information for [pvc-bfec008e-712a-480a-aa11-44edf3f205ac]. Error : [rpc error: code = Internal desc = remote call failed with response &{[] {400 Invalid value in 'filesetName'} { 0 }}: GET request https://GUI:443/scalemgmt/v2/filesystems/fs0/filesets/pvc-bfec008e-712a-480a-aa11-44edf3f205ac, user: csi-storage-gui-user, param: , response: &{400 Bad Request 400 HTTP/1.1 1 1 map[Access-Control-Allow-Credentials:[true] Access-Control-Allow-Headers:[Authorization,Content-Type,Accept] Access-Control-Allow-Methods:[GET, POST, PUT, DELETE] Cache-Control:[no-cache="set-cookie, set-cookie2"] Content-Language:[en-US] Content-Length:[89] Content-Type:[application/json] Date:[Sat, 30 Dec 2023 03:26:43 GMT] Expires:[Thu, 01 Dec 1994 16:00:00 GMT] Set-Cookie:[JSESSIONID=0000iwu__xLnn7Uj2ivQxTzYnfW:1eb0a1d7-f9fd-424d-ada5-ecd252ed2642; Path=/; Secure; HttpOnly; SameSite=Strict] Strict-Transport-Security:[max-age=31536000] X-Content-Type-Options:[nosniff]] 0xc0000ba880 89 [] true false map[] 0xc000510500 0xc00036e2c0}] [root@OCP ]#

Expected behavior

Clone should be created for the shallow copy volume

Logs

/scale-csi/D.1079 csisnap.tar.gz

IBM / ibm-spectrum-scale-csi