Jiva Volume is not getting mounted on the pod and no error is raised

Jiva is freshely installed but when tring to create the first JV the pod ends mounting directly the host /var FS but not the volume:

Pod:

apiVersion: v1
kind: Pod
metadata:
  name: task-pv-pod
spec:
  volumes:
    - name: task-pv-storage
      persistentVolumeClaim:
        claimName: task-pv-claim
  containers:
    - name: task-pv-container
      image: nginx
      ports:
        - containerPort: 80
          name: "http-server"
      volumeMounts:
        - mountPath: "/usr/share/nginx/html"
          name: task-pv-storage

Persistent Volume Claim:

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
pv.kubernetes.io/bind-completed: "yes"
pv.kubernetes.io/bound-by-controller: "yes"
volume.beta.kubernetes.io/storage-provisioner: jiva.csi.openebs.io
volume.kubernetes.io/selected-node: t4srv101b4.ad04.eni.intranet
volume.kubernetes.io/storage-provisioner: jiva.csi.openebs.io
creationTimestamp: "2022-08-23T10:59:25Z"
finalizers:
- kubernetes.io/pvc-protection
name: test-jiva
namespace: default
resourceVersion: "27081023"
selfLink: /api/v1/namespaces/default/persistentvolumeclaims/test-jiva
uid: 79264c75-4275-453d-9cd8-36e01cb9c7f5
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
  storage: 10Gi
storageClassName: replicated
volumeMode: Filesystem
volumeName: pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5
status:
accessModes:
- ReadWriteOnce
capacity:
storage: 10Gi
phase: Bound

Evidence from container's shell:

root@task-pv-pod:/# df -h
Filesystem            Size  Used Avail Use% Mounted on
overlay                38G  7.3G   31G  20% /
tmpfs                  64M     0   64M   0% /dev
tmpfs                 3.9G     0  3.9G   0% /sys/fs/cgroup
shm                    64M     0   64M   0% /dev/shm
/dev/mapper/rhel-var   38G  7.3G   31G  20% /etc/hosts
tmpfs                 7.7G   12K  7.7G   1% /run/secrets/kubernetes.io/serviceaccount
tmpfs                 3.9G     0  3.9G   0% /proc/acpi
tmpfs                 3.9G     0  3.9G   0% /proc/scsi
tmpfs                 3.9G     0  3.9G   0% /sys/firmware
root@task-pv-pod:/# mount
overlay on / type overlay (rw,relatime,lowerdir=/var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/511/fs:/var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/510/fs:/var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/509/fs:/var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/508/fs:/var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/507/fs:/var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/506/fs,upperdir=/var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/523/fs,workdir=/var/snap/microk8s/common/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/523/work)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev type tmpfs (rw,nosuid,size=65536k,mode=755)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666)
mqueue on /dev/mqueue type mqueue (rw,nosuid,nodev,noexec,relatime)
sysfs on /sys type sysfs (ro,nosuid,nodev,noexec,relatime)
tmpfs on /sys/fs/cgroup type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (ro,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/blkio type cgroup (ro,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/freezer type cgroup (ro,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (ro,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/rdma type cgroup (ro,nosuid,nodev,noexec,relatime,rdma)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (ro,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/perf_event type cgroup (ro,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/devices type cgroup (ro,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/cpuset type cgroup (ro,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (ro,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/pids type cgroup (ro,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/memory type cgroup (ro,nosuid,nodev,noexec,relatime,memory)
shm on /dev/shm type tmpfs (rw,nosuid,nodev,noexec,relatime,size=65536k)
/dev/mapper/rhel-var on /etc/hosts type xfs (rw,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota)
/dev/mapper/rhel-var on /dev/termination-log type xfs (rw,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota)
/dev/mapper/rhel-var on /etc/hostname type xfs (rw,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota)
/dev/mapper/rhel-var on /etc/resolv.conf type xfs (rw,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota)
/dev/mapper/rhel-var on /usr/share/nginx/html type xfs (rw,relatime,attr2,inode64,logbufs=8,logbsize=32k,noquota)
tmpfs on /run/secrets/kubernetes.io/serviceaccount type tmpfs (ro,relatime,size=8043240k)
proc on /proc/bus type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/fs type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/irq type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/sys type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/sysrq-trigger type proc (ro,nosuid,nodev,noexec,relatime)
tmpfs on /proc/acpi type tmpfs (ro,relatime)
tmpfs on /proc/kcore type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/keys type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/timer_list type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/sched_debug type tmpfs (rw,nosuid,size=65536k,mode=755)
tmpfs on /proc/scsi type tmpfs (ro,relatime)
tmpfs on /sys/firmware type tmpfs (ro,relatime)
root@task-pv-pod:/#

Evidence from host shell:

[root@t4srv101b4 ~]# mount |grep aa8e5673-9d1f-4dbe-85f8-3c1c4249542a
tmpfs on /var/snap/microk8s/common/var/lib/kubelet/pods/aa8e5673-9d1f-4dbe-85f8-3c1c4249542a/volumes/kubernetes.io~projected/kube-api-access-s5tgp type tmpfs (rw,relatime,size=8043240k)
[root@t4srv101b4 ~]# lsscsi
[0:0:0:0]    disk    VMware   Virtual disk     2.0   /dev/sda
[0:0:1:0]    disk    VMware   Virtual disk     2.0   /dev/sdb
[0:0:2:0]    disk    VMware   Virtual disk     2.0   /dev/sdc
[0:0:3:0]    disk    VMware   Virtual disk     2.0   /dev/sdd
[0:0:4:0]    disk    VMware   Virtual disk     2.0   /dev/sde
[0:0:5:0]    disk    VMware   Virtual disk     2.0   /dev/sdf
[0:0:6:0]    disk    VMware   Virtual disk     2.0   /dev/sdg
[3:0:0:0]    cd/dvd  NECVMWar VMware SATA CD00 1.00  /dev/sr0
[35:0:0:0]   disk    OPENEBS  JIVA             0.1   /dev/sdj
[root@t4srv101b4 ~]# mount |grep sdj
[root@t4srv101b4 ~]#

Jiva Operators logs:

time="2022-08-16T17:22:31Z" level=info msg="Go Version: go1.17.6"
time="2022-08-16T17:22:31Z" level=info msg="Go OS/Arch: linux/amd64"
time="2022-08-16T17:22:31Z" level=info msg="Version of jiva-operator: develop-dev"
time="2022-08-16T17:22:31Z" level=info msg="starting manager"
<<redacted>
time="2022-08-23T11:01:47Z" level=info msg="not able to get controller pod ip for volume pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5: expected 1 controller pod got 0"
time="2022-08-23T11:01:48Z" level=info msg="start bootstraping jiva componentsJivaVolume: pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5"
time="2022-08-23T11:01:48Z" level=info msg="Creating a new serviceService.NamespaceopenebsService.Namepvc-79264c75-4275-453d-9cd8-36e01cb9c7f5-jiva-ctrl-svc"
time="2022-08-23T11:01:49Z" level=info msg="Updating JivaVolume with iscsi specISCSISpec{10.152.183.221 3260 iqn.2016-09.com.openebs.jiva:pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5}"
time="2022-08-23T11:01:49Z" level=info msg="Creating a new deploymentDeploy.NamespaceopenebsDeploy.Namepvc-79264c75-4275-453d-9cd8-36e01cb9c7f5-jiva-ctrl"
time="2022-08-23T11:01:49Z" level=info msg="Creating a new StatefulsetStatefulset.NamespaceopenebsSts.Namepvc-79264c75-4275-453d-9cd8-36e01cb9c7f5-jiva-rep"
time="2022-08-23T11:01:49Z" level=info msg="Creating a new pod disruption budgetPdb.NamespaceopenebsPdb.Namepvc-79264c75-4275-453d-9cd8-36e01cb9c7f5-pdb"
time="2022-08-23T11:01:49Z" level=info msg="not able to get controller pod ip for volume pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5: expected 1 controller pod got 0"
time="2022-08-23T11:01:50Z" level=info msg="start bootstraping jiva componentsJivaVolume: pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5"
time="2022-08-23T11:01:50Z" level=info msg="Updating JivaVolume with iscsi specISCSISpec{10.152.183.221 3260 iqn.2016-09.com.openebs.jiva:pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5}"
time="2022-08-23T11:01:50Z" level=error msg="failed to bootstrap volume pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5, due to error: failed to update JivaVolume with service info, err: Operation cannot be fulfilled on jivavolumes.openebs.io \"pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5\": the object has been modified; please apply your changes to the latest version and try again"
time="2022-08-23T11:01:50Z" level=error msg="failed to update JivaVolume, err: Operation cannot be fulfilled on jivavolumes.openebs.io \"pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5\": the object has been modified; please apply your changes to the latest version and try againfailed to update JivaVolume phase"
time="2022-08-23T11:01:50Z" level=info msg="not able to get controller pod ip for volume pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5: expected 1 controller pod got 0"
time="2022-08-23T11:01:53Z" level=error msg="failed to update JivaVolume, err: Operation cannot be fulfilled on jivavolumes.openebs.io \"pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5\": the object has been modified; please apply your changes to the latest version and try againfailed to update status"
time="2022-08-23T11:01:55Z" level=error msg="failed to update JivaVolume, err: Operation cannot be fulfilled on jivavolumes.openebs.io \"pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5\": the object has been modified; please apply your changes to the latest version and try againfailed to update status"
time="2022-08-23T11:02:04Z" level=error msg="failed to update JivaVolume, err: Operation cannot be fulfilled on jivavolumes.openebs.io \"pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5\": the object has been modified; please apply your changes to the latest version and try againfailed to update status"

Pv controller logs:

[root@t4srv101b4 ~]# kubectl -n openebs logs pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5-jiva-ctrl-67ff7f6xwnq5 jiva-controller
time="2022-08-23T11:01:51Z" level=info msg="RPC_READ_TIMEOUT env not set"
time="2022-08-23T11:01:51Z" level=info msg="RPC_WRITE_TIMEOUT env not set"
time="2022-08-23T11:01:51Z" level=info msg="REPLICATION_FACTOR: 3, RPC_READ_TIMEOUT: 0s, RPC_WRITE_TIMEOUT: 0s"
time="2022-08-23T11:01:51Z" level=info msg="Starting controller with frontendIP: , and clusterIP: 10.152.183.221"
time="2022-08-23T11:01:51Z" level=info msg="resetting controller"
time="2022-08-23T11:01:51Z" level=info msg="Listening on :9501"
time="2022-08-23T11:01:53Z" level=info msg="Register Replica, Address: 172.16.204.224 UUID: 771be914e8f69feec00c59682a954ada81db8f5a Uptime: 17.103004ms State: closed Type: Backend RevisionCount: 1"
time="2022-08-23T11:01:53Z" level=warning msg="No of yet to be registered replicas are less than 3 , No of registered replicas: 1"
172.16.204.192 - - [23/Aug/2022:11:01:53 +0000] "POST /v1/register HTTP/1.1" 200 0
time="2022-08-23T11:01:53Z" level=info msg="Register Replica, Address: 172.16.67.153 UUID: e461a5cfb6e95b5a50c48250092438c80a4c9448 Uptime: 22.890005ms State: closed Type: Backend RevisionCount: 1"
time="2022-08-23T11:01:53Z" level=info msg="Replica 172.16.204.224 signalled to start, registered replicas: map[string]types.RegReplica{\"172.16.204.224\":types.RegReplica{Address:\"172.16.204.224\", UUID:\"771be914e8f69feec00c59682a954ada81db8f5a\", UpTime:17103004, RevCount:1, RepType:\"Backend\", RepState:\"closed\"}, \"172.16.67.153\":types.RegReplica{Address:\"172.16.67.153\", UUID:\"e461a5cfb6e95b5a50c48250092438c80a4c9448\", UpTime:22890005, RevCount:1, RepType:\"Backend\", RepState:\"closed\"}}"
172.16.67.128 - - [23/Aug/2022:11:01:53 +0000] "POST /v1/register HTTP/1.1" 200 0
time="2022-08-23T11:01:53Z" level=info msg="resetting controller"
time="2022-08-23T11:01:53Z" level=info msg="Connecting to remote: 172.16.204.224:9502"
time="2022-08-23T11:01:53Z" level=info msg="Opening: 172.16.204.224:9502"
time="2022-08-23T11:01:53Z" level=info msg="check if replica tcp://172.16.204.224:9502 is already added"
time="2022-08-23T11:01:53Z" level=info msg="check if any WO replica available"
time="2022-08-23T11:01:53Z" level=info msg="Set replica mode of 172.16.204.224:9502 to : WO"
time="2022-08-23T11:01:53Z" level=info msg="Adding backend: tcp://172.16.204.224:9502"
time="2022-08-23T11:01:53Z" level=info msg="replicator reset false"
time="2022-08-23T11:01:53Z" level=info msg="buildreadwriters: prev: 0 0 cur: 1 0"
time="2022-08-23T11:01:53Z" level=info msg="Start monitoring tcp://172.16.204.224:9502"
time="2022-08-23T11:01:53Z" level=info msg="Get backend tcp://172.16.204.224:9502 clone status"
time="2022-08-23T11:01:53Z" level=error msg="Waiting for replica to update CloneStatus to Completed/NA, retry after 2s"
time="2022-08-23T11:01:55Z" level=info msg="Get backend tcp://172.16.204.224:9502 clone status"
time="2022-08-23T11:01:55Z" level=info msg="Set replica mode of 172.16.204.224:9502 to : RW"
time="2022-08-23T11:01:55Z" level=info msg="Set backend tcp://172.16.204.224:9502 replica mode to RW"
time="2022-08-23T11:01:55Z" level=info msg="Set replica tcp://172.16.204.224:9502 to mode RW"
time="2022-08-23T11:01:55Z" level=info msg="addr tcp://172.16.204.224:9502 m: RW prev: WO in setmode"
time="2022-08-23T11:01:55Z" level=info msg="replicator reset false"
time="2022-08-23T11:01:55Z" level=info msg="buildreadwriters: prev: 0 0 cur: 1 1"
time="2022-08-23T11:01:55Z" level=info msg="Get backend tcp://172.16.204.224:9502 revision counter 1"
time="2022-08-23T11:01:55Z" level=info msg="sending add signal to 172.16.67.153"
time="2022-08-23T11:01:55Z" level=info msg="Create Replica for address tcp://172.16.67.153:9502"
time="2022-08-23T11:01:55Z" level=info msg="Update volume status"
time="2022-08-23T11:01:55Z" level=info msg="Previously Volume RO: true, Currently: true, Total Replicas: 1, RW replicas: 1, Total backends: 1"
time="2022-08-23T11:01:55Z" level=info msg="prevCheckpoint: , currCheckpoint: "
time="2022-08-23T11:01:55Z" level=info msg="Start SCSI target"
time="2022-08-23T11:01:55Z" level=info msg="SCSI device created"
time="2022-08-23T11:01:55Z" level=info msg="iSCSI service listening on: 0.0.0.0:3260"
172.16.204.192 - - [23/Aug/2022:11:01:53 +0000] "POST /v1/volumes/cHZjLTc5MjY0Yzc1LTQyNzUtNDUzZC05Y2Q4LTM2ZTAxY2I5YzdmNQ==?action=start HTTP/1.1" 200 1045
time="2022-08-23T11:01:55Z" level=info msg="check if replica tcp://172.16.67.153:9502 is already added"
time="2022-08-23T11:01:55Z" level=info msg="check if any WO replica available"
time="2022-08-23T11:01:55Z" level=info msg="verify replication factor"
time="2022-08-23T11:01:55Z" level=info msg="Connecting to remote: 172.16.67.153:9502"
time="2022-08-23T11:01:55Z" level=info msg="Opening: 172.16.67.153:9502"
time="2022-08-23T11:01:55Z" level=info msg="Create Replica for address tcp://172.16.98.67:9502"
time="2022-08-23T11:01:55Z" level=info msg="check if replica tcp://172.16.98.67:9502 is already added"
time="2022-08-23T11:01:55Z" level=info msg="check if any WO replica available"
time="2022-08-23T11:01:55Z" level=info msg="verify replication factor"
time="2022-08-23T11:01:55Z" level=info msg="Connecting to remote: 172.16.98.67:9502"
time="2022-08-23T11:01:55Z" level=info msg="Opening: 172.16.98.67:9502"
time="2022-08-23T11:01:55Z" level=info msg="check if replica tcp://172.16.67.153:9502 is already added"
time="2022-08-23T11:01:55Z" level=info msg="check if any WO replica available"
time="2022-08-23T11:01:55Z" level=info msg="Snapshot: 172.16.204.224:9502 b212e6f0-02ff-448b-a0c9-f0b71bf2217d UserCreated false Created at 2022-08-23T11:01:55Z"
time="2022-08-23T11:01:55Z" level=info msg="successfully taken snapshots cnt 1"
time="2022-08-23T11:01:55Z" level=info msg="Snapshot: 172.16.67.153:9502 b212e6f0-02ff-448b-a0c9-f0b71bf2217d UserCreated false Created at 2022-08-23T11:01:55Z"
time="2022-08-23T11:01:55Z" level=info msg="Set replica mode of 172.16.67.153:9502 to : WO"
time="2022-08-23T11:01:55Z" level=info msg="Adding backend: tcp://172.16.67.153:9502"
time="2022-08-23T11:01:55Z" level=info msg="replicator reset false"
time="2022-08-23T11:01:55Z" level=info msg="buildreadwriters: prev: 0 1 cur: 2 1"
time="2022-08-23T11:01:55Z" level=info msg="Previously Volume RO: true, Currently: true, Total Replicas: 2, RW replicas: 1, Total backends: 2"
time="2022-08-23T11:01:55Z" level=info msg="prevCheckpoint: , currCheckpoint: "
time="2022-08-23T11:01:55Z" level=info msg="Start monitoring tcp://172.16.67.153:9502"
172.16.67.128 - - [23/Aug/2022:11:01:55 +0000] "POST /v1/replicas HTTP/1.1" 200 435
time="2022-08-23T11:01:55Z" level=info msg="check if replica tcp://172.16.98.67:9502 is already added"
time="2022-08-23T11:01:55Z" level=info msg="check if any WO replica available"
time="2022-08-23T11:01:55Z" level=info msg="Check if Replica: tcp://172.16.98.67:9502 has greater revision count"
time="2022-08-23T11:01:55Z" level=info msg="Get backend tcp://172.16.67.153:9502 revision counter 1"
time="2022-08-23T11:01:55Z" level=info msg="Revision count: 1 of New Replica: tcp://172.16.98.67:9502, Revision count: 1 of WO Replica: tcp://172.16.67.153:9502"
time="2022-08-23T11:01:55Z" level=warning msg="can have only one WO replica at a time, found WO replica: tcp://172.16.67.153:9502"
time="2022-08-23T11:01:55Z" level=info msg="addReplicaNoLock tcp://172.16.98.67:9502 from addReplica failed can only have one WO replica at a time, found WO Replica: tcp://172.16.67.153:9502"
time="2022-08-23T11:01:55Z" level=error msg="Error in request: can only have one WO replica at a time, found WO Replica: tcp://172.16.67.153:9502"
10.110.157.140 - - [23/Aug/2022:11:01:55 +0000] "POST /v1/replicas HTTP/1.1" 500 230
172.16.67.128 - - [23/Aug/2022:11:01:55 +0000] "GET /v1/replicas/dGNwOi8vMTcyLjE2LjY3LjE1Mzo5NTAy HTTP/1.1" 200 435
time="2022-08-23T11:01:55Z" level=info msg="Prepare Rebuild Replica for id tcp://172.16.67.153:9502"
time="2022-08-23T11:01:55Z" level=info msg="Synchronizing volume-head-001.img.meta@tcp://172.16.204.224:9502 to volume-head-001.img.meta@172.16.67.153:9700"
time="2022-08-23T11:01:58Z" level=info msg="Done synchronizing volume-head-001.img.meta to volume-head-001.img.meta@172.16.67.153:9700"
172.16.67.128 - - [23/Aug/2022:11:01:55 +0000] "POST /v1/replicas/dGNwOi8vMTcyLjE2LjY3LjE1Mzo5NTAy?action=preparerebuild HTTP/1.1" 200 238
172.16.67.128 - - [23/Aug/2022:11:01:58 +0000] "GET /v1/replicas/dGNwOi8vMTcyLjE2LjY3LjE1Mzo5NTAy HTTP/1.1" 200 435
time="2022-08-23T11:01:58Z" level=info msg="Verify Rebuild Replica for id tcp://172.16.67.153:9502"
time="2022-08-23T11:01:58Z" level=info msg="chain [volume-head-001.img volume-snap-b212e6f0-02ff-448b-a0c9-f0b71bf2217d.img] from rw replica tcp://172.16.204.224:9502, indx: 1"
time="2022-08-23T11:01:58Z" level=info msg="chain [volume-head-001.img volume-snap-b212e6f0-02ff-448b-a0c9-f0b71bf2217d.img] from wo replica tcp://172.16.67.153:9502, indx: 1"
time="2022-08-23T11:01:58Z" level=info msg="Get backend tcp://172.16.204.224:9502 revision counter 1"
time="2022-08-23T11:01:58Z" level=info msg="rw replica tcp://172.16.204.224:9502 revision counter 1"
time="2022-08-23T11:01:58Z" level=info msg="Set replica mode of 172.16.67.153:9502 to : RW"
time="2022-08-23T11:01:58Z" level=info msg="Set backend tcp://172.16.67.153:9502 replica mode to RW"
time="2022-08-23T11:01:58Z" level=info msg="Set revision counter of 172.16.67.153:9502 to : 1"
172.16.67.128 - - [23/Aug/2022:11:01:58 +0000] "POST /v1/replicas/dGNwOi8vMTcyLjE2LjY3LjE1Mzo5NTAy?action=verifyrebuild HTTP/1.1" 200 435
time="2022-08-23T11:01:58Z" level=info msg="Set backend tcp://172.16.67.153:9502 revision counter to 1"
time="2022-08-23T11:01:58Z" level=info msg="WO replica tcp://172.16.67.153:9502's chain verified, update replica mode to RW"
time="2022-08-23T11:01:58Z" level=info msg="Set replica tcp://172.16.67.153:9502 to mode RW"
time="2022-08-23T11:01:58Z" level=info msg="addr tcp://172.16.67.153:9502 m: RW prev: WO in setmode"
time="2022-08-23T11:01:58Z" level=info msg="replicator reset false"
time="2022-08-23T11:01:58Z" level=info msg="buildreadwriters: prev: 0 1 cur: 2 2"
time="2022-08-23T11:01:58Z" level=info msg="Previously Volume RO: true, Currently: false, Total Replicas: 2, RW replicas: 2, Total backends: 2"
time="2022-08-23T11:01:58Z" level=info msg="prevCheckpoint: , currCheckpoint: "
time="2022-08-23T11:02:01Z" level=error msg="Read msg.Version failed, Error: EOF"
time="2022-08-23T11:02:01Z" level=error msg="Error reading from wire: EOF, RemoteAddr: 172.16.98.67:9503"
time="2022-08-23T11:02:01Z" level=info msg="Exiting rpc reader, RemoteAddr:172.16.98.67:9503"
time="2022-08-23T11:02:03Z" level=info msg="Create Replica for address tcp://172.16.98.67:9502"
time="2022-08-23T11:02:03Z" level=info msg="check if replica tcp://172.16.98.67:9502 is already added"
time="2022-08-23T11:02:03Z" level=info msg="check if any WO replica available"
time="2022-08-23T11:02:03Z" level=info msg="verify replication factor"
time="2022-08-23T11:02:03Z" level=info msg="Connecting to remote: 172.16.98.67:9502"
time="2022-08-23T11:02:03Z" level=info msg="Opening: 172.16.98.67:9502"
time="2022-08-23T11:02:03Z" level=info msg="check if replica tcp://172.16.98.67:9502 is already added"
time="2022-08-23T11:02:03Z" level=info msg="check if any WO replica available"
time="2022-08-23T11:02:03Z" level=info msg="Snapshot: 172.16.67.153:9502 4ede9bde-8043-4656-b86a-0c638e46b11c UserCreated false Created at 2022-08-23T11:02:03Z"
time="2022-08-23T11:02:03Z" level=info msg="Snapshot: 172.16.204.224:9502 4ede9bde-8043-4656-b86a-0c638e46b11c UserCreated false Created at 2022-08-23T11:02:03Z"
time="2022-08-23T11:02:03Z" level=info msg="successfully taken snapshots cnt 2"
time="2022-08-23T11:02:03Z" level=info msg="Snapshot: 172.16.98.67:9502 4ede9bde-8043-4656-b86a-0c638e46b11c UserCreated false Created at 2022-08-23T11:02:03Z"
time="2022-08-23T11:02:03Z" level=info msg="Set replica mode of 172.16.98.67:9502 to : WO"
time="2022-08-23T11:02:03Z" level=info msg="Adding backend: tcp://172.16.98.67:9502"
time="2022-08-23T11:02:03Z" level=info msg="replicator reset false"
time="2022-08-23T11:02:03Z" level=info msg="buildreadwriters: prev: 0 2 cur: 3 2"
time="2022-08-23T11:02:03Z" level=info msg="Previously Volume RO: false, Currently: false, Total Replicas: 3, RW replicas: 2, Total backends: 3"
time="2022-08-23T11:02:03Z" level=info msg="prevCheckpoint: , currCheckpoint: "
time="2022-08-23T11:02:03Z" level=info msg="Start monitoring tcp://172.16.98.67:9502"
10.110.157.140 - - [23/Aug/2022:11:02:03 +0000] "POST /v1/replicas HTTP/1.1" 200 434
10.110.157.140 - - [23/Aug/2022:11:02:03 +0000] "GET /v1/replicas/dGNwOi8vMTcyLjE2Ljk4LjY3Ojk1MDI= HTTP/1.1" 200 434
time="2022-08-23T11:02:03Z" level=info msg="Prepare Rebuild Replica for id tcp://172.16.98.67:9502"
time="2022-08-23T11:02:03Z" level=info msg="Synchronizing volume-head-002.img.meta@tcp://172.16.204.224:9502 to volume-head-001.img.meta@172.16.98.67:9700"
time="2022-08-23T11:02:03Z" level=info msg="Exiting rpc loop for 172.16.98.67:9503 with err EOF"
time="2022-08-23T11:02:03Z" level=info msg="Closing read on RPC connection"
time="2022-08-23T11:02:03Z" level=info msg="Closing write on RPC connection"
time="2022-08-23T11:02:03Z" level=info msg="Exiting rpc writer, RemoteAddr:172.16.98.67:9503"
time="2022-08-23T11:02:03Z" level=info msg="Done synchronizing volume-head-002.img.meta to volume-head-001.img.meta@172.16.98.67:9700"
10.110.157.140 - - [23/Aug/2022:11:02:03 +0000] "POST /v1/replicas/dGNwOi8vMTcyLjE2Ljk4LjY3Ojk1MDI=?action=preparerebuild HTTP/1.1" 200 291
time="2022-08-23T11:02:04Z" level=info msg="connection establishing at: 172.16.98.127:3260"
time="2022-08-23T11:02:04Z" level=info msg="Target is connected to initiator: 10.110.157.140:17301"
time="2022-08-23T11:02:04Z" level=error msg="read BHS failed:EOF"
time="2022-08-23T11:02:04Z" level=warning msg="iscsi connection[0] closed"
time="2022-08-23T11:02:04Z" level=info msg="connection establishing at: 172.16.98.127:3260"
time="2022-08-23T11:02:04Z" level=info msg="Target is connected to initiator: 10.110.157.140:35608"
time="2022-08-23T11:02:04Z" level=info msg="Discovery request received from initiator: iqn.1994-05.com.redhat:e431f0a16835, Session type: Discovery, ISID: 0x23d000000"
time="2022-08-23T11:02:04Z" level=warning msg="unexpected connection state: full feature"
time="2022-08-23T11:02:04Z" level=error msg="read BHS failed:read tcp 172.16.98.127:3260->10.110.157.140:35608: read: connection reset by peer"
time="2022-08-23T11:02:04Z" level=warning msg="iscsi connection[0] closed"
time="2022-08-23T11:02:04Z" level=info msg="connection establishing at: 172.16.98.127:3260"
time="2022-08-23T11:02:04Z" level=info msg="Target is connected to initiator: 10.110.157.140:57446"
time="2022-08-23T11:02:04Z" level=info msg="Login request received from initiator: iqn.1994-05.com.redhat:e431f0a16835, Session type: Normal, Target name:iqn.2016-09.com.openebs.jiva:pvc-79264c75-4275-453d-9cd8-36e01cb9c7f5, ISID: 0x23d000006"
time="2022-08-23T11:02:04Z" level=error msg="rsa: 0h, sa:false not supported"
time="2022-08-23T11:02:04Z" level=warning msg="opcode: a3h err: check condition"
time="2022-08-23T11:02:05Z" level=warning msg="Closing RPC conn with replica: 172.16.98.67:9503"
10.110.157.140 - - [23/Aug/2022:11:02:13 +0000] "GET /v1/replicas/dGNwOi8vMTcyLjE2Ljk4LjY3Ojk1MDI= HTTP/1.1" 200 434
time="2022-08-23T11:02:13Z" level=info msg="Verify Rebuild Replica for id tcp://172.16.98.67:9502"
time="2022-08-23T11:02:13Z" level=info msg="chain [volume-head-002.img volume-snap-4ede9bde-8043-4656-b86a-0c638e46b11c.img volume-snap-b212e6f0-02ff-448b-a0c9-f0b71bf2217d.img] from rw replica tcp://172.16.204.224:9502, indx: 2"
time="2022-08-23T11:02:13Z" level=info msg="chain [volume-head-001.img volume-snap-4ede9bde-8043-4656-b86a-0c638e46b11c.img volume-snap-b212e6f0-02ff-448b-a0c9-f0b71bf2217d.img] from wo replica tcp://172.16.98.67:9502, indx: 2"
time="2022-08-23T11:02:13Z" level=info msg="Get backend tcp://172.16.204.224:9502 revision counter 85"
time="2022-08-23T11:02:13Z" level=info msg="rw replica tcp://172.16.204.224:9502 revision counter 85"
time="2022-08-23T11:02:13Z" level=info msg="Set replica mode of 172.16.98.67:9502 to : RW"
time="2022-08-23T11:02:13Z" level=info msg="Set backend tcp://172.16.98.67:9502 replica mode to RW"
time="2022-08-23T11:02:13Z" level=info msg="Set revision counter of 172.16.98.67:9502 to : 85"
time="2022-08-23T11:02:13Z" level=info msg="Set backend tcp://172.16.98.67:9502 revision counter to 85"
time="2022-08-23T11:02:13Z" level=info msg="WO replica tcp://172.16.98.67:9502's chain verified, update replica mode to RW"
time="2022-08-23T11:02:13Z" level=info msg="Set replica tcp://172.16.98.67:9502 to mode RW"
time="2022-08-23T11:02:13Z" level=info msg="addr tcp://172.16.98.67:9502 m: RW prev: WO in setmode"
time="2022-08-23T11:02:13Z" level=info msg="replicator reset false"
time="2022-08-23T11:02:13Z" level=info msg="buildreadwriters: prev: 0 2 cur: 3 3"
time="2022-08-23T11:02:13Z" level=info msg="Previously Volume RO: false, Currently: false, Total Replicas: 3, RW replicas: 3, Total backends: 3"
time="2022-08-23T11:02:13Z" level=info msg="successfully set checkpoint cnt 3"
time="2022-08-23T11:02:13Z" level=info msg="prevCheckpoint: , currCheckpoint: volume-snap-4ede9bde-8043-4656-b86a-0c638e46b11c.img"
10.110.157.140 - - [23/Aug/2022:11:02:13 +0000] "POST /v1/replicas/dGNwOi8vMTcyLjE2Ljk4LjY3Ojk1MDI=?action=verifyrebuild HTTP/1.1" 200 434
172.16.204.192 - - [23/Aug/2022:11:02:55 +0000] "GET /v1/checkpoint HTTP/1.1" 200 161

openebs-archive / jiva-operator

Jiva Volume is not getting mounted on the pod and no error is raised #197