NVMeF Luns go to read only after few minutes of iozone run

[veereshuk]

Steps: -> Unload all RDMA drivers, load cxgb4 driver, assign ip to

interfaces. -> load SIW driver, then check if the devices is listed in the ibv_devices.

[root@host1 ~]# ibv_devices device node GUID

---

siw_eno1            0cc47acc4b2c0000
siw_ens6f4d1        00074304b1b80000
siw_lo              7369775f6c6f0000
siw_ens6f4          00074304b1b00000

-> run below script to create targets.

[root@host2]# cat nvme_new_target.sh mount -t configfs none /sys/kernel/config rmmod nvmet_rdma nvmet nvme null_blk

sleep 1

IPPORT="4420" # 4420 is the reserved NVME/Fabrics RDMA port IPADDR="102.1.1.146" # the ipaddress of your target rdma interface IPADDR1="102.2.2.146" # the ipaddress of your target rdma interface NAME="nvme-ram" # this can be whatever you want

DEV="/dev/ram" # you can use any block device

DEV="/dev/nvme_disk/vol" # you can use any block device

modprobe nvmet modprobe nvmet-rdma

use_srq=1

for i in 0 1 2 3 4 5 6 7 8 9; do mkdir /sys/kernel/config/nvmet/subsystems/${NAME}${i} mkdir /sys/kernel/config/nvmet/subsystems/${NAME}${i}/namespaces/1 echo -n ${DEV}${i} >/sys/kernel/config/nvmet/subsystems/${NAME}${i}/namespaces/1/device_path echo 1 > /sys/kernel/config/nvmet/subsystems/${NAME}${i}/attr_allow_any_host echo 1 > /sys/kernel/config/nvmet/subsystems/${NAME}${i}/namespaces/1/enable done

mkdir /sys/kernel/config/nvmet/ports/1 echo "ipv4" > /sys/kernel/config/nvmet/ports/1/addr_adrfam echo "rdma" > /sys/kernel/config/nvmet/ports/1/addr_trtype echo $IPPORT > /sys/kernel/config/nvmet/ports/1/addr_trsvcid echo $IPADDR > /sys/kernel/config/nvmet/ports/1/addr_traddr

mkdir /sys/kernel/config/nvmet/ports/2 echo "ipv4" > /sys/kernel/config/nvmet/ports/2/addr_adrfam echo "rdma" > /sys/kernel/config/nvmet/ports/2/addr_trtype echo $IPPORT > /sys/kernel/config/nvmet/ports/2/addr_trsvcid echo $IPADDR1 > /sys/kernel/config/nvmet/ports/2/addr_traddr

for i in 0 1 2 3 4 ; do ln -s /sys/kernel/config/nvmet/subsystems/${NAME}${i} /sys/kernel/config/nvmet/ports/1/subsystems/${NAME}${i} done

for i in 5 6 7 8 9; do ln -s /sys/kernel/config/nvmet/subsystems/${NAME}${i} /sys/kernel/config/nvmet/ports/2/subsystems/${NAME}${i} done

-> Then login to target from initiator with below script.

[root@host1]# cat nvme_new_login.sh modprobe nvme modprobe nvme-rdma for i in $(seq 0 4) ; do echo "traddr=102.1.1.146,transport=rdma,trsvcid=4420,nqn=nvme-ram${i}" >/dev/nvme-fabrics ; done

modprobe nvme-rdma for i in $(seq 5 9) ; do echo "traddr=102.2.2.146,transport=rdma,trsvcid=4420,nqn=nvme-ram${i}" >/dev/nvme-fabrics ; done

-> after 10 to 15 minutes , the LUNS go into read only mode.

-> attached is the snipet of the initiator debug logs.

OS: RHEL7.3 kernel 4.13.2 Driver:inbox driver initiator.txt

[veereshuk]

tried with only one LUN, attached the debug logs. dmesg_initator.txt dmesg_target.txt