search

SCST-project / scst

SCST is a SCSI target software stack that allows to export any block device or file via iSCSI, FC or RDMA (SRP).
http://scst.sourceforge.net
96 stars 35 forks source link

Not work with RDMA on Ubuntu 22.04 #168

Open AleksovAnry opened 1 year ago

AleksovAnry commented 1 year ago

Works fine over TCP, not over RDMA (iSER)

root@b:~# rping -c -a 192.168.0.11 -C 10 -v
ping data: rdma-ping-0: ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqr
ping data: rdma-ping-1: BCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrs
ping data: rdma-ping-2: CDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrst
ping data: rdma-ping-3: DEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstu
ping data: rdma-ping-4: EFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuv
ping data: rdma-ping-5: FGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw
ping data: rdma-ping-6: GHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwx
ping data: rdma-ping-7: HIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxy
ping data: rdma-ping-8: IJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz
ping data: rdma-ping-9: JKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyzA
client DISCONNECT EVENT...
root@b:~# iscsiadm --op=new --op=delete -m discovery -t st -p 192.168.0.11 -I iser
iscsiadm: Connection to discovery portal 192.168.0.11 failed: encountered connection failure
iscsiadm: Connection to discovery portal 192.168.0.11 failed: encountered connection failure
iscsiadm: Connection to discovery portal 192.168.0.11 failed: encountered connection failure
iscsiadm: Connection to discovery portal 192.168.0.11 failed: encountered connection failure
iscsiadm: Connection to discovery portal 192.168.0.11 failed: encountered connection failure
iscsiadm: Connection to discovery portal 192.168.0.11 failed: encountered connection failure
iscsiadm: connection login retries (reopen_max) 5 exceeded
iscsiadm: Could not perform SendTargets discovery: iSCSI PDU timed out
root@b:~# lsmod|grep rdma
rpcrdma               303104  0
sunrpc                585728  1 rpcrdma
rdma_ucm               28672  0
rdma_cm               122880  5 rpcrdma,ib_srp,ib_iser,ib_isert,rdma_ucm
iw_cm                  49152  1 rdma_cm
ib_cm                 131072  3 rdma_cm,ib_ipoib,ib_srp
ib_uverbs             163840  18 rdma_ucm,mlx5_ib
ib_core               393216  12 rdma_cm,ib_ipoib,rpcrdma,ib_srp,iw_cm,ib_iser,ib_umad,ib_isert,rdma_ucm,ib_uverbs,mlx5_ib,ib_cm
root@a:~#  lsmod|grep scst
scst_vdisk            122880  0
scst_disk              20480  0
iscsi_scst            110592  4
scst                 3584000  4 scst_disk,scst_vdisk,ib_srpt,iscsi_scst
dlm                   319488  1 scst
libcrc32c              16384  5 scst_vdisk,btrfs,iscsi_scst,drbd,raid456
root@a:~# modinfo scst
filename:       /lib/modules/5.15.0-73-generic/extra/scst.ko
revision:       
version:        3.8.0-pre
description:    SCSI target core
license:        GPL
author:         Vladislav Bolkhovitin
srcversion:     6F9499305804C63A6A2A446
depends:        dlm
retpoline:      Y
name:           scst
vermagic:       5.15.0-73-generic SMP mod_unload modversions 
sig_id:         PKCS#7
signer:         SCST Kernel Modules
sig_key:        0A:68:C5:29:44:DF:15:2D:6C:A5:02:43:23:D7:C9:1D:BE:84:2F:F2
sig_hashalgo:   sha512
signature:      08:32:7F:DB:0D:3E:B4:37:53:BF:E5:29:14:4B:46:DF:FC:ED:AD:36:
        10:48:F9:01:D9:0C:9C:F2:1F:2B:8A:DE:03:11:9D:2C:29:EF:90:D9:
        2D:45:EE:10:B0:97:AF:67:22:F7:FF:84:88:8F:71:84:F5:03:B2:62:
        AF:7C:DD:89:14:D3:55:28:7D:9C:5B:B7:D5:78:4C:4F:8F:D4:9A:FF:
        27:16:64:48:EC:40:AC:8F:B5:81:7A:C4:16:F3:E7:E5:D4:3E:49:04:
        65:D0:0A:34:6E:A6:65:2A:9A:12:48:39:A7:8A:E2:16:ED:E3:48:E6:
        27:74:23:F1:84:EB:87:40:C5:BB:A3:D6:FD:F9:BE:4D:84:88:6E:F2:
        3D:37:AF:6E:2B:87:09:D2:6E:6B:C3:55:11:6A:33:DC:32:8B:2C:21:
        41:5E:4E:EE:48:6F:5B:C5:36:D7:5A:5C:F2:7E:ED:C9:70:D9:B7:26:
        E2:E4:56:BA:7B:0B:94:C4:F4:8B:D6:D6:EA:3C:8C:06:3C:78:A1:BC:
        56:6E:72:E8:1E:47:7D:67:39:52:CE:D8:07:82:66:ED:17:EE:5E:D5:
        B5:52:6D:46:A5:AC:1B:65:01:0B:0B:BE:B5:A1:B8:13:61:89:30:1A:
        66:AE:69:BD:E8:52:EF:83:B7:DF:C7:96:90:2C:4E:21:CF:D7:4F:9F:
        4D:C2:EF:52:3E:A1:8B:CB:94:D2:65:F9:EC:F9:1A:2E:81:99:2C:A0:
        DB:46:A9:F3:BD:21:92:4E:56:DC:18:E7:37:3C:34:09:2F:51:D2:1A:
        79:13:3F:27:EC:26:14:B0:C5:5D:98:0A:C9:0C:44:FD:83:D3:18:95:
        AC:7D:20:89:89:C4:4D:92:22:83:8C:A6:79:97:4B:3A:E3:C1:00:63:
        83:52:68:5E:01:BA:4F:15:5A:27:3F:6B:E8:EB:FC:AF:34:C7:CB:B1:
        7B:C5:19:05:29:50:FF:A5:4E:D9:86:CF:0A:B4:98:26:3C:AD:6B:3D:
        54:80:33:D1:C7:E0:E7:4D:C3:08:31:93:CD:A4:77:5F:1B:66:7A:33:
        79:15:54:56:4D:11:2D:89:4B:C3:AE:71:64:09:46:2A:B1:20:E1:CB:
        A7:5A:B9:18:28:66:52:68:6B:19:81:5F:31:28:DB:AD:AB:E0:B9:B4:
        7A:24:43:33:3C:74:56:5F:A7:E8:12:EE:AF:89:85:4B:AA:A4:18:2C:
        DB:94:14:92:FE:FB:25:96:2B:FD:32:18:E4:26:F1:3E:B4:3B:6B:52:
        3C:D4:82:6B:B5:55:84:28:E2:CF:0A:B3:B9:EA:C5:DA:26:6A:76:E3:
        8A:77:90:A6:44:08:F7:28:A0:7A:28:A3
parm:           alua_invariant_check:Enables a run-time ALUA state invariant check. (default: false) (bool)
parm:           scst_threads:SCSI target threads count (int)
parm:           scst_max_cmd_mem:Maximum memory allowed to be consumed by all SCSI commands of all devices at any given time in MB (int)
parm:           scst_max_dev_cmd_mem:Maximum memory allowed to be consumed by all SCSI commands of a device at any given time in MB (int)
parm:           forcibly_close_sessions:If enabled, close the sessions associated with an access control group (ACG) when an ACG is deleted via sysfs instead of returning -EBUSY. (default: false) (bool)
parm:           auto_cm_assignment:Enables the copy managers auto registration. (default: true) (bool)
root@b:~# cat /etc/scst.conf 
# Automatically generated by SCST Configurator v3.8.0-pre.

HANDLER vdisk_blockio {
    DEVICE nv0 {
        filename /dev/drbd0
        size 2000337846272
        write_through 1
    }

    DEVICE rv0 {
        filename /dev/drbd1
        size 6651088613376
        write_through 1
    }
}

TARGET_DRIVER copy_manager {
    TARGET copy_manager_tgt {
        LUN 0 nv0
        LUN 1 rv0
    }
}

TARGET_DRIVER iscsi {
    enabled 1

    TARGET iqn.2023-06.pcs-a.hafs.set.local:b {
        QueuedCommands 128
        allowed_portal 192.168.0.12
        enabled 1
        rel_tgt_id 2

        LUN 0 nv0
        LUN 1 rv0
    }
}

DEVICE_GROUP esos {
    DEVICE nv0
    DEVICE rv0

    TARGET_GROUP local {
        group_id 257
        state nonoptimized

        TARGET iqn.2023-06.pcs-a.hafs.set.local:b
    }

    TARGET_GROUP remote {
        group_id 256
        state active

        TARGET iqn.2023-06.pcs-a.hafs.set.local:a {
            rel_tgt_id 1
        }
    }
}
lnocturno commented 1 year ago

Hi,

Could you try adding RDMAExtensions Yes to your iSCSI target:

TARGET_DRIVER iscsi {
    enabled 1

    TARGET iqn.2023-06.pcs-a.hafs.set.local:b {
        QueuedCommands 128
        allowed_portal 192.168.0.12
        enabled 1
        rel_tgt_id 2
        RDMAExtensions Yes

        LUN 0 nv0
        LUN 1 rv0
    }
}

Thanks, Gleb

AleksovAnry commented 1 year ago

Hello Gleb! Unfortunately it no helps...