search

openzfs / zfs

OpenZFS on Linux and FreeBSD
https://openzfs.github.io/openzfs-docs
Other
10.66k stars 1.76k forks source link

vdev_id not properly detecting multipath disk configuration #11095

Closed gdevenyi closed 3 years ago

gdevenyi commented 4 years ago

System information

Type Version/Name
Distribution Name Ubuntu
Distribution Version 20.04.1
Linux Kernel 5.4.0-51-generic
Architecture x86_64
ZFS Version 0.8.5-1
SPL Version 0.8.5-1

Describe the problem you're observing

I have a system with 2x LSI SAS3008 PCI-Express Fusion-MPT SAS-3 connecting to two IO controllers on a Supermicro 90-disk JBOD: https://www.supermicro.com/en/products/chassis/4U/946/SC946ED-R2KJBOD in a crossover dual-channel config

LSI0 --> IO Controller 1 channel A
     |
     --> IO Controller 2 channel A

LSI1 --> IO Controller 1 channel B
     |
     --> IO controller 2 channel B

This is intended to provide cross-failover protection in case of card or IO controller failure. I currently have this configured with "multibus" in multipathd, and it is properly detecting and setting up 90 aggregated paths:

Snip of multipath -ll, some addresses greeked

35000cca26f633fd8 dm-31 HGST,HUH721212AL4200
size=11T features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
  |- 0:0:46:0   sdht 134:48  active ready running
  |- 15:0:46:0  sdck 69:128  active ready running
  |- 0:0:139:0  sdlf 67:464  active ready running
  `- 15:0:139:0 sdfw 131:32  active ready running
35000cca27a3c9d58 dm-30 HGST,HUH721212AL4200
size=11T features='0' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
  |- 0:0:45:0   sdhs 134:32  active ready running
  |- 15:0:45:0  sdcj 69:112  active ready running
  |- 0:0:138:0  sdle 67:448  active ready running
  `- 15:0:138:0 sdfv 131:16  active ready running

When attempting to use vdev_id.conf using a standard multipath config:

multipath yes

#       PCI_ID  HBA PORT  CHANNEL NAME
channel 5e:00.0 1         A
channel 5e:00.0 0         B
channel af:00.0 1         A
channel af:00.0 0         B

I get the following /dev/disk/by-vdev entries:

# ls -l
total 0
lrwxrwxrwx 1 root root 10 Oct 21 19:35 A7 -> ../../sdek
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B0 -> ../../dm-43
lrwxrwxrwx 1 root root 10 Oct 21 19:38 B1 -> ../../dm-1
lrwxrwxrwx 1 root root 10 Oct 21 19:38 B10 -> ../../dm-2
lrwxrwxrwx 1 root root 10 Oct 21 19:38 B11 -> ../../dm-3
lrwxrwxrwx 1 root root 10 Oct 21 19:38 B12 -> ../../dm-8
lrwxrwxrwx 1 root root 10 Oct 21 19:38 B13 -> ../../dm-9
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B14 -> ../../dm-49
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B15 -> ../../dm-11
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B16 -> ../../dm-12
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B17 -> ../../dm-52
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B18 -> ../../dm-53
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B19 -> ../../dm-15
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B2 -> ../../dm-82
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B20 -> ../../dm-56
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B21 -> ../../dm-18
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B22 -> ../../dm-58
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B23 -> ../../dm-20
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B24 -> ../../dm-60
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B25 -> ../../dm-22
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B26 -> ../../dm-62
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B27 -> ../../dm-63
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B28 -> ../../dm-64
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B29 -> ../../dm-26
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B3 -> ../../dm-66
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B30 -> ../../dm-28
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B31 -> ../../dm-29
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B32 -> ../../dm-30
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B33 -> ../../dm-70
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B34 -> ../../dm-71
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B35 -> ../../dm-72
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B36 -> ../../dm-73
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B37 -> ../../dm-74
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B38 -> ../../dm-36
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B4 -> ../../dm-37
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B5 -> ../../dm-85
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B6 -> ../../dm-86
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B7 -> ../../dm-40
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B8 -> ../../dm-80
lrwxrwxrwx 1 root root 11 Oct 21 19:38 B9 -> ../../dm-81

Note that this is missing many entries, currently totaling only 40.

When testing vdev_id for bugs, it does appear to throw an error:

# sh -x /lib/udev/vdev_id -d /dev/dm-1
+ PATH=/bin:/sbin:/usr/bin:/usr/sbin
+ CONFIG=/etc/zfs/vdev_id.conf
+ PHYS_PER_PORT=
+ DEV=
+ MULTIPATH=
+ TOPOLOGY=
+ BAY=
+ getopts c:d:eg:mp:h OPTION
+ DEV=/dev/dm-1
+ getopts c:d:eg:mp:h OPTION
+ [ ! -r /etc/zfs/vdev_id.conf ]
+ [ -z /dev/dm-1 -a -z  ]
+ [ -z  ]
+ awk $1 == "topology" {print $2; exit} /etc/zfs/vdev_id.conf
+ TOPOLOGY=
+ [ -z  ]
+ awk $1 == "slot" {print $2; exit} /etc/zfs/vdev_id.conf
+ BAY=
+ TOPOLOGY=sas_direct
+ [  = yes -a sas_direct = sas_direct ]
+ alias_handler
+ local DM_PART=
+ echo
+ grep -q -E p[0-9][0-9]*$
+ ID_VDEV=
+ [ -z  ]
+ BAY=bay
+ sas_handler
+ [ -z  ]
+ awk $1 == "phys_per_port"                     {print $2; exit} /etc/zfs/vdev_id.conf
+ PHYS_PER_PORT=
+ PHYS_PER_PORT=4
+ echo 4
+ grep -q -E ^[0-9]+$
+ [ -z  ]
+ awk $1 == "multipath"                         {print $2; exit} /etc/zfs/vdev_id.conf
+ MULTIPATH_MODE=yes
+ [ yes = yes ]
+ [ -z  ]
+ ls -l --full-time /dev/mapper
+ awk /\//dev/dm-1$/{print $9}
awk: cmd. line:1: /\//dev/dm-1$/{print $9}
awk: cmd. line:1:               ^ unterminated regexp
+ DM_NAME=
+ [  != partition ]
+ echo
+ awk -Fp /p/{print "-part"$2}
+ PART=
+ echo
+ sed s/p[0-9][0-9]*$//
+ DM_NAME=
+ [ -z  ]
+ return
+ ID_VDEV=
+ [ -n  ]

It returns the same error for mapper entries that do not get vdevs generated:

# sh -x /lib/udev/vdev_id -d /dev/dm-4
+ PATH=/bin:/sbin:/usr/bin:/usr/sbin
+ CONFIG=/etc/zfs/vdev_id.conf
+ PHYS_PER_PORT=
+ DEV=
+ MULTIPATH=
+ TOPOLOGY=
+ BAY=
+ getopts c:d:eg:mp:h OPTION
+ DEV=/dev/dm-4
+ getopts c:d:eg:mp:h OPTION
+ [ ! -r /etc/zfs/vdev_id.conf ]
+ [ -z /dev/dm-4 -a -z  ]
+ [ -z  ]
+ awk $1 == "topology" {print $2; exit} /etc/zfs/vdev_id.conf
+ TOPOLOGY=
+ [ -z  ]
+ awk $1 == "slot" {print $2; exit} /etc/zfs/vdev_id.conf
+ BAY=
+ TOPOLOGY=sas_direct
+ [  = yes -a sas_direct = sas_direct ]
+ alias_handler
+ local DM_PART=
+ echo
+ grep -q -E p[0-9][0-9]*$
+ ID_VDEV=
+ [ -z  ]
+ BAY=bay
+ sas_handler
+ [ -z  ]
+ awk $1 == "phys_per_port"                     {print $2; exit} /etc/zfs/vdev_id.conf
+ PHYS_PER_PORT=
+ PHYS_PER_PORT=4
+ echo 4
+ grep -q -E ^[0-9]+$
+ [ -z  ]
+ awk $1 == "multipath"                         {print $2; exit} /etc/zfs/vdev_id.conf
+ MULTIPATH_MODE=yes
+ [ yes = yes ]
+ [ -z  ]
+ ls -l --full-time /dev/mapper
+ awk /\//dev/dm-4$/{print $9}
awk: cmd. line:1: /\//dev/dm-4$/{print $9}
awk: cmd. line:1:               ^ unterminated regexp
+ DM_NAME=
+ [  != partition ]
+ echo
+ awk -Fp /p/{print "-part"$2}
+ PART=
+ echo
+ sed s/p[0-9][0-9]*$//
+ DM_NAME=
+ [ -z  ]
+ return
+ ID_VDEV=
+ [ -n  ]

I'm unclear from the documentation what a number of the terms are for the vdev_id.conf file in the context of real-world hardware configurations (how do I determine number of phys, how do I know if switch or direct, is a HBA port the actual physical wire, what does "CHANNEL NAME" refer to?), so I'm having much difficulty guessing as to what settings might need to be changed.

Finally, I offer the output of sas_discover from https://github.com/stanford-rc/sasutils which appears to have mapped the entire SAS topology, it does look a bit odd compared to what I expect to see, this may be be due to the internal wiring of the JBOD

hostname
|--host15 
|  |--4x--expander-15:0  
|  |      |--1x--end_device-15:0:12
|  |      |      `--enclosure  
|  |      |--1x--end_device-15:0:0
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:1
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:2
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:3
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:4
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:5
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:6
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:7
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:8
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:9
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:10
|  |      |  `--  1 x disk
|  |      |--1x--end_device-15:0:11
|  |      |  `--  1 x disk
|  |      |--8x--expander-15:2  
|  |      |  |-- 39 x end_device -- disk
|  |      |  `--  1 x end_device -- enclosure  
|  |      `--8x--expander-15:1  
|  |         |-- 39 x end_device -- disk
|  |         `--  1 x end_device -- enclosure  
|  `--4x--expander-15:3  
|         |--1x--end_device-15:3:12
|         |      `--enclosure  
|         |--1x--end_device-15:3:0
|         |  `--  1 x disk
|         |--1x--end_device-15:3:1
|         |  `--  1 x disk
|         |--1x--end_device-15:3:2
|         |  `--  1 x disk
|         |--1x--end_device-15:3:3
|         |  `--  1 x disk
|         |--1x--end_device-15:3:4
|         |  `--  1 x disk
|         |--1x--end_device-15:3:5
|         |  `--  1 x disk
|         |--1x--end_device-15:3:6
|         |  `--  1 x disk
|         |--1x--end_device-15:3:7
|         |  `--  1 x disk
|         |--1x--end_device-15:3:8
|         |  `--  1 x disk
|         |--1x--end_device-15:3:9
|         |  `--  1 x disk
|         |--1x--end_device-15:3:10
|         |  `--  1 x disk
|         |--1x--end_device-15:3:11
|         |  `--  1 x disk
|         |--8x--expander-15:5  
|         |  |-- 39 x end_device -- disk
|         |  `--  1 x end_device -- enclosure  
|         `--8x--expander-15:4  
|            |-- 39 x end_device -- disk
|            `--  1 x end_device -- enclosure  
`--host0 
   |--4x--expander-0:3  
   |      |--1x--end_device-0:3:12
   |      |      `--enclosure  
   |      |--1x--end_device-0:3:0
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:1
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:2
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:3
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:4
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:5
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:6
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:7
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:8
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:9
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:10
   |      |  `--  1 x disk
   |      |--1x--end_device-0:3:11
   |      |  `--  1 x disk
   |      |--8x--expander-0:5  
   |      |  |-- 39 x end_device -- disk
   |      |  `--  1 x end_device -- enclosure  
   |      `--8x--expander-0:4  
   |         |-- 39 x end_device -- disk
   |         `--  1 x end_device -- enclosure  
   `--4x--expander-0:0  
          |--1x--end_device-0:0:12
          |      `--enclosure  
          |--1x--end_device-0:0:0
          |  `--  1 x disk
          |--1x--end_device-0:0:1
          |  `--  1 x disk
          |--1x--end_device-0:0:2
          |  `--  1 x disk
          |--1x--end_device-0:0:3
          |  `--  1 x disk
          |--1x--end_device-0:0:4
          |  `--  1 x disk
          |--1x--end_device-0:0:5
          |  `--  1 x disk
          |--1x--end_device-0:0:6
          |  `--  1 x disk
          |--1x--end_device-0:0:7
          |  `--  1 x disk
          |--1x--end_device-0:0:8
          |  `--  1 x disk
          |--1x--end_device-0:0:9
          |  `--  1 x disk
          |--1x--end_device-0:0:10
          |  `--  1 x disk
          |--1x--end_device-0:0:11
          |  `--  1 x disk
          |--8x--expander-0:2  
          |  |-- 39 x end_device -- disk
          |  `--  1 x end_device -- enclosure  
          `--8x--expander-0:1  
             |-- 39 x end_device -- disk
             `--  1 x end_device -- enclosure

It looks like internally the IO controllers each have 1 expander going to 12 disks, and cascading 2 more expanders with 39 disks each, a very odd config, but each of the 90 disks is exposed properly on each physical wiring path, as expected.

gdevenyi commented 4 years ago

Seems I'm running the test-version of vdev_id wrong, so here's the right way for the same devices:

# sh -x /lib/udev/vdev_id -d dm-0
+ PATH=/bin:/sbin:/usr/bin:/usr/sbin
+ CONFIG=/etc/zfs/vdev_id.conf
+ PHYS_PER_PORT=
+ DEV=
+ MULTIPATH=
+ TOPOLOGY=
+ BAY=
+ getopts c:d:eg:mp:h OPTION
+ DEV=dm-0
+ getopts c:d:eg:mp:h OPTION
+ [ ! -r /etc/zfs/vdev_id.conf ]
+ [ -z dm-0 -a -z  ]
+ [ -z  ]
+ awk $1 == "topology" {print $2; exit} /etc/zfs/vdev_id.conf
+ TOPOLOGY=
+ [ -z  ]
+ awk $1 == "slot" {print $2; exit} /etc/zfs/vdev_id.conf
+ BAY=
+ TOPOLOGY=sas_direct
+ [  = yes -a sas_direct = sas_direct ]
+ alias_handler
+ local DM_PART=
+ echo
+ grep -q -E p[0-9][0-9]*$
+ ID_VDEV=
+ [ -z  ]
+ BAY=bay
+ sas_handler
+ [ -z  ]
+ awk $1 == "phys_per_port"                     {print $2; exit} /etc/zfs/vdev_id.conf
+ PHYS_PER_PORT=
+ PHYS_PER_PORT=4
+ echo 4
+ grep -q -E ^[0-9]+$
+ [ -z  ]
+ awk $1 == "multipath"                         {print $2; exit} /etc/zfs/vdev_id.conf
+ MULTIPATH_MODE=yes
+ [ yes = yes ]
+ [ -z  ]
+ ls -l --full-time /dev/mapper
+ awk /\/dm-0$/{print $9}
+ DM_NAME=35000cca26f7c3ca0
+ [  != partition ]
+ echo 35000cca26f7c3ca0
+ awk -Fp /p/{print "-part"$2}
+ PART=
+ echo 35000cca26f7c3ca0
+ sed s/p[0-9][0-9]*$//
+ DM_NAME=35000cca26f7c3ca0
+ [ -z 35000cca26f7c3ca0 ]
+ multipath -ll 35000cca26f7c3ca0
+ awk /running/{gsub("^[|]"," "); print $3 ; exit}
+ DEV=sdc
+ [ -z sdc ]
+ echo sdc
+ grep -q ^/devices/
+ udevadm info -q path -p /sys/block/sdc
+ sys_path=/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:0/end_device-0:0:0/target0:0:0/0:0:0:0/block/sdc
+ echo /devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:0/end_device-0:0:0/target0:0:0/0:0:0:0/block/sdc
+ tr /  
+ set -- devices pci0000:5d 0000:5d:00.0 0000:5e:00.0 host0 port-0:0 expander-0:0 port-0:0:0 end_device-0:0:0 target0:0:0 0:0:0:0 block sdc
+ num_dirs=13
+ scsi_host_dir=/sys
+ i=1
+ [ 1 -le 13 ]
+ eval echo ${1}
+ echo devices
+ d=devices
+ scsi_host_dir=/sys/devices
+ echo devices
+ grep -q -E ^host[0-9]+$
+ i=2
+ [ 2 -le 13 ]
+ eval echo ${2}
+ echo pci0000:5d
+ d=pci0000:5d
+ scsi_host_dir=/sys/devices/pci0000:5d
+ echo pci0000:5d
+ grep -q -E ^host[0-9]+$
+ i=3
+ [ 3 -le 13 ]
+ eval echo ${3}
+ echo 0000:5d:00.0
+ d=0000:5d:00.0
+ scsi_host_dir=/sys/devices/pci0000:5d/0000:5d:00.0
+ echo 0000:5d:00.0
+ grep -q -E ^host[0-9]+$
+ i=4
+ [ 4 -le 13 ]
+ eval echo ${4}
+ echo 0000:5e:00.0
+ d=0000:5e:00.0
+ scsi_host_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0
+ echo 0000:5e:00.0
+ grep -q -E ^host[0-9]+$
+ i=5
+ [ 5 -le 13 ]
+ eval echo ${5}
+ echo host0
+ d=host0
+ scsi_host_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0
+ echo host0
+ grep -q -E ^host[0-9]+$
+ break
+ [ 5 = 13 ]
+ eval echo ${4}
+ echo 0000:5e:00.0
+ awk -F: {print $2":"$3}
+ PCI_ID=5e:00.0
+ port_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0
+ j=6
+ i=6
+ [ 6 -le 6 ]
+ eval echo ${6}
+ echo port-0:0
+ port_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0
+ i=7
+ [ 7 -le 6 ]
+ + + head -1
lsawk -d -F: /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/phy-0:0 {print $NF} /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/phy-0:1
 /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/phy-0:2 /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/phy-0:3
+ PHY=0
+ [ -z 0 ]
+ PORT=0
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0
+ [ 7 -lt 13 ]
+ eval echo ${7}
+ echo expander-0:0
+ d=expander-0:0
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0
+ echo expander-0:0
+ grep -q ^end_device
+ i=8
+ [ 8 -lt 13 ]
+ eval echo ${8}
+ echo port-0:0:0
+ d=port-0:0:0
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:0
+ echo port-0:0:0
+ grep -q ^end_device
+ i=9
+ [ 9 -lt 13 ]
+ eval echo ${9}
+ echo end_device-0:0:0
+ d=end_device-0:0:0
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:0/end_device-0:0:0
+ echo end_device-0:0:0
+ grep -q ^end_device
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:0/end_device-0:0:0/sas_device/end_device-0:0:0
+ break
+ SLOT=
+ cat /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:0/end_device-0:0:0/sas_device/end_device-0:0:0/bay_identifier
+ SLOT=0
+ [ -z 0 ]
+ map_channel 5e:00.0 0
+ local MAPPED_CHAN=
+ local PCI_ID=5e:00.0
+ local PORT=0
+ awk $1 == "channel" &&                        $2 == "5e:00.0" && $3 == 0                      { print $4; exit } /etc/zfs/vdev_id.conf
+ MAPPED_CHAN=B
+ printf %s B
+ CHAN=B
+ map_slot 0 B
+ local LINUX_SLOT=0
+ local CHANNEL=B
+ local MAPPED_SLOT=
+ awk $1 == "slot" && $2 == 0 &&                        $4 ~ /^B$|^$/ { print $3; exit } /etc/zfs/vdev_id.conf
+ MAPPED_SLOT=
+ [ -z  ]
+ MAPPED_SLOT=0
+ printf %d 0
+ SLOT=0
+ [ -z B ]
+ echo B0
+ ID_VDEV=B0
+ [ -n B0 ]
+ echo ID_VDEV=B0
ID_VDEV=B0
+ echo ID_VDEV_PATH=disk/by-vdev/B0
ID_VDEV_PATH=disk/by-vdev/B0
# sh -x /lib/udev/vdev_id -d dm-4
+ PATH=/bin:/sbin:/usr/bin:/usr/sbin
+ CONFIG=/etc/zfs/vdev_id.conf
+ PHYS_PER_PORT=
+ DEV=
+ MULTIPATH=
+ TOPOLOGY=
+ BAY=
+ getopts c:d:eg:mp:h OPTION
+ DEV=dm-4
+ getopts c:d:eg:mp:h OPTION
+ [ ! -r /etc/zfs/vdev_id.conf ]
+ [ -z dm-4 -a -z  ]
+ [ -z  ]
+ awk $1 == "topology" {print $2; exit} /etc/zfs/vdev_id.conf
+ TOPOLOGY=
+ [ -z  ]
+ awk $1 == "slot" {print $2; exit} /etc/zfs/vdev_id.conf
+ BAY=
+ TOPOLOGY=sas_direct
+ [  = yes -a sas_direct = sas_direct ]
+ alias_handler
+ local DM_PART=
+ echo
+ grep -q -E p[0-9][0-9]*$
+ ID_VDEV=
+ [ -z  ]
+ BAY=bay
+ sas_handler
+ [ -z  ]
+ awk $1 == "phys_per_port"                     {print $2; exit} /etc/zfs/vdev_id.conf
+ PHYS_PER_PORT=
+ PHYS_PER_PORT=4
+ echo 4
+ grep -q -E ^[0-9]+$
+ [ -z  ]
+ awk $1 == "multipath"                         {print $2; exit} /etc/zfs/vdev_id.conf
+ MULTIPATH_MODE=yes
+ [ yes = yes ]
+ [ -z  ]
+ ls -l --full-time /dev/mapper
+ awk /\/dm-4$/{print $9}
+ DM_NAME=35000cca26f81db84
+ [  != partition ]
+ echo 35000cca26f81db84
+ awk -Fp /p/{print "-part"$2}
+ PART=
+ echo 35000cca26f81db84
+ sed s/p[0-9][0-9]*$//
+ DM_NAME=35000cca26f81db84
+ [ -z 35000cca26f81db84 ]
+ multipath -ll 35000cca26f81db84
+ awk /running/{gsub("^[|]"," "); print $3 ; exit}
+ DEV=sdo
+ [ -z sdo ]
+ echo sdo
+ grep -q ^/devices/
+ udevadm info -q path -p /sys/block/sdo
+ sys_path=/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:13/expander-0:1/port-0:1:0/end_device-0:1:0/target0:0:13/0:0:13:0/block/sdo
+ echo /devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:13/expander-0:1/port-0:1:0/end_device-0:1:0/target0:0:13/0:0:13:0/block/sdo
+ tr /  
+ set -- devices pci0000:5d 0000:5d:00.0 0000:5e:00.0 host0 port-0:0 expander-0:0 port-0:0:13 expander-0:1 port-0:1:0 end_device-0:1:0 target0:0:13 0:0:13:0 block sdo
+ num_dirs=15
+ scsi_host_dir=/sys
+ i=1
+ [ 1 -le 15 ]
+ eval echo ${1}
+ echo devices
+ d=devices
+ scsi_host_dir=/sys/devices
+ echo devices
+ grep -q -E ^host[0-9]+$
+ i=2
+ [ 2 -le 15 ]
+ eval echo ${2}
+ echo pci0000:5d
+ d=pci0000:5d
+ scsi_host_dir=/sys/devices/pci0000:5d
+ echo pci0000:5d
+ grep -q -E ^host[0-9]+$
+ i=3
+ [ 3 -le 15 ]
+ eval echo ${3}
+ echo 0000:5d:00.0
+ d=0000:5d:00.0
+ scsi_host_dir=/sys/devices/pci0000:5d/0000:5d:00.0
+ echo 0000:5d:00.0
+ grep -q -E ^host[0-9]+$
+ i=4
+ [ 4 -le 15 ]
+ eval echo ${4}
+ echo 0000:5e:00.0
+ d=0000:5e:00.0
+ scsi_host_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0
+ echo 0000:5e:00.0
+ grep -q -E ^host[0-9]+$
+ i=5
+ [ 5 -le 15 ]
+ eval echo ${5}
+ echo host0
+ d=host0
+ scsi_host_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0
+ echo host0
+ grep -q -E ^host[0-9]+$
+ break
+ [ 5 = 15 ]
+ eval echo ${4}
+ echo 0000:5e:00.0
+ awk -F: {print $2":"$3}
+ PCI_ID=5e:00.0
+ port_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0
+ j=6
+ i=6
+ [ 6 -le 6 ]
+ eval echo ${6}
+ echo port-0:0
+ port_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0
+ i=7
+ [ 7 -le 6 ]
+ ls -d /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/phy-0:0 /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/phy-0:1 /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/phy-0:2 /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/phy-0:3
+ head -1
+ awk -F: {print $NF}
+ PHY=0
+ [ -z 0 ]
+ PORT=0
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0
+ [ 7 -lt 15 ]
+ eval echo ${7}
+ echo expander-0:0
+ d=expander-0:0
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0
+ echo expander-0:0
+ grep -q ^end_device
+ i=8
+ [ 8 -lt 15 ]
+ eval echo ${8}
+ echo port-0:0:13
+ d=port-0:0:13
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:13
+ echo port-0:0:13
+ grep -q ^end_device
+ i=9
+ [ 9 -lt 15 ]
+ eval echo ${9}
+ echo expander-0:1
+ d=expander-0:1
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:13/expander-0:1
+ echo expander-0:1
+ grep -q ^end_device
+ i=10
+ [ 10 -lt 15 ]
+ eval echo ${10}
+ echo port-0:1:0
+ d=port-0:1:0
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:13/expander-0:1/port-0:1:0
+ echo port-0:1:0
+ grep -q ^end_device
+ i=11
+ [ 11 -lt 15 ]
+ eval echo ${11}
+ echo end_device-0:1:0
+ d=end_device-0:1:0
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:13/expander-0:1/port-0:1:0/end_device-0:1:0
+ echo end_device-0:1:0
+ grep -q ^end_device
+ end_device_dir=/sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:13/expander-0:1/port-0:1:0/end_device-0:1:0/sas_device/end_device-0:1:0
+ break
+ SLOT=
+ cat /sys/devices/pci0000:5d/0000:5d:00.0/0000:5e:00.0/host0/port-0:0/expander-0:0/port-0:0:13/expander-0:1/port-0:1:0/end_device-0:1:0/sas_device/end_device-0:1:0/bay_identifier
+ SLOT=0
+ [ -z 0 ]
+ map_channel 5e:00.0 0
+ local MAPPED_CHAN=
+ local PCI_ID=5e:00.0
+ local PORT=0
+ awk $1 == "channel" &&                        $2 == "5e:00.0" && $3 == 0                      { print $4; exit } /etc/zfs/vdev_id.conf
+ MAPPED_CHAN=B
+ printf %s B
+ CHAN=B
+ map_slot 0 B
+ local LINUX_SLOT=0
+ local CHANNEL=B
+ local MAPPED_SLOT=
+ awk $1 == "slot" && $2 == 0 &&                        $4 ~ /^B$|^$/ { print $3; exit } /etc/zfs/vdev_id.conf
+ MAPPED_SLOT=
+ [ -z  ]
+ MAPPED_SLOT=0
+ printf %d 0
+ SLOT=0
+ [ -z B ]
+ echo B0
+ ID_VDEV=B0
+ [ -n B0 ]
+ echo ID_VDEV=B0
ID_VDEV=B0
+ echo ID_VDEV_PATH=disk/by-vdev/B0
ID_VDEV_PATH=disk/by-vdev/B0

It looks like something about the ID stuff is breaking down and the symlinks are clobbering each other. I will investigate the identifier config.

gdevenyi commented 4 years ago

Changing the slot to any of the other options doesn't change the returned vdev name.

gdevenyi commented 3 years ago

Adding slot id to the configuration was able to properly find 90 unique slot identifiers, although with a missing slot 12.