search

omniosorg / pkg5

Image Packaging System
https://omnios.org/
Other
11 stars 55 forks source link

bhyve pci passthru doesn't work when having multiple VMs #331

Closed sjorge closed 3 years ago

sjorge commented 3 years ago

While waiting on the freeradius package I started looking at some other things on my TODO list... I setup 2 VMs with bhyve that each get a USB controller.

However on the VM with ppt0 sees the actual device.

zadm configuration

{
   "acpi" : "on",
   "autoboot" : "false",
   "bootargs" : "",
   "bootdisk" : {
      "blocksize" : "4K",
      "path" : "rpool/vms/rcon1/disk0",
      "size" : "10G",
      "sparse" : "false"
   },
   "bootorder" : "cd",
   "bootrom" : "BHYVE_RELEASE",
   "brand" : "bhyve",
   "diskif" : "nvme",
   "fs-allowed" : "",
   "hostbridge" : "i440fx",
   "hostid" : "",
   "ip-type" : "exclusive",
   "limitpriv" : "default",
   "net" : [
      {
         "global-nic" : "ixgbe0",
         "mac-addr" : "00:22:06:eb:5c:9f",
         "physical" : "rcon1",
         "vlan-id" : "110"
      }
   ],
   "netif" : "virtio-net",
   "pool" : "",
   "ppt" : [
      "ppt0"
   ],
   "ram" : "1G",
   "rng" : "on",
   "scheduling-class" : "",
   "type" : "generic",
   "vcpus" : "1,sockets=1,cores=1,threads=1",
   "vga" : "off",
   "vnc" : "off",
   "xhci" : "off",
   "zonename" : "rcon1",
   "zonepath" : "/zones/rcon1"
}
{
   "acpi" : "on",
   "autoboot" : "false",
   "bootargs" : "",
   "bootdisk" : {
      "blocksize" : "4K",
      "path" : "rpool/vms/rcon2/disk0",
      "size" : "10G",
      "sparse" : "false"
   },
   "bootorder" : "cd",
   "bootrom" : "BHYVE_RELEASE",
   "brand" : "bhyve",
   "diskif" : "nvme",
   "fs-allowed" : "",
   "hostbridge" : "i440fx",
   "hostid" : "",
   "ip-type" : "exclusive",
   "limitpriv" : "default",
   "net" : [
      {
         "global-nic" : "ixgbe0",
         "mac-addr" : "00:22:06:ae:a3:64",
         "physical" : "rcon2",
         "vlan-id" : "110"
      }
   ],
   "netif" : "virtio-net",
   "pool" : "",
   "ppt" : [
      "ppt1"
   ],
   "ram" : "1G",
   "rng" : "on",
   "scheduling-class" : "",
   "type" : "generic",
   "vcpus" : "1,sockets=1,cores=1,threads=1",
   "vga" : "off",
   "vnc" : "off",
   "xhci" : "off",
   "zonename" : "rcon2",
   "zonepath" : "/zones/rcon2"
}

ppt devices

{
        "devices": [
                {"dev":"/dev/ppt0","path":"/pci@5e,0/pci8086,2032@2/pci12d8,8608@0/pci12d8,8608@1/pci1912,15@0","vendor-id":"1912","device-id":"15","subsystem-vendor-id":"1912","subsystem-id":"15","revision-id":"2","label":"Renesas Technology Corp. uPD720202 USB 3.0 Host Controller"},
                {"dev":"/dev/ppt1","path":"/pci@5e,0/pci8086,2032@2/pci12d8,8608@0/pci12d8,8608@5/pci1912,15@0","vendor-id":"1912","device-id":"15","subsystem-vendor-id":"1912","subsystem-id":"15","revision-id":"2","label":"Renesas Technology Corp. uPD720202 USB 3.0 Host Controller"},
                {"dev":"/dev/ppt2","path":"/pci@5e,0/pci8086,2032@2/pci12d8,8608@0/pci12d8,8608@7/pci1912,15@0","vendor-id":"1912","device-id":"15","subsystem-vendor-id":"1912","subsystem-id":"15","revision-id":"2","label":"Renesas Technology Corp. uPD720202 USB 3.0 Host Controller"},
                {"dev":"/dev/ppt3","path":"/pci@5e,0/pci8086,2032@2/pci12d8,8608@0/pci12d8,8608@9/pci1912,15@0","vendor-id":"1912","device-id":"15","subsystem-vendor-id":"1912","subsystem-id":"15","revision-id":"2","label":"Renesas Technology Corp. uPD720202 USB 3.0 Host Controller"},
                {"dev":"/dev/ppt4","path":"/pci@5e,0/pci8086,2030@0/pci8086,37c0@0/pci8086,37c5@3/pci15d9,37d2@0","vendor-id":"8086","device-id":"37d2","subsystem-vendor-id":"15d9","subsystem-id":"37d2","revision-id":"9","label":"Intel Corporation Ethernet Connection X722 for 10GBASE-T"},
                {"dev":"/dev/ppt5","path":"/pci@5e,0/pci8086,2030@0/pci8086,37c0@0/pci8086,37c5@3/pci15d9,37d2@0,1","vendor-id":"8086","device-id":"37d2","subsystem-vendor-id":"15d9","subsystem-id":"37d2","revision-id":"9","label":"Intel Corporation Ethernet Connection X722 for 10GBASE-T"}
        ]
}

resulted bhyve.cfg's

#
# Generated from zone configuration
#
acpi_tables=false
memory.size=1G
memory.wired=true
x86.strictmsr=true
x86.vmexit_on_hlt=true
uuid=2ac52335-162b-c06b-d9c0-e0eff49d4b59
smbios.manufacturer=OmniOS
smbios.product=OmniOS HVM
smbios.version=1.0
smbios.serial=2ac52335-162b-c06b-d9c0-e0eff49d4b59
smbios.sku=001
smbios.family=Virtual Machine
cpus=1
sockets=1
cores=1
threads=1
lpc.bootrom=/usr/share/bhyve/firmware/BHYVE_RELEASE.fd
lpc.com1.path=/dev/zconsole
pci.0.0.0.device=hostbridge
pci.0.0.0.model=i440fx
pci.0.1.0.device=lpc
pci.0.4.0.device=nvme
pci.0.4.0.path=/dev/zvol/rdsk/rpool/vms/rcon1/disk0
pci.0.6.0.device=virtio-net
pci.0.6.0.backend=dlpi
pci.0.6.0.vnic=rcon1
pci.0.9.0.device=passthru
pci.0.9.0.path=/dev/ppt0
pci.0.10.0.device=virtio-rnd
name=rcon1

# Generated from zone configuration
#
acpi_tables=false
memory.size=1G
memory.wired=true
x86.strictmsr=true
x86.vmexit_on_hlt=true
uuid=17f57668-b2b9-c188-dac6-e44919140731
smbios.manufacturer=OmniOS
smbios.product=OmniOS HVM
smbios.version=1.0
smbios.serial=17f57668-b2b9-c188-dac6-e44919140731
smbios.sku=001
smbios.family=Virtual Machine
cpus=1
sockets=1
cores=1
threads=1
lpc.bootrom=/usr/share/bhyve/firmware/BHYVE_RELEASE.fd
lpc.com1.path=/dev/zconsole
pci.0.0.0.device=hostbridge
pci.0.0.0.model=i440fx
pci.0.1.0.device=lpc
pci.0.4.0.device=nvme
pci.0.4.0.path=/dev/zvol/rdsk/rpool/vms/rcon2/disk0
pci.0.6.0.device=virtio-net
pci.0.6.0.backend=dlpi
pci.0.6.0.vnic=rcon2
pci.0.9.1.device=passthru
pci.0.9.1.path=/dev/ppt1
pci.0.10.0.device=virtio-rnd
name=rcon2

And here we see our problem! The non working VM has:

pci.0.9.1.device=passthru
pci.0.9.1.path=/dev/ppt1

There is nothing in function 0 (0.9.0) so bhyve ignores this device as it always needs to start at function 0.

sjorge commented 3 years ago

Reopened here, as the zone config for the broken zone is:

device:
        match: /dev/ppt1

Which is correct, so it's not zadm not generating it correctly.

sjorge commented 3 years ago 
root@jupiter:/usr/lib/brand/bhyve# diff -u boot.orig boot
--- boot.orig   Sun Jul  4 13:47:44 2021
+++ boot        Sun Jul  4 13:47:53 2021
@@ -370,8 +370,8 @@
     k = int(m.group(1))
     if ppt == 0:
         args.append('-S')
-    args.extend(['-s', '{0}:{1},passthru,/dev/ppt{1}'.format(
-        PPT_SLOT, k)])
+    args.extend(['-s', '{0}:{1},passthru,/dev/ppt{2}'.format(
+        PPT_SLOT, ppt, k)])
     ppt += 1

 # RNG

Should fix it, let me do some tests