search

TritonDataCenter / smartos-live

For more information, please see http://smartos.org/ For any questions that aren't answered there, please join the SmartOS discussion list: https://smartos.topicbox.com/groups/smartos-discuss
1.57k stars 245 forks source link

bhyve passthrough "Assertion failed: error == 0, file pci_emul.c, line 517, function modify_bar_registration" #901

Closed sjorge closed 4 years ago

sjorge commented 4 years ago

I got a "Assertion failed: error == 0, file pci_emul.c, line 517, function modify_bar_registration\n" when trying to pass my non working i40e nics to a VM to at least use them and play with ppt some more.

[root@boron /zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b]# pptadm list -a
DEV        VENDOR DEVICE PATH
/dev/ppt0  8086   37d2   /pci@5e,0/pci8086,2030@0/pci8086,37c0@0/pci8086,37c5@3/pci15d9,37d2@0
/dev/ppt1  8086   37d2   /pci@5e,0/pci8086,2030@0/pci8086,37c0@0/pci8086,37c5@3/pci15d9,37d2@0,1
[root@boron /mnt]# cat /mnt/bootfs/etc/ppt_matches
pci10de,15f0
pciex8086,37d2
[root@boron /mnt]# cat /mnt/bootfs/etc/ppt_aliases
ppt "pciex8086,37d2"
[root@boron /mnt]# prtconf -dD | grep -B 4 pciex8086,37d2
    pci, instance #2 (driver name: npe)
        pci8086,2030 (pciex8086,2030) [Intel Corporation Sky Lake-E PCI Express Root Port A], instance #6 (driver name: pcieb)
            pci8086,37c0 (pciex8086,37c0) [Intel Corporation unknown device], instance #8 (driver name: pcieb)
                pci8086,37c5 (pciex8086,37c5) [Intel Corporation unknown device], instance #9 (driver name: pcieb)
                    pci15d9,37d2 (pciex8086,37d2) [Intel Corporation Ethernet Connection X722 for 10GBASE-T], instance #0 (driver name: ppt)
                    pci15d9,37d2 (pciex8086,37d2) [Intel Corporation Ethernet Connection X722 for 10GBASE-T], instance #1 (driver name: ppt)

vmadm get

{
  "zonename": "eceec5eb-22c7-e65b-fd6b-81ebdb36406b",
  "autoboot": false,
  "brand": "bhyve",
  "limit_priv": "default,-file_link_any,-net_access,-proc_fork,-proc_info,-proc_session",
  "v": 1,
  "create_timestamp": "2018-09-03T17:02:42.132Z",
  "cpu_shares": 100,
  "max_lwps": 2000,
  "max_msg_ids": 4096,
  "max_sem_ids": 4096,
  "max_shm_ids": 4096,
  "max_shm_memory": 3328,
  "billing_id": "00000000-0000-0000-0000-000000000000",
  "owner_uuid": "00000000-0000-0000-0000-000000000000",
  "resolvers": [
    "10.23.10.1"
  ],
  "alias": "wino",
  "max_physical_memory": 5376,
  "max_locked_memory": 5376,
  "max_swap": 5376,
  "ram": 4096,
  "vnc_port": 5901,
  "nics": [
    {
      "interface": "net0",
      "mac": "82:fc:0c:c2:e4:6f",
      "vlan_id": 10,
      "nic_tag": "trunk",
      "ip": "dhcp",
      "ips": [
        "dhcp",
        "addrconf"
      ],
      "model": "virtio",
      "primary": true
    }
  ],
  "device": null,
  "disks": [
    {
      "path": "/dev/zvol/rdsk/zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b/disk0",
      "boot": true,
      "model": "virtio",
      "media": "disk",
      "pci_slot": "0:4:0",
      "uuid": "740e692f-1295-ed54-8fd0-97b823b04ccb",
      "zfs_filesystem": "zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b/disk0",
      "zpool": "zones",
      "size": 30720,
      "compression": "lz4",
      "refreservation": 31683,
      "block_size": 8192
    }
  ],
  "bhyve_extra_opts": "-c sockets=1,cores=2,threads=2",
  "vcpus": 4,
  "flexible_disk_size": 38400,
  "bootrom": "uefi",
  "pci_devices": [
    {
      "path": "/devices/pci@5e,0/pci8086,2030@0/pci8086,37c0@0/pci8086,37c5@3/pci15d9,37d2@0",
      "model": "passthru",
      "pci_slot": "0:8:0"
    },
    {
      "path": "/devices/pci@5e,0/pci8086,2030@0/pci8086,37c0@0/pci8086,37c5@3/pci15d9,37d2@0,1",
      "model": "passthru",
      "pci_slot": "0:8:1"
    }
  ],
  "com1": "/dev/zconsole",
  "com2": "socket,/tmp/vm.ttyb",
  "zlog_mode": "g--",
  "zlog_name": "platform.log",
  "uuid": "eceec5eb-22c7-e65b-fd6b-81ebdb36406b",
  "zone_state": "installed",
  "zonepath": "/zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b",
  "hvm": true,
  "zoneid": null,
  "zonedid": 5,
  "last_modified": "2020-02-08T22:56:28.000Z",
  "exit_status": -134,
  "exit_timestamp": "2020-02-08T22:56:28.934Z",
  "firewall_enabled": false,
  "server_uuid": "00000000-0000-0000-0000-ac1f6b4137cc",
  "platform_buildstamp": "20200206T061259Z",
  "state": "stopped",
  "customer_metadata": {},
  "internal_metadata": {},
  "routes": {},
  "tags": {},
  "quota": 10,
  "zfs_root_recsize": 131072,
  "zfs_filesystem": "zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b",
  "zpool": "zones",
  "snapshots": [],
  "free_space": 7680
}

This is a windows 10 VM

sjorge commented 4 years ago

platform.log

{"event":"open","stream":"logfile","time":"2020-02-08T23:13:34.373710000Z"}
{"log":"add_arg: argv[2]='-U'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550172000Z"}
{"log":"add_arg: argv[3]='eceec5eb-22c7-e65b-fd6b-81ebdb36406b'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550264000Z"}
{"log":"add_arg: argv[4]='-B'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550295000Z"}
{"log":"add_arg: argv[5]='1,manufacturer=Joyent,product=SmartDC HVM,version=7.20200206T061259Z,serial=eceec5eb-22c7-e65b-fd6b-81ebdb36406b,sku=001,family=Virtual Machine'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550331000Z"}
{"log":"add_arg: argv[6]='-s'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550374000Z"}
{"log":"add_arg: argv[7]='31,lpc'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550400000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_bootrom=uefi'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550427000Z"}
{"log":"add_arg: argv[8]='-l'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550452000Z"}
{"log":"add_arg: argv[9]='bootrom,/usr/share/bhyve/uefi-rom.bin'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550489000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_com1=/dev/zconsole'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550516000Z"}
{"log":"add_arg: argv[10]='-l'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550542000Z"}
{"log":"add_arg: argv[11]='com1,/dev/zconsole'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550568000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_com2=socket,/tmp/vm.ttyb'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550595000Z"}
{"log":"add_arg: argv[12]='-l'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550643000Z"}
{"log":"add_arg: argv[13]='com2,socket,/tmp/vm.ttyb'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550671000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_hostbridge=<null>'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550698000Z"}
{"log":"add_arg: argv[14]='-s'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550723000Z"}
{"log":"add_arg: argv[15]='0,hostbridge,model=i440fx'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550749000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_vcpus=4'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550784000Z"}
{"log":"add_arg: argv[16]='-c'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550810000Z"}
{"log":"add_arg: argv[17]='4'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550835000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_ram=4096'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550861000Z"}
{"log":"add_arg: argv[18]='-m'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550906000Z"}
{"log":"add_arg: argv[19]='4096'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550932000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device_resources=_dev_zvol_rdsk_zones_eceec5eb_22c7_e65b_fd6b_81ebdb36406b_disk0 _devices_pci_5e_0_pci8086_2030_0_pci8086_37c0_0_pci8086_37c5_3_pci15d9_37d2_0 _devices_pci_5e_0_pci8086_2030_0_pci8086_37c0_0_pci8086_37c5_3_pci15d9_37d2_0_1 '\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.550973000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__dev_zvol_rdsk_zones_eceec5eb_22c7_e65b_fd6b_81ebdb36406b_disk0_path=/dev/zvol/rdsk/zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b/disk0'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551017000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__dev_zvol_rdsk_zones_eceec5eb_22c7_e65b_fd6b_81ebdb36406b_disk0_model=virtio'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551049000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__dev_zvol_rdsk_zones_eceec5eb_22c7_e65b_fd6b_81ebdb36406b_disk0_boot=true'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551080000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__dev_zvol_rdsk_zones_eceec5eb_22c7_e65b_fd6b_81ebdb36406b_disk0_pci_slot=0:4:0'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551121000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__dev_zvol_rdsk_zones_eceec5eb_22c7_e65b_fd6b_81ebdb36406b_disk0_model=virtio'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551152000Z"}
{"log":"add_arg: argv[20]='-s'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551177000Z"}
{"log":"add_arg: argv[21]='0:4:0,virtio-blk,/dev/zvol/rdsk/zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b/disk0'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551216000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__devices_pci_5e_0_pci8086_2030_0_pci8086_37c0_0_pci8086_37c5_3_pci15d9_37d2_0_path=/dev/ppt0'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551248000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__devices_pci_5e_0_pci8086_2030_0_pci8086_37c0_0_pci8086_37c5_3_pci15d9_37d2_0_model=passthru'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551280000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__devices_pci_5e_0_pci8086_2030_0_pci8086_37c0_0_pci8086_37c5_3_pci15d9_37d2_0_pci_slot=0:8:0'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551321000Z"}
{"log":"add_arg: argv[22]='-S'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551346000Z"}
{"log":"add_arg: argv[23]='-s'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551370000Z"}
{"log":"add_arg: argv[24]='0:8:0,passthru,/dev/ppt0'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551396000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__devices_pci_5e_0_pci8086_2030_0_pci8086_37c0_0_pci8086_37c5_3_pci15d9_37d2_0_1_path=/dev/ppt1'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551437000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__devices_pci_5e_0_pci8086_2030_0_pci8086_37c0_0_pci8086_37c5_3_pci15d9_37d2_0_1_model=passthru'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551469000Z"}
{"log":"get_zcfg_var: '_ZONECFG_device__devices_pci_5e_0_pci8086_2030_0_pci8086_37c0_0_pci8086_37c5_3_pci15d9_37d2_0_1_pci_slot=0:8:1'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551501000Z"}
{"log":"add_arg: argv[25]='-s'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551544000Z"}
{"log":"add_arg: argv[26]='0:8:1,passthru,/dev/ppt1'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551571000Z"}
{"log":"get_zcfg_var: '_ZONECFG_net_resources=net0 '\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551597000Z"}
{"log":"get_zcfg_var: '_ZONECFG_net_net0_primary=true'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551632000Z"}
{"log":"add_arg: argv[27]='-s'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551657000Z"}
{"log":"add_arg: argv[28]='6:0,virtio-net-viona,net0'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551683000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_bhyve_extra_opts=-c sockets=1,cores=2,threads=2'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551712000Z"}
{"log":"add_arg: argv[29]='-c'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551737000Z"}
{"log":"add_arg: argv[30]='sockets=1,cores=2,threads=2'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551789000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_vnc_port=5901'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551816000Z"}
{"log":"get_zcfg_var: '_ZONECFG_attr_vnc_password=<null>'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551843000Z"}
{"log":"add_arg: argv[31]='-s'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551868000Z"}
{"log":"add_arg: argv[32]='30:0,fbuf,vga=off,unix=/tmp/vm.vnc'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551906000Z"}
{"log":"add_arg: argv[33]='-s'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551931000Z"}
{"log":"add_arg: argv[34]='30:1,xhci,tablet'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551956000Z"}
{"log":"add_arg: argv[35]='SYSbhyve-5'\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.551982000Z"}
{"log":"packing nvlist:\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.552007000Z"}
{"log":"nvlist version: 0\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.552040000Z"}
{"log":"\tbhyve_args = bhyve -H -U eceec5eb-22c7-e65b-fd6b-81ebdb36406b -B 1,manufacturer=Joyent,product=SmartDC HVM,version=7.20200206T061259Z,serial=eceec5eb-22c7-e65b-fd6b-81ebdb36406b,sku=001,family=Virtual Machine -s 31,lpc -l bootrom,/usr/share/bhyve/uefi-rom.bin -l com1,/dev/zconsole -l com2,socket,/tmp/vm.ttyb -s 0,hostbridge,model=i440fx -c 4 -m 4096 -s 0:4:0,virtio-blk,/dev/zvol/rdsk/zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b/disk0 -S -s 0:8:0,passthru,/dev/ppt0 -s 0:8:1,passthru,/dev/ppt1 -s 6:0,virtio-net-viona,net0 -c sockets=1,cores=2,threads=2 -s 30:0,fbuf,vga=off,unix=/tmp/vm.vnc -s 30:1,xhci,tablet SYSbhyve-5\n","stream":"zoneadmd","time":"2020-02-08T23:13:35.552106000Z"}
{"log":"Configuration from /var/run/bhyve/zhyve.cmd:\nnvlist version: 0\n","stream":"stdout","time":"2020-02-08T23:13:35.591151000Z"}
{"log":"\tbhyve_args = bhyve -H -U eceec5eb-22c7-e65b-fd6b-81ebdb36406b -B 1,manufacturer=Joyent,product=SmartDC HVM,version=7.20200206T061259Z,serial=eceec5eb-22c7-e65b-fd6b-81ebdb36406b,sku=001,family=Virtual Machine -s 31,lpc -l bootrom,/usr/share/bhyve/uefi-rom.bin -l com1,/dev/zconsole -l com2,socket,/tmp/vm.ttyb -s 0,hostbridge,model=i440fx -c 4 -m 4096 -s 0:4:0,virtio-blk,/dev/zvol/rdsk/zones/eceec5eb-22c7-e65b-fd6b-81ebdb36406b/disk0 -S -s 0:8:0,passthru,/dev/ppt0 -s 0:8:1,passthru,/dev/ppt1 -s 6:0,virtio-net-viona,net0","stream":"stdout","time":"2020-02-08T23:13:35.591311000Z"}
{"log":" -c sockets=1,cores=2,threads=2 -s 30:0,fbuf,vga=off,unix=/tmp/vm.vnc -s 30:1,xhci,tablet SYSbhyve-5\n","stream":"stdout","time":"2020-02-08T23:13:35.591391000Z"}
{"log":"Unhandled ps2 keyboard command 0x02\n","stream":"stderr","time":"2020-02-08T23:13:39.473196000Z"}
{"log":"Unhandled ps2 keyboard command 0x02\n","stream":"stderr","time":"2020-02-08T23:13:42.790741000Z"}
{"log":"fbuf frame buffer base: fffffc7feae00000 [sz 16777216]\r\n","stream":"stdout","time":"2020-02-08T23:13:46.295013000Z"}
{"log":"Assertion failed: error == 0, file pci_emul.c, line 517, function modify_bar_registration\n","stream":"stderr","time":"2020-02-08T23:13:46.295122000Z"}
{"event":"close","stream":"logfile","time":"2020-02-08T23:13:48.258231000Z"}

console.log

{"event":"open","stream":"logfile","time":"2020-02-08T23:13:34.402799000Z"}
{"log":"\u001b[2J\u001b[01;01H\u001b[=3h\u001b[2J\u001b[01;01H\u001b[2J\u001b[01;01H\u001b[=3h\u001b[2J\u001b[01;01H\u001b[2J\u001b[01;01H\u001b[=3h\u001b[2J\u001b[01;01H\u001b[2J\u001b[01;01H\u001b[=3h\u001b[2J\u001b[01;01H\u001b[2J\u001b[01;01H\u001b[=3h\u001b[2J\u001b[01;01H\u001b[0m\u001b[39m\u001b[49m\u001b[2J\u001b[01;01H\u001b[=3h\u001b[2J\u001b[01;01H\u001b[0m\u001b[39m\u001b[49m\u001b[2J\u001b[01;01H","stream":"console","time":"2020-02-08T23:13:48.251410000Z"}
{"event":"close","stream":"logfile","time":"2020-02-08T23:13:48.251564000Z"}
sjorge commented 4 years ago

@hrosenfeld as requested

sjorge commented 4 years ago

I did find:

A known problem - if a PCI device has 2 or more memory BARs, the bhyve PCI emulation gets a little upset at Windows setting the base address to 0xFF's to get the size while the BAR is still enabled. I have a fix for this.

https://forums.freebsd.org/threads/vm-bhyve-windows-2012-r2-and-passthru.60832/

Will try with a illumos or freebsd vm tomorrow.

sjorge commented 4 years ago

Looks indeed to be windows that wants to remap the bar for devices with multiple bars!

FreeBSD 12 VM works fine, and it does list 2 bars!

ixl0@pci0:0:8:0:        class=0x020000 card=0x37d215d9 chip=0x37d28086 rev=0x09 hdr=0x00
    vendor     = 'Intel Corporation'
    device     = 'Ethernet Connection X722 for 10GBASE-T'
    class      = network
    subclass   = ethernet
    bar   [10] = type Prefetchable Memory, range 64, base 0xc1000000, size 16777216, enabled
    bar   [1c] = type Prefetchable Memory, range 64, base 0xc2000000, size 32768, enabled
    cap 01[40] = powerspec 3  supports D0 D3  current D0
    cap 05[50] = MSI supports 1 message, 64 bit, vector masks
    cap 11[70] = MSI-X supports 8 messages, enabled
                 Table in map 0x1c[0x0], PBA in map 0x1c[0x1000]
    cap 10[a0] = PCI-Express 2 endpoint max data 256(512) FLR RO
                 link x1(x1) speed 2.5(2.5) ASPM disabled(L0s/L1)
    cap 03[e0] = VPD
    VPD ident  = 'Example VPD'
ixl1@pci0:0:8:1:        class=0x020000 card=0x37d215d9 chip=0x37d28086 rev=0x09 hdr=0x00
    vendor     = 'Intel Corporation'
    device     = 'Ethernet Connection X722 for 10GBASE-T'
    class      = network
    subclass   = ethernet
    bar   [10] = type Prefetchable Memory, range 64, base 0xc3000000, size 16777216, enabled
    bar   [1c] = type Prefetchable Memory, range 64, base 0xc4000000, size 32768, enabled
    cap 01[40] = powerspec 3  supports D0 D3  current D0
    cap 05[50] = MSI supports 1 message, 64 bit, vector masks
    cap 11[70] = MSI-X supports 8 messages, enabled
                 Table in map 0x1c[0x0], PBA in map 0x1c[0x1000]
    cap 10[a0] = PCI-Express 2 endpoint max data 256(512) FLR RO
                 link x1(x1) speed 2.5(2.5) ASPM disabled(L0s/L1)
    cap 03[e0] = VPD
    VPD ident  = 'Example VPD'
papertigers commented 4 years ago 
    pi_bar = [
        {
            type = 3 (PCIBAR_MEM64)
            size = 0x1000000
            addr = 0xc1000000
        },
        {
            type = 4 (PCIBAR_MEMHI64)
            size = 0
            addr = 0
        },
        {
            type = 0 (PCIBAR_NONE)
            size = 0
            addr = 0
        },
        {
            type = 3 (PCIBAR_MEM64)
            size = 0x8000
            addr = 0xffffffffffff8000
        },
        {
            type = 4 (PCIBAR_MEMHI64)
            size = 0
            addr = 0
        },
        {
            type = 0 (PCIBAR_NONE)
            size = 0
            addr = 0
        },
    ]

From your core file here's what I see for BARs

papertigers commented 4 years ago

Also the call to unregister_mem() ends up seeing an ENOENT from mmio_rb_lookup(&mmio_rb_root, memp->base, &entry);


  8  89087            unregister_mem:return 
              bhyve`unregister_mem+0x109
              bhyve`unregister_bar+0xd
              bhyve`update_bar_address+0x50
              bhyve`pci_cfgrw+0x28b
              bhyve`pci_emul_ecfg_handler+0x69
              bhyve`mem_write+0x37
              bhyve`emulate_mov+0xda
              bhyve`vmm_emulate_instruction+0x1b4
              bhyve`emulate_mem_cb+0x34
              bhyve`access_memory+0x125
              bhyve`emulate_mem+0x28
              bhyve`vmexit_inst_emul+0x40
              bhyve`vm_loop+0x9a
              bhyve`fbsdrun_start_thread+0x62
              libc.so.1`_thrp_setup+0x6c
              libc.so.1`_lwp_start
unregister_mem return: 2
sjorge commented 4 years ago

The macros in usr/contrib/freebsd/sys/tree.h that are using in unregister_mem, are the same as upstream... so they did not change anything there.

sjorge commented 4 years ago

FreeBSD bug got closed as already fixed in: https://svnweb.freebsd.org/base?view=revision&revision=348779 Looks we already pulled this in though?

sjorge commented 4 years ago

Some more digging, it looks like we only partially pulled this in? The changes to pci_passthru.c are not all there for us?

https://svnweb.freebsd.org/base/head/usr.sbin/bhyve/pci_passthru.c?r1=348779&r2=348778&pathrev=348779

sjorge commented 4 years ago

Adding all of the diff to pci_passthru.c on top of bhyve-sync branch, make it work! I just dropped the change in cfginit() as it looks like we do not need it as ppt? does the diff.