search

joeknock90 / Single-GPU-Passthrough

1.49k stars 75 forks source link

Black screen when trying to start VM. #12

Closed KalleStruik closed 4 years ago

KalleStruik commented 4 years ago

After starting the VM all my displays tun off and never come on again. All the displays are connected directly to my GPU. All the extra devices (including the Spice and QXL display) have been removed true virt-manager. SSHing into my main pc and running the start.sh file manually gives no errors. Trying to start the vm after this also seems to work but I dont get any display. Running virsh list shows the VM as running. I have tried to shut it down using virsh shutdown win10 but this seems to not do anything either. (I suspect that the VM is completly unresponsive but I dont know how I could check this.) I have attached some files that might be usefull to figure out the issue. If there is anything else that I did not provide feel free to tell me.

My start.sh:

#!/bin/bash
# Helpful to read output when debugging
set -x

# Stop display manager
systemctl stop sddm.service

# Unbind VTconsoles
echo 0 > /sys/class/vtconsole/vtcon0/bind
echo 0 > /sys/class/vtconsole/vtcon1/bind

# Unbind EFI-Framebuffer
echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind

# Avoid a Race condition by waiting 2 seconds. This can be calibrated to be shorter or longer if required for your system
sleep 5 # Changed to 5 because of the Race condition

# Unload all Nvidia drivers
modprobe -r nvidia_drm
modprobe -r nvidia_modeset
modprobe -r nvidia_uvm
modprobe -r nvidia
# Looks like these might need to be unloaded on Ryzen Systems. Not sure yet.
modprobe -r ipmi_devintf
modprobe -r ipmi_msghandler

# Unbind the GPU from display driver
virsh nodedev-detach pci_0000_20_00_0
virsh nodedev-detach pci_0000_20_00_1

# Load VFIO Kernel Module  
modprobe vfio-pci

The log file in /var/log/libvirt/qemu/win.log:

2020-04-20 17:25:32.795+0000: starting up libvirt version: 5.10.0, qemu version: 4.2.0, kernel: 5.6.4-zen1-1-zen, hostname: kalle-pc.home
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/bin \
HOME=/var/lib/libvirt/qemu/domain-1-win10 \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-1-win10/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-1-win10/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-1-win10/.config \
QEMU_AUDIO_DRV=none \
/usr/bin/qemu-system-x86_64 \
-name guest=win10,debug-threads=on \
-S \
-object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-1-win10/master-key.aes \
-blockdev '{"driver":"file","filename":"/usr/share/ovmf/x64/OVMF_CODE.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/win10_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \
-machine pc-q35-4.2,accel=kvm,usb=off,vmport=off,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format \
-cpu EPYC-IBPB,x2apic=on,tsc-deadline=on,hypervisor=on,tsc-adjust=on,arch-capabilities=on,xsaves=on,cmp-legacy=on,perfctr-core=on,virt-ssbd=on,rdctl-no=on,skip-l1dfl-vmentry=on,mds-no=on,monitor=off,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1fff \
-m 8192 \
-overcommit mem-lock=off \
-smp 8,sockets=8,cores=1,threads=1 \
-uuid 6a049f2b-9136-470a-aecb-c0d59bdcd358 \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=32,server,nowait \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 \
-device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
-device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
-device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
-device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \
-device pcie-root-port,port=0x8,chassis=6,id=pci.6,bus=pcie.0,addr=0x1 \
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/images/win10.qcow2","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":false,"driver":"qcow2","file":"libvirt-2-storage","backing":null}' \
-device ide-hd,bus=ide.0,drive=libvirt-2-format,id=sata0-0-0,bootindex=1 \
-blockdev '{"driver":"file","filename":"/home/kalle/Downloads/Firefox/Win10_1909_EnglishInternational_x64.iso","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"}' \
-device ide-cd,bus=ide.1,drive=libvirt-1-format,id=sata0-0-1,bootindex=2 \
-netdev tap,fd=34,id=hostnet0 \
-device e1000e,netdev=hostnet0,id=net0,mac=52:54:00:0d:91:6b,bus=pci.1,addr=0x0 \
-device vfio-pci,host=0000:20:00.0,id=hostdev0,bus=pci.5,addr=0x0,romfile=/var/lib/libvirt/vbios/Zotac-Mini-1070-Patched.bin \
-device vfio-pci,host=0000:20:00.1,id=hostdev1,bus=pci.6,addr=0x0 \
-device virtio-balloon-pci,id=balloon0,bus=pci.4,addr=0x0 \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on

The full XML file from sudo virsh edit win10:

<domain type='kvm'>
  <name>win10</name>
  <uuid>6a049f2b-9136-470a-aecb-c0d59bdcd358</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://microsoft.com/win/10"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit='KiB'>8388608</memory>
  <currentMemory unit='KiB'>8388608</currentMemory>
  <vcpu placement='static'>8</vcpu>
  <os>
    <type arch='x86_64' machine='pc-q35-4.2'>hvm</type>
    <loader readonly='yes' type='pflash'>/usr/share/ovmf/x64/OVMF_CODE.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/win10_VARS.fd</nvram>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv>
      <relaxed state='on'/>
      <vapic state='on'/>
      <spinlocks state='on' retries='8191'/>
    </hyperv>
    <vmport state='off'/>
  </features>
  <cpu mode='host-model' check='none'/>
  <clock offset='localtime'>
    <timer name='rtc' tickpolicy='catchup'/>
    <timer name='pit' tickpolicy='delay'/>
    <timer name='hpet' present='no'/>
    <timer name='hypervclock' present='yes'/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled='no'/>
    <suspend-to-disk enabled='no'/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2'/>
      <source file='/var/lib/libvirt/images/win10.qcow2'/>
      <target dev='sda' bus='sata'/>
      <boot order='1'/>
      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
    </disk>
    <disk type='file' device='cdrom'>
      <driver name='qemu' type='raw'/>
      <source file='/home/kalle/Downloads/Firefox/Win10_1909_EnglishInternational_x64.iso'/>
      <target dev='sdb' bus='sata'/>
      <readonly/>
      <boot order='2'/>
      <address type='drive' controller='0' bus='0' target='0' unit='1'/>
    </disk>
    <controller type='usb' index='0' model='qemu-xhci' ports='15'>
      <address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
    </controller>
    <controller type='sata' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pcie-root'/>
    <controller type='pci' index='1' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='1' port='0x10'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
    </controller>
    <controller type='pci' index='2' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='2' port='0x11'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
    </controller>
    <controller type='pci' index='3' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='3' port='0x12'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
    </controller>
    <controller type='pci' index='4' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='4' port='0x13'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
    </controller>
    <controller type='pci' index='5' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='5' port='0x14'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
    </controller>
    <controller type='pci' index='6' model='pcie-root-port'>
      <model name='pcie-root-port'/>
      <target chassis='6' port='0x8'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
    </controller>
    <controller type='virtio-serial' index='0'>
      <address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
    </controller>
    <interface type='network'>
      <mac address='52:54:00:0d:91:6b'/>
      <source network='default'/>
      <model type='e1000e'/>
      <address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
    </interface>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x20' slot='0x00' function='0x0'/>
      </source>
      <rom file='/var/lib/libvirt/vbios/Zotac-Mini-1070-Patched.bin'/>
      <address type='pci' domain='0x0000' bus='0x05' slot='0x00' function='0x0'/>
    </hostdev>
    <hostdev mode='subsystem' type='pci' managed='yes'>
      <source>
        <address domain='0x0000' bus='0x20' slot='0x00' function='0x1'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x06' slot='0x00' function='0x0'/>
    </hostdev>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
    </memballoon>
  </devices>
</domain>
joeknock90 commented 4 years ago

Nevermind I see the make and model from your patched rom.

Hmmm.....

Anything in dmesg?

KalleStruik commented 4 years ago

I dont see anything myself, but I have attached it in case you see anything.

Output from the start commands from ssh:

[kalle@kalle-pc ~]$ sudo /etc/libvirt/hooks/qemu.d/win10/
prepare/ release/ 
[kalle@kalle-pc ~]$ sudo /etc/libvirt/hooks/qemu.d/win10/prepare/begin/start.sh 
[sudo] password for kalle: 
+ systemctl stop sddm.service
+ echo 0
+ echo 0
+ echo efi-framebuffer.0
+ sleep 5
+ modprobe -r nvidia_drm
+ modprobe -r nvidia_modeset
+ modprobe -r nvidia_uvm
+ modprobe -r nvidia
+ modprobe -r ipmi_devintf
+ modprobe -r ipmi_msghandler
+ virsh nodedev-detach pci_0000_20_00_0
Device pci_0000_20_00_0 detached

+ virsh nodedev-detach pci_0000_20_00_1
Device pci_0000_20_00_1 detached

+ modprobe vfio-pci
[kalle@kalle-pc ~]$ sudo virsh start win10
Domain win10 started

Dmesg output:

[ 2657.231081] [drm] [nvidia-drm] [GPU ID 0x00002000] Unloading driver
[ 2657.353251] nvidia-modeset: Unloading
[ 2657.365778] nvidia-nvlink: Unregistered the Nvlink Core, major device number 237
[ 2657.493282] VFIO - User Level meta-driver version: 0.3
[ 2657.504655] vfio-pci 0000:20:00.0: vgaarb: changed VGA decodes: olddecodes=none,decodes=io+mem:owns=io+mem
[ 2657.718666] kauditd_printk_skb: 23 callbacks suppressed
[ 2657.718669] audit: type=1106 audit(1587406827.689:196): pid=3943 uid=0 auid=1000 ses=4 msg='op=PAM:session_close grantors=pam_limits,pam_unix,pam_permit acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/3 res=success'
[ 2657.718769] audit: type=1104 audit(1587406827.689:197): pid=3943 uid=0 auid=1000 ses=4 msg='op=PAM:setcred grantors=pam_unix,pam_permit acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/3 res=success'
[ 2673.087378] audit: type=1101 audit(1587406843.058:198): pid=4066 uid=1000 auid=1000 ses=4 msg='op=PAM:accounting grantors=pam_unix,pam_permit,pam_time acct="kalle" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/3 res=success'
[ 2673.087526] audit: type=1110 audit(1587406843.058:199): pid=4066 uid=0 auid=1000 ses=4 msg='op=PAM:setcred grantors=pam_unix,pam_permit,pam_env acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/3 res=success'
[ 2673.087784] audit: type=1105 audit(1587406843.058:200): pid=4066 uid=0 auid=1000 ses=4 msg='op=PAM:session_open grantors=pam_limits,pam_unix,pam_permit acct="root" exe="/usr/bin/sudo" hostname=? addr=? terminal=/dev/pts/3 res=success'
[ 2678.186446] audit: type=2502 audit(1587406848.157:201): pid=651 uid=0 auid=4294967295 ses=4294967295 msg='virt=kvm vm="win10" uuid=6a049f2b-9136-470a-aecb-c0d59bdcd358 vm-ctx=+65534:+992 img-ctx=+65534:+992 model=dac exe="/usr/bin/libvirtd" hostname=? addr=? terminal=? res=success'
[ 2678.207935] audit: type=1130 audit(1587406848.179:202): pid=1 uid=0 auid=4294967295 ses=4294967295 msg='unit=virtlogd comm="systemd" exe="/usr/lib/systemd/systemd" hostname=? addr=? terminal=? res=success'
[ 2678.224826] virbr0: port 2(vnet0) entered blocking state
[ 2678.224840] virbr0: port 2(vnet0) entered disabled state
[ 2678.224909] device vnet0 entered promiscuous mode
[ 2678.224932] audit: type=1700 audit(1587406848.195:203): dev=vnet0 prom=256 old_prom=0 auid=4294967295 uid=0 gid=0 ses=4294967295
[ 2678.225066] virbr0: port 2(vnet0) entered blocking state
[ 2678.225068] virbr0: port 2(vnet0) entered listening state
[ 2678.225138] audit: type=2501 audit(1587406848.196:204): pid=651 uid=0 auid=4294967295 ses=4294967295 msg='virt=kvm resrc=net reason=open vm="win10" uuid=6a049f2b-9136-470a-aecb-c0d59bdcd358 net=52:54:00:0d:91:6b path="/dev/net/tun" rdev=0A:C8 exe="/usr/bin/libvirtd" hostname=? addr=? terminal=? res=success'
[ 2678.267026] audit: type=2501 audit(1587406848.238:205): pid=651 uid=0 auid=4294967295 ses=4294967295 msg='virt=kvm resrc=cgroup reason=deny vm="win10" uuid=6a049f2b-9136-470a-aecb-c0d59bdcd358 cgroup="/sys/fs/cgroup/devices/machine.slice/machine-qemu\x2d1\x2dwin10.scope/" class=all exe="/usr/bin/libvirtd" hostname=? addr=? terminal=? res=success'
[ 2678.267045] audit: type=2501 audit(1587406848.238:206): pid=651 uid=0 auid=4294967295 ses=4294967295 msg='virt=kvm resrc=cgroup reason=allow vm="win10" uuid=6a049f2b-9136-470a-aecb-c0d59bdcd358 cgroup="/sys/fs/cgroup/devices/machine.slice/machine-qemu\x2d1\x2dwin10.scope/" class=major category=pty maj=88 acl=rw exe="/usr/bin/libvirtd" hostname=? addr=? terminal=? res=success'
[ 2678.267079] audit: type=2501 audit(1587406848.238:207): pid=651 uid=0 auid=4294967295 ses=4294967295 msg='virt=kvm resrc=cgroup reason=allow vm="win10" uuid=6a049f2b-9136-470a-aecb-c0d59bdcd358 cgroup="/sys/fs/cgroup/devices/machine.slice/machine-qemu\x2d1\x2dwin10.scope/" class=path path="/dev/null" rdev=01:03 acl=rw exe="/usr/bin/libvirtd" hostname=? addr=? terminal=? res=success'
[ 2678.267107] audit: type=2501 audit(1587406848.238:208): pid=651 uid=0 auid=4294967295 ses=4294967295 msg='virt=kvm resrc=cgroup reason=allow vm="win10" uuid=6a049f2b-9136-470a-aecb-c0d59bdcd358 cgroup="/sys/fs/cgroup/devices/machine.slice/machine-qemu\x2d1\x2dwin10.scope/" class=path path="/dev/full" rdev=01:07 acl=rw exe="/usr/bin/libvirtd" hostname=? addr=? terminal=? res=success'
[ 2678.267136] audit: type=2501 audit(1587406848.238:209): pid=651 uid=0 auid=4294967295 ses=4294967295 msg='virt=kvm resrc=cgroup reason=allow vm="win10" uuid=6a049f2b-9136-470a-aecb-c0d59bdcd358 cgroup="/sys/fs/cgroup/devices/machine.slice/machine-qemu\x2d1\x2dwin10.scope/" class=path path="/dev/zero" rdev=01:05 acl=rw exe="/usr/bin/libvirtd" hostname=? addr=? terminal=? res=success'
[ 2678.267164] audit: type=2501 audit(1587406848.238:210): pid=651 uid=0 auid=4294967295 ses=4294967295 msg='virt=kvm resrc=cgroup reason=allow vm="win10" uuid=6a049f2b-9136-470a-aecb-c0d59bdcd358 cgroup="/sys/fs/cgroup/devices/machine.slice/machine-qemu\x2d1\x2dwin10.scope/" class=path path="/dev/random" rdev=01:08 acl=rw exe="/usr/bin/libvirtd" hostname=? addr=? terminal=? res=success'
[ 2679.625730] vfio-pci 0000:20:00.0: vfio_ecap_init: hiding ecap 0x19@0x900
[ 2679.629142] vfio-pci 0000:20:00.0: No more image in the PCI ROM
[ 2680.277251] virbr0: port 2(vnet0) entered learning state
[ 2682.324884] virbr0: port 2(vnet0) entered forwarding state
[ 2682.324891] virbr0: topology change detected, propagating
KalleStruik commented 4 years ago

I got it to work. After connecting through a VNC server from my laptop I was able to check the device manager and it ended up being a code 43. I have attached my fixed vm config virsh edit win10 for any other user coming across the same issue. Thank you very much for your time trying to help me.

<domain type="kvm">
  <name>win10</name>
  <uuid>6a049f2b-9136-470a-aecb-c0d59bdcd358</uuid>
  <metadata>
    <libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0">
      <libosinfo:os id="http://microsoft.com/win/10"/>
    </libosinfo:libosinfo>
  </metadata>
  <memory unit="KiB">8388608</memory>
  <currentMemory unit="KiB">8388608</currentMemory>
  <vcpu placement="static">8</vcpu>
  <os>
    <type arch="x86_64" machine="pc-q35-4.2">hvm</type>
    <loader readonly="yes" type="pflash">/usr/share/ovmf/x64/OVMF_CODE.fd</loader>
    <nvram>/var/lib/libvirt/qemu/nvram/win10_VARS.fd</nvram>
  </os>
  <features>
    <acpi/>
    <apic/>
    <hyperv>
      <relaxed state="on"/>
      <vapic state="on"/>
      <spinlocks state="on" retries="8191"/>
      <vpindex state="on"/>
      <runtime state="on"/>
      <synic state="on"/>
      <stimer state="on"/>
      <reset state="on"/>
      <vendor_id state="on" value="1234567890ab"/>
      <frequencies state="on"/>
      <reenlightenment state="on"/>
      <tlbflush state="on"/>
      <ipi state="off"/>
      <evmcs state="off"/>
    </hyperv>
    <kvm>
      <hidden state="on"/>
    </kvm>
    <vmport state="off"/>
    <ioapic driver="kvm"/>
  </features>
  <cpu mode="host-model" check="partial">
    <topology sockets="1" cores="4" threads="2"/>
    <feature policy="require" name="topoext"/>
    <feature policy="require" name="invtsc"/>
  </cpu>
  <clock offset="localtime">
    <timer name="rtc" tickpolicy="catchup"/>
    <timer name="pit" tickpolicy="delay"/>
    <timer name="hpet" present="no"/>
    <timer name="hypervclock" present="yes"/>
  </clock>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>destroy</on_crash>
  <pm>
    <suspend-to-mem enabled="no"/>
    <suspend-to-disk enabled="no"/>
  </pm>
  <devices>
    <emulator>/usr/bin/qemu-system-x86_64</emulator>
    <disk type="file" device="disk">
      <driver name="qemu" type="qcow2"/>
      <source file="/var/lib/libvirt/images/win10.qcow2"/>
      <target dev="sda" bus="sata"/>
      <boot order="1"/>
      <address type="drive" controller="0" bus="0" target="0" unit="0"/>
    </disk>
    <disk type="file" device="cdrom">
      <driver name="qemu" type="raw"/>
      <source file="/home/kalle/Downloads/Firefox/Win10_1909_EnglishInternational_x64.iso"/>
      <target dev="sdb" bus="sata"/>
      <readonly/>
      <boot order="2"/>
      <address type="drive" controller="0" bus="0" target="0" unit="1"/>
    </disk>
    <controller type="usb" index="0" model="qemu-xhci" ports="15">
      <address type="pci" domain="0x0000" bus="0x02" slot="0x00" function="0x0"/>
    </controller>
    <controller type="sata" index="0">
      <address type="pci" domain="0x0000" bus="0x00" slot="0x1f" function="0x2"/>
    </controller>
    <controller type="pci" index="0" model="pcie-root"/>
    <controller type="pci" index="1" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="1" port="0x10"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x0" multifunction="on"/>
    </controller>
    <controller type="pci" index="2" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="2" port="0x11"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x1"/>
    </controller>
    <controller type="pci" index="3" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="3" port="0x12"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x2"/>
    </controller>
    <controller type="pci" index="4" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="4" port="0x13"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x3"/>
    </controller>
    <controller type="pci" index="5" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="5" port="0x14"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x02" function="0x4"/>
    </controller>
    <controller type="pci" index="6" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="6" port="0x8"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x0" multifunction="on"/>
    </controller>
    <controller type="pci" index="7" model="pcie-root-port">
      <model name="pcie-root-port"/>
      <target chassis="7" port="0x9"/>
      <address type="pci" domain="0x0000" bus="0x00" slot="0x01" function="0x1"/>
    </controller>
    <controller type="pci" index="8" model="pcie-to-pci-bridge">
      <model name="pcie-pci-bridge"/>
      <address type="pci" domain="0x0000" bus="0x05" slot="0x00" function="0x0"/>
    </controller>
    <controller type="virtio-serial" index="0">
      <address type="pci" domain="0x0000" bus="0x03" slot="0x00" function="0x0"/>
    </controller>
    <interface type="network">
      <mac address="52:54:00:0d:91:6b"/>
      <source network="default"/>
      <model type="e1000e"/>
      <address type="pci" domain="0x0000" bus="0x01" slot="0x00" function="0x0"/>
    </interface>
    <input type="mouse" bus="ps2"/>
    <input type="keyboard" bus="ps2"/>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x20" slot="0x00" function="0x0"/>
      </source>
      <rom file="/var/lib/libvirt/vbios/Zotac-Mini-1070-Patched.bin"/>
      <address type="pci" domain="0x0000" bus="0x06" slot="0x00" function="0x0"/>
    </hostdev>
    <hostdev mode="subsystem" type="pci" managed="yes">
      <source>
        <address domain="0x0000" bus="0x20" slot="0x00" function="0x1"/>
      </source>
      <address type="pci" domain="0x0000" bus="0x07" slot="0x00" function="0x0"/>
    </hostdev>
    <hostdev mode="subsystem" type="usb" managed="yes">
      <source>
        <vendor id="0x1e7d"/>
        <product id="0x2dca"/>
      </source>
      <address type="usb" bus="0" port="1"/>
    </hostdev>
    <hostdev mode="subsystem" type="usb" managed="yes">
      <source>
        <vendor id="0x0951"/>
        <product id="0x16d2"/>
      </source>
      <address type="usb" bus="0" port="2"/>
    </hostdev>
    <hostdev mode="subsystem" type="usb" managed="yes">
      <source>
        <vendor id="0x0d8c"/>
        <product id="0x0014"/>
      </source>
      <address type="usb" bus="0" port="3"/>
    </hostdev>
    <memballoon model="virtio">
      <address type="pci" domain="0x0000" bus="0x04" slot="0x00" function="0x0"/>
    </memballoon>
  </devices>
</domain>