Black Screen after VM Boot - vfio-pci driver is loaded

[franchioping]

Sorry for any spelling mistakes, i'm not English After booting the VM i simply get a black screen. it doesnt crash, i simply get no output, according to "sudo virsh list" the VM is running lspci -nnk showed the vfio-pci driver is loaded as expected.

Some Info that I don't think changes anything but ill feel like a dumb ass if its the problem:

• The disk is a physical disk with windows pre-installed from a previous setup (same pc, but bare-metal).
• I am running gnome on Arch linux

in the log bellow (/var/log/libvirt/qemu/win10.log) the VM was killed by me, after 10 minutes of a black screen.

2021-07-14 01:56:58.259+0000: starting up libvirt version: 7.5.0, qemu version: 6.0.0, kernel: 5.12.15-arch1-1, hostname: Arch-Dt
LC_ALL=C \
PATH=/usr/local/sbin:/usr/local/bin:/usr/bin:/var/lib/snapd/snap/bin \
HOME=/var/lib/libvirt/qemu/domain-2-win10 \
XDG_DATA_HOME=/var/lib/libvirt/qemu/domain-2-win10/.local/share \
XDG_CACHE_HOME=/var/lib/libvirt/qemu/domain-2-win10/.cache \
XDG_CONFIG_HOME=/var/lib/libvirt/qemu/domain-2-win10/.config \
/usr/bin/qemu-system-x86_64 \
-name guest=win10,debug-threads=on \
-S \
-object '{"qom-type":"secret","id":"masterKey0","format":"raw","file":"/var/lib/libvirt/qemu/domain-2-win10/master-key.aes"}' \
-blockdev '{"driver":"file","filename":"/usr/share/edk2-ovmf/x64/OVMF_CODE.fd","node-name":"libvirt-pflash0-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash0-format","read-only":true,"driver":"raw","file":"libvirt-pflash0-storage"}' \
-blockdev '{"driver":"file","filename":"/var/lib/libvirt/qemu/nvram/win10_VARS.fd","node-name":"libvirt-pflash1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-pflash1-format","read-only":false,"driver":"raw","file":"libvirt-pflash1-storage"}' \
-machine pc-q35-6.0,accel=kvm,usb=off,vmport=off,dump-guest-core=off,pflash0=libvirt-pflash0-format,pflash1=libvirt-pflash1-format,memory-backend=pc.ram \
-cpu host,migratable=on,hv-time,hv-relaxed,hv-vapic,hv-spinlocks=0x1fff,hv-vendor-id=whatever,kvm=off \
-m 8192 \
-object '{"qom-type":"memory-backend-ram","id":"pc.ram","size":8589934592}' \
-overcommit mem-lock=off \
-smp 6,sockets=1,dies=1,cores=3,threads=2 \
-uuid 7c614128-ac98-455c-9e7e-40eb15fbd44c \
-display none \
-no-user-config \
-nodefaults \
-chardev socket,id=charmonitor,fd=31,server=on,wait=off \
-mon chardev=charmonitor,id=monitor,mode=control \
-rtc base=localtime,driftfix=slew \
-global kvm-pit.lost_tick_policy=delay \
-no-hpet \
-no-shutdown \
-global ICH9-LPC.disable_s3=1 \
-global ICH9-LPC.disable_s4=1 \
-boot strict=on \
-device pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2 \
-device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1 \
-device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2 \
-device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3 \
-device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4 \
-device pcie-root-port,port=0x8,chassis=6,id=pci.6,bus=pcie.0,multifunction=on,addr=0x1 \
-device pcie-root-port,port=0x9,chassis=7,id=pci.7,bus=pcie.0,addr=0x1.0x1 \
-device pcie-root-port,port=0xa,chassis=8,id=pci.8,bus=pcie.0,addr=0x1.0x2 \
-device qemu-xhci,p2=15,p3=15,id=usb,bus=pci.2,addr=0x0 \
-device virtio-serial-pci,id=virtio-serial0,bus=pci.3,addr=0x0 \
-blockdev '{"driver":"host_device","filename":"/dev/disk/by-id/wwn-0x5002538e407b13de","aio":"native","node-name":"libvirt-2-storage","cache":{"direct":true,"no-flush":false},"auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-2-format","read-only":false,"cache":{"direct":true,"no-flush":false},"driver":"raw","file":"libvirt-2-storage"}' \
-device ide-hd,bus=ide.0,drive=libvirt-2-format,id=sata0-0-0,bootindex=1,write-cache=on \
-blockdev '{"driver":"file","filename":"/home/franchioping/Downloads/virtio-win-0.1.185.iso","node-name":"libvirt-1-storage","auto-read-only":true,"discard":"unmap"}' \
-blockdev '{"node-name":"libvirt-1-format","read-only":true,"driver":"raw","file":"libvirt-1-storage"}' \
-device ide-cd,bus=ide.1,drive=libvirt-1-format,id=sata0-0-1 \
-netdev tap,fd=33,id=hostnet0 \
-device e1000e,netdev=hostnet0,id=net0,mac=52:54:00:62:73:1e,bus=pci.1,addr=0x0 \
-chardev pty,id=charserial0 \
-device isa-serial,chardev=charserial0,id=serial0 \
-device virtio-mouse-pci,id=input0,bus=pci.7,addr=0x0 \
-device virtio-keyboard-pci,id=input1,bus=pci.8,addr=0x0 \
-audiodev id=audio1,driver=spice \
-device vfio-pci,host=0000:01:00.0,id=hostdev0,bus=pci.5,addr=0x0,romfile=/home/franchioping/Documents/VM/rom.bin \
-device vfio-pci,host=0000:01:00.1,id=hostdev1,bus=pci.6,addr=0x0 \
-device virtio-balloon-pci,id=balloon0,bus=pci.4,addr=0x0 \
-object input-linux,id=mouse1,evdev=/dev/input/by-id/usb-Logitech_G203_Prodigy_Gaming_Mouse_058238593330-event-mouse \
-object input-linux,id=kbd1,evdev=/dev/input/by-id/usb-Corsair_Corsair_Gaming_K55_RGB_Keyboard_AF6B11215E65CC21F5001C0140048000-event-kbd,grab_all=on,repeat=on \
-device ich9-intel-hda,bus=pcie.0,addr=0x1b \
-device hda-micro,audiodev=hda \
-audiodev pa,id=hda,server=/run/user/1000/pulse/native \
-sandbox on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny \
-msg timestamp=on
char device redirected to /dev/pts/0 (label charserial0)
audio: Could not init 'spice' audio driver
audio: warning: Using timer based audio emulation
2021-07-14T02:06:00.604572Z qemu-system-x86_64: terminating on signal 15 from pid 8235 (/usr/bin/libvirtd)
pulseaudio: Failed to disconnect! err=-15
pulseaudio: Failed to disconnect! err=-15
2021-07-14 02:06:02.206+0000: shutting down, reason=destroyed

gpu that i want to passtrough to the vm, while logged into gnome (lspci -nnk):

01:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP106 [GeForce GTX 1060 6GB] [10de:1c03] (rev a1)
    Subsystem: Gigabyte Technology Co., Ltd Device [1458:3771]
    Kernel driver in use: nvidia
    Kernel modules: nouveau, nvidia_drm, nvidia
01:00.1 Audio device [0403]: NVIDIA Corporation GP106 High Definition Audio Controller [10de:10f1] (rev a1)
    Subsystem: Gigabyte Technology Co., Ltd Device [1458:3771]
    Kernel driver in use: snd_hda_intel
    Kernel modules: snd_hda_intel

IOMMU groups:

IOMMU Group 0 00:00.0 Host bridge [0600]: Intel Corporation 8th Gen Core Processor Host Bridge/DRAM Registers [8086:3ec2] (rev 07)
IOMMU Group 1 00:01.0 PCI bridge [0604]: Intel Corporation 6th-10th Gen Core Processor PCIe Controller (x16) [8086:1901] (rev 07)
IOMMU Group 1 01:00.0 VGA compatible controller [0300]: NVIDIA Corporation GP106 [GeForce GTX 1060 6GB] [10de:1c03] (rev a1)
IOMMU Group 1 01:00.1 Audio device [0403]: NVIDIA Corporation GP106 High Definition Audio Controller [10de:10f1] (rev a1)
IOMMU Group 2 00:14.0 USB controller [0c03]: Intel Corporation Cannon Lake PCH USB 3.1 xHCI Host Controller [8086:a36d] (rev 10)
IOMMU Group 2 00:14.2 RAM memory [0500]: Intel Corporation Cannon Lake PCH Shared SRAM [8086:a36f] (rev 10)
IOMMU Group 3 00:14.3 Network controller [0280]: Intel Corporation Cannon Lake PCH CNVi WiFi [8086:a370] (rev 10)
IOMMU Group 4 00:16.0 Communication controller [0780]: Intel Corporation Cannon Lake PCH HECI Controller [8086:a360] (rev 10)
IOMMU Group 5 00:17.0 SATA controller [0106]: Intel Corporation Cannon Lake PCH SATA AHCI Controller [8086:a352] (rev 10)
IOMMU Group 6 00:1b.0 PCI bridge [0604]: Intel Corporation Cannon Lake PCH PCI Express Root Port #17 [8086:a340] (rev f0)
IOMMU Group 7 00:1c.0 PCI bridge [0604]: Intel Corporation Cannon Lake PCH PCI Express Root Port #1 [8086:a338] (rev f0)
IOMMU Group 8 00:1d.0 PCI bridge [0604]: Intel Corporation Cannon Lake PCH PCI Express Root Port #9 [8086:a330] (rev f0)
IOMMU Group 9 00:1f.0 ISA bridge [0601]: Intel Corporation Z390 Chipset LPC/eSPI Controller [8086:a305] (rev 10)
IOMMU Group 9 00:1f.3 Audio device [0403]: Intel Corporation Cannon Lake PCH cAVS [8086:a348] (rev 10)
IOMMU Group 9 00:1f.4 SMBus [0c05]: Intel Corporation Cannon Lake PCH SMBus Controller [8086:a323] (rev 10)
IOMMU Group 9 00:1f.5 Serial bus controller [0c80]: Intel Corporation Cannon Lake PCH SPI Controller [8086:a324] (rev 10)
IOMMU Group 9 00:1f.6 Ethernet controller [0200]: Intel Corporation Ethernet Connection (7) I219-V [8086:15bc] (rev 10)

The Start.sh and Stop.sh are the same as in the guide, and after logging in from ssh and force shutting down the VM, every thing comes back as it should

I've been stuck at this for hours, any help is appreciated!

Edit: tried to format text better, very clearly failed, pleas some1 explain to me how to make the logs look good

[QaidVoid]

I don't see any issue on the log.. I have never tried passing physical disk (don't have extra disk) so I don't know how that works.. Have you tried passing through without including the physical disk? You'd get dropped to EFI shell (which would confirm that passthrough works) if you don't add disk/installer..

As for the audio related errors, you can't use default sound device set in virt-manager, which uses spice.. This is not the cause of black screen issue though..

[franchioping]

I don't see any issue on the log.. I have never tried passing physical disk (don't have extra disk) so I don't know how that works.. Have you tried passing through without including the physical disk? You'd get dropped to EFI shell (which would confirm that passthrough works) if you don't add disk/installer..

As for the audio related errors, you can't use default sound device set in virt-manager, which uses spice.. This is not the cause of black screen issue though..

I just tried without any storage connected to the VM, still same thing. What do you mean the default sound device?

Forgot to mention this but i was fully using the vm in qemu before passing the gpu

Edit: should i remove "Serial 1"?

Edit2: htop screenshot

[franchioping]

Should the rom file be .bin or .rom?

[franchioping]

Got it running! I was using the wrong vbios, right gpu, diferent revision

[QaidVoid]

What do you mean the default sound device?

I meant the sound device (Sound ich9) which is set my virt-manager when creating VM.

Got it running! I was using the wrong vbios, right gpu, diferent revision

Great. Have fun.