[vfio-users] GPU performance degradation using KVM and Nvidia

[Tobias B.]

Hello,

when I use KVM in combination with a Nvidia GPU I get far lower fps 
values than expected. Sometimes even 60% lower than native performance. 
I don't use Hugepages and I pin my vCPUs according to the real pCPUs, 
like 1-3,5-7. Physical core #0 is assigned to the host and #1,#2,#3 
entirely to guest. But sometimes the guest stutters, when the host is 
busy doing cpu intensive tasks on it's cores.

Kind regards and thanks in advance.


### CPU info ###
> cat /proc/cpuinfo | grep -iP "processor|core id"
processor	: 0
core id		: 0
processor	: 1
core id		: 1
processor	: 2
core id		: 2
processor	: 3
core id		: 3
processor	: 4
core id		: 0
processor	: 5
core id		: 1
processor	: 6
core id		: 2
processor	: 7
core id		: 3

### libvirt config script ###
<domain type='kvm' 
xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
   <name>win8.1</name>
   <uuid>faee0180-cd1b-49ef-ad33-26aa1bd663ac</uuid>
   <memory unit='KiB'>8290304</memory>
   <currentMemory unit='KiB'>8290304</currentMemory>
   <vcpu placement='static' cpuset='1-3,5-7'>6</vcpu>
   <os>
     <type arch='x86_64' machine='pc-i440fx-2.4'>hvm</type>
     <loader readonly='yes' 
type='pflash'>/usr/share/ovmf/x64/ovmf_code_x64.bin</loader>
     <nvram>/var/lib/libvirt/qemu/nvram/win8.1_VARS.fd</nvram>
     <boot dev='hd'/>
     <bootmenu enable='no'/>
   </os>
   <features>
     <acpi/>
     <apic/>
     <pae/>
     <kvm>
       <hidden state='on'/>
     </kvm>
     <vmport state='off'/>
   </features>
   <cpu mode='host-passthrough'>
     <topology sockets='1' cores='3' threads='2'/>
   </cpu>
   <clock offset='localtime'>
     <timer name='rtc' tickpolicy='catchup'/>
     <timer name='pit' tickpolicy='delay'/>
     <timer name='hpet' present='no'/>
   </clock>
   <on_poweroff>destroy</on_poweroff>
   <on_reboot>restart</on_reboot>
   <on_crash>restart</on_crash>
   <pm>
     <suspend-to-mem enabled='no'/>
     <suspend-to-disk enabled='no'/>
   </pm>
   <devices>
     <emulator>/usr/sbin/qemu-system-x86_64</emulator>
     <disk type='file' device='disk'>
       <driver name='qemu' type='qcow2' cache='none' io='native'/>
       <source file='/home/tobias/Dokumente/test.qcow2'/>
       <target dev='vda' bus='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x02' 
function='0x0'/>
     </disk>
     <disk type='block' device='cdrom'>
       <driver name='qemu' type='raw'/>
       <target dev='sda' bus='sata'/>
       <readonly/>
       <address type='drive' controller='0' bus='0' target='0' unit='0'/>
     </disk>
     <controller type='pci' index='0' model='pci-root'/>
     <controller type='virtio-serial' index='0'>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x05' 
function='0x0'/>
     </controller>
     <controller type='sata' index='0'>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x07' 
function='0x0'/>
     </controller>
     <controller type='usb' index='0' model='ich9-ehci1'>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x06' 
function='0x7'/>
     </controller>
     <controller type='usb' index='0' model='ich9-uhci1'>
       <master startport='0'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x06' 
function='0x0' multifunction='on'/>
     </controller>
     <controller type='usb' index='0' model='ich9-uhci2'>
       <master startport='2'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x06' 
function='0x1'/>
     </controller>
     <controller type='usb' index='0' model='ich9-uhci3'>
       <master startport='4'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x06' 
function='0x2'/>
     </controller>
     <interface type='network'>
       <mac address='52:54:00:38:2c:b5'/>
       <source network='default'/>
       <model type='virtio'/>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x03' 
function='0x0'/>
     </interface>
     <hostdev mode='subsystem' type='usb' managed='yes'>
       <source>
         <vendor id='0x041e'/>
         <product id='0x30d3'/>
       </source>
     </hostdev>
     <redirdev bus='usb' type='spicevmc'>
     </redirdev>
     <memballoon model='virtio'>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x09' 
function='0x0'/>
     </memballoon>
     <rng model='virtio'>
       <backend model='random'>/dev/random</backend>
       <address type='pci' domain='0x0000' bus='0x00' slot='0x08' 
function='0x0'/>
     </rng>
   </devices>
   <qemu:commandline>
     <qemu:arg value='-device'/>
     <qemu:arg value='vfio-pci,host=02:00.0,multifunction=on,x-vga=on'/>
     <qemu:arg value='-device'/>
     <qemu:arg value='vfio-pci,host=02:00.1'/>
   </qemu:commandline>
</domain>