[vfio-users] Cannot register eventfd with MSI/MSI-X interrupts

[Alex Williamson]

On Mon, 1 Aug 2016 10:24:52 +0000
chris thompson <the_cartographer at hotmail.com> wrote:

>  Hi,
>  
> I'm trying to talk to my PCI-e card (USB3 host controller) from userland, and have been successful accessing the PCI Config registers and device memory, and able to register eventfd file descriptors for legacy interrupts (IRQx) and Error interrupts - including stimulating these via the ioctl calls. Unfortunately the MSI and MSI-X interrupts return EINVAL (invalid argument) when trying to register them.
>  
> Sample from the code:
> printf("\nInterrupts:\n");
> int * irq_fds[VFIO_PCI_NUM_IRQS];
> int num_irqs[VFIO_PCI_NUM_IRQS];
> for (i = 0; i < device_info.num_irqs; i++) {
>   struct vfio_irq_info irq = { .argsz = sizeof(irq) };
>   irq.index = i;
>   ioctl(device, VFIO_DEVICE_GET_IRQ_INFO, &irq);
>   printf("argsz 0x%x, flags 0x%x, index 0x%x, count 0x%x,\n", irq.argsz, irq.flags, irq.index, irq.count);
>   num_irqs[i] = irq.count;
>   irq_fds[i]=(int*)malloc(irq.count * sizeof(int));
>   int j;
>   for(j=0; j<irq.count; j++)
>   {
>     irq_fds[i][j] = eventfd(0, 0);
>     printf("config IRQ %u : %u\n", i, j);
>     struct vfio_irq_set *irq_setup;
>     unsigned int size = sizeof(struct vfio_irq_set) + sizeof(int);
>     irq_setup = malloc(size);
>     irq_setup->argsz = size;
>     irq_setup->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER ;
>     irq_setup->index = i;
>     irq_setup->start = j;
>     irq_setup->count = 1;
>     *(int*)(irq_setup->data)=irq_fds[i][j];
>     int ret = ioctl(device, VFIO_DEVICE_SET_IRQS, irq_setup);
>     if(ret){ printf("failed to register interrupt %u : %u, error %i %s\n", i, j, ret, strerror(errno)); }
>     free(irq_setup);
>   }
> }
>  
> Which gives the output:
>  
> Interrupts:
> argsz 0x10, flags 0x7, index 0x0, count 0x1,
> config IRQ 0 : 0
> argsz 0x10, flags 0x9, index 0x1, count 0x40,
> config IRQ 1 : 0
> failed to register interrupt 1 : 0, error -1 Invalid argument
> config IRQ 1 : 1
> failed to register interrupt 1 : 1, error -1 Invalid argument
> config IRQ 1 : 2
> ...
> 
> Does anyone know a likely reason that I can't register these? From what I've read my IOMMUs all support IRQ remapping:
> [    0.016443] dmar: IOMMU 0: reg_base_addr fed90000 ver 1:0 cap c0000020e60262 ecap f0101a
> [    0.016450] dmar: IOMMU 1: reg_base_addr fed91000 ver 1:0 cap c9008020660262 ecap f0105a
> 
> The hardware is an Intel i5 2500 in an HP 8200 Elite small form-factor PC, running Ubuntu 14.04 on the 3.13 kernel.


include/uapi/linux/vfio.h:
 * The NORESIZE flag indicates that the interrupt lines within the index
 * are setup as a set and new subindexes cannot be enabled without first
 * disabling the entire index.  This is used for interrupts like PCI MSI
 * and MSI-X where the driver may only use a subset of the available
 * indexes, but VFIO needs to enable a specific number of vectors
 * upfront.  In the case of MSI-X, where the user can enable MSI-X and
 * then add and unmask vectors, it's up to userspace to make the decision
 * whether to allocate the maximum supported number of vectors or tear
 * down setup and incrementally increase the vectors as each is enabled.

struct vfio_irq_info {
        __u32   argsz;
        __u32   flags;
#define VFIO_IRQ_INFO_EVENTFD           (1 << 0)
#define VFIO_IRQ_INFO_MASKABLE          (1 << 1)
#define VFIO_IRQ_INFO_AUTOMASKED        (1 << 2)
#define VFIO_IRQ_INFO_NORESIZE          (1 << 3)   <----
        __u32   index;          /* IRQ index */
        __u32   count;          /* Number of IRQs within this index */
};