[libvirt] [PATCH 08/10] hostdev: Clean up delayed VFIO devices

[Andrea Bolognani]

When we determine that a VFIO device can be safely reattached to the
host, we have to also reattach all devices we might have delayed
earlier in order to clean up properly.

The new function virHostdevExpandPCIDeviceListIOMMUGroups() helps by
adding inactive devices that are part of the same IOMMU group to the
working set.

Resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1272300
---
 src/util/virhostdev.c | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/src/util/virhostdev.c b/src/util/virhostdev.c
index 0f2916b..1042ab3 100644
--- a/src/util/virhostdev.c
+++ b/src/util/virhostdev.c
@@ -809,6 +809,84 @@ virHostdevPreparePCIDevices(virHostdevManagerPtr hostdev_mgr,
     return ret;
 }
 
+/* virHostdevExpandPCIDeviceListIOMMUGroups:
+ * @list: list of PCI devices
+ * @otherList: list of PCI devices used to expand @list
+ *
+ * Expand IOMMU groups in @list by adding all devices in @otherList that
+ * are in the same IOMMU group as a device in @list.
+ *
+ * No device is removed from @otherList.
+ *
+ * Returns: 0 on success, <0 on failure
+ */
+static int
+virHostdevExpandPCIDeviceListIOMMUGroups(virPCIDeviceListPtr list,
+                                         virPCIDeviceListPtr otherList)
+{
+    virPCIDeviceListPtr addList = NULL;
+    virPCIDeviceListPtr tmpList = NULL;
+    size_t i;
+    int ret = -1;
+
+    /* addList will temporarily store all devices we're going to add */
+    if (!(addList = virPCIDeviceListNew()))
+        goto cleanup;
+
+    for (i = 0; i < virPCIDeviceListCount(list); i++) {
+        virPCIDevicePtr dev = virPCIDeviceListGet(list, i);
+        size_t j;
+
+        /* Skip non-VFIO devices */
+        if (virPCIDeviceGetStubDriver(dev) != VIR_PCI_STUB_DRIVER_VFIO)
+            continue;
+
+        /* Get the list of devices in the same IOMMU group */
+        virObjectUnref(tmpList);
+        if (!(tmpList = virPCIDeviceGetIOMMUGroupList(dev)))
+            goto cleanup;
+
+        for (j = 0; j < virPCIDeviceListCount(tmpList); j++) {
+            virPCIDevicePtr tmpDev = virPCIDeviceListGet(tmpList, j);
+            virPCIDevicePtr otherDev;
+
+            /* Devices that are present in the original list or we have already
+             * decided we want to add must be skipped to avoid duplicates */
+            if (virPCIDeviceListFind(list, tmpDev))
+                continue;
+            if (virPCIDeviceListFind(addList, tmpDev))
+                continue;
+
+            /* Devices that do not appear in otherList should not be included.
+             * We retrieve the device stored in otherList instead of using the
+             * one in tmpList because it might contain extra information,
+             * eg. what stub driver should be used when performing device
+             * assignment */
+            if (!(otherDev = virPCIDeviceListFind(otherList, tmpDev)))
+                continue;
+
+            /* Make a copy of the device so we can add it later */
+            if (virPCIDeviceListAddCopy(addList, otherDev) < 0)
+                goto cleanup;
+        }
+    }
+
+    /* Move all devices we've copied earlier to the list */
+    while (virPCIDeviceListCount(addList) > 0) {
+        virPCIDevicePtr addDev = virPCIDeviceListStealIndex(addList, 0);
+        if (virPCIDeviceListAdd(list, addDev) < 0)
+            goto cleanup;
+    }
+
+    ret = 0;
+
+ cleanup:
+    virObjectUnref(tmpList);
+    virObjectUnref(addList);
+
+    return ret;
+}
+
 /*
  * Pre-condition: inactivePCIHostdevs & activePCIHostdevs
  * are locked
@@ -967,6 +1045,14 @@ virHostdevReAttachPCIDevices(virHostdevManagerPtr hostdev_mgr,
      * from activePCIHostdevs and have been added to inactivePCIHostdevs;
      * pcidevs contains the devices that we can safely process right now */
 
+    /* We need to reset and reattach not only them, but also all inactive
+     * devices that are in the same IOMMU group if VFIO is in use.
+     * virHostdevExpandPCIDeviceListIOMMUGroups() takes care of updating
+     * the device list appropriately */
+    if (virHostdevExpandPCIDeviceListIOMMUGroups(pcidevs,
+                                                 hostdev_mgr->inactivePCIHostdevs) < 0)
+        goto cleanup;
+
     /* Loop 3: perform a PCI Reset on all devices */
     for (i = 0; i < virPCIDeviceListCount(pcidevs); i++) {
         virPCIDevicePtr dev = virPCIDeviceListGet(pcidevs, i);
-- 
2.5.0