[libvirt] [PATCH v2 9/9] Wait for iommmu device to go away before reprobing the host driver

[Shivaprasad G Bhat]

There could be a delay of 1 or 2 seconds before the vfio-pci driver
is unbound and the device file /dev/vfio/<iommu> is actually
removed. If the file exists, the host driver probing the device
can lead to crash. So, wait and avoid the crash.

Signed-off-by: Shivaprasad G Bhat <sbhat at linux.vnet.ibm.com>
---
 src/util/virpci.c  |   42 ++++++++++++++++++++++++++++++++++++++++++
 tests/virpcimock.c |   14 ++++++++++++--
 2 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/src/util/virpci.c b/src/util/virpci.c
index 0bb465b..68fd54c 100644
--- a/src/util/virpci.c
+++ b/src/util/virpci.c
@@ -1098,6 +1098,43 @@ virPCIIsAKnownStub(char *driver)
     return ret;
 }
 
+#define VFIO_UNBIND_TIMEOUT 10
+
+/* It is not safe to initiate host driver probe if the vfio driver has not
+ * completely unbound the device. Usual wait time is 1 to 2 seconds.
+ * So, return if the unbind didn't complete in 10 seconds.
+ */
+static int
+virPCIWaitForVFIOUnbindCompletion(virPCIDevicePtr dev)
+{
+    int retry = 0;
+    int ret = -1;
+    char *path = NULL;
+
+    if (!(path = virPCIDeviceGetIOMMUGroupDev(dev)))
+        goto cleanup;
+
+    while (retry++ < VFIO_UNBIND_TIMEOUT) {
+        if (!virFileExists(path))
+            break;
+         sleep(1);
+    }
+
+    if (virFileExists(path)) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       _("VFIO unbind not completed even after %d seconds"
+                         " for device %s"), retry, dev->name);
+        goto cleanup;
+    }
+
+    ret = 0;
+cleanup :
+    VIR_FREE(path);
+    return ret;
+
+}
+
+
 static int virPCIDeviceReprobeHostDriver(virPCIDevicePtr dev, char *driver, char *drvdir)
 {
     char *path = NULL;
@@ -1119,6 +1156,7 @@ static int virPCIDeviceReprobeHostDriver(virPCIDevicePtr dev, char *driver, char
             goto cleanup;
         }
     }
+
     result = 0;
  cleanup:
     VIR_FREE(path);
@@ -1266,6 +1304,10 @@ virPCIDeviceUnbindFromStub(virPCIDevicePtr dev,
         /* This device is the last to unbind from vfio. As we explicitly
          * add a missing device in the list to inactiveList, we will just
          * go through the list. */
+
+        if (virPCIWaitForVFIOUnbindCompletion(dev) < 0)
+            goto cleanup;
+
         while (inactiveDevs && (i < virPCIDeviceListCount(inactiveDevs))) {
             virPCIDevicePtr pcidev = virPCIDeviceListGet(inactiveDevs, i);
             if (dev->iommuGroup == pcidev->iommuGroup) {
diff --git a/tests/virpcimock.c b/tests/virpcimock.c
index 926a548..9ee46d9 100644
--- a/tests/virpcimock.c
+++ b/tests/virpcimock.c
@@ -52,6 +52,7 @@ static DIR * (*realopendir)(const char *name);
 char *fakesysfsdir;
 
 # define PCI_SYSFS_PREFIX "/sys/bus/pci/"
+# define ROOT_DEV_PREFIX "/dev/"
 
 # define STDERR(...)                                                    \
     fprintf(stderr, "%s %zu: ", __FUNCTION__, (size_t) __LINE__);       \
@@ -248,6 +249,13 @@ getrealpath(char **newpath,
             errno = ENOMEM;
             return -1;
         }
+    } else if (STRPREFIX(path, ROOT_DEV_PREFIX)) {
+        if (virAsprintfQuiet(newpath, "%s/%s",
+                             fakesysfsdir,
+                             path) < 0) {
+            errno = ENOMEM;
+            return -1;
+        }
     } else {
         if (VIR_STRDUP_QUIET(*newpath, path) < 0)
             return -1;
@@ -995,7 +1003,8 @@ access(const char *path, int mode)
 
     init_syms();
 
-    if (STRPREFIX(path, PCI_SYSFS_PREFIX)) {
+    if (STRPREFIX(path, PCI_SYSFS_PREFIX) ||
+        STRPREFIX(path, ROOT_DEV_PREFIX)) {
         char *newpath;
         if (getrealpath(&newpath, path) < 0)
             return -1;
@@ -1014,7 +1023,8 @@ __lxstat(int ver, const char *path, struct stat *sb)
 
     init_syms();
 
-    if (STRPREFIX(path, PCI_SYSFS_PREFIX)) {
+    if (STRPREFIX(path, PCI_SYSFS_PREFIX) ||
+        STRPREFIX(path, ROOT_DEV_PREFIX)) {
         char *newpath;
         if (getrealpath(&newpath, path) < 0)
             return -1;