[libvirt PATCHv3 10/12] qemu: add code for handling virtiofsd

Masayoshi Mizuma msys.mizuma at gmail.com
Tue Feb 4 03:16:29 UTC 2020


On Thu, Jan 30, 2020 at 06:06:26PM +0100, Ján Tomko wrote:
> Start virtiofsd for each <filesystem> device using it.
> 
> Pre-create the socket for communication with QEMU and pass it
> to virtiofsd.
> 
> Note that virtiofsd needs to run as root.
> 
> https://bugzilla.redhat.com/show_bug.cgi?id=1694166
> 
> Introduced by QEMU commit a43efa34c7d7b628cbf1ec0fe60043e5c91043ea
> 
> Signed-off-by: Ján Tomko <jtomko at redhat.com>
> ---
>  po/POTFILES.in            |   1 +
>  src/qemu/Makefile.inc.am  |   2 +
>  src/qemu/qemu_domain.c    |   5 +-
>  src/qemu/qemu_domain.h    |   2 +-
>  src/qemu/qemu_extdevice.c |  20 ++-
>  src/qemu/qemu_virtiofs.c  | 290 ++++++++++++++++++++++++++++++++++++++
>  src/qemu/qemu_virtiofs.h  |  38 +++++
>  tests/qemuxml2argvtest.c  |  11 ++
>  8 files changed, 366 insertions(+), 3 deletions(-)
>  create mode 100644 src/qemu/qemu_virtiofs.c
>  create mode 100644 src/qemu/qemu_virtiofs.h
> 
> diff --git a/po/POTFILES.in b/po/POTFILES.in
> index c18e21615f..813fb24199 100644
> --- a/po/POTFILES.in
> +++ b/po/POTFILES.in
> @@ -168,6 +168,7 @@
>  @SRCDIR@/src/qemu/qemu_tpm.c
>  @SRCDIR@/src/qemu/qemu_vhost_user.c
>  @SRCDIR@/src/qemu/qemu_vhost_user_gpu.c
> + at SRCDIR@/src/qemu/qemu_virtiofs.c
>  @SRCDIR@/src/remote/remote_daemon.c
>  @SRCDIR@/src/remote/remote_daemon_config.c
>  @SRCDIR@/src/remote/remote_daemon_dispatch.c
> diff --git a/src/qemu/Makefile.inc.am b/src/qemu/Makefile.inc.am
> index d04a87e659..7a205b4da6 100644
> --- a/src/qemu/Makefile.inc.am
> +++ b/src/qemu/Makefile.inc.am
> @@ -67,6 +67,8 @@ QEMU_DRIVER_SOURCES = \
>  	qemu/qemu_vhost_user.h \
>  	qemu/qemu_vhost_user_gpu.c \
>  	qemu/qemu_vhost_user_gpu.h \
> +	qemu/qemu_virtiofs.c \
> +	qemu/qemu_virtiofs.h \
>  	qemu/qemu_checkpoint.c \
>  	qemu/qemu_checkpoint.h \
>  	qemu/qemu_backup.c \
> diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
> index b5d5812ff8..3064c33ca8 100644
> --- a/src/qemu/qemu_domain.c
> +++ b/src/qemu/qemu_domain.c
> @@ -1440,8 +1440,11 @@ qemuDomainFSPrivateNew(void)
>  
>  
>  static void
> -qemuDomainFSPrivateDispose(void *obj G_GNUC_UNUSED)
> +qemuDomainFSPrivateDispose(void *obj)
>  {
> +    qemuDomainFSPrivatePtr priv = obj;
> +
> +    g_free(priv->vhostuser_fs_sock);
>  }

How about adding close the logfile fd here?
That is because virtlogd keeps opening the fd even after the guest is
shutdowned.

  # virsh shutdown guest
  Domain guest is being shutdown
  
  # virsh list
   Id   Name   State
  --------------------
  
  # virsh start guest
  error: Failed to start domain guest
  error: Cannot open log file: '/var/log/libvirt/qemu/guest-fs0-virtiofsd.log': Device or resource busy
  
  # lsof | grep /var/log/libvirt/qemu/guest-fs0-virtiofsd.log
  virtlogd   5133                   root   19w      REG              253,0        0   35189059 /var/log/libvirt/qemu/guest-fs0-virtiofsd.log
  #

Sample patch to close logfile fd:

diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index b3b5cb2054..7d5b7775fa 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -1439,6 +1439,7 @@ qemuDomainFSPrivateDispose(void *obj)
     qemuDomainFSPrivatePtr priv = obj;

     g_free(priv->vhostuser_fs_sock);
+    VIR_FORCE_CLOSE(priv->logfd);
 }

 static virClassPtr qemuDomainVideoPrivateClass;
diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 83150e4e6d..e4b3ac471f 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -569,6 +569,7 @@ struct _qemuDomainFSPrivate {
     virObject parent;

     char *vhostuser_fs_sock;
+    int logfd;
 };


diff --git a/src/qemu/qemu_virtiofs.c b/src/qemu/qemu_virtiofs.c
index 4ea8f23fd5..17532bac20 100644
--- a/src/qemu/qemu_virtiofs.c
+++ b/src/qemu/qemu_virtiofs.c
@@ -217,7 +217,7 @@ qemuVirtioFSStart(virLogManagerPtr logManager,
         goto cleanup;

     rc = virCommandRun(cmd, NULL);
-    logfd = -1;
+    QEMU_DOMAIN_FS_PRIVATE(fs)->logfd = logfd;

     if (rc < 0) {
         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",

--
Thanks,
Masa

>  
>  static virClassPtr qemuDomainVideoPrivateClass;
> diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
> index c581b3a162..83150e4e6d 100644
> --- a/src/qemu/qemu_domain.h
> +++ b/src/qemu/qemu_domain.h
> @@ -568,7 +568,7 @@ typedef qemuDomainFSPrivate *qemuDomainFSPrivatePtr;
>  struct _qemuDomainFSPrivate {
>      virObject parent;
>  
> -    int dummy;
> +    char *vhostuser_fs_sock;
>  };
>  
>  
> diff --git a/src/qemu/qemu_extdevice.c b/src/qemu/qemu_extdevice.c
> index 7f3bb104d9..5103d4921c 100644
> --- a/src/qemu/qemu_extdevice.c
> +++ b/src/qemu/qemu_extdevice.c
> @@ -20,11 +20,13 @@
>  
>  #include <config.h>
>  
> +#include "qemu_command.h"
>  #include "qemu_extdevice.h"
>  #include "qemu_vhost_user_gpu.h"
>  #include "qemu_domain.h"
>  #include "qemu_tpm.h"
>  #include "qemu_slirp.h"
> +#include "qemu_virtiofs.h"
>  
>  #include "viralloc.h"
>  #include "virlog.h"
> @@ -153,7 +155,7 @@ qemuExtDevicesCleanupHost(virQEMUDriverPtr driver,
>  int
>  qemuExtDevicesStart(virQEMUDriverPtr driver,
>                      virDomainObjPtr vm,
> -                    virLogManagerPtr logManager G_GNUC_UNUSED,
> +                    virLogManagerPtr logManager,
>                      bool incomingMigration)
>  {
>      virDomainDefPtr def = vm->def;
> @@ -183,6 +185,15 @@ qemuExtDevicesStart(virQEMUDriverPtr driver,
>              return -1;
>      }
>  
> +    for (i = 0; i < def->nfss; i++) {
> +        virDomainFSDefPtr fs = def->fss[i];
> +
> +        if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS) {
> +            if (qemuVirtioFSStart(logManager, driver, vm, fs) < 0)
> +                return -1;
> +        }
> +    }
> +
>      return 0;
>  }
>  
> @@ -214,6 +225,13 @@ qemuExtDevicesStop(virQEMUDriverPtr driver,
>          if (slirp)
>              qemuSlirpStop(slirp, vm, driver, net, false);
>      }
> +
> +    for (i = 0; i < def->nfss; i++) {
> +        virDomainFSDefPtr fs = def->fss[i];
> +
> +        if (fs->fsdriver == VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS)
> +            qemuVirtioFSStop(driver, vm, fs);
> +    }
>  }
>  
>  
> diff --git a/src/qemu/qemu_virtiofs.c b/src/qemu/qemu_virtiofs.c
> new file mode 100644
> index 0000000000..4aa8eaed2c
> --- /dev/null
> +++ b/src/qemu/qemu_virtiofs.c
> @@ -0,0 +1,290 @@
> +/*
> + * qemu_virtiofs.c: virtiofs support
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library.  If not, see
> + * <http://www.gnu.org/licenses/>.
> + */
> +
> +#include <config.h>
> +
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <fcntl.h>
> +
> +#include "logging/log_manager.h"
> +#include "virlog.h"
> +#include "qemu_command.h"
> +#include "qemu_conf.h"
> +#include "qemu_extdevice.h"
> +#include "qemu_security.h"
> +#include "qemu_virtiofs.h"
> +#include "virpidfile.h"
> +
> +#define VIR_FROM_THIS VIR_FROM_QEMU
> +
> +
> +char *
> +qemuVirtioFSCreatePidFilename(virQEMUDriverConfigPtr cfg,
> +                              const virDomainDef *def,
> +                              const char *alias)
> +{
> +    g_autofree char *shortName = NULL;
> +    g_autofree char *name = NULL;
> +
> +    if (!(shortName = virDomainDefGetShortName(def)))
> +        return NULL;
> +
> +    name = g_strdup_printf("%s-%s-virtiofsd", shortName, alias);
> +
> +    return virPidFileBuildPath(cfg->stateDir, name);
> +}
> +
> +
> +char *
> +qemuVirtioFSCreateSocketFilename(virDomainObjPtr vm,
> +                                 const char *alias)
> +{
> +    qemuDomainObjPrivatePtr priv = vm->privateData;
> +
> +    return virFileBuildPath(priv->libDir, alias, "-virtiofsd.sock");
> +}
> +
> +
> +static char *
> +qemuVirtioFSCreateLogFilename(virQEMUDriverConfigPtr cfg,
> +                              const virDomainDef *def,
> +                              const char *alias)
> +{
> +    g_autofree char *name = NULL;
> +
> +    name = g_strdup_printf("%s-%s", def->name, alias);
> +
> +    return virFileBuildPath(cfg->logDir, name, "-virtiofsd.log");
> +}
> +
> +
> +static int
> +qemuVirtioFSOpenChardev(virQEMUDriverPtr driver,
> +                        virDomainObjPtr vm,
> +                        const char *socket_path)
> +{
> +    virDomainChrSourceDefPtr chrdev = virDomainChrSourceDefNew(NULL);
> +    virDomainChrDef chr = { .source = chrdev };
> +    VIR_AUTOCLOSE fd = -1;
> +    int ret = -1;
> +
> +    chrdev->type = VIR_DOMAIN_CHR_TYPE_UNIX;
> +    chrdev->data.nix.listen = true;
> +    chrdev->data.nix.path = g_strdup(socket_path);
> +
> +    if (qemuSecuritySetDaemonSocketLabel(driver->securityManager, vm->def) < 0)
> +        goto cleanup;
> +    fd = qemuOpenChrChardevUNIXSocket(chrdev);
> +    if (fd < 0) {
> +        ignore_value(qemuSecurityClearSocketLabel(driver->securityManager, vm->def));
> +        goto cleanup;
> +    }
> +    if (qemuSecurityClearSocketLabel(driver->securityManager, vm->def) < 0)
> +        goto cleanup;
> +
> +    if (qemuSecuritySetChardevLabel(driver, vm, &chr) < 0)
> +        goto cleanup;
> +
> +    ret = fd;
> +    fd = -1;
> +
> + cleanup:
> +    virObjectUnref(chrdev);
> +    return ret;
> +}
> +
> +static virCommandPtr
> +qemuVirtioFSBuildCommandLine(virQEMUDriverConfigPtr cfg,
> +                             virDomainFSDefPtr fs,
> +                             int *fd)
> +{
> +    g_autoptr(virCommand) cmd = NULL;
> +    g_auto(virBuffer) opts = VIR_BUFFER_INITIALIZER;
> +
> +    if (!(cmd = virCommandNew(fs->binary)))
> +        return NULL;
> +
> +    virCommandAddArgFormat(cmd, "--fd=%d", *fd);
> +    virCommandPassFD(cmd, *fd, VIR_COMMAND_PASS_FD_CLOSE_PARENT);
> +    *fd = -1;
> +
> +    virCommandAddArg(cmd, "-o");
> +    virBufferAsprintf(&opts, "source=%s,", fs->src->path);
> +    if (fs->cache)
> +        virBufferAsprintf(&opts, "cache=%s,", virDomainFSCacheModeTypeToString(fs->cache));
> +
> +    if (fs->xattr == VIR_TRISTATE_SWITCH_ON)
> +        virBufferAddLit(&opts, "xattr,");
> +    else if (fs->xattr == VIR_TRISTATE_SWITCH_OFF)
> +        virBufferAddLit(&opts, "no_xattr,");
> +
> +    if (fs->flock == VIR_TRISTATE_SWITCH_ON)
> +        virBufferAddLit(&opts, "flock,");
> +    else if (fs->flock == VIR_TRISTATE_SWITCH_OFF)
> +        virBufferAddLit(&opts, "no_flock,");
> +
> +    if (fs->posix_lock == VIR_TRISTATE_SWITCH_ON)
> +        virBufferAddLit(&opts, "posix_lock,");
> +    else if (fs->posix_lock == VIR_TRISTATE_SWITCH_OFF)
> +        virBufferAddLit(&opts, "no_posix_lock,");
> +
> +    virBufferTrim(&opts, ",", -1);
> +
> +    virCommandAddArgBuffer(cmd, &opts);
> +    if (cfg->virtiofsdDebug)
> +        virCommandAddArg(cmd, "-d");
> +
> +    return g_steal_pointer(&cmd);
> +}
> +
> +int
> +qemuVirtioFSStart(virLogManagerPtr logManager,
> +                  virQEMUDriverPtr driver,
> +                  virDomainObjPtr vm,
> +                  virDomainFSDefPtr fs)
> +{
> +    g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
> +    g_autoptr(virCommand) cmd = NULL;
> +    g_autofree char *socket_path = NULL;
> +    g_autofree char *pidfile = NULL;
> +    g_autofree char *logpath = NULL;
> +    pid_t pid = (pid_t) -1;
> +    VIR_AUTOCLOSE fd = -1;
> +    VIR_AUTOCLOSE logfd = -1;
> +    int ret = -1;
> +    int rc;
> +
> +    if (!(pidfile = qemuVirtioFSCreatePidFilename(cfg, vm->def, fs->info.alias)))
> +        goto cleanup;
> +
> +    if (!(socket_path = qemuVirtioFSCreateSocketFilename(vm, fs->info.alias)))
> +        goto cleanup;
> +
> +    if ((fd = qemuVirtioFSOpenChardev(driver, vm, socket_path)) < 0)
> +        goto cleanup;
> +
> +    logpath = qemuVirtioFSCreateLogFilename(cfg, vm->def, fs->info.alias);
> +
> +    if (cfg->stdioLogD) {
> +        if ((logfd = virLogManagerDomainOpenLogFile(logManager,
> +                                                    "qemu",
> +                                                    vm->def->uuid,
> +                                                    vm->def->name,
> +                                                    logpath,
> +                                                    0,
> +                                                    NULL, NULL)) < 0)
> +            goto cleanup;
> +    } else {
> +        if ((logfd = open(logpath, O_WRONLY | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR)) < 0) {
> +            virReportSystemError(errno, _("failed to create logfile %s"),
> +                                 logpath);
> +            goto cleanup;
> +        }
> +        if (virSetCloseExec(logfd) < 0) {
> +            virReportSystemError(errno, _("failed to set close-on-exec flag on %s"),
> +                                 logpath);
> +            goto error;
> +        }
> +    }
> +
> +    if (!(cmd = qemuVirtioFSBuildCommandLine(cfg, fs, &fd)))
> +        goto cleanup;
> +
> +    virCommandSetPidFile(cmd, pidfile);
> +    virCommandSetOutputFD(cmd, &logfd);
> +    virCommandSetErrorFD(cmd, &logfd);
> +    virCommandNonblockingFDs(cmd);
> +    virCommandDaemonize(cmd);
> +
> +    if (qemuExtDeviceLogCommand(driver, vm, cmd, "virtiofsd") < 0)
> +        goto cleanup;
> +
> +    rc = virCommandRun(cmd, NULL);
> +    logfd = -1;
> +
> +    if (rc < 0) {
> +        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> +                       _("Could not start 'virtiofsd'"));
> +        goto error;
> +    }
> +
> +    rc = virPidFileReadPath(pidfile, &pid);
> +    if (rc < 0) {
> +        virReportSystemError(-rc,
> +                             _("Unable to read virtiofsd pidfile '%s'"),
> +                             pidfile);
> +        goto error;
> +    }
> +
> +    if (virProcessKill(pid, 0) != 0) {
> +        virReportSystemError(errno, "%s",
> +                             _("virtiofsd died unexpectedly"));
> +        goto error;
> +    }
> +
> +    QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock = g_steal_pointer(&socket_path);
> +    ret = 0;
> +
> + cleanup:
> +    if (socket_path)
> +        unlink(socket_path);
> +    return ret;
> +
> + error:
> +    if (pid != -1)
> +        virProcessKillPainfully(pid, true);
> +    if (pidfile)
> +        unlink(pidfile);
> +    goto cleanup;
> +}
> +
> +
> +void
> +qemuVirtioFSStop(virQEMUDriverPtr driver,
> +                    virDomainObjPtr vm,
> +                    virDomainFSDefPtr fs)
> +{
> +    g_autoptr(virQEMUDriverConfig) cfg = virQEMUDriverGetConfig(driver);
> +    g_autofree char *pidfile = NULL;
> +    virErrorPtr orig_err;
> +    pid_t pid = -1;
> +    int rc;
> +
> +    virErrorPreserveLast(&orig_err);
> +
> +    if (!(pidfile = qemuVirtioFSCreatePidFilename(cfg, vm->def, fs->info.alias)))
> +        goto cleanup;
> +
> +    rc = virPidFileReadPathIfAlive(pidfile, &pid, NULL);
> +    if (rc >= 0 && pid != (pid_t) -1)
> +        virProcessKillPainfully(pid, true);
> +
> +    if (unlink(pidfile) < 0 &&
> +        errno != ENOENT) {
> +        virReportSystemError(errno,
> +                             _("Unable to remove stale pidfile %s"),
> +                             pidfile);
> +    }
> +
> +    if (QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock)
> +        unlink(QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock);
> +
> + cleanup:
> +    virErrorRestore(&orig_err);
> +}
> diff --git a/src/qemu/qemu_virtiofs.h b/src/qemu/qemu_virtiofs.h
> new file mode 100644
> index 0000000000..49db807b19
> --- /dev/null
> +++ b/src/qemu/qemu_virtiofs.h
> @@ -0,0 +1,38 @@
> +/*
> + * qemu_virtiofs.h: virtiofs support
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library.  If not, see
> + * <http://www.gnu.org/licenses/>.
> + */
> +
> +#pragma once
> +
> +
> +char *
> +qemuVirtioFSCreatePidFilename(virQEMUDriverConfigPtr cfg,
> +                              const virDomainDef *def,
> +                              const char *alias);
> +char *
> +qemuVirtioFSCreateSocketFilename(virDomainObjPtr vm,
> +                                 const char *alias);
> +
> +int
> +qemuVirtioFSStart(virLogManagerPtr logManager,
> +                  virQEMUDriverPtr driver,
> +                  virDomainObjPtr vm,
> +                  virDomainFSDefPtr fs);
> +void
> +qemuVirtioFSStop(virQEMUDriverPtr driver,
> +                 virDomainObjPtr vm,
> +                 virDomainFSDefPtr fs);
> diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c
> index a36183bf34..f268d336a6 100644
> --- a/tests/qemuxml2argvtest.c
> +++ b/tests/qemuxml2argvtest.c
> @@ -496,6 +496,17 @@ testCompareXMLToArgv(const void *data)
>          }
>      }
>  
> +    for (i = 0; i < vm->def->nfss; i++) {
> +        virDomainFSDefPtr fs = vm->def->fss[i];
> +        char *s;
> +
> +        if (fs->fsdriver != VIR_DOMAIN_FS_DRIVER_TYPE_VIRTIOFS)
> +            continue;
> +
> +        s = g_strdup_printf("/tmp/lib/domain--1-guest/fs%zu.vhost-fs.sock", i);
> +        QEMU_DOMAIN_FS_PRIVATE(fs)->vhostuser_fs_sock = s;
> +    }
> +
>      if (vm->def->vsock) {
>          virDomainVsockDefPtr vsock = vm->def->vsock;
>          qemuDomainVsockPrivatePtr vsockPriv =
> -- 
> 2.21.0
> 





More information about the libvir-list mailing list