[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: [libvirt] [PATCH 5/6] qemu: RDMA migration support




On 09/17/2014 10:53 AM, Jiri Denemark wrote:
> From: "Michael R. Hines" <mrhines us ibm com>
> 
> This patch adds support for RDMA protocol in migration URIs.
> 
> USAGE: $ virsh migrate --live --migrateuri rdma://hostname domain qemu+ssh://hostname/system
> 
> Since libvirt runs QEMU in a pretty restricted environment, several
> files needs to be added to cgroup_device_acl (in qemu.conf) for QEMU to
> be able to access the host's infiniband hardware. Full documenation of
> the feature can be found on QEMU wiki:
> http://wiki.qemu.org/Features/RDMALiveMigration
> 
> Signed-off-by: Michael R. Hines <mrhines us ibm com>
> Signed-off-by: Jiri Denemark <jdenemar redhat com>
> ---
> 
> Notes:
>     The question is whether the IB devices should be added to
>     cgroup_device_acl by default or not...
>     
>     Version 3:
>     - moved capabilities code into a separate patch
>     - got rid of migration URI hacks
>     - removed hacks that disabled IPv6 with RDMA
>     - moved refactoring into a dedicated patch
>     - documented IB devices which need to be added to cgroup acl in
>       qemu.conf
>     - forbid RDMA migrations unless memory hard limit is set until we
>       have a better plan for setting limits for mlock
>     - set QEMU's RLIMIT_MEMLOCK to memory hard_limit before starting
>       RDMA migration
>     - check if RDMA migration is supported by source QEMU before trying
>       to migrate
> 
>  src/qemu/qemu.conf        |  8 ++++++++
>  src/qemu/qemu_command.c   |  8 ++++++++
>  src/qemu/qemu_migration.c | 39 +++++++++++++++++++++++++++++++++++++--
>  3 files changed, 53 insertions(+), 2 deletions(-)
> 

My morning Coverity run found a memory leak...

> diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf
> index 79bba36..92ca715 100644
> --- a/src/qemu/qemu.conf
> +++ b/src/qemu/qemu.conf
> @@ -274,6 +274,14 @@
>  #    "/dev/ptmx", "/dev/kvm", "/dev/kqemu",
>  #    "/dev/rtc","/dev/hpet", "/dev/vfio/vfio"
>  #]
> +#
> +# RDMA migration requires the following extra files to be added to the list:
> +#   "/dev/infiniband/rdma_cm",
> +#   "/dev/infiniband/issm0",
> +#   "/dev/infiniband/issm1",
> +#   "/dev/infiniband/umad0",
> +#   "/dev/infiniband/umad1",
> +#   "/dev/infiniband/uverbs0"
>  
>  
>  # The default format for Qemu/KVM guest save images is raw; that is, the
> diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
> index a892d99..fceed62 100644
> --- a/src/qemu/qemu_command.c
> +++ b/src/qemu/qemu_command.c
> @@ -9399,6 +9399,14 @@ qemuBuildCommandLine(virConnectPtr conn,
>                  goto error;
>              }
>              virCommandAddArg(cmd, migrateFrom);
> +        } else if (STRPREFIX(migrateFrom, "rdma")) {
> +            if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {
> +                virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
> +                               _("incoming RDMA migration is not supported "
> +                                 "with this QEMU binary"));
> +                goto error;
> +            }
> +            virCommandAddArg(cmd, migrateFrom);
>          } else if (STREQ(migrateFrom, "stdio")) {
>              if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) {
>                  virCommandAddArgFormat(cmd, "fd:%d", migrateFd);
> diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
> index d0e2653..b59e94d 100644
> --- a/src/qemu/qemu_migration.c
> +++ b/src/qemu/qemu_migration.c
> @@ -56,6 +56,7 @@
>  #include "virhook.h"
>  #include "virstring.h"
>  #include "virtypedparam.h"
> +#include "virprocess.h"
>  
>  #define VIR_FROM_THIS VIR_FROM_QEMU
>  
> @@ -2653,6 +2654,13 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
>                                         QEMU_MIGRATION_COOKIE_NBD)))
>          goto cleanup;
>  
> +    if (STREQ(protocol, "rdma") && !vm->def->mem.hard_limit) {
> +        virReportError(VIR_ERR_OPERATION_INVALID, "%s",
> +                       _("cannot start RDMA migration with no memory hard "
> +                         "limit set"));
> +        goto cleanup;
> +    }
> +
>      if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
>          goto cleanup;
>      qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PREPARE);
> @@ -2696,6 +2704,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
>                                      QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
>          goto stop;
>  
> +    if (STREQ(protocol, "rdma") &&
> +        virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {
> +        goto stop;
> +    }
> +
>      if (mig->lockState) {
>          VIR_DEBUG("Received lockstate %s", mig->lockState);
>          VIR_FREE(priv->lockState);
> @@ -2926,7 +2939,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver,
>          if (!(uri = qemuMigrationParseURI(uri_in, &well_formed_uri)))
>              goto cleanup;
>  
> -        if (STRNEQ(uri->scheme, "tcp")) {
> +        if (STRNEQ(uri->scheme, "tcp") &&
> +            STRNEQ(uri->scheme, "rdma")) {
>              virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED,
>                             _("unsupported scheme %s in migration URI %s"),
>                             uri->scheme, uri_in);
> @@ -3545,6 +3559,11 @@ qemuMigrationRun(virQEMUDriverPtr driver,
>  
>      switch (spec->destType) {
>      case MIGRATION_DEST_HOST:
> +        if (STREQ(spec->dest.host.protocol, "rdma") &&
> +            virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {
> +            qemuDomainObjExitMonitor(driver, vm);
> +            goto cleanup;
> +        }
>          ret = qemuMonitorMigrateToHost(priv->mon, migrate_flags,
>                                         spec->dest.host.protocol,
>                                         spec->dest.host.name,
> @@ -3717,7 +3736,23 @@ static int doNativeMigrate(virQEMUDriverPtr driver,
>      if (!(uribits = qemuMigrationParseURI(uri, NULL)))
>          return -1;
>  
> -    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD))
> +    if (STREQ(uribits->scheme, "rdma")) {
> +        if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {
> +            virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
> +                           _("outgoing RDMA migration is not supported "
> +                             "with this QEMU binary"));

Need to virURIFree(uribits); or add a cleanup label below and jump there

> +            return -1;
> +        }
> +        if (!vm->def->mem.hard_limit) {
> +            virReportError(VIR_ERR_OPERATION_INVALID, "%s",
> +                           _("cannot start RDMA migration with no memory hard "
> +                             "limit set"));
> +            return -1;

Same here

> +        }
> +    }
> +
> +    if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) &&
> +        STRNEQ(uribits->scheme, "rdma"))
>          spec.destType = MIGRATION_DEST_CONNECT_HOST;
>      else
>          spec.destType = MIGRATION_DEST_HOST;
> 


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]