[libvirt] [PATCH 5/6] qemu: RDMA migration support
John Ferlan
jferlan at redhat.com
Tue Sep 23 10:27:19 UTC 2014
On 09/17/2014 10:53 AM, Jiri Denemark wrote:
> From: "Michael R. Hines" <mrhines at us.ibm.com>
>
> This patch adds support for RDMA protocol in migration URIs.
>
> USAGE: $ virsh migrate --live --migrateuri rdma://hostname domain qemu+ssh://hostname/system
>
> Since libvirt runs QEMU in a pretty restricted environment, several
> files needs to be added to cgroup_device_acl (in qemu.conf) for QEMU to
> be able to access the host's infiniband hardware. Full documenation of
> the feature can be found on QEMU wiki:
> http://wiki.qemu.org/Features/RDMALiveMigration
>
> Signed-off-by: Michael R. Hines <mrhines at us.ibm.com>
> Signed-off-by: Jiri Denemark <jdenemar at redhat.com>
> ---
>
> Notes:
> The question is whether the IB devices should be added to
> cgroup_device_acl by default or not...
>
> Version 3:
> - moved capabilities code into a separate patch
> - got rid of migration URI hacks
> - removed hacks that disabled IPv6 with RDMA
> - moved refactoring into a dedicated patch
> - documented IB devices which need to be added to cgroup acl in
> qemu.conf
> - forbid RDMA migrations unless memory hard limit is set until we
> have a better plan for setting limits for mlock
> - set QEMU's RLIMIT_MEMLOCK to memory hard_limit before starting
> RDMA migration
> - check if RDMA migration is supported by source QEMU before trying
> to migrate
>
> src/qemu/qemu.conf | 8 ++++++++
> src/qemu/qemu_command.c | 8 ++++++++
> src/qemu/qemu_migration.c | 39 +++++++++++++++++++++++++++++++++++++--
> 3 files changed, 53 insertions(+), 2 deletions(-)
>
My morning Coverity run found a memory leak...
> diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf
> index 79bba36..92ca715 100644
> --- a/src/qemu/qemu.conf
> +++ b/src/qemu/qemu.conf
> @@ -274,6 +274,14 @@
> # "/dev/ptmx", "/dev/kvm", "/dev/kqemu",
> # "/dev/rtc","/dev/hpet", "/dev/vfio/vfio"
> #]
> +#
> +# RDMA migration requires the following extra files to be added to the list:
> +# "/dev/infiniband/rdma_cm",
> +# "/dev/infiniband/issm0",
> +# "/dev/infiniband/issm1",
> +# "/dev/infiniband/umad0",
> +# "/dev/infiniband/umad1",
> +# "/dev/infiniband/uverbs0"
>
>
> # The default format for Qemu/KVM guest save images is raw; that is, the
> diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
> index a892d99..fceed62 100644
> --- a/src/qemu/qemu_command.c
> +++ b/src/qemu/qemu_command.c
> @@ -9399,6 +9399,14 @@ qemuBuildCommandLine(virConnectPtr conn,
> goto error;
> }
> virCommandAddArg(cmd, migrateFrom);
> + } else if (STRPREFIX(migrateFrom, "rdma")) {
> + if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {
> + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
> + _("incoming RDMA migration is not supported "
> + "with this QEMU binary"));
> + goto error;
> + }
> + virCommandAddArg(cmd, migrateFrom);
> } else if (STREQ(migrateFrom, "stdio")) {
> if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD)) {
> virCommandAddArgFormat(cmd, "fd:%d", migrateFd);
> diff --git a/src/qemu/qemu_migration.c b/src/qemu/qemu_migration.c
> index d0e2653..b59e94d 100644
> --- a/src/qemu/qemu_migration.c
> +++ b/src/qemu/qemu_migration.c
> @@ -56,6 +56,7 @@
> #include "virhook.h"
> #include "virstring.h"
> #include "virtypedparam.h"
> +#include "virprocess.h"
>
> #define VIR_FROM_THIS VIR_FROM_QEMU
>
> @@ -2653,6 +2654,13 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
> QEMU_MIGRATION_COOKIE_NBD)))
> goto cleanup;
>
> + if (STREQ(protocol, "rdma") && !vm->def->mem.hard_limit) {
> + virReportError(VIR_ERR_OPERATION_INVALID, "%s",
> + _("cannot start RDMA migration with no memory hard "
> + "limit set"));
> + goto cleanup;
> + }
> +
> if (qemuMigrationJobStart(driver, vm, QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
> goto cleanup;
> qemuMigrationJobSetPhase(driver, vm, QEMU_MIGRATION_PHASE_PREPARE);
> @@ -2696,6 +2704,11 @@ qemuMigrationPrepareAny(virQEMUDriverPtr driver,
> QEMU_ASYNC_JOB_MIGRATION_IN) < 0)
> goto stop;
>
> + if (STREQ(protocol, "rdma") &&
> + virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {
> + goto stop;
> + }
> +
> if (mig->lockState) {
> VIR_DEBUG("Received lockstate %s", mig->lockState);
> VIR_FREE(priv->lockState);
> @@ -2926,7 +2939,8 @@ qemuMigrationPrepareDirect(virQEMUDriverPtr driver,
> if (!(uri = qemuMigrationParseURI(uri_in, &well_formed_uri)))
> goto cleanup;
>
> - if (STRNEQ(uri->scheme, "tcp")) {
> + if (STRNEQ(uri->scheme, "tcp") &&
> + STRNEQ(uri->scheme, "rdma")) {
> virReportError(VIR_ERR_ARGUMENT_UNSUPPORTED,
> _("unsupported scheme %s in migration URI %s"),
> uri->scheme, uri_in);
> @@ -3545,6 +3559,11 @@ qemuMigrationRun(virQEMUDriverPtr driver,
>
> switch (spec->destType) {
> case MIGRATION_DEST_HOST:
> + if (STREQ(spec->dest.host.protocol, "rdma") &&
> + virProcessSetMaxMemLock(vm->pid, vm->def->mem.hard_limit << 10) < 0) {
> + qemuDomainObjExitMonitor(driver, vm);
> + goto cleanup;
> + }
> ret = qemuMonitorMigrateToHost(priv->mon, migrate_flags,
> spec->dest.host.protocol,
> spec->dest.host.name,
> @@ -3717,7 +3736,23 @@ static int doNativeMigrate(virQEMUDriverPtr driver,
> if (!(uribits = qemuMigrationParseURI(uri, NULL)))
> return -1;
>
> - if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD))
> + if (STREQ(uribits->scheme, "rdma")) {
> + if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_RDMA)) {
> + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
> + _("outgoing RDMA migration is not supported "
> + "with this QEMU binary"));
Need to virURIFree(uribits); or add a cleanup label below and jump there
> + return -1;
> + }
> + if (!vm->def->mem.hard_limit) {
> + virReportError(VIR_ERR_OPERATION_INVALID, "%s",
> + _("cannot start RDMA migration with no memory hard "
> + "limit set"));
> + return -1;
Same here
> + }
> + }
> +
> + if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_MIGRATE_QEMU_FD) &&
> + STRNEQ(uribits->scheme, "rdma"))
> spec.destType = MIGRATION_DEST_CONNECT_HOST;
> else
> spec.destType = MIGRATION_DEST_HOST;
>
More information about the libvir-list
mailing list