[libvirt] [PATCH v5 2/5] vz: add migration backbone code
Nikolay Shirokovskiy
nshirokovskiy at parallels.com
Fri Sep 11 09:05:54 UTC 2015
> As mentioned in the previous review, I really want to see VZ
> provide the full set of drive callbacks required by the V3
> migration protocol, not just those you happen to need to have
> today. ie add Begin, Finish & Confirm. This in turn makes it
> quite easy to support non-P2P mode which is desirable because
> in P2P mode there is no practical way todo interactive auth
> for the destination connection, but in non-P2P mode the client
> app opens the destination connection, so can do auth if needed.
Ok, I just wanted this to be a matter of a different patchset.
>
> I believe that the following changes to this patch should do
> all that is required. NB, I have only compile tested this,
> not actually run it, since I don't have an active VZ install,
> only the SDK install.
For me the patch you suggested is overkill in part of p2p.
vzDomainMigratePerform3P2PParams follows virDomainMigrateVersion3Full too
close, while vz p2p migration is much simplier that qemu or some abstract case.
1. Confirm phase is noop and this even won't change in future as vz migration
is practically unmanaged direct one. So why call it in p2p at all?
2. Finish phase function is to return migrated domain only(in vz case). Thus it
have meaning only for direct managed case due its API definition.
3. Calling begin phase in p2p is excessive too. We don't use generated dxml, so
this phase is only a stub for direct managed case.
Even if we support changing domain xml in future it is more practical
to use specific functions and not general API which cases a lot
of preparation work and has rather uncomfortable way of passing
parameters.
I'd better just add a small patch to existent vzDomainMigratePerform3Params which
branches on how to get miguri and session_uuid.
As to supporting all API versions. I resent a second version of the patch
that should cover the case of toURI API so that implementing only
version3 params protocol functions would be enough. But the patch
does not cover the case of migration{N} API.
So I can issue another patch of this sort or go the helpers way as you
suggested to Joao Martins. Please take a look at mentioned patch.
>
> This should work functionally identically with & without the
> PEER2PEER flag being set by the caller.
>
>
> diff --git a/src/vz/vz_driver.c b/src/vz/vz_driver.c
> index 36c64e9..b160a3b 100644
> --- a/src/vz/vz_driver.c
> +++ b/src/vz/vz_driver.c
> @@ -1398,6 +1398,7 @@ vzParseCookie(const char *xml, unsigned char *session_uuid)
>
> #define VZ_MIGRATION_PARAMETERS \
> VIR_MIGRATE_PARAM_URI, VIR_TYPED_PARAM_STRING, \
> + VIR_MIGRATE_PARAM_DEST_XML, VIR_TYPED_PARAM_STRING, \
> NULL
>
> static char*
> @@ -1425,49 +1426,6 @@ vzCreateMigrationURI(void)
> }
>
> static int
> -vzDomainMigratePrepare3Params(virConnectPtr conn,
> - virTypedParameterPtr params,
> - int nparams,
> - const char *cookiein ATTRIBUTE_UNUSED,
> - int cookieinlen ATTRIBUTE_UNUSED,
> - char **cookieout,
> - int *cookieoutlen,
> - char **uri_out,
> - unsigned int flags)
> -{
> - vzConnPtr privconn = conn->privateData;
> - const char *miguri = NULL;
> - int ret = -1;
> -
> - virCheckFlags(VZ_MIGRATION_FLAGS, -1);
> -
> - if (virTypedParamsValidate(params, nparams, VZ_MIGRATION_PARAMETERS) < 0)
> - goto cleanup;
> -
> - if (virTypedParamsGetString(params, nparams,
> - VIR_MIGRATE_PARAM_URI, &miguri) < 0)
> - goto cleanup;
> -
> - /* This check is of par with direct managed imlementation */
> - if (miguri == NULL)
> - *uri_out = vzCreateMigrationURI();
> -
> - if (!(*cookieout = vzFormatCookie(privconn->session_uuid)))
> - goto cleanup;
> - *cookieoutlen = strlen(*cookieout) + 1;
> - ret = 0;
> -
> - cleanup:
> - if (ret != 0) {
> - VIR_FREE(*cookieout);
> - *cookieoutlen = 0;
> - VIR_FREE(*uri_out);
> - }
> -
> - return ret;
> -}
> -
> -static int
> vzConnectSupportsFeature(virConnectPtr conn ATTRIBUTE_UNUSED, int feature)
> {
> switch (feature) {
> @@ -1479,7 +1437,7 @@ vzConnectSupportsFeature(virConnectPtr conn ATTRIBUTE_UNUSED, int feature)
> }
> }
>
> -virURIPtr
> +static virURIPtr
> vzParseVzURI(const char *uri_str)
> {
> virURIPtr uri = NULL;
> @@ -1521,64 +1479,128 @@ vzParseVzURI(const char *uri_str)
> return uri;
> }
>
> +static char *
> +vzDomainMigrateBegin3Params(virDomainPtr domain,
> + virTypedParameterPtr params,
> + int nparams,
> + char **cookieout ATTRIBUTE_UNUSED,
> + int *cookieoutlen ATTRIBUTE_UNUSED,
> + unsigned int flags ATTRIBUTE_UNUSED)
> +{
> + virDomainObjPtr privdom;
> + char *ret = NULL;
> +
> + if (!(privdom = vzDomObjFromDomain(domain)))
> + goto cleanup;
> +
> + if (virTypedParamsValidate(params, nparams, VZ_MIGRATION_PARAMETERS) < 0)
> + goto cleanup;
> +
> + if (virTypedParamsGet(params, nparams, VIR_MIGRATE_PARAM_DEST_XML)) {
> + virReportError(VIR_ERR_OPERATION_UNSUPPORTED, "%s",
> + _("Changing destination XML is not supported"));
> + goto cleanup;
> + }
> +
> + /* We don't really need the XML, but the Begin API requires it
> + * be returned anyway
> + */
> + ret = virDomainDefFormat(privdom->def, VIR_DOMAIN_XML_MIGRATABLE);
> +
> + cleanup:
> + if (privdom)
> + virObjectUnlock(privdom);
> + return ret;
> +
> +}
> +
> static int
> -vzDomainMigratePerform3Params(virDomainPtr domain,
> - const char *dconnuri,
> +vzDomainMigratePrepare3Params(virConnectPtr conn,
> virTypedParameterPtr params,
> int nparams,
> const char *cookiein ATTRIBUTE_UNUSED,
> int cookieinlen ATTRIBUTE_UNUSED,
> - char **cookieout ATTRIBUTE_UNUSED,
> - int *cookieoutlen ATTRIBUTE_UNUSED,
> + char **cookieout,
> + int *cookieoutlen,
> + char **uri_out,
> unsigned int flags)
> {
> - int ret = -1;
> - virDomainObjPtr dom = NULL;
> - virConnectPtr dconn = NULL;
> - virURIPtr vzuri = NULL;
> - unsigned char session_uuid[VIR_UUID_BUFLEN];
> - vzConnPtr privconn = domain->conn->privateData;
> - char *cookie = NULL;
> - int cookielen = 0;
> + vzConnPtr privconn = conn->privateData;
> const char *miguri = NULL;
> + int ret = -1;
>
> virCheckFlags(VZ_MIGRATION_FLAGS, -1);
>
> if (virTypedParamsValidate(params, nparams, VZ_MIGRATION_PARAMETERS) < 0)
> goto cleanup;
>
> - if (!(dom = vzDomObjFromDomain(domain)))
> + if (virTypedParamsGetString(params, nparams,
> + VIR_MIGRATE_PARAM_URI, &miguri) < 0)
> goto cleanup;
>
> - dconn = virConnectOpen(dconnuri);
> - if (dconn == NULL) {
> - virReportError(VIR_ERR_OPERATION_FAILED,
> - _("Failed to connect to remote libvirt URI %s: %s"),
> - dconnuri, virGetLastErrorMessage());
> + /* This check is of par with direct managed imlementation */
> + if (miguri == NULL)
> + *uri_out = vzCreateMigrationURI();
> +
> + if (!(*cookieout = vzFormatCookie(privconn->session_uuid)))
> goto cleanup;
> + *cookieoutlen = strlen(*cookieout) + 1;
> + ret = 0;
> +
> + cleanup:
> + if (ret != 0) {
> + VIR_FREE(*cookieout);
> + *cookieoutlen = 0;
> + VIR_FREE(*uri_out);
> }
>
> - if (virDomainMigratePrepare3Params(dconn, params, nparams, NULL, 0,
> - &cookie, &cookielen,
> - (char**)&miguri, flags) < 0)
> + return ret;
> +}
> +
> +
> +static int
> +vzDomainMigratePerform3DirectParams(virDomainPtr domain,
> + virTypedParameterPtr params,
> + int nparams,
> + const char *cookiein,
> + int cookieinlen ATTRIBUTE_UNUSED,
> + char **cookieout ATTRIBUTE_UNUSED,
> + int *cookieoutlen ATTRIBUTE_UNUSED,
> + unsigned int flags)
> +{
> + int ret = -1;
> + virDomainObjPtr dom = NULL;
> + virConnectPtr dconn = NULL;
> + virURIPtr vzuri = NULL;
> + unsigned char session_uuid[VIR_UUID_BUFLEN];
> + vzConnPtr privconn = domain->conn->privateData;
> + char *defmiguri = NULL;
> + const char *miguri = NULL;
> +
> + virCheckFlags(VZ_MIGRATION_FLAGS, -1);
> +
> + if (virTypedParamsValidate(params, nparams, VZ_MIGRATION_PARAMETERS) < 0)
> goto cleanup;
>
> - if (vzParseCookie(cookie, session_uuid) < 0)
> + if (!(dom = vzDomObjFromDomain(domain)))
> goto cleanup;
>
> - if (miguri == NULL &&
> - virTypedParamsGetString(params, nparams,
> + if (virTypedParamsGetString(params, nparams,
> VIR_MIGRATE_PARAM_URI, &miguri) < 0)
> - goto cleanup;
> + goto cleanup;
>
> if (miguri == NULL) {
> virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> _("domainMigratePrepare3Params did not set miguri"));
> + goto cleanup;
> }
>
> if (!(vzuri = vzParseVzURI(miguri)))
> goto cleanup;
>
> + if (vzParseCookie(cookiein, session_uuid) < 0)
> + goto cleanup;
> +
> if (prlsdkMigrate(dom, vzuri, session_uuid) < 0)
> goto cleanup;
>
> @@ -1592,11 +1614,336 @@ vzDomainMigratePerform3Params(virDomainPtr domain,
> virObjectUnlock(dom);
> virObjectUnref(dconn);
> virURIFree(vzuri);
> - VIR_FREE(cookie);
> + VIR_FREE(defmiguri);
> +
> + return ret;
> +}
> +
> +static int
> +vzDomainMigratePerform3P2PParams(virDomainPtr domain,
> + const char *dconnuri,
> + virTypedParameterPtr params,
> + int nparams,
> + unsigned int flags)
> +{
> + virDomainPtr ddomain = NULL;
> + char *uri_out = NULL;
> + char *cookiein = NULL;
> + char *cookieout = NULL;
> + char *dom_xml = NULL;
> + int cookieinlen = 0;
> + int cookieoutlen = 0;
> + int ret;
> + virErrorPtr orig_err = NULL;
> + int cancelled = 1;
> + bool notify_source = true;
> + unsigned int destflags;
> + int state;
> + virTypedParameterPtr tmp;
> + const char *uri = NULL;
> + virConnectPtr dconn = NULL;
> +
> + VIR_DOMAIN_DEBUG(domain,
> + "params=%p, nparams=%d, flags=%x",
> + params, nparams, flags);
> + VIR_TYPED_PARAMS_DEBUG(params, nparams);
> +
> + if (virTypedParamsCopy(&tmp, params, nparams) < 0)
> + return NULL;
> + params = tmp;
> +
> + if ((dconn = virConnectOpen(dconnuri)) == NULL)
> + goto done;
> +
> + VIR_DEBUG("Begin3 %p", domain->conn);
> + dom_xml = domain->conn->driver->domainMigrateBegin3Params
> + (domain, params, nparams, &cookieout, &cookieoutlen,
> + flags);
> + if (!dom_xml)
> + goto done;
> +
> + ret = virDomainGetState(domain, &state, NULL, 0);
> + if (ret == 0 && state == VIR_DOMAIN_PAUSED)
> + flags |= VIR_MIGRATE_PAUSED;
> +
> + destflags = flags & ~(VIR_MIGRATE_ABORT_ON_ERROR |
> + VIR_MIGRATE_AUTO_CONVERGE);
> +
> + VIR_DEBUG("Prepare3 %p flags=%x", dconn, destflags);
> + cookiein = cookieout;
> + cookieinlen = cookieoutlen;
> + cookieout = NULL;
> + cookieoutlen = 0;
> + if (virTypedParamsReplaceString(¶ms, &nparams,
> + VIR_MIGRATE_PARAM_DEST_XML,
> + dom_xml) < 0)
> + goto done;
> + ret = dconn->driver->domainMigratePrepare3Params
> + (dconn, params, nparams, cookiein, cookieinlen,
> + &cookieout, &cookieoutlen, &uri_out, destflags);
> + if (ret == -1) {
> + goto done;
> + }
> +
> + /* Did domainMigratePrepare3 change URI? */
> + if (uri_out) {
> + uri = uri_out;
> + if (virTypedParamsReplaceString(¶ms, &nparams,
> + VIR_MIGRATE_PARAM_URI,
> + uri_out) < 0) {
> + cancelled = 1;
> + orig_err = virSaveLastError();
> + goto finish;
> + }
> + } else if (virTypedParamsGetString(params, nparams,
> + VIR_MIGRATE_PARAM_URI, &uri) <= 0) {
> + virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> + _("domainMigratePrepare3 did not set uri"));
> + cancelled = 1;
> + orig_err = virSaveLastError();
> + goto finish;
> + }
> +
> + if (flags & VIR_MIGRATE_OFFLINE) {
> + VIR_DEBUG("Offline migration, skipping Perform phase");
> + VIR_FREE(cookieout);
> + cookieoutlen = 0;
> + cancelled = 0;
> + goto finish;
> + }
> +
> + /* Perform the migration. The driver isn't supposed to return
> + * until the migration is complete. The src VM should remain
> + * running, but in paused state until the destination can
> + * confirm migration completion.
> + */
> + VIR_DEBUG("Perform3 %p uri=%s", domain->conn, uri);
> + VIR_FREE(cookiein);
> + cookiein = cookieout;
> + cookieinlen = cookieoutlen;
> + cookieout = NULL;
> + cookieoutlen = 0;
> + /* dconnuri not relevant in non-P2P modes, so left NULL here */
> + ret = domain->conn->driver->domainMigratePerform3Params
> + (domain, NULL, params, nparams, cookiein, cookieinlen,
> + &cookieout, &cookieoutlen, flags & ~(VIR_MIGRATE_PEER2PEER));
> +
> + /* Perform failed. Make sure Finish doesn't overwrite the error */
> + if (ret < 0) {
> + orig_err = virSaveLastError();
> + /* Perform failed so we don't need to call confirm to let source know
> + * about the failure.
> + */
> + notify_source = false;
> + }
> +
> + /* If Perform returns < 0, then we need to cancel the VM
> + * startup on the destination
> + */
> + cancelled = ret < 0 ? 1 : 0;
> +
> + finish:
> + /*
> + * The status code from the source is passed to the destination.
> + * The dest can cleanup if the source indicated it failed to
> + * send all migration data. Returns NULL for ddomain if
> + * the dest was unable to complete migration.
> + */
> + VIR_DEBUG("Finish3 %p ret=%d", dconn, ret);
> + VIR_FREE(cookiein);
> + cookiein = cookieout;
> + cookieinlen = cookieoutlen;
> + cookieout = NULL;
> + cookieoutlen = 0;
> + if (virTypedParamsGetString(params, nparams,
> + VIR_MIGRATE_PARAM_DEST_NAME, NULL) <= 0 &&
> + virTypedParamsReplaceString(¶ms, &nparams,
> + VIR_MIGRATE_PARAM_DEST_NAME,
> + domain->name) < 0) {
> + ddomain = NULL;
> + } else {
> + ddomain = dconn->driver->domainMigrateFinish3Params
> + (dconn, params, nparams, cookiein, cookieinlen,
> + &cookieout, &cookieoutlen, destflags, cancelled);
> + }
> +
> + if (cancelled) {
> + if (ddomain) {
> + VIR_ERROR(_("finish step ignored that migration was cancelled"));
> + } else {
> + /* If Finish reported a useful error, use it instead of the
> + * original "migration unexpectedly failed" error.
> + *
> + * This is ugly but we can't do better with the APIs we have. We
> + * only replace the error if Finish was called with cancelled == 1
> + * and reported a real error (old libvirt would report an error
> + * from RPC instead of MIGRATE_FINISH_OK), which only happens when
> + * the domain died on destination. To further reduce a possibility
> + * of false positives we also check that Perform returned
> + * VIR_ERR_OPERATION_FAILED.
> + */
> + if (orig_err &&
> + orig_err->domain == VIR_FROM_QEMU &&
> + orig_err->code == VIR_ERR_OPERATION_FAILED) {
> + virErrorPtr err = virGetLastError();
> + if (err &&
> + err->domain == VIR_FROM_QEMU &&
> + err->code != VIR_ERR_MIGRATE_FINISH_OK) {
> + virFreeError(orig_err);
> + orig_err = NULL;
> + }
> + }
> + }
> + }
> +
> + /* If ddomain is NULL, then we were unable to start
> + * the guest on the target, and must restart on the
> + * source. There is a small chance that the ddomain
> + * is NULL due to an RPC failure, in which case
> + * ddomain could in fact be running on the dest.
> + * The lock manager plugins should take care of
> + * safety in this scenario.
> + */
> + cancelled = ddomain == NULL ? 1 : 0;
> +
> + /* If finish3 set an error, and we don't have an earlier
> + * one we need to preserve it in case confirm3 overwrites
> + */
> + if (!orig_err)
> + orig_err = virSaveLastError();
> +
> + /*
> + * If cancelled, then src VM will be restarted, else it will be killed.
> + * Don't do this if migration failed on source and thus it was already
> + * cancelled there.
> + */
> + if (notify_source) {
> + VIR_DEBUG("Confirm3 %p ret=%d domain=%p", domain->conn, ret, domain);
> + VIR_FREE(cookiein);
> + cookiein = cookieout;
> + cookieinlen = cookieoutlen;
> + cookieout = NULL;
> + cookieoutlen = 0;
> + ret = domain->conn->driver->domainMigrateConfirm3Params
> + (domain, params, nparams, cookiein, cookieinlen,
> + flags, cancelled);
> + /* If Confirm3 returns -1, there's nothing more we can
> + * do, but fortunately worst case is that there is a
> + * domain left in 'paused' state on source.
> + */
> + if (ret < 0) {
> + VIR_WARN("Guest %s probably left in 'paused' state on source",
> + domain->name);
> + }
> + }
> +
> + done:
> + if (orig_err) {
> + virSetError(orig_err);
> + virFreeError(orig_err);
> + }
> + VIR_FREE(dom_xml);
> + VIR_FREE(uri_out);
> + VIR_FREE(cookiein);
> + VIR_FREE(cookieout);
> + virTypedParamsFree(params, nparams);
> + virObjectUnref(dconn);
> + ret = ddomain ? 0 : -1;
> + virObjectUnref(ddomain);
> + return ret;
> +}
>
> +
> +static int
> +vzDomainMigratePerform3Params(virDomainPtr domain,
> + const char *dconnuri,
> + virTypedParameterPtr params,
> + int nparams,
> + const char *cookiein,
> + int cookieinlen,
> + char **cookieout,
> + int *cookieoutlen,
> + unsigned int flags)
> +{
> + virCheckFlags(VZ_MIGRATION_FLAGS, -1);
> +
> + if (virTypedParamsValidate(params, nparams, VZ_MIGRATION_PARAMETERS) < 0)
> + return -1;
> +
> + if (flags & VIR_MIGRATE_PEER2PEER) {
> + return vzDomainMigratePerform3P2PParams(domain,
> + dconnuri,
> + params,
> + nparams,
> + flags);
> + } else {
> + return vzDomainMigratePerform3DirectParams(domain,
> + params,
> + nparams,
> + cookiein,
> + cookieinlen,
> + cookieout,
> + cookieoutlen,
> + flags);
> + }
> +}
> +
> +
> +static virDomainPtr
> +vzDomainMigrateFinish3Params(virConnectPtr dconn,
> + virTypedParameterPtr params,
> + int nparams,
> + const char *cookiein ATTRIBUTE_UNUSED,
> + int cookieinlen ATTRIBUTE_UNUSED,
> + char **cookieout ATTRIBUTE_UNUSED,
> + int *cookieoutlen ATTRIBUTE_UNUSED,
> + unsigned int flags ATTRIBUTE_UNUSED,
> + int cancelled ATTRIBUTE_UNUSED)
> +{
> + vzConnPtr privconn = dconn->privateData;
> + virDomainPtr ret = NULL;
> + virDomainObjPtr dom;
> + const char *name;
> +
> + if (virTypedParamsGetString(params, nparams,
> + VIR_MIGRATE_PARAM_DEST_NAME,
> + &name) < 0)
> + return NULL;
> +
> + vzDriverLock(privconn);
> + dom = virDomainObjListFindByName(privconn->domains, name);
> + vzDriverUnlock(privconn);
> +
> + if (dom == NULL) {
> + virReportError(VIR_ERR_NO_DOMAIN,
> + _("no domain with matching name '%s'"), name);
> + goto cleanup;
> + }
> +
> + ret = virGetDomain(dconn, dom->def->name, dom->def->uuid);
> + if (ret)
> + ret->id = dom->def->id;
> +
> + cleanup:
> + virDomainObjEndAPI(&dom);
> return ret;
> +
> }
>
> +
> +static int
> +vzDomainMigrateConfirm3Params(virDomainPtr domain ATTRIBUTE_UNUSED,
> + virTypedParameterPtr params ATTRIBUTE_UNUSED,
> + int nparams ATTRIBUTE_UNUSED,
> + const char *cookiein ATTRIBUTE_UNUSED,
> + int cookieinlen ATTRIBUTE_UNUSED,
> + unsigned int flags ATTRIBUTE_UNUSED,
> + int cancelled ATTRIBUTE_UNUSED)
> +{
> + return 0;
> +}
> +
> +
> static virHypervisorDriver vzDriver = {
> .name = "vz",
> .connectOpen = vzConnectOpen, /* 0.10.0 */
> @@ -1651,8 +1998,11 @@ static virHypervisorDriver vzDriver = {
> .domainInterfaceStats = vzDomainInterfaceStats, /* 1.2.17 */
> .domainMemoryStats = vzDomainMemoryStats, /* 1.2.17 */
> .connectSupportsFeature = vzConnectSupportsFeature, /* 1.2.20 */
> + .domainMigrateBegin3Params = vzDomainMigrateBegin3Params, /* 1.2.20 */
> .domainMigratePrepare3Params = vzDomainMigratePrepare3Params, /* 1.2.20 */
> .domainMigratePerform3Params = vzDomainMigratePerform3Params, /* 1.2.20 */
> + .domainMigrateFinish3Params = vzDomainMigrateFinish3Params, /* 1.2.20 */
> + .domainMigrateConfirm3Params = vzDomainMigrateConfirm3Params, /* 1.2.20 */
> };
>
> static virConnectDriver vzConnectDriver = {
>
>
>
> Regards,
> Daniel
>
More information about the libvir-list
mailing list