[PATCH 02/12] migration/migration-pin: pin migration pid by given cpumap
Jiang Jiacheng
jiangjiacheng at huawei.com
Tue Jan 3 13:08:21 UTC 2023
From: zhengchuan <zhengchuan at huawei.com>
Secondly, we start to pin migration pid by given cpumap, add
virDomainPinMigrationThread() for migration pin.
Signed-off-by:zhengchuan<zhengchuan at huawei.com>
---
include/libvirt/libvirt-domain.h | 4 ++
src/conf/domain_conf.c | 9 ++++
src/conf/domain_conf.h | 11 ++++
src/conf/virconftypes.h | 2 +
src/driver-hypervisor.h | 6 +++
src/libvirt-domain.c | 65 +++++++++++++++++++++++
src/libvirt_private.syms | 1 +
src/libvirt_public.syms | 1 +
src/qemu/qemu_driver.c | 91 ++++++++++++++++++++++++++++++++
src/remote/remote_driver.c | 1 +
src/remote/remote_protocol.x | 13 ++++-
src/remote_protocol-structs | 5 ++
src/util/vircgroup.c | 3 ++
src/util/vircgroup.h | 1 +
14 files changed, 212 insertions(+), 1 deletion(-)
diff --git a/include/libvirt/libvirt-domain.h b/include/libvirt/libvirt-domain.h
index e11f2795f1..26af8292d3 100644
--- a/include/libvirt/libvirt-domain.h
+++ b/include/libvirt/libvirt-domain.h
@@ -2424,6 +2424,10 @@ int virDomainPinEmulator (virDomainPtr domain,
int maplen,
unsigned int flags);
+int virDomainPinMigrationThread (virDomainPtr domain,
+ unsigned char *cpumap,
+ int maplen);
+
int virDomainGetEmulatorPinInfo (virDomainPtr domain,
unsigned char *cpumaps,
int maplen,
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index 6c088ff295..a94a4c1c88 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -30299,3 +30299,12 @@ virDomainDefHasSpiceGraphics(const virDomainDef *def)
return false;
}
+
+void
+virDomainMigrationIDDefFree(virDomainMigrationIDDef *def)
+{
+ if (!def)
+ return;
+ virBitmapFree(def->cpumask);
+ VIR_FREE(def);
+}
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index 1404c55053..a8384a4f6e 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -27,6 +27,7 @@
#include <libxml/xpath.h>
#include "internal.h"
+#include "viralloc.h"
#include "virconftypes.h"
#include "capabilities.h"
#include "cpu_conf.h"
@@ -4353,3 +4354,13 @@ virDomainObjGetMessages(virDomainObj *vm,
bool
virDomainDefHasSpiceGraphics(const virDomainDef *def);
+
+struct _virDomainMigrationIDDef {
+ bool autofill;
+ int thread_id;
+ virBitmap *cpumask;
+ virDomainThreadSchedParam sched;
+};
+
+void
+virDomainMigrationIDDefFree(virDomainMigrationIDDef *def);
diff --git a/src/conf/virconftypes.h b/src/conf/virconftypes.h
index 154805091a..984c2181d8 100644
--- a/src/conf/virconftypes.h
+++ b/src/conf/virconftypes.h
@@ -255,3 +255,5 @@ typedef struct _virDomainXMLOption virDomainXMLOption;
typedef struct _virDomainXMLPrivateDataCallbacks virDomainXMLPrivateDataCallbacks;
typedef struct _virDomainXenbusControllerOpts virDomainXenbusControllerOpts;
+
+typedef struct _virDomainMigrationIDDef virDomainMigrationIDDef;
diff --git a/src/driver-hypervisor.h b/src/driver-hypervisor.h
index 618f116012..1d7f606c59 100644
--- a/src/driver-hypervisor.h
+++ b/src/driver-hypervisor.h
@@ -1441,6 +1441,11 @@ typedef int
int seconds,
unsigned int flags);
+typedef int
+(*virDrvDomainPinMigrationThread)(virDomainPtr domain,
+ unsigned char *cpumap,
+ int maplen);
+
typedef char *
(*virDrvDomainMigrateGetMigrationPids)(virDomainPtr domain,
unsigned int flags);
@@ -1716,5 +1721,6 @@ struct _virHypervisorDriver {
virDrvDomainAuthorizedSSHKeysSet domainAuthorizedSSHKeysSet;
virDrvDomainGetMessages domainGetMessages;
virDrvDomainStartDirtyRateCalc domainStartDirtyRateCalc;
+ virDrvDomainPinMigrationThread domainPinMigrationThread;
virDrvDomainMigrateGetMigrationPids domainMigrateGetMigrationPids;
};
diff --git a/src/libvirt-domain.c b/src/libvirt-domain.c
index fabfb2dd7f..d35a8237ae 100644
--- a/src/libvirt-domain.c
+++ b/src/libvirt-domain.c
@@ -14011,3 +14011,68 @@ virDomainMigrateGetMigrationPids(virDomainPtr domain,
virDispatchError(domain->conn);
return NULL;
}
+
+/**
+ * virDomainPinMigrationThread:
+ * @domain: a domain object
+ * @cpumap: pointer to a bit map of real CPUs (in 8-bit bytes) (IN)
+ * Each bit set to 1 means that corresponding CPU is usable.
+ * Bytes are stored in little-endian order: CPU0-7, 8-15...
+ * In each byte, lowest CPU number is least significant bit.
+ * @maplen: number of bytes in cpumap, from 1 up to size of CPU map in
+ * underlying virtualization system (Xen...).
+ * If maplen < size, missing bytes are set to zero.
+ * If maplen > size, failure code is returned.
+ *
+ * Allocate the real CPUs to the migrationThread which will be launched
+ * at the beginning of migration.
+ * This allocation can be handled whether before the migration, or during
+ * the migration which performs as an instant change.
+ *
+ * This function may require privileged access to the hypervisor.
+ *
+ * Returns 0 in case of success, -1 in case of failure.
+ *
+ * Since: 9.1.0
+ */
+int
+virDomainPinMigrationThread(virDomainPtr domain,
+ unsigned char *cpumap,
+ int maplen)
+{
+ virConnectPtr conn;
+ g_autofree char *str = NULL;
+
+ VIR_DOMAIN_DEBUG(domain, "migration: cpumap=%p, maplen=%d",
+ cpumap, maplen);
+
+ virResetLastError();
+
+ virCheckDomainReturn(domain, -1);
+ conn = domain->conn;
+
+ virCheckNonNullArgGoto(cpumap, error);
+ virCheckPositiveArgGoto(maplen, error);
+
+ str = virBitmapDataFormat(cpumap, maplen);
+ VIR_INFO("Begin to PinMigrationThread(domain=%s): new CPU Affinity:%s",
+ NULLSTR(domain->name), NULLSTR(str));
+
+ if (conn->driver->domainPinMigrationThread) {
+ int ret;
+ ret = conn->driver->domainPinMigrationThread(domain, cpumap, maplen);
+ if (ret < 0) {
+ VIR_ERROR(_("Failed to PinMigrationThread(domain=%s), ret=%d"),
+ NULLSTR(domain->name), ret);
+ goto error;
+ }
+ VIR_INFO("Success to PinMigrationThread(domain=%s)", NULLSTR(domain->name));
+ return ret;
+ }
+
+ virReportUnsupportedError();
+
+ error:
+ virDispatchError(domain->conn);
+ return -1;
+}
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index ae746a2d51..ad19d37d95 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -516,6 +516,7 @@ virDomainMemoryModelTypeToString;
virDomainMemoryRemove;
virDomainMemorySourceTypeFromString;
virDomainMemorySourceTypeToString;
+virDomainMigrationIDDefFree;
virDomainMouseModeTypeFromString;
virDomainMouseModeTypeToString;
virDomainNetAllocateActualDevice;
diff --git a/src/libvirt_public.syms b/src/libvirt_public.syms
index f11fe1b26b..61561f2367 100644
--- a/src/libvirt_public.syms
+++ b/src/libvirt_public.syms
@@ -929,6 +929,7 @@ LIBVIRT_8.5.0 {
LIBVIRT_9.1.0 {
global:
+ virDomainPinMigrationThread;
virDomainMigrateGetMigrationPids;
} LIBVIRT_8.5.0;
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index b9dc5f29f5..7ae81f459f 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -20678,6 +20678,96 @@ qemuDomainMigrateGetMigrationPids(virDomainPtr dom,
return ret;
}
+static int
+qemuDomainPinMigrationThread(virDomainPtr dom,
+ unsigned char *cpumap,
+ int maplen)
+{
+ int ret = -1;
+ virQEMUDriver *driver = dom->conn->privateData;
+ virQEMUDriverConfig *cfg = NULL;
+ virCaps *caps = NULL;
+ virDomainObj *vm = NULL;
+ virBitmap *pcpumap = NULL;
+ virCgroup *cgroup_migthread = NULL;
+ qemuDomainObjPrivate *priv = NULL;
+ int migration_id = 0;
+ virDomainMigrationIDDef *migration = NULL;
+
+ cfg = virQEMUDriverGetConfig(driver);
+
+ if (!(caps = virQEMUDriverGetCapabilities(driver, false)))
+ goto cleanup;
+
+ if (!(vm = qemuDomainObjFromDomain(dom)))
+ goto cleanup;
+
+ priv = vm->privateData;
+
+ if (virDomainPinMigrationThreadEnsureACL(dom->conn, vm->def) < 0)
+ goto cleanup;
+
+ if (virDomainObjBeginJob(vm, VIR_JOB_QUERY) < 0)
+ goto cleanup;
+
+ if (!(pcpumap = virBitmapNewData(cpumap, maplen)))
+ goto release;
+
+ if (virBitmapIsAllClear(pcpumap)) {
+ virReportError(VIR_ERR_INVALID_ARG, "%s",
+ _("Empty migration thread cpumap list for pinning"));
+ goto release;
+ }
+
+ virBitmapFree(priv->pcpumap);
+ priv->pcpumap = pcpumap;
+
+ migration = g_new0(virDomainMigrationIDDef, 1);
+
+ if (priv->migrationPids != NULL) {
+ if (virStrToLong_i(priv->migrationPids, NULL, 10, &(migration->thread_id)) != 0) {
+ VIR_ERROR(_("migrationPid trans failure, migrationPids = %s"), priv->migrationPids);
+ goto endjob;
+ }
+
+ if (pcpumap) {
+ if (virCgroupHasController(priv->cgroup,
+ VIR_CGROUP_CONTROLLER_CPUSET)) {
+ if (virCgroupNewThread(priv->cgroup, VIR_CGROUP_THREAD_MIGRATION_THREAD,
+ migration_id, false, &cgroup_migthread) < 0)
+ goto endjob;
+ if (virDomainCgroupSetupCpusetCpus(cgroup_migthread, pcpumap) < 0) {
+ virReportError(VIR_ERR_OPERATION_INVALID,
+ _("failed to set cpuset.cpus in cgroup"
+ " for migration thread %d"), migration_id);
+ goto endjob;
+ }
+ }
+
+ if (virProcessSetAffinity(migration->thread_id, pcpumap, false) < 0)
+ goto endjob;
+ }
+ }
+
+ ret = 0;
+ goto endjob;
+
+ release:
+ virBitmapFree(pcpumap);
+
+ endjob:
+ virDomainObjEndJob(vm);
+
+ cleanup:
+ if (cgroup_migthread)
+ virCgroupFree(cgroup_migthread);
+ virDomainMigrationIDDefFree(migration);
+ virDomainObjEndAPI(&vm);
+ virObjectUnref(caps);
+ virObjectUnref(cfg);
+ return ret;
+}
+
static virHypervisorDriver qemuHypervisorDriver = {
.name = QEMU_DRIVER_NAME,
@@ -20927,6 +21017,7 @@ static virHypervisorDriver qemuHypervisorDriver = {
.domainGetMessages = qemuDomainGetMessages, /* 7.1.0 */
.domainStartDirtyRateCalc = qemuDomainStartDirtyRateCalc, /* 7.2.0 */
.domainSetLaunchSecurityState = qemuDomainSetLaunchSecurityState, /* 8.0.0 */
+ .domainPinMigrationThread = qemuDomainPinMigrationThread, /* 9.1.0 */
.domainMigrateGetMigrationPids = qemuDomainMigrateGetMigrationPids, /* 9.1.0 */
};
diff --git a/src/remote/remote_driver.c b/src/remote/remote_driver.c
index 9d8d7142b7..1c7cac6b29 100644
--- a/src/remote/remote_driver.c
+++ b/src/remote/remote_driver.c
@@ -8651,6 +8651,7 @@ static virHypervisorDriver hypervisor_driver = {
.domainGetMessages = remoteDomainGetMessages, /* 7.1.0 */
.domainStartDirtyRateCalc = remoteDomainStartDirtyRateCalc, /* 7.2.0 */
.domainSetLaunchSecurityState = remoteDomainSetLaunchSecurityState, /* 8.0.0 */
+ .domainPinMigrationThread = remoteDomainPinMigrationThread, /* 9.1.0 */
.domainMigrateGetMigrationPids = remoteDomainMigrateGetMigrationPids, /* 9.1.0 */
};
diff --git a/src/remote/remote_protocol.x b/src/remote/remote_protocol.x
index d1b799db13..f86172b636 100644
--- a/src/remote/remote_protocol.x
+++ b/src/remote/remote_protocol.x
@@ -1291,6 +1291,11 @@ struct remote_domain_pin_iothread_args {
unsigned int flags;
};
+struct remote_domain_pin_migration_thread_args {
+ remote_nonnull_domain dom;
+ opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */
+};
+
struct remote_domain_add_iothread_args {
remote_nonnull_domain dom;
unsigned int iothread_id;
@@ -6976,5 +6981,11 @@ enum remote_procedure {
* @generate: both
* @acl: domain:migrate
*/
- REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443
+ REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443,
+
+ /**
+ * @generate: both
+ * @acl: domain:write
+ */
+ REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444
};
diff --git a/src/remote_protocol-structs b/src/remote_protocol-structs
index da4d799f42..9ef27913f1 100644
--- a/src/remote_protocol-structs
+++ b/src/remote_protocol-structs
@@ -3268,6 +3268,10 @@ struct remote_domain_event_memory_device_size_change_msg {
remote_nonnull_string alias;
uint64_t size;
};
+struct remote_domain_pin_migration_thread_args {
+ remote_nonnull_domain dom;
+ opaque cpumap<REMOTE_CPUMAP_MAX>; /* (unsigned char *) */
+};
struct remote_domain_migrate_get_migration_pids_args {
remote_nonnull_domain dom;
u_int flags;
@@ -3719,4 +3723,5 @@ enum remote_procedure {
REMOTE_PROC_DOMAIN_RESTORE_PARAMS = 441,
REMOTE_PROC_DOMAIN_ABORT_JOB_FLAGS = 442,
REMOTE_PROC_DOMAIN_MIGRATE_GET_MIGRATION_PIDS = 443,
+ REMOTE_PROC_DOMAIN_PIN_MIGRATION_THREAD = 444,
};
diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c
index 73675b4478..d4307c0d72 100644
--- a/src/util/vircgroup.c
+++ b/src/util/vircgroup.c
@@ -1123,6 +1123,9 @@ virCgroupNewThread(virCgroup *domain,
case VIR_CGROUP_THREAD_IOTHREAD:
name = g_strdup_printf("iothread%d", id);
break;
+ case VIR_CGROUP_THREAD_MIGRATION_THREAD:
+ name = g_strdup_printf("migthread%d", id);
+ break;
case VIR_CGROUP_THREAD_LAST:
virReportError(VIR_ERR_INTERNAL_ERROR,
_("unexpected name value %d"), nameval);
diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h
index 690f09465c..3c5c23965f 100644
--- a/src/util/vircgroup.h
+++ b/src/util/vircgroup.h
@@ -53,6 +53,7 @@ typedef enum {
VIR_CGROUP_THREAD_VCPU = 0,
VIR_CGROUP_THREAD_EMULATOR,
VIR_CGROUP_THREAD_IOTHREAD,
+ VIR_CGROUP_THREAD_MIGRATION_THREAD,
VIR_CGROUP_THREAD_LAST
} virCgroupThreadName;
--
2.33.0
More information about the libvir-list
mailing list