[libvirt] [PATCH v3 28/30] qemu: Allow forcing VFIO when computing memlock limit
Michal Privoznik
mprivozn at redhat.com
Mon Dec 2 14:26:51 UTC 2019
With NVMe disks, one can start a blockjob with a NVMe disk
that is not visible in domain XML (at least right away). Usually,
it's fairly easy to override this limitation of
qemuDomainGetMemLockLimitBytes() - for instance for hostdevs we
temporarily add the device to domain def, let the function
calculate the limit and then remove the device. But it's not so
easy with virStorageSourcePtr - in some cases they don't
necessarily are attached to a disk. And even if they are it's
done later in the process and frankly, I find it too complicated
to be able to use the simple trick we use with hostdevs.
Signed-off-by: Michal Privoznik <mprivozn at redhat.com>
---
src/qemu/qemu_command.c | 2 +-
src/qemu/qemu_domain.c | 46 ++++++++++++++++++++++++++---------------
src/qemu/qemu_domain.h | 6 ++++--
src/qemu/qemu_hotplug.c | 12 +++++------
tests/qemumemlocktest.c | 2 +-
5 files changed, 41 insertions(+), 27 deletions(-)
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 97d11c1d5f..7cc6a55c75 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -10463,7 +10463,7 @@ qemuBuildCommandLine(virQEMUDriverPtr driver,
/* In some situations, eg. VFIO passthrough, QEMU might need to lock a
* significant amount of memory, so we need to set the limit accordingly */
- virCommandSetMaxMemLock(cmd, qemuDomainGetMemLockLimitBytes(def));
+ virCommandSetMaxMemLock(cmd, qemuDomainGetMemLockLimitBytes(def, false));
if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_MSG_TIMESTAMP) &&
cfg->logTimestamp)
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
index e81d986e9e..be4aef049d 100644
--- a/src/qemu/qemu_domain.c
+++ b/src/qemu/qemu_domain.c
@@ -11875,12 +11875,14 @@ ppc64VFIODeviceIsNV2Bridge(const char *device)
/**
* getPPC64MemLockLimitBytes:
* @def: domain definition
+ * @forceVFIO: force VFIO usage
*
* A PPC64 helper that calculates the memory locking limit in order for
* the guest to operate properly.
*/
static unsigned long long
-getPPC64MemLockLimitBytes(virDomainDefPtr def)
+getPPC64MemLockLimitBytes(virDomainDefPtr def,
+ bool forceVFIO)
{
unsigned long long memKB = 0;
unsigned long long baseLimit = 0;
@@ -11971,7 +11973,7 @@ getPPC64MemLockLimitBytes(virDomainDefPtr def)
passthroughLimit = maxMemory +
128 * (1ULL<<30) / 512 * nPCIHostBridges +
8192;
- } else if (usesVFIO) {
+ } else if (usesVFIO || forceVFIO) {
/* For regular (non-NVLink2 present) VFIO passthrough, the value
* of passthroughLimit is:
*
@@ -12009,16 +12011,20 @@ getPPC64MemLockLimitBytes(virDomainDefPtr def)
/**
* qemuDomainGetMemLockLimitBytes:
* @def: domain definition
+ * @forceVFIO: force VFIO calculation
*
* Calculate the memory locking limit that needs to be set in order for
* the guest to operate properly. The limit depends on a number of factors,
* including certain configuration options and less immediately apparent ones
* such as the guest architecture or the use of certain devices.
+ * The @forceVFIO argument can be used to tell this function will use VFIO even
+ * though @def doesn't indicates so right now.
*
* Returns: the memory locking limit, or 0 if setting the limit is not needed
*/
unsigned long long
-qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
+qemuDomainGetMemLockLimitBytes(virDomainDefPtr def,
+ bool forceVFIO)
{
unsigned long long memKB = 0;
bool usesVFIO = false;
@@ -12039,7 +12045,7 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
return VIR_DOMAIN_MEMORY_PARAM_UNLIMITED;
if (ARCH_IS_PPC64(def->os.arch) && def->virtType == VIR_DOMAIN_VIRT_KVM)
- return getPPC64MemLockLimitBytes(def);
+ return getPPC64MemLockLimitBytes(def, forceVFIO);
/* For device passthrough using VFIO the guest memory and MMIO memory
* regions need to be locked persistent in order to allow DMA.
@@ -12059,18 +12065,20 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
*
* Note that this may not be valid for all platforms.
*/
- for (i = 0; i < def->nhostdevs; i++) {
- if (virHostdevIsVFIODevice(def->hostdevs[i]) ||
- virHostdevIsMdevDevice(def->hostdevs[i])) {
- usesVFIO = true;
- break;
+ if (!forceVFIO) {
+ for (i = 0; i < def->nhostdevs; i++) {
+ if (virHostdevIsVFIODevice(def->hostdevs[i]) ||
+ virHostdevIsMdevDevice(def->hostdevs[i])) {
+ usesVFIO = true;
+ break;
+ }
}
- }
- if (virDomainDefHasNVMeDisk(def))
- usesVFIO = true;
+ if (virDomainDefHasNVMeDisk(def))
+ usesVFIO = true;
+ }
- if (usesVFIO)
+ if (usesVFIO || forceVFIO)
memKB = virDomainDefGetMemoryTotal(def) + 1024 * 1024;
done:
@@ -12081,9 +12089,12 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
/**
* qemuDomainAdjustMaxMemLock:
* @vm: domain
+ * @forceVFIO: apply VFIO requirements even if vm's def doesn't require it
*
* Adjust the memory locking limit for the QEMU process associated to @vm, in
- * order to comply with VFIO or architecture requirements.
+ * order to comply with VFIO or architecture requirements. If @forceVFIO is
+ * true then the limit is changed even if nothing in @vm's definition indicates
+ * so.
*
* The limit will not be changed unless doing so is needed; the first time
* the limit is changed, the original (default) limit is stored in @vm and
@@ -12093,12 +12104,13 @@ qemuDomainGetMemLockLimitBytes(virDomainDefPtr def)
* Returns: 0 on success, <0 on failure
*/
int
-qemuDomainAdjustMaxMemLock(virDomainObjPtr vm)
+qemuDomainAdjustMaxMemLock(virDomainObjPtr vm,
+ bool forceVFIO)
{
unsigned long long bytes = 0;
int ret = -1;
- bytes = qemuDomainGetMemLockLimitBytes(vm->def);
+ bytes = qemuDomainGetMemLockLimitBytes(vm->def, forceVFIO);
if (bytes) {
/* If this is the first time adjusting the limit, save the current
@@ -12147,7 +12159,7 @@ qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
int ret = 0;
vm->def->hostdevs[vm->def->nhostdevs++] = hostdev;
- if (qemuDomainAdjustMaxMemLock(vm) < 0)
+ if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
ret = -1;
vm->def->hostdevs[--(vm->def->nhostdevs)] = NULL;
diff --git a/src/qemu/qemu_domain.h b/src/qemu/qemu_domain.h
index 65bd83aece..65d137462d 100644
--- a/src/qemu/qemu_domain.h
+++ b/src/qemu/qemu_domain.h
@@ -955,8 +955,10 @@ bool qemuDomainSupportsPCI(virDomainDefPtr def,
void qemuDomainUpdateCurrentMemorySize(virDomainObjPtr vm);
-unsigned long long qemuDomainGetMemLockLimitBytes(virDomainDefPtr def);
-int qemuDomainAdjustMaxMemLock(virDomainObjPtr vm);
+unsigned long long qemuDomainGetMemLockLimitBytes(virDomainDefPtr def,
+ bool forceVFIO);
+int qemuDomainAdjustMaxMemLock(virDomainObjPtr vm,
+ bool forceVFIO);
int qemuDomainAdjustMaxMemLockHostdev(virDomainObjPtr vm,
virDomainHostdevDefPtr hostdev);
diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c
index 053523f8c5..61deaffdd9 100644
--- a/src/qemu/qemu_hotplug.c
+++ b/src/qemu/qemu_hotplug.c
@@ -1646,7 +1646,7 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
if (teardowndevice &&
qemuDomainNamespaceTeardownHostdev(vm, hostdev) < 0)
VIR_WARN("Unable to remove host device from /dev");
- if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm) < 0)
+ if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm, false) < 0)
VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
if (releaseaddr)
@@ -2383,7 +2383,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
if (virDomainMemoryInsert(vm->def, mem) < 0)
goto cleanup;
- if (qemuDomainAdjustMaxMemLock(vm) < 0)
+ if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
goto removedef;
qemuDomainObjEnterMonitor(driver, vm);
@@ -2454,7 +2454,7 @@ qemuDomainAttachMemory(virQEMUDriverPtr driver,
/* reset the mlock limit */
virErrorPreserveLast(&orig_err);
- ignore_value(qemuDomainAdjustMaxMemLock(vm));
+ ignore_value(qemuDomainAdjustMaxMemLock(vm, false));
virErrorRestore(&orig_err);
goto audit;
@@ -2857,7 +2857,7 @@ qemuDomainAttachMediatedDevice(virQEMUDriverPtr driver,
ret = 0;
cleanup:
if (ret < 0) {
- if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm) < 0)
+ if (teardownmemlock && qemuDomainAdjustMaxMemLock(vm, false) < 0)
VIR_WARN("Unable to reset maximum locked memory on hotplug fail");
if (teardowncgroup && qemuTeardownHostdevCgroup(vm, hostdev) < 0)
VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
@@ -4356,7 +4356,7 @@ qemuDomainRemoveMemoryDevice(virQEMUDriverPtr driver,
ignore_value(qemuProcessRefreshBalloonState(driver, vm, QEMU_ASYNC_JOB_NONE));
/* decrease the mlock limit after memory unplug if necessary */
- ignore_value(qemuDomainAdjustMaxMemLock(vm));
+ ignore_value(qemuDomainAdjustMaxMemLock(vm, false));
return 0;
}
@@ -4483,7 +4483,7 @@ qemuDomainRemoveHostDevice(virQEMUDriverPtr driver,
qemuDomainRemovePCIHostDevice(driver, vm, hostdev);
/* QEMU might no longer need to lock as much memory, eg. we just
* detached the last VFIO device, so adjust the limit here */
- if (qemuDomainAdjustMaxMemLock(vm) < 0)
+ if (qemuDomainAdjustMaxMemLock(vm, false) < 0)
VIR_WARN("Failed to adjust locked memory limit");
break;
case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_USB:
diff --git a/tests/qemumemlocktest.c b/tests/qemumemlocktest.c
index ef3bfa0345..23a144a8ce 100644
--- a/tests/qemumemlocktest.c
+++ b/tests/qemumemlocktest.c
@@ -42,7 +42,7 @@ testCompareMemLock(const void *data)
goto cleanup;
}
- ret = virTestCompareToULL(info->memlock, qemuDomainGetMemLockLimitBytes(def));
+ ret = virTestCompareToULL(info->memlock, qemuDomainGetMemLockLimitBytes(def, false));
cleanup:
virDomainDefFree(def);
--
2.23.0
More information about the libvir-list
mailing list