[libvirt] [PATCH 1/1] qemu: host NUMA hugepage policy without guest NUMA

Sam Bobroff sam.bobroff at au1.ibm.com
Wed Oct 12 04:04:53 UTC 2016


At the moment, guests that are backed by hugepages in the host are
only able to use policy to control the placement of those hugepages
on a per-(guest-)CPU basis. Policy applied globally is ignored.

Such guests would use <memoryBacking><hugepages/></memoryBacking> and
a <numatune> block with <memory mode=... nodeset=.../> but no <memnode
.../> elements.

This patch corrects this by, in this specific case, changing the QEMU
command line from "-mem-prealloc -mem-path=..." (which cannot
specify NUMA policy) to "-object memory-backend-file ..." (which can).

Note: This is not visible to the guest and does not appear to create
a migration incompatibility.

Signed-off-by: Sam Bobroff <sam.bobroff at au1.ibm.com>
---
There was some discussion leading up to this patch, here:

https://www.redhat.com/archives/libvir-list/2016-October/msg00033.html

During that discussion it seemed that fixing this issue would cause
migration incompatibility but after some careful testing, it appears
that it only does when a memory-backend object is attached to a guest
NUMA node and that is not the case here. If only one is created, and
used globally (not attached via mem=<id>), the migration data does not
seem to be changed and so it seems reasonable to try something like
this patch.

This patch does work for my test cases but I don't claim a deep
understanding of the libvirt code so this is at least partly a RFC.
Comments welcome :-)

Cheers,
Sam.

 src/qemu/qemu_command.c | 55 ++++++++++++++++++++++++++++++++++++++++---------
 src/qemu/qemu_command.h |  2 +-
 2 files changed, 46 insertions(+), 11 deletions(-)

diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
index 0804133..c28c8f2 100644
--- a/src/qemu/qemu_command.c
+++ b/src/qemu/qemu_command.c
@@ -3143,7 +3143,7 @@ qemuBuildMemoryBackendStr(unsigned long long size,
                           int guestNode,
                           virBitmapPtr userNodeset,
                           virBitmapPtr autoNodeset,
-                          virDomainDefPtr def,
+                          const virDomainDefPtr def,
                           virQEMUCapsPtr qemuCaps,
                           virQEMUDriverConfigPtr cfg,
                           const char **backendType,
@@ -7129,12 +7129,18 @@ qemuBuildSmpCommandLine(virCommandPtr cmd,
 
 static int
 qemuBuildMemPathStr(virQEMUDriverConfigPtr cfg,
-                    const virDomainDef *def,
+                    const virDomainDefPtr def,
                     virQEMUCapsPtr qemuCaps,
-                    virCommandPtr cmd)
+                    virCommandPtr cmd,
+                    virBitmapPtr auto_nodeset)
 {
     const long system_page_size = virGetSystemPageSizeKB();
     char *mem_path = NULL;
+    virBitmapPtr nodemask = NULL;
+    const char *backendType = NULL;
+    char *backendStr = NULL;
+    virJSONValuePtr props;
+    int rv = -1;
 
     /*
      *  No-op if hugepages were not requested.
@@ -7159,18 +7165,47 @@ qemuBuildMemPathStr(virQEMUDriverConfigPtr cfg,
     if (qemuGetHupageMemPath(cfg, def->mem.hugepages[0].size, &mem_path) < 0)
         return -1;
 
-    virCommandAddArgList(cmd, "-mem-prealloc", "-mem-path", mem_path, NULL);
+    if (virDomainNumatuneMaybeGetNodeset(def->numa, auto_nodeset,
+                                         &nodemask, -1) < 0)
+        return -1;
+    if (nodemask && virQEMUCapsGet(qemuCaps, QEMU_CAPS_OBJECT_MEMORY_FILE)) {
+        props = virJSONValueNewObject();
+        if (qemuBuildMemoryBackendStr(virDomainDefGetMemoryInitial(def),
+                                      0, -1, NULL, auto_nodeset,
+                                      def, qemuCaps, cfg, &backendType,
+                                      &props, false) < 0)
+            goto cleanup;
+        if (!(backendStr = virQEMUBuildObjectCommandlineFromJSON(backendType,
+                                                                 "mem",
+                                                                 props)))
+            goto cleanup;
+        virCommandAddArgList(cmd, "-object", backendStr, NULL);
+        rv = 0;
+cleanup:
+        VIR_FREE(backendStr);
+        VIR_FREE(props);
+    }
+    else {
+        if (nodemask || 1)
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("Memory file backend objects are "
+                             "unsupported by QEMU binary. Global NUMA "
+                             "hugepage policy will be ignored."));
+        virCommandAddArgList(cmd, "-mem-prealloc", "-mem-path", mem_path, NULL);
+        rv = 0;
+    }
     VIR_FREE(mem_path);
 
-    return 0;
+    return rv;
 }
 
 
 static int
 qemuBuildMemCommandLine(virCommandPtr cmd,
                         virQEMUDriverConfigPtr cfg,
-                        const virDomainDef *def,
-                        virQEMUCapsPtr qemuCaps)
+                        const virDomainDefPtr def,
+                        virQEMUCapsPtr qemuCaps,
+                        virBitmapPtr auto_nodeset)
 {
     if (qemuDomainDefValidateMemoryHotplug(def, qemuCaps, NULL) < 0)
         return -1;
@@ -7194,7 +7229,7 @@ qemuBuildMemCommandLine(virCommandPtr cmd,
      * there is no numa node specified.
      */
     if (!virDomainNumaGetNodeCount(def->numa) &&
-        qemuBuildMemPathStr(cfg, def, qemuCaps, cmd) < 0)
+        qemuBuildMemPathStr(cfg, def, qemuCaps, cmd, auto_nodeset) < 0)
         return -1;
 
     if (def->mem.locked && !virQEMUCapsGet(qemuCaps, QEMU_CAPS_REALTIME_MLOCK)) {
@@ -7331,7 +7366,7 @@ qemuBuildNumaArgStr(virQEMUDriverConfigPtr cfg,
     }
 
     if (!needBackend &&
-        qemuBuildMemPathStr(cfg, def, qemuCaps, cmd) < 0)
+        qemuBuildMemPathStr(cfg, def, qemuCaps, cmd, auto_nodeset) < 0)
         goto cleanup;
 
     for (i = 0; i < ncells; i++) {
@@ -9381,7 +9416,7 @@ qemuBuildCommandLine(virQEMUDriverPtr driver,
     if (!migrateURI && !snapshot && qemuDomainAlignMemorySizes(def) < 0)
         goto error;
 
-    if (qemuBuildMemCommandLine(cmd, cfg, def, qemuCaps) < 0)
+    if (qemuBuildMemCommandLine(cmd, cfg, def, qemuCaps, nodeset) < 0)
         goto error;
 
     if (qemuBuildSmpCommandLine(cmd, def) < 0)
diff --git a/src/qemu/qemu_command.h b/src/qemu/qemu_command.h
index 2f2a6ff..1e97d76 100644
--- a/src/qemu/qemu_command.h
+++ b/src/qemu/qemu_command.h
@@ -126,7 +126,7 @@ int qemuBuildMemoryBackendStr(unsigned long long size,
                               int guestNode,
                               virBitmapPtr userNodeset,
                               virBitmapPtr autoNodeset,
-                              virDomainDefPtr def,
+                              const virDomainDefPtr def,
                               virQEMUCapsPtr qemuCaps,
                               virQEMUDriverConfigPtr cfg,
                               const char **backendType,
-- 
2.10.0.297.gf6727b0




More information about the libvir-list mailing list