[libvirt] [PATCH RESEND 1/2] NUMA: cleanup for numa related codes

Gao feng gaofeng at cn.fujitsu.com
Wed Mar 20 03:35:08 UTC 2013


Intend to reduce the redundant code,use virNumaSetupMemoryPolicy
to replace virLXCControllerSetupNUMAPolicy and
qemuProcessInitNumaMemoryPolicy.

This patch also moves the numa related codes to the
file virnuma.c and virnuma.h

Signed-off-by: Gao feng <gaofeng at cn.fujitsu.com>
---
 src/conf/domain_conf.c   |  31 ++++--------
 src/conf/domain_conf.h   |  25 +---------
 src/libvirt_private.syms |   9 ++--
 src/lxc/lxc_controller.c | 116 +------------------------------------------
 src/qemu/qemu_cgroup.c   |   4 +-
 src/qemu/qemu_driver.c   |   6 +--
 src/qemu/qemu_process.c  | 123 +--------------------------------------------
 src/util/virnuma.c       | 126 +++++++++++++++++++++++++++++++++++++++++++++++
 src/util/virnuma.h       |  30 +++++++++++
 9 files changed, 182 insertions(+), 288 deletions(-)

diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
index a1cfc76..fa70329 100644
--- a/src/conf/domain_conf.c
+++ b/src/conf/domain_conf.c
@@ -690,11 +690,6 @@ VIR_ENUM_IMPL(virDomainTimerMode, VIR_DOMAIN_TIMER_MODE_LAST,
               "paravirt",
               "smpsafe");
 
-VIR_ENUM_IMPL(virDomainNumatuneMemMode, VIR_DOMAIN_NUMATUNE_MEM_LAST,
-              "strict",
-              "preferred",
-              "interleave");
-
 VIR_ENUM_IMPL(virDomainStartupPolicy, VIR_DOMAIN_STARTUP_POLICY_LAST,
               "default",
               "mandatory",
@@ -709,12 +704,6 @@ VIR_ENUM_IMPL(virDomainDiskTray, VIR_DOMAIN_DISK_TRAY_LAST,
               "closed",
               "open");
 
-VIR_ENUM_IMPL(virDomainNumatuneMemPlacementMode,
-              VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_LAST,
-              "default",
-              "static",
-              "auto");
-
 VIR_ENUM_IMPL(virDomainRNGModel,
               VIR_DOMAIN_RNG_MODEL_LAST,
               "virtio");
@@ -9852,7 +9841,7 @@ virDomainDefParseXML(virCapsPtr caps,
                     int placement_mode = 0;
                     if (placement) {
                         if ((placement_mode =
-                             virDomainNumatuneMemPlacementModeTypeFromString(placement)) < 0) {
+                             virNumaTuneMemPlacementModeTypeFromString(placement)) < 0) {
                             virReportError(VIR_ERR_XML_ERROR,
                                            _("Unsupported memory placement "
                                              "mode '%s'"), placement);
@@ -9862,18 +9851,18 @@ virDomainDefParseXML(virCapsPtr caps,
                         VIR_FREE(placement);
                     } else if (def->numatune.memory.nodemask) {
                         /* Defaults to "static" if nodeset is specified. */
-                        placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC;
+                        placement_mode = VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_STATIC;
                     } else {
                         /* Defaults to "placement" of <vcpu> if nodeset is
                          * not specified.
                          */
                         if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_STATIC)
-                            placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC;
+                            placement_mode = VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_STATIC;
                         else
-                            placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO;
+                            placement_mode = VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO;
                     }
 
-                    if (placement_mode == VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC &&
+                    if (placement_mode == VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_STATIC &&
                         !def->numatune.memory.nodemask) {
                         virReportError(VIR_ERR_XML_ERROR, "%s",
                                        _("nodeset for NUMA memory tuning must be set "
@@ -9882,13 +9871,13 @@ virDomainDefParseXML(virCapsPtr caps,
                     }
 
                     /* Ignore 'nodeset' if 'placement' is 'auto' finally */
-                    if (placement_mode == VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)
+                    if (placement_mode == VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO)
                         virBitmapFree(def->numatune.memory.nodemask);
 
                     /* Copy 'placement' of <numatune> to <vcpu> if its 'placement'
                      * is not specified and 'placement' of <numatune> is specified.
                      */
-                    if (placement_mode == VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO &&
+                    if (placement_mode == VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO &&
                         !def->cpumask)
                         def->placement_mode = VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO;
 
@@ -9907,7 +9896,7 @@ virDomainDefParseXML(virCapsPtr caps,
          * and 'placement' of <vcpu> is 'auto'.
          */
         if (def->placement_mode == VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) {
-            def->numatune.memory.placement_mode = VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO;
+            def->numatune.memory.placement_mode = VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO;
             def->numatune.memory.mode = VIR_DOMAIN_NUMATUNE_MEM_STRICT;
         }
     }
@@ -14818,7 +14807,7 @@ virDomainDefFormatInternal(virDomainDefPtr def,
         virBufferAsprintf(buf, "    <memory mode='%s' ", mode);
 
         if (def->numatune.memory.placement_mode ==
-            VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) {
+            VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_STATIC) {
             nodemask = virBitmapFormat(def->numatune.memory.nodemask);
             if (nodemask == NULL) {
                 virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
@@ -14829,7 +14818,7 @@ virDomainDefFormatInternal(virDomainDefPtr def,
             virBufferAsprintf(buf, "nodeset='%s'/>\n", nodemask);
             VIR_FREE(nodemask);
         } else if (def->numatune.memory.placement_mode) {
-            placement = virDomainNumatuneMemPlacementModeTypeToString(def->numatune.memory.placement_mode);
+            placement = virNumaTuneMemPlacementModeTypeToString(def->numatune.memory.placement_mode);
             virBufferAsprintf(buf, "placement='%s'/>\n", placement);
         }
         virBufferAddLit(buf, "  </numatune>\n");
diff --git a/src/conf/domain_conf.h b/src/conf/domain_conf.h
index bfc37a0..6d856a3 100644
--- a/src/conf/domain_conf.h
+++ b/src/conf/domain_conf.h
@@ -47,6 +47,7 @@
 # include "device_conf.h"
 # include "virbitmap.h"
 # include "virstoragefile.h"
+# include "virnuma.h"
 
 /* forward declarations of all device types, required by
  * virDomainDeviceDef
@@ -1605,14 +1606,6 @@ enum virDomainCpuPlacementMode {
     VIR_DOMAIN_CPU_PLACEMENT_MODE_LAST
 };
 
-enum virDomainNumatuneMemPlacementMode {
-    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_DEFAULT = 0,
-    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC,
-    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO,
-
-    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_LAST
-};
-
 typedef struct _virDomainTimerCatchupDef virDomainTimerCatchupDef;
 typedef virDomainTimerCatchupDef *virDomainTimerCatchupDefPtr;
 struct _virDomainTimerCatchupDef {
@@ -1701,18 +1694,6 @@ virDomainVcpuPinDefPtr virDomainVcpuPinFindByVcpu(virDomainVcpuPinDefPtr *def,
                                                   int nvcpupin,
                                                   int vcpu);
 
-typedef struct _virDomainNumatuneDef virDomainNumatuneDef;
-typedef virDomainNumatuneDef *virDomainNumatuneDefPtr;
-struct _virDomainNumatuneDef {
-    struct {
-        virBitmapPtr nodemask;
-        int mode;
-        int placement_mode; /* enum virDomainNumatuneMemPlacementMode */
-    } memory;
-
-    /* Future NUMA tuning related stuff should go here. */
-};
-
 typedef struct _virBlkioDeviceWeight virBlkioDeviceWeight;
 typedef virBlkioDeviceWeight *virBlkioDeviceWeightPtr;
 struct _virBlkioDeviceWeight {
@@ -1802,7 +1783,7 @@ struct _virDomainDef {
         virDomainVcpuPinDefPtr emulatorpin;
     } cputune;
 
-    virDomainNumatuneDef numatune;
+    virNumaTuneDef numatune;
 
     /* These 3 are based on virDomainLifeCycleAction enum flags */
     int onReboot;
@@ -2397,8 +2378,6 @@ VIR_ENUM_DECL(virDomainGraphicsSpicePlaybackCompression)
 VIR_ENUM_DECL(virDomainGraphicsSpiceStreamingMode)
 VIR_ENUM_DECL(virDomainGraphicsSpiceClipboardCopypaste)
 VIR_ENUM_DECL(virDomainGraphicsSpiceMouseMode)
-VIR_ENUM_DECL(virDomainNumatuneMemMode)
-VIR_ENUM_DECL(virDomainNumatuneMemPlacementMode)
 VIR_ENUM_DECL(virDomainHyperv)
 VIR_ENUM_DECL(virDomainRNGModel)
 VIR_ENUM_DECL(virDomainRNGBackend)
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index dc01bfa..8890859 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -252,10 +252,6 @@ virDomainNetRemove;
 virDomainNetTypeToString;
 virDomainNostateReasonTypeFromString;
 virDomainNostateReasonTypeToString;
-virDomainNumatuneMemModeTypeFromString;
-virDomainNumatuneMemModeTypeToString;
-virDomainNumatuneMemPlacementModeTypeFromString;
-virDomainNumatuneMemPlacementModeTypeToString;
 virDomainObjAssignDef;
 virDomainObjCopyPersistentDef;
 virDomainObjGetPersistentDef;
@@ -1557,7 +1553,12 @@ virNodeSuspendGetTargetMask;
 
 
 # util/virnuma.h
+virDomainNumatuneMemModeTypeFromString;
+virDomainNumatuneMemModeTypeToString;
+virNumaTuneMemPlacementModeTypeFromString;
+virNumaTuneMemPlacementModeTypeToString;
 virNumaGetAutoPlacementAdvice;
+virNumaSetupMemoryPolicy;
 
 # util/virobject.h
 virClassForObject;
diff --git a/src/lxc/lxc_controller.c b/src/lxc/lxc_controller.c
index 405205c..f19f8c1 100644
--- a/src/lxc/lxc_controller.c
+++ b/src/lxc/lxc_controller.c
@@ -46,11 +46,6 @@
 # include <cap-ng.h>
 #endif
 
-#if WITH_NUMACTL
-# define NUMA_VERSION1_COMPATIBILITY 1
-# include <numa.h>
-#endif
-
 #include "virerror.h"
 #include "virlog.h"
 #include "virutil.h"
@@ -469,113 +464,6 @@ cleanup:
     return ret;
 }
 
-#if WITH_NUMACTL
-static int virLXCControllerSetupNUMAPolicy(virLXCControllerPtr ctrl,
-                                           virBitmapPtr nodemask)
-{
-    nodemask_t mask;
-    int mode = -1;
-    int node = -1;
-    int ret = -1;
-    int i = 0;
-    int maxnode = 0;
-    bool warned = false;
-    virDomainNumatuneDef numatune = ctrl->def->numatune;
-    virBitmapPtr tmp_nodemask = NULL;
-
-    if (numatune.memory.placement_mode ==
-        VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) {
-        if (!numatune.memory.nodemask)
-            return 0;
-        VIR_DEBUG("Set NUMA memory policy with specified nodeset");
-        tmp_nodemask = numatune.memory.nodemask;
-    } else if (numatune.memory.placement_mode ==
-               VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) {
-        VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad");
-        tmp_nodemask = nodemask;
-    } else {
-        return 0;
-    }
-
-    VIR_DEBUG("Setting NUMA memory policy");
-
-    if (numa_available() < 0) {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-                       "%s", _("Host kernel is not aware of NUMA."));
-        return -1;
-    }
-
-    maxnode = numa_max_node() + 1;
-
-    /* Convert nodemask to NUMA bitmask. */
-    nodemask_zero(&mask);
-    i = -1;
-    while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) {
-        if (i > NUMA_NUM_NODES) {
-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-                           _("Host cannot support NUMA node %d"), i);
-            return -1;
-        }
-        if (i > maxnode && !warned) {
-            VIR_WARN("nodeset is out of range, there is only %d NUMA "
-                     "nodes on host", maxnode);
-            warned = true;
-        }
-        nodemask_set(&mask, i);
-    }
-
-    mode = ctrl->def->numatune.memory.mode;
-
-    if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) {
-        numa_set_bind_policy(1);
-        numa_set_membind(&mask);
-        numa_set_bind_policy(0);
-    } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) {
-        int nnodes = 0;
-        for (i = 0; i < NUMA_NUM_NODES; i++) {
-            if (nodemask_isset(&mask, i)) {
-                node = i;
-                nnodes++;
-            }
-        }
-
-        if (nnodes != 1) {
-            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-                           "%s", _("NUMA memory tuning in 'preferred' mode "
-                                   "only supports single node"));
-            goto cleanup;
-        }
-
-        numa_set_bind_policy(0);
-        numa_set_preferred(node);
-    } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) {
-        numa_set_interleave_mask(&mask);
-    } else {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
-                       _("Unable to set NUMA policy %s"),
-                       virDomainNumatuneMemModeTypeToString(mode));
-        goto cleanup;
-    }
-
-    ret = 0;
-
-cleanup:
-    return ret;
-}
-#else
-static int virLXCControllerSetupNUMAPolicy(virLXCControllerPtr ctrl,
-                                           virBitmapPtr nodemask ATTRIBUTE_UNUSED)
-{
-    if (ctrl->def->numatune.memory.nodemask) {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                       _("NUMA policy is not available on this platform"));
-        return -1;
-    }
-
-    return 0;
-}
-#endif
-
 
 /*
  * To be run while still single threaded
@@ -638,7 +526,7 @@ static int virLXCControllerGetNumadAdvice(virLXCControllerPtr ctrl,
     if ((ctrl->def->placement_mode ==
          VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) ||
         (ctrl->def->numatune.memory.placement_mode ==
-         VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) {
+         VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO)) {
         nodeset = virNumaGetAutoPlacementAdvice(ctrl->def->vcpus,
                                                 ctrl->def->mem.cur_balloon);
         if (!nodeset)
@@ -675,7 +563,7 @@ static int virLXCControllerSetupResourceLimits(virLXCControllerPtr ctrl,
     int ret = -1;
 
     if (virLXCControllerGetNumadAdvice(ctrl, &nodemask) < 0 ||
-        virLXCControllerSetupNUMAPolicy(ctrl, nodemask) < 0)
+        virNumaSetupMemoryPolicy(ctrl->def->numatune, nodemask) < 0)
         goto cleanup;
 
     if (virLXCControllerSetupCpuAffinity(ctrl) < 0)
diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index 9d6e88b..c9b4ca2 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -423,12 +423,12 @@ int qemuSetupCgroup(virQEMUDriverPtr driver,
 
     if ((vm->def->numatune.memory.nodemask ||
          (vm->def->numatune.memory.placement_mode ==
-          VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) &&
+          VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO)) &&
         vm->def->numatune.memory.mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT &&
         qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPUSET)) {
         char *mask = NULL;
         if (vm->def->numatune.memory.placement_mode ==
-            VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)
+            VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO)
             mask = virBitmapFormat(nodemask);
         else
             mask = virBitmapFormat(vm->def->numatune.memory.nodemask);
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 9cd9e44..f057d38 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -7648,7 +7648,7 @@ qemuDomainSetNumaParameters(virDomainPtr dom,
                 virBitmapFree(vm->def->numatune.memory.nodemask);
 
                 vm->def->numatune.memory.placement_mode =
-                    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC;
+                    VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_STATIC;
                 vm->def->numatune.memory.nodemask = virBitmapNewCopy(nodeset);
             }
 
@@ -7657,7 +7657,7 @@ qemuDomainSetNumaParameters(virDomainPtr dom,
 
                 persistentDef->numatune.memory.nodemask = nodeset;
                 persistentDef->numatune.memory.placement_mode =
-                    VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC;
+                    VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_STATIC;
                 nodeset = NULL;
             }
             virBitmapFree(nodeset);
@@ -7667,7 +7667,7 @@ qemuDomainSetNumaParameters(virDomainPtr dom,
     if (flags & VIR_DOMAIN_AFFECT_CONFIG) {
         if (!persistentDef->numatune.memory.placement_mode)
             persistentDef->numatune.memory.placement_mode =
-                VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO;
+                VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO;
         if (virDomainSaveConfig(cfg->configDir, persistentDef) < 0)
             ret = -1;
     }
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index 2465938..fcb3c50 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -45,11 +45,6 @@
 #include "qemu_bridge_filter.h"
 #include "qemu_migration.h"
 
-#if WITH_NUMACTL
-# define NUMA_VERSION1_COMPATIBILITY 1
-# include <numa.h>
-#endif
-
 #include "datatypes.h"
 #include "virlog.h"
 #include "virerror.h"
@@ -1791,120 +1786,6 @@ qemuProcessDetectVcpuPIDs(virQEMUDriverPtr driver,
 }
 
 
-/*
- * Set NUMA memory policy for qemu process, to be run between
- * fork/exec of QEMU only.
- */
-#if WITH_NUMACTL
-static int
-qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm,
-                                virBitmapPtr nodemask)
-{
-    nodemask_t mask;
-    int mode = -1;
-    int node = -1;
-    int ret = -1;
-    int i = 0;
-    int maxnode = 0;
-    bool warned = false;
-    virDomainNumatuneDef numatune = vm->def->numatune;
-    virBitmapPtr tmp_nodemask = NULL;
-
-    if (numatune.memory.placement_mode ==
-        VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_STATIC) {
-        if (!numatune.memory.nodemask)
-            return 0;
-        VIR_DEBUG("Set NUMA memory policy with specified nodeset");
-        tmp_nodemask = numatune.memory.nodemask;
-    } else if (numatune.memory.placement_mode ==
-               VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO) {
-        VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad");
-        tmp_nodemask = nodemask;
-    } else {
-        return 0;
-    }
-
-    if (numa_available() < 0) {
-        virReportError(VIR_ERR_INTERNAL_ERROR,
-                       "%s", _("Host kernel is not aware of NUMA."));
-        return -1;
-    }
-
-    maxnode = numa_max_node() + 1;
-    /* Convert nodemask to NUMA bitmask. */
-    nodemask_zero(&mask);
-    i = -1;
-    while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) {
-        if (i > NUMA_NUM_NODES) {
-            virReportError(VIR_ERR_INTERNAL_ERROR,
-                           _("Host cannot support NUMA node %d"), i);
-            return -1;
-        }
-        if (i > maxnode && !warned) {
-            VIR_WARN("nodeset is out of range, there is only %d NUMA "
-                     "nodes on host", maxnode);
-            warned = true;
-        }
-        nodemask_set(&mask, i);
-    }
-
-    mode = numatune.memory.mode;
-
-    if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) {
-        numa_set_bind_policy(1);
-        numa_set_membind(&mask);
-        numa_set_bind_policy(0);
-    } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) {
-        int nnodes = 0;
-        for (i = 0; i < NUMA_NUM_NODES; i++) {
-            if (nodemask_isset(&mask, i)) {
-                node = i;
-                nnodes++;
-            }
-        }
-
-        if (nnodes != 1) {
-            virReportError(VIR_ERR_INTERNAL_ERROR,
-                           "%s", _("NUMA memory tuning in 'preferred' mode "
-                                   "only supports single node"));
-            goto cleanup;
-        }
-
-        numa_set_bind_policy(0);
-        numa_set_preferred(node);
-    } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) {
-        numa_set_interleave_mask(&mask);
-    } else {
-        /* XXX: Shouldn't go here, as we already do checking when
-         * parsing domain XML.
-         */
-        virReportError(VIR_ERR_XML_ERROR,
-                       "%s", _("Invalid mode for memory NUMA tuning."));
-        goto cleanup;
-    }
-
-    ret = 0;
-
-cleanup:
-    return ret;
-}
-#else
-static int
-qemuProcessInitNumaMemoryPolicy(virDomainObjPtr vm,
-                                virBitmapPtr nodemask ATTRIBUTE_UNUSED)
-{
-    if (vm->def->numatune.memory.nodemask) {
-        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
-                       _("libvirt is compiled without NUMA tuning support"));
-
-        return -1;
-    }
-
-    return 0;
-}
-#endif
-
-
 /* Helper to prepare cpumap for affinity setting, convert
  * NUMA nodeset into cpuset if @nodemask is not NULL, otherwise
  * just return a new allocated bitmap.
@@ -2654,7 +2535,7 @@ static int qemuProcessHook(void *data)
         qemuProcessInitCpuAffinity(h->driver, h->vm, h->nodemask) < 0)
         goto cleanup;
 
-    if (qemuProcessInitNumaMemoryPolicy(h->vm, h->nodemask) < 0)
+    if (virNumaSetupMemoryPolicy(h->vm->def->numatune, h->nodemask) < 0)
         goto cleanup;
 
     ret = 0;
@@ -3608,7 +3489,7 @@ int qemuProcessStart(virConnectPtr conn,
     if ((vm->def->placement_mode ==
          VIR_DOMAIN_CPU_PLACEMENT_MODE_AUTO) ||
         (vm->def->numatune.memory.placement_mode ==
-         VIR_DOMAIN_NUMATUNE_MEM_PLACEMENT_MODE_AUTO)) {
+         VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO)) {
         nodeset = virNumaGetAutoPlacementAdvice(vm->def->vcpus,
                                                 vm->def->mem.cur_balloon);
         if (!nodeset)
diff --git a/src/util/virnuma.c b/src/util/virnuma.c
index f6a6eb2..bace06f 100644
--- a/src/util/virnuma.c
+++ b/src/util/virnuma.c
@@ -21,12 +21,30 @@
 
 #include <config.h>
 
+#if WITH_NUMACTL
+# define NUMA_VERSION1_COMPATIBILITY 1
+# include <numa.h>
+#endif
+
 #include "virnuma.h"
 #include "vircommand.h"
 #include "virerror.h"
+#include "virlog.h"
 
 #define VIR_FROM_THIS VIR_FROM_NONE
 
+VIR_ENUM_IMPL(virDomainNumatuneMemMode,
+              VIR_DOMAIN_NUMATUNE_MEM_LAST,
+              "strict",
+              "preferred",
+              "interleave");
+
+VIR_ENUM_IMPL(virNumaTuneMemPlacementMode,
+              VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_LAST,
+              "default",
+              "static",
+              "auto");
+
 #if HAVE_NUMAD
 char *
 virNumaGetAutoPlacementAdvice(unsigned short vcpus,
@@ -59,3 +77,111 @@ virNumaGetAutoPlacementAdvice(unsigned short vcpus ATTRIBUTE_UNUSED,
     return NULL;
 }
 #endif
+
+#if WITH_NUMACTL
+int
+virNumaSetupMemoryPolicy(virNumaTuneDef numatune,
+                         virBitmapPtr nodemask)
+{
+    nodemask_t mask;
+    int mode = -1;
+    int node = -1;
+    int ret = -1;
+    int i = 0;
+    int maxnode = 0;
+    bool warned = false;
+    virBitmapPtr tmp_nodemask = NULL;
+
+    if (numatune.memory.placement_mode ==
+        VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_STATIC) {
+        if (!numatune.memory.nodemask)
+            return 0;
+        VIR_DEBUG("Set NUMA memory policy with specified nodeset");
+        tmp_nodemask = numatune.memory.nodemask;
+    } else if (numatune.memory.placement_mode ==
+               VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO) {
+        VIR_DEBUG("Set NUMA memory policy with advisory nodeset from numad");
+        tmp_nodemask = nodemask;
+    } else {
+        return 0;
+    }
+
+    if (numa_available() < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR,
+                       "%s", _("Host kernel is not aware of NUMA."));
+        return -1;
+    }
+
+    maxnode = numa_max_node() + 1;
+    /* Convert nodemask to NUMA bitmask. */
+    nodemask_zero(&mask);
+    i = -1;
+    while ((i = virBitmapNextSetBit(tmp_nodemask, i)) >= 0) {
+        if (i > NUMA_NUM_NODES) {
+            virReportError(VIR_ERR_INTERNAL_ERROR,
+                           _("Host cannot support NUMA node %d"), i);
+            return -1;
+        }
+        if (i > maxnode && !warned) {
+            VIR_WARN("nodeset is out of range, there is only %d NUMA "
+                     "nodes on host", maxnode);
+            warned = true;
+        }
+        nodemask_set(&mask, i);
+    }
+
+    mode = numatune.memory.mode;
+
+    if (mode == VIR_DOMAIN_NUMATUNE_MEM_STRICT) {
+        numa_set_bind_policy(1);
+        numa_set_membind(&mask);
+        numa_set_bind_policy(0);
+    } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_PREFERRED) {
+        int nnodes = 0;
+        for (i = 0; i < NUMA_NUM_NODES; i++) {
+            if (nodemask_isset(&mask, i)) {
+                node = i;
+                nnodes++;
+            }
+        }
+
+        if (nnodes != 1) {
+            virReportError(VIR_ERR_INTERNAL_ERROR,
+                           "%s", _("NUMA memory tuning in 'preferred' mode "
+                                   "only supports single node"));
+            goto cleanup;
+        }
+
+        numa_set_bind_policy(0);
+        numa_set_preferred(node);
+    } else if (mode == VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE) {
+        numa_set_interleave_mask(&mask);
+    } else {
+        /* XXX: Shouldn't go here, as we already do checking when
+         * parsing domain XML.
+         */
+        virReportError(VIR_ERR_XML_ERROR,
+                       "%s", _("Invalid mode for memory NUMA tuning."));
+        goto cleanup;
+    }
+
+    ret = 0;
+
+cleanup:
+    return ret;
+}
+#else
+int
+virNumaSetupMemoryPolicy(virNumaTuneDef numatune,
+                         virBitmapPtr nodemask ATTRIBUTE_UNUSED)
+{
+    if (numatune.memory.nodemask) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                       _("libvirt is compiled without NUMA tuning support"));
+
+        return -1;
+    }
+
+    return 0;
+}
+#endif
diff --git a/src/util/virnuma.h b/src/util/virnuma.h
index d3d7d3e..9ff8e69 100644
--- a/src/util/virnuma.h
+++ b/src/util/virnuma.h
@@ -22,7 +22,37 @@
 #ifndef __VIR_NUMA_H__
 # define __VIR_NUMA_H__
 
+# include "internal.h"
+# include "virbitmap.h"
+# include "virutil.h"
+
+enum virNumaTuneMemPlacementMode {
+    VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_DEFAULT = 0,
+    VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_STATIC,
+    VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_AUTO,
+
+    VIR_NUMA_TUNE_MEM_PLACEMENT_MODE_LAST
+};
+
+VIR_ENUM_DECL(virNumaTuneMemPlacementMode)
+
+VIR_ENUM_DECL(virDomainNumatuneMemMode)
+
+typedef struct _virNumaTuneDef virNumaTuneDef;
+typedef virNumaTuneDef *virNumaTuneDefPtr;
+struct _virNumaTuneDef {
+    struct {
+        virBitmapPtr nodemask;
+        int mode;
+        int placement_mode; /* enum virNumaTuneMemPlacementMode */
+    } memory;
+
+    /* Future NUMA tuning related stuff should go here. */
+};
+
 char *virNumaGetAutoPlacementAdvice(unsigned short vcups,
                                     unsigned long long balloon);
 
+int virNumaSetupMemoryPolicy(virNumaTuneDef numatune,
+                             virBitmapPtr nodemask);
 #endif /* __VIR_NUMA_H__ */
-- 
1.7.11.7




More information about the libvir-list mailing list