[libvirt] [PATCH] Add NUMA memory and CPU thread siblings to capabilities
Dusty Mabe
dustymabe at gmail.com
Thu Nov 8 20:31:42 UTC 2012
---
docs/schemas/capability.rng | 15 +++
src/conf/capabilities.c | 65 ++++++++++---
src/conf/capabilities.h | 7 +-
src/nodeinfo.c | 155 +++++++++++++++++++++++++++++-
src/test/test_driver.c | 2 +-
src/xen/xend_internal.c | 4 +-
tests/capabilityschemadata/caps-test3.xml | 88 +++++++++++++++++
7 files changed, 319 insertions(+), 17 deletions(-)
create mode 100644 tests/capabilityschemadata/caps-test3.xml
diff --git a/docs/schemas/capability.rng b/docs/schemas/capability.rng
index 8c928bc..1ab36ed 100644
--- a/docs/schemas/capability.rng
+++ b/docs/schemas/capability.rng
@@ -176,6 +176,10 @@
</attribute>
<optional>
+ <ref name='memory'/>
+ </optional>
+
+ <optional>
<element name='cpus'>
<attribute name='num'>
<ref name='unsignedInt'/>
@@ -188,11 +192,22 @@
</element>
</define>
+ <define name='memory'>
+ <element name='memory'>
+ <ref name='scaledInteger'/>
+ </element>
+ </define>
+
<define name='cpu'>
<element name='cpu'>
<attribute name='id'>
<ref name='unsignedInt'/>
</attribute>
+ <optional>
+ <attribute name='thread_siblings'>
+ <text/>
+ </attribute>
+ </optional>
</element>
</define>
diff --git a/src/conf/capabilities.c b/src/conf/capabilities.c
index a8ee2cf..1b514ef 100644
--- a/src/conf/capabilities.c
+++ b/src/conf/capabilities.c
@@ -73,10 +73,16 @@ virCapabilitiesNew(const char *arch,
static void
virCapabilitiesFreeHostNUMACell(virCapsHostNUMACellPtr cell)
{
+ int i;
if (cell == NULL)
return;
+ if (cell->threadSiblings)
+ for (i=0; i < cell->ncpus; i++)
+ VIR_FREE(cell->threadSiblings[i]);
+
VIR_FREE(cell->cpus);
+ VIR_FREE(cell->threadSiblings);
VIR_FREE(cell);
}
@@ -253,7 +259,10 @@ int
virCapabilitiesAddHostNUMACell(virCapsPtr caps,
int num,
int ncpus,
- const int *cpus)
+ unsigned long long mem,
+ const int *cpus,
+ const virBitmapPtr *threadSiblings
+)
{
virCapsHostNUMACellPtr cell;
@@ -272,8 +281,20 @@ virCapabilitiesAddHostNUMACell(virCapsPtr caps,
cpus,
ncpus * sizeof(*cpus));
+ if (threadSiblings) {
+ if (VIR_ALLOC_N(cell->threadSiblings, ncpus) < 0) {
+ VIR_FREE(cell->cpus);
+ VIR_FREE(cell);
+ return -1;
+ }
+ memcpy(cell->threadSiblings,
+ threadSiblings,
+ ncpus * sizeof(*threadSiblings));
+ }
+
cell->ncpus = ncpus;
cell->num = num;
+ cell->mem = mem;
caps->host.numaCell[caps->host.nnumaCell++] = cell;
@@ -695,6 +716,7 @@ virCapabilitiesFormatXML(virCapsPtr caps)
virBuffer xml = VIR_BUFFER_INITIALIZER;
int i, j, k;
char host_uuid[VIR_UUID_STRING_BUFLEN];
+ char *str;
virBufferAddLit(&xml, "<capabilities>\n\n");
virBufferAddLit(&xml, " <host>\n");
@@ -754,22 +776,43 @@ virCapabilitiesFormatXML(virCapsPtr caps)
}
if (caps->host.nnumaCell) {
- virBufferAddLit(&xml, " <topology>\n");
- virBufferAsprintf(&xml, " <cells num='%zu'>\n",
+ virBufferAdjustIndent(&xml, 4);
+ virBufferAddLit(&xml, "<topology>\n");
+ virBufferAsprintf(&xml, " <cells num='%zu'>\n",
caps->host.nnumaCell);
for (i = 0 ; i < caps->host.nnumaCell ; i++) {
- virBufferAsprintf(&xml, " <cell id='%d'>\n",
+ virBufferAsprintf(&xml, " <cell id='%d'>\n",
caps->host.numaCell[i]->num);
- virBufferAsprintf(&xml, " <cpus num='%d'>\n",
+
+ /* Print out the numacell memory total if it is available */
+ if (caps->host.numaCell[i]->mem)
+ virBufferAsprintf(&xml, " <memory unit='KiB'>%llu</memory>\n",
+ caps->host.numaCell[i]->mem);
+
+ virBufferAsprintf(&xml, " <cpus num='%d'>\n",
caps->host.numaCell[i]->ncpus);
- for (j = 0 ; j < caps->host.numaCell[i]->ncpus ; j++)
- virBufferAsprintf(&xml, " <cpu id='%d'/>\n",
+ for (j = 0 ; j < caps->host.numaCell[i]->ncpus ; j++) {
+ virBufferAsprintf(&xml, " <cpu id='%d'",
caps->host.numaCell[i]->cpus[j]);
- virBufferAddLit(&xml, " </cpus>\n");
- virBufferAddLit(&xml, " </cell>\n");
+
+ /* Print out thread siblings if they were populated */
+ if (caps->host.numaCell[i]->threadSiblings) {
+ str = virBitmapFormat(caps->host.numaCell[i]->threadSiblings[j]);
+ if (str) {
+ virBufferAsprintf(&xml, " thread_siblings='%s'", str);
+ VIR_FREE(str);
+ }
+ }
+
+ virBufferAsprintf(&xml, "/>\n");
+ }
+
+ virBufferAddLit(&xml, " </cpus>\n");
+ virBufferAddLit(&xml, " </cell>\n");
}
- virBufferAddLit(&xml, " </cells>\n");
- virBufferAddLit(&xml, " </topology>\n");
+ virBufferAddLit(&xml, " </cells>\n");
+ virBufferAddLit(&xml, "</topology>\n");
+ virBufferAdjustIndent(&xml, -4);
}
for (i = 0; i < caps->host.nsecModels; i++) {
diff --git a/src/conf/capabilities.h b/src/conf/capabilities.h
index 99056f8..ded0d57 100644
--- a/src/conf/capabilities.h
+++ b/src/conf/capabilities.h
@@ -88,7 +88,9 @@ typedef virCapsHostNUMACell *virCapsHostNUMACellPtr;
struct _virCapsHostNUMACell {
int num;
int ncpus;
+ unsigned long long mem; /* in kibibytes */
int *cpus;
+ virBitmapPtr *threadSiblings;
};
typedef struct _virCapsHostSecModel virCapsHostSecModel;
@@ -200,7 +202,10 @@ extern int
virCapabilitiesAddHostNUMACell(virCapsPtr caps,
int num,
int ncpus,
- const int *cpus);
+ unsigned long long mem,
+ const int *cpus,
+ const virBitmapPtr *threadSiblings
+);
extern int
diff --git a/src/nodeinfo.c b/src/nodeinfo.c
index 36fbd66..cc42c43 100644
--- a/src/nodeinfo.c
+++ b/src/nodeinfo.c
@@ -56,6 +56,7 @@
#ifdef __linux__
# define CPUINFO_PATH "/proc/cpuinfo"
# define SYSFS_SYSTEM_PATH "/sys/devices/system"
+# define SYSFS_CPU_PATH "/sys/devices/system/cpu"
# define PROCSTAT_PATH "/proc/stat"
# define MEMINFO_PATH "/proc/meminfo"
# define SYSFS_MEMORY_SHARED_PATH "/sys/kernel/mm/ksm"
@@ -77,6 +78,7 @@ static int linuxNodeGetMemoryStats(FILE *meminfo,
int cellNum,
virNodeMemoryStatsPtr params,
int *nparams);
+static unsigned long long nodeGetCellMemory(int cell);
/* Return the positive decimal contents of the given
* DIR/cpu%u/FILE, or -1 on error. If MISSING_OK and the
@@ -125,6 +127,66 @@ cleanup:
return value;
}
+/**
+ * virNodeGetThreadSiblingsList
+ * @dir: directory where cpu0-N files are located.
+ * @cpu: the specific cpu to get the siblings for.
+ *
+ * Will open the "thread_siblings_list" file for the cpu and
+ * return a string representing the contents. The contents of the
+ * file is a string represnting the cpus that are siblings; like
+ * 1,9 or 1-4.
+ *
+ * Returns NULL on failure, char * on success
+ *
+ * Note: Responsibility of caller to free string
+ */
+static char *virNodeGetThreadSiblingsList(const char *dir, unsigned int cpu)
+{
+ char *path;
+ FILE *pathfp;
+ char *str = NULL;
+ int strsize;
+
+ if (virAsprintf(&path, "%s/cpu%u/topology/thread_siblings_list",
+ dir, cpu) < 0) {
+ virReportOOMError();
+ return str;
+ }
+
+ pathfp = fopen(path, "r");
+ if (pathfp == NULL) {
+ virReportSystemError(errno, _("cannot open %s"), path);
+ VIR_FREE(path);
+ return str;
+ }
+
+ /* Detect how large of a string we need to make */
+ fseek(pathfp, 0L, SEEK_END);
+ strsize = ftell(pathfp);
+ fseek(pathfp, 0L, SEEK_SET);
+
+
+ if (VIR_ALLOC_N(str, strsize) < 0) {
+ virReportOOMError();
+ str = NULL;
+ goto cleanup;
+ }
+
+ if (fgets(str, strsize, pathfp) == NULL) {
+ virReportSystemError(errno, _("cannot read from %s"), path);
+ VIR_FREE(str);
+ str = NULL;
+ goto cleanup;
+ }
+
+cleanup:
+ VIR_FORCE_FCLOSE(pathfp);
+ VIR_FREE(path);
+
+ return str;
+}
+
static unsigned long
virNodeCountThreadSiblings(const char *dir, unsigned int cpu)
{
@@ -1313,9 +1375,14 @@ nodeCapsInitNUMA(virCapsPtr caps)
int n;
unsigned long *mask = NULL;
unsigned long *allonesmask = NULL;
+ unsigned long long memory;
int *cpus = NULL;
int ret = -1;
int max_n_cpus = NUMA_MAX_N_CPUS;
+ char *str = NULL;
+ virBitmapPtr *threadSiblings = NULL;
+
+
if (numa_available() < 0)
return 0;
@@ -1343,29 +1410,56 @@ nodeCapsInitNUMA(virCapsPtr caps)
continue;
}
+ /* Detect the amount of memory in the numa cell */
+ memory = nodeGetCellMemory(n);
+ if (memory == 0)
+ goto cleanup;
+
for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++)
if (MASK_CPU_ISSET(mask, i))
ncpus++;
+ /* Create some memory for the array of cpus. */
if (VIR_ALLOC_N(cpus, ncpus) < 0)
goto cleanup;
- for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++)
- if (MASK_CPU_ISSET(mask, i))
+ /* Create some memory for the array of siblings. */
+ if (VIR_ALLOC_N(threadSiblings, ncpus) < 0)
+ goto cleanup;
+
+ for (ncpus = 0, i = 0 ; i < max_n_cpus ; i++) {
+ if (MASK_CPU_ISSET(mask, i)) {
+
+ /* Get the string of thread siblings for this cpu */
+ if (!(str = virNodeGetThreadSiblingsList(SYSFS_CPU_PATH, i)))
+ goto cleanup;
+
+ /* Convert the string to a bitmap to be stored */
+ if (virBitmapParse(str, 0, &threadSiblings[ncpus], max_n_cpus) < 0)
+ goto cleanup;
+
cpus[ncpus++] = i;
+ VIR_FREE(str);
+ }
+ }
if (virCapabilitiesAddHostNUMACell(caps,
n,
ncpus,
- cpus) < 0)
+ memory,
+ cpus,
+ threadSiblings) < 0)
goto cleanup;
+ VIR_FREE(threadSiblings);
VIR_FREE(cpus);
}
ret = 0;
cleanup:
+ VIR_FREE(str);
+ VIR_FREE(threadSiblings);
VIR_FREE(cpus);
VIR_FREE(mask);
VIR_FREE(allonesmask);
@@ -1441,6 +1535,54 @@ cleanup:
return freeMem;
}
+/**
+ * nodeGetCellMemory
+ * @cell: The number of the numa cell to get memory info for.
+ *
+ * Will call the numa_node_size64() function from libnuma to get
+ * the amount of total memory in bytes. It is then converted to
+ * KiB and returned.
+ *
+ * Returns 0 on failure, amount of memory in KiB on success.
+ */
+static unsigned long long
+nodeGetCellMemory(int cell)
+{
+ long long mem;
+ unsigned long long memKiB = 0;
+ int maxCell;
+
+ if (numa_available() < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ "%s", _("NUMA not supported on this host"));
+ goto cleanup;
+ }
+
+ /* Make sure the provided cell number is valid. */
+ maxCell = numa_max_node();
+ if (cell > maxCell) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("cell %d out of range (0-%d)"),
+ cell, maxCell);
+ goto cleanup;
+ }
+
+ /* Get the amount of memory(bytes) in the node */
+ mem = numa_node_size64(cell, NULL);
+ if (mem < 0) {
+ virReportError(VIR_ERR_INTERNAL_ERROR,
+ _("Failed to query NUMA total memory for node: %d"),
+ cell);
+ goto cleanup;
+ }
+
+ /* Convert the memory from bytes to KiB */
+ memKiB = mem >> 10;
+
+cleanup:
+ return memKiB;
+}
+
#else
int nodeCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) {
return 0;
@@ -1462,4 +1604,11 @@ unsigned long long nodeGetFreeMemory(virConnectPtr conn ATTRIBUTE_UNUSED)
_("NUMA memory information not available on this platform"));
return 0;
}
+
+static unsigned long long nodeGetCellMemory(int cell)
+{
+ virReportError(VIR_ERR_NO_SUPPORT, "%s",
+ _("NUMA memory information not available on this platform"));
+ return 0;
+}
#endif
diff --git a/src/test/test_driver.c b/src/test/test_driver.c
index c974e1a..9d715a0 100644
--- a/src/test/test_driver.c
+++ b/src/test/test_driver.c
@@ -174,7 +174,7 @@ testBuildCapabilities(virConnectPtr conn) {
for (i = 0; i < privconn->numCells; i++) {
if (virCapabilitiesAddHostNUMACell(caps, i, privconn->cells[i].numCpus,
- privconn->cells[i].cpus) < 0)
+ 0, privconn->cells[i].cpus, NULL) < 0)
goto no_memory;
}
diff --git a/src/xen/xend_internal.c b/src/xen/xend_internal.c
index 922c571..d18eecb 100644
--- a/src/xen/xend_internal.c
+++ b/src/xen/xend_internal.c
@@ -1167,7 +1167,9 @@ sexpr_to_xend_topology(const struct sexpr *root,
if (virCapabilitiesAddHostNUMACell(caps,
cell,
nb_cpus,
- cpuNums) < 0)
+ 0,
+ cpuNums,
+ NULL) < 0)
goto memory_error;
}
VIR_FREE(cpuNums);
diff --git a/tests/capabilityschemadata/caps-test3.xml b/tests/capabilityschemadata/caps-test3.xml
new file mode 100644
index 0000000..724ced4
--- /dev/null
+++ b/tests/capabilityschemadata/caps-test3.xml
@@ -0,0 +1,88 @@
+<capabilities>
+
+ <host>
+ <uuid>35383339-3134-5553-4531-30314e394a50</uuid>
+ <cpu>
+ <arch>x86_64</arch>
+ <model>Westmere</model>
+ <vendor>Intel</vendor>
+ <topology sockets='1' cores='6' threads='2'/>
+ <feature name='rdtscp'/>
+ <feature name='pdpe1gb'/>
+ <feature name='dca'/>
+ <feature name='pdcm'/>
+ <feature name='xtpr'/>
+ <feature name='tm2'/>
+ <feature name='est'/>
+ <feature name='smx'/>
+ <feature name='vmx'/>
+ <feature name='ds_cpl'/>
+ <feature name='monitor'/>
+ <feature name='dtes64'/>
+ <feature name='pclmuldq'/>
+ <feature name='pbe'/>
+ <feature name='tm'/>
+ <feature name='ht'/>
+ <feature name='ss'/>
+ <feature name='acpi'/>
+ <feature name='ds'/>
+ <feature name='vme'/>
+ </cpu>
+ <power_management>
+ <suspend_disk/>
+ </power_management>
+ <migration_features>
+ <live/>
+ <uri_transports>
+ <uri_transport>tcp</uri_transport>
+ </uri_transports>
+ </migration_features>
+ <topology>
+ <cells num='2'>
+ <cell id='0'>
+ <memory unit='KiB'>12572412</memory>
+ <cpus num='12'>
+ <cpu id='0' thread_siblings='0,12'/>
+ <cpu id='2' thread_siblings='2,14'/>
+ <cpu id='4' thread_siblings='4,16'/>
+ <cpu id='6' thread_siblings='6,18'/>
+ <cpu id='8' thread_siblings='8,20'/>
+ <cpu id='10' thread_siblings='10,22'/>
+ <cpu id='12' thread_siblings='0,12'/>
+ <cpu id='14' thread_siblings='2,14'/>
+ <cpu id='16' thread_siblings='4,16'/>
+ <cpu id='18' thread_siblings='6,18'/>
+ <cpu id='20' thread_siblings='8,20'/>
+ <cpu id='22' thread_siblings='10,22'/>
+ </cpus>
+ </cell>
+ <cell id='1'>
+ <memory unit='KiB'>12582908</memory>
+ <cpus num='12'>
+ <cpu id='1' thread_siblings='1,13'/>
+ <cpu id='3' thread_siblings='3,15'/>
+ <cpu id='5' thread_siblings='5,17'/>
+ <cpu id='7' thread_siblings='7,19'/>
+ <cpu id='9' thread_siblings='9,21'/>
+ <cpu id='11' thread_siblings='11,23'/>
+ <cpu id='13' thread_siblings='1,13'/>
+ <cpu id='15' thread_siblings='3,15'/>
+ <cpu id='17' thread_siblings='5,17'/>
+ <cpu id='19' thread_siblings='7,19'/>
+ <cpu id='21' thread_siblings='9,21'/>
+ <cpu id='23' thread_siblings='11,23'/>
+ </cpus>
+ </cell>
+ </cells>
+ </topology>
+ <secmodel>
+ <model>none</model>
+ <doi>0</doi>
+ </secmodel>
+ <secmodel>
+ <model>dac</model>
+ <doi>0</doi>
+ </secmodel>
+ </host>
+
+</capabilities>
--
1.7.11.7
More information about the libvir-list
mailing list