[libvirt] PATCH: Add NUMA info to QEMU driver

Daniel P. Berrange berrange at redhat.com
Thu May 15 21:10:38 UTC 2008


This patch includes NUMA topology info in the QEMU driver capabilities 
XML output. It also implements the free memory driver APIs. This is done
with the LGPL'd  numactl library. The configure script probes for it and
only enables this functionality if it is found. The numactl library has
been around for quite a while - RHEL-3 vintage at least


 configure.in      |   39 ++++++++++++++++++++++++++++++
 libvirt.spec.in   |    3 ++
 src/Makefile.am   |    2 +
 src/qemu_conf.c   |   69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 src/qemu_driver.c |   65 ++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 178 insertions(+)

Regards,
Daniel

Index: src/qemu_conf.c
===================================================================
RCS file: /data/cvs/libvirt/src/qemu_conf.c,v
retrieving revision 1.64
diff -u -p -r1.64 qemu_conf.c
--- src/qemu_conf.c	15 May 2008 20:07:34 -0000	1.64
+++ src/qemu_conf.c	15 May 2008 21:07:42 -0000
@@ -42,6 +42,10 @@
 #include <libxml/xpath.h>
 #include <libxml/uri.h>
 
+#if HAVE_NUMACTL
+#include <numa.h>
+#endif
+
 #include "libvirt/virterror.h"
 
 #include "qemu_conf.h"
@@ -49,6 +53,7 @@
 #include "buf.h"
 #include "conf.h"
 #include "util.h"
+#include "memory.h"
 #include <verify.h>
 
 #define qemudLog(level, msg...) fprintf(stderr, msg)
@@ -389,6 +394,67 @@ qemudCapsInitGuest(virCapsPtr caps,
     return 0;
 }
 
+#if HAVE_NUMACTL
+#define MAX_CPUS 4096
+#define MAX_CPUS_MASK_SIZE (sizeof(unsigned long))
+#define MAX_CPUS_MASK_LEN (MAX_CPUS / MAX_CPUS_MASK_SIZE)
+#define MAX_CPUS_MASK_BYTES (MAX_CPUS / 8)
+static int
+qemudCapsInitNUMA(virCapsPtr caps)
+{
+    int n, i;
+    unsigned long *mask = NULL;
+    int ncpus;
+    int *cpus = NULL;
+    int ret = -1;
+
+    fprintf(stderr, "Add numa\n");
+
+    if (numa_available() < 0)
+        return 0;
+
+    fprintf(stderr, "Start\n");
+    if (VIR_ALLOC_N(mask, MAX_CPUS_MASK_LEN) < 0)
+        goto cleanup;
+
+    for (n = 0 ; n <= numa_max_node() ; n++) {
+        fprintf(stderr, "Do node %d\n", n);
+
+        if (numa_node_to_cpus(n, mask, MAX_CPUS_MASK_BYTES) < 0)
+            goto cleanup;
+
+        for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++)
+            if ((mask[(i / MAX_CPUS_MASK_SIZE)] >> (i % MAX_CPUS_MASK_SIZE)) & 1)
+                ncpus++;
+
+        if (VIR_ALLOC_N(cpus, ncpus) < 0)
+            goto cleanup;
+
+        for (ncpus = 0, i = 0 ; i < MAX_CPUS ; i++)
+            if ((mask[(i / MAX_CPUS_MASK_SIZE)] >> (i % MAX_CPUS_MASK_SIZE)) & 1)
+                cpus[ncpus++] = i;
+
+        fprintf(stderr, "Do node %d %d\n", n, ncpus);
+        if (virCapabilitiesAddHostNUMACell(caps,
+                                           n,
+                                           ncpus,
+                                           cpus) < 0)
+            goto cleanup;
+
+        VIR_FREE(cpus);
+    }
+
+    ret = 0;
+
+cleanup:
+    VIR_FREE(cpus);
+    VIR_FREE(mask);
+    return ret;
+}
+#else
+static int qemudCapsInitNUMA(virCapsPtr caps ATTRIBUTE_UNUSED) { return 0; }
+#endif
+
 virCapsPtr qemudCapsInit(void) {
     struct utsname utsname;
     virCapsPtr caps;
@@ -401,6 +467,9 @@ virCapsPtr qemudCapsInit(void) {
                                    0, 0)) == NULL)
         goto no_memory;
 
+    if (qemudCapsInitNUMA(caps) < 0)
+        goto no_memory;
+
     for (i = 0 ; i < (sizeof(arch_info_hvm)/sizeof(arch_info_hvm[0])) ; i++)
         if (qemudCapsInitGuest(caps,
                                utsname.machine,
Index: src/qemu_driver.c
===================================================================
RCS file: /data/cvs/libvirt/src/qemu_driver.c,v
retrieving revision 1.74
diff -u -p -r1.74 qemu_driver.c
--- src/qemu_driver.c	15 May 2008 16:11:40 -0000	1.74
+++ src/qemu_driver.c	15 May 2008 21:07:50 -0000
@@ -47,6 +47,10 @@
 #include <sys/wait.h>
 #include <libxml/uri.h>
 
+#if HAVE_NUMACTL
+#include <numa.h>
+#endif
+
 #include "libvirt/virterror.h"
 
 #include "event.h"
@@ -1605,6 +1609,62 @@ static char *qemudGetCapabilities(virCon
 }
 
 
+#if HAVE_NUMACTL
+static int
+qemudNodeGetCellsFreeMemory(virConnectPtr conn,
+                            unsigned long long *freeMems,
+                            int startCell,
+                            int maxCells)
+{
+    int n, lastCell, numCells;
+    fprintf(stderr, "Foo  %d %d\n", startCell, maxCells);
+    if (numa_available() < 0) {
+        qemudReportError(conn, NULL, NULL, VIR_ERR_NO_SUPPORT,
+                         "%s", _("NUMA not supported on this host"));
+        return -1;
+    }
+    lastCell = startCell + maxCells - 1;
+    if (lastCell > numa_max_node())
+        lastCell = numa_max_node();
+
+    for (numCells = 0, n = startCell ; n <= lastCell ; n++) {
+        long long mem;
+        if (numa_node_size64(n, &mem) < 0) {
+            qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                             "%s", _("Failed to query NUMA free memory"));
+            return -1;
+        }
+        fprintf(stderr, "baro %d %llu\n", n, mem);
+        freeMems[numCells++] = mem;
+    }
+    return numCells;
+}
+
+static unsigned long long
+qemudNodeGetFreeMemory (virConnectPtr conn)
+{
+    unsigned long long freeMem = 0;
+    int n;
+    if (numa_available() < 0) {
+        qemudReportError(conn, NULL, NULL, VIR_ERR_NO_SUPPORT,
+                         "%s", _("NUMA not supported on this host"));
+        return -1;
+    }
+
+    for (n = 0 ; n <= numa_max_node() ; n++) {
+        long long mem;
+        if (numa_node_size64(n, &mem) < 0) {
+            qemudReportError(conn, NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                             "%s", _("Failed to query NUMA free memory"));
+            return -1;
+        }
+        freeMem += mem;
+    }
+
+    return freeMem;
+}
+
+#endif
 
 static int qemudGetProcessInfo(unsigned long long *cpuTime, int pid) {
     char proc[PATH_MAX];
@@ -3168,8 +3228,13 @@ static virDriver qemuDriver = {
     NULL, /* domainMigrateFinish */
     qemudDomainBlockStats, /* domainBlockStats */
     qemudDomainInterfaceStats, /* domainInterfaceStats */
+#if HAVE_NUMACTL
+    qemudNodeGetCellsFreeMemory, /* nodeGetCellsFreeMemory */
+    qemudNodeGetFreeMemory,  /* getFreeMemory */
+#else
     NULL, /* nodeGetCellsFreeMemory */
     NULL, /* getFreeMemory */
+#endif
 };
 
 static virNetworkDriver qemuNetworkDriver = {
Index: src/Makefile.am
===================================================================
RCS file: /data/cvs/libvirt/src/Makefile.am,v
retrieving revision 1.79
diff -u -p -r1.79 Makefile.am
--- src/Makefile.am	29 Apr 2008 15:38:13 -0000	1.79
+++ src/Makefile.am	15 May 2008 21:07:57 -0000
@@ -9,6 +9,7 @@ INCLUDES = \
 	   $(GNUTLS_CFLAGS) \
 	   $(SASL_CFLAGS) \
 	   $(SELINUX_CFLAGS) \
+	   $(NUMACTL_CFLAGS) \
 	   -DBINDIR=\""$(libexecdir)"\" \
 	   -DSBINDIR=\""$(sbindir)"\" \
 	   -DSYSCONF_DIR="\"$(sysconfdir)\"" \
@@ -100,6 +101,7 @@ endif
 
 libvirt_la_SOURCES = $(CLIENT_SOURCES) $(SERVER_SOURCES)
 libvirt_la_LIBADD = $(LIBXML_LIBS) $(GNUTLS_LIBS) $(SASL_LIBS) $(SELINUX_LIBS) \
+                    $(NUMACTL_LIBS) \
 		    @CYGWIN_EXTRA_LIBADD@ ../gnulib/lib/libgnu.la
 libvirt_la_LDFLAGS = -Wl,--version-script=$(srcdir)/libvirt_sym.version \
                      -version-info @LIBVIRT_VERSION_INFO@ \
Index: configure.in
===================================================================
RCS file: /data/cvs/libvirt/configure.in,v
retrieving revision 1.143
diff -u -p -r1.143 configure.in
--- configure.in	5 May 2008 19:58:56 -0000	1.143
+++ configure.in	15 May 2008 21:08:05 -0000
@@ -534,6 +534,40 @@ AM_CONDITIONAL(HAVE_SELINUX, [test "$wit
 AC_SUBST(SELINUX_CFLAGS)
 AC_SUBST(SELINUX_LIBS)
 
+dnl NUMA lib
+AC_ARG_WITH(numactl,
+  [  --with-numactl         use numactl for host topology info],
+  [],
+  [with_numactl=check])
+
+NUMACTL_CFLAGS=
+NUMACTL_LIBS=
+if test "$with_qemu" = "yes" -a "$with_numactl" != "no"; then
+  old_cflags="$CFLAGS"
+  old_libs="$LIBS"
+  if test "$with_numactl" = "check"; then
+    AC_CHECK_HEADER([numa.h],[],[with_numactl=no])
+    AC_CHECK_LIB(numa, numa_available,[],[with_numactl=no])
+    if test "$with_numactl" != "no"; then
+      with_numactl="yes"
+    fi
+  else
+    AC_CHECK_HEADER([numa.h],[],
+       [AC_MSG_ERROR([You must install the numactl development package in order to compile libvirt])])
+    AC_CHECK_LIB(numa, numa_available,[],
+       [AC_MSG_ERROR([You must install the numactl development package in order to compile and run libvirt])])
+  fi
+  CFLAGS="$old_cflags"
+  LIBS="$old_libs"
+fi
+if test "$with_numactl" = "yes"; then
+  NUMACTL_LIBS="-lnuma"
+  AC_DEFINE_UNQUOTED(HAVE_NUMACTL, 1, [whether Numactl is available for security])
+fi
+AM_CONDITIONAL(HAVE_NUMACTL, [test "$with_numactl" != "no"])
+AC_SUBST(NUMACTL_CFLAGS)
+AC_SUBST(NUMACTL_LIBS)
+
 dnl virsh libraries
 AC_CHECK_HEADERS([readline/readline.h])
 
@@ -1001,6 +1035,11 @@ AC_MSG_NOTICE([  selinux: $SELINUX_CFLAG
 else
 AC_MSG_NOTICE([  selinux: no])
 fi
+if test "$with_numactl" = "yes" ; then
+AC_MSG_NOTICE([  numactl: $NUMACTL_CFLAGS $NUMACTL_LIBS])
+else
+AC_MSG_NOTICE([  numactl: no])
+fi
 AC_MSG_NOTICE([])
 AC_MSG_NOTICE([Miscellaneous])
 AC_MSG_NOTICE([])
Index: libvirt.spec.in
===================================================================
RCS file: /data/cvs/libvirt/libvirt.spec.in,v
retrieving revision 1.83
diff -u -p -r1.83 libvirt.spec.in
--- libvirt.spec.in	8 Apr 2008 16:45:57 -0000	1.83
+++ libvirt.spec.in	15 May 2008 21:08:14 -0000
@@ -67,6 +67,9 @@ BuildRequires: dnsmasq
 BuildRequires: bridge-utils
 BuildRequires: qemu
 BuildRequires: cyrus-sasl-devel
+%if %{with_qemu}
+BuildRequires: numactl-devel
+%endif
 %if %{with_polkit}
 BuildRequires: PolicyKit-devel >= 0.6
 %endif

-- 
|: Red Hat, Engineering, Boston   -o-   http://people.redhat.com/berrange/ :|
|: http://libvirt.org  -o-  http://virt-manager.org  -o-  http://ovirt.org :|
|: http://autobuild.org       -o-         http://search.cpan.org/~danberr/ :|
|: GnuPG: 7D3B9505  -o-  F3C9 553F A1DA 4AC2 5648 23C1 B3DF F742 7D3B 9505 :|




More information about the libvir-list mailing list