[libvirt] [PATCH] Fix QEMU cpu affinity at startup to include all threads

[Daniel Veillard]

On Thu, Mar 25, 2010 at 05:27:16PM +0000, Daniel P. Berrange wrote:
> The QEMU cpu affinity is used in NUMA scenarios to ensure that
> guest memory is allocated from a specific node. Normally memory
> is allocate on demand in vCPU threads, but when using hugepages
> the initial thread leader allocates memory upfront. libvirt was
> not setting affinity of the thread leader, or I/O threads. This
> patch changes the code to set the process affinity in between
> the fork()/exec() of QEMU. This ensures that every single QEMU
> thread gets the affinity
> 
> * src/qemu/qemu_driver.c: Set affinity on entire QEMU process
>   at startup
> ---
>  src/qemu/qemu_driver.c |   29 ++++++++++++++++-------------
>  1 files changed, 16 insertions(+), 13 deletions(-)
> 
> diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
> index 257f914..2598deb 100644
> --- a/src/qemu/qemu_driver.c
> +++ b/src/qemu/qemu_driver.c
> @@ -1701,6 +1701,9 @@ qemuDetectVcpuPIDs(struct qemud_driver *driver,
>      return 0;
>  }
>  
> +/*
> + * To be run between fork/exec of QEMU only
> + */
>  static int
>  qemudInitCpuAffinity(virDomainObjPtr vm)
>  {
> @@ -1708,7 +1711,8 @@ qemudInitCpuAffinity(virDomainObjPtr vm)
>      virNodeInfo nodeinfo;
>      unsigned char *cpumap;
>      int cpumaplen;
> -    qemuDomainObjPrivatePtr priv = vm->privateData;
> +
> +    DEBUG0("Setting CPU affinity");
>  
>      if (nodeGetInfo(NULL, &nodeinfo) < 0)
>          return -1;
> @@ -1740,14 +1744,14 @@ qemudInitCpuAffinity(virDomainObjPtr vm)
>              VIR_USE_CPU(cpumap, i);
>      }
>  
> -    /* The XML config only gives a per-VM affinity, so we apply
> -     * the same mapping to all vCPUs */
> -    for (i = 0 ; i < priv->nvcpupids ; i++) {
> -        if (virProcessInfoSetAffinity(priv->vcpupids[i],
> -                                      cpumap, cpumaplen, maxcpu) < 0) {
> -            VIR_FREE(cpumap);
> -            return -1;
> -        }
> +    /* We are assuming we are running between fork/exec of QEMU, so
> +     * that getpid() gives the QEMU process ID and we know that
> +     * no threads are running.
> +     */
> +    if (virProcessInfoSetAffinity(getpid(),
> +                                  cpumap, cpumaplen, maxcpu) < 0) {
> +        VIR_FREE(cpumap);
> +        return -1;
>      }
>      VIR_FREE(cpumap);
>  
> @@ -2653,6 +2657,9 @@ struct qemudHookData {
>  static int qemudSecurityHook(void *data) {
>      struct qemudHookData *h = data;
>  
> +    if (qemudInitCpuAffinity(h->vm) < 0)
> +        return -1;
> +
>      if (qemuAddToCgroup(h->driver, h->vm->def) < 0)
>          return -1;
>  
> @@ -2943,10 +2950,6 @@ static int qemudStartVMDaemon(virConnectPtr conn,
>      if (qemuDetectVcpuPIDs(driver, vm) < 0)
>          goto abort;
>  
> -    DEBUG0("Setting CPU affinity");
> -    if (qemudInitCpuAffinity(vm) < 0)
> -        goto abort;
> -
>      DEBUG0("Setting any required VM passwords");
>      if (qemuInitPasswords(conn, driver, vm, qemuCmdFlags) < 0)
>          goto abort;

  ACK,

Daniel

-- 
Daniel Veillard      | libxml Gnome XML XSLT toolkit  http://xmlsoft.org/
daniel at veillard.com  | Rpmfind RPM search engine http://rpmfind.net/
http://veillard.com/ | virtualization library  http://libvirt.org/