[PATCH 1/2] util: Separate numactl and stubs into different source files

Daniel P. Berrangé berrange at redhat.com
Mon Mar 20 11:50:16 UTC 2023


On Mon, Mar 20, 2023 at 12:39:55PM +0100, Martin Kletzander wrote:
> Fix the build, mainly with clang's optimizations and demonstrate a
> separation of function implementations.  This patch does it only for
> functions that differ in implementation based on WITH_NUMACTL but
> ideally this would be done for the WITH_NUMAD and __linux__ parts as
> well.

I don't think we should do this, as it is addressing just one specific
example that happens to hit us today. We've hit thue same problem a few
years ago in different functions.

The problem with interprocedural analysis is a general one that can affect
any part of the code base. I don't think we want wait to hit the problem
again in yet another piece of the code in the future, as it is really
horrible to debug and diagnose.

The use of the -fsemantic-interposition arg gives us a general solution
to the problem with CLang for the long term, that makes our mocking of
functions reliable. 

> 
> Signed-off-by: Martin Kletzander <mkletzan at redhat.com>
> ---
>  po/POTFILES                |   2 +
>  src/util/meson.build       |  10 +
>  src/util/virnuma.c         | 427 +------------------------------------
>  src/util/virnuma_common.h  |  57 +++++
>  src/util/virnuma_numactl.c | 357 +++++++++++++++++++++++++++++++
>  src/util/virnuma_stubs.c   | 124 +++++++++++
>  6 files changed, 551 insertions(+), 426 deletions(-)
>  create mode 100644 src/util/virnuma_common.h
>  create mode 100644 src/util/virnuma_numactl.c
>  create mode 100644 src/util/virnuma_stubs.c
> 
> diff --git a/po/POTFILES b/po/POTFILES
> index fa769a8a954f..da81806c86ac 100644
> --- a/po/POTFILES
> +++ b/po/POTFILES
> @@ -304,6 +304,8 @@ src/util/virnetdevvportprofile.c
>  src/util/virnetlink.c
>  src/util/virnodesuspend.c
>  src/util/virnuma.c
> +src/util/virnuma_numactl.c
> +src/util/virnuma_stubs.c
>  src/util/virnvme.c
>  src/util/virobject.c
>  src/util/virpci.c
> diff --git a/src/util/meson.build b/src/util/meson.build
> index c81500ea04af..c80237cf8f50 100644
> --- a/src/util/meson.build
> +++ b/src/util/meson.build
> @@ -111,6 +111,16 @@ util_sources = [
>    'virxml.c',
>  ]
>  
> +if conf.has('WITH_NUMACTL')
> +  util_sources += [
> +    'virnuma_numactl.c',
> +  ]
> +else
> +  util_sources += [
> +    'virnuma_stubs.c',
> +  ]
> +endif
> +
>  util_public_sources = files(
>    'virerror.c',
>    'virevent.c',
> diff --git a/src/util/virnuma.c b/src/util/virnuma.c
> index 4a15bf32c83f..3abe15b16880 100644
> --- a/src/util/virnuma.c
> +++ b/src/util/virnuma.c
> @@ -21,23 +21,11 @@
>  
>  #include <config.h>
>  
> -#define NUMA_MAX_N_CPUS 4096
> -
> -#if WITH_NUMACTL
> -# define NUMA_VERSION1_COMPATIBILITY 1
> -# include <numa.h>
> -
> -# if LIBNUMA_API_VERSION > 1
> -#  undef NUMA_MAX_N_CPUS
> -#  define NUMA_MAX_N_CPUS (numa_all_cpus_ptr->size)
> -# endif
> -
> -#endif /* WITH_NUMACTL */
> -
>  #include <sys/types.h>
>  #include <dirent.h>
>  
>  #include "virnuma.h"
> +#include "virnuma_common.h"
>  #include "vircommand.h"
>  #include "virerror.h"
>  #include "virlog.h"
> @@ -87,419 +75,6 @@ virNumaGetAutoPlacementAdvice(unsigned short vcpus G_GNUC_UNUSED,
>  }
>  #endif /* !WITH_NUMAD */
>  
> -#if WITH_NUMACTL
> -int
> -virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode,
> -                         virBitmap *nodeset)
> -{
> -    nodemask_t mask;
> -    int bit = 0;
> -    size_t i;
> -    int maxnode = 0;
> -
> -    if (!nodeset)
> -        return 0;
> -
> -    if (!virNumaNodesetIsAvailable(nodeset))
> -        return -1;
> -
> -    maxnode = numa_max_node();
> -    maxnode = maxnode < NUMA_NUM_NODES ? maxnode : NUMA_NUM_NODES;
> -
> -    /* Convert nodemask to NUMA bitmask. */
> -    nodemask_zero(&mask);
> -    bit = -1;
> -    while ((bit = virBitmapNextSetBit(nodeset, bit)) >= 0) {
> -        if (bit > maxnode) {
> -            virReportError(VIR_ERR_INTERNAL_ERROR,
> -                           _("NUMA node %d is out of range"), bit);
> -            return -1;
> -        }
> -        nodemask_set(&mask, bit);
> -    }
> -
> -    switch (mode) {
> -    case VIR_DOMAIN_NUMATUNE_MEM_STRICT:
> -        numa_set_bind_policy(1);
> -        numa_set_membind(&mask);
> -        numa_set_bind_policy(0);
> -        break;
> -
> -    case VIR_DOMAIN_NUMATUNE_MEM_PREFERRED:
> -    {
> -# ifdef WITH_NUMACTL_SET_PREFERRED_MANY
> -        struct bitmask *bitmask = NULL;
> -# endif
> -        int G_GNUC_UNUSED node = -1;
> -        int nnodes = 0;
> -        bool has_preferred_many = false;
> -
> -# ifdef WITH_NUMACTL_SET_PREFERRED_MANY
> -        if (numa_has_preferred_many() > 0) {
> -            has_preferred_many = true;
> -        }
> -# endif
> -
> -        for (i = 0; i < NUMA_NUM_NODES; i++) {
> -            if (nodemask_isset(&mask, i)) {
> -                node = i;
> -                nnodes++;
> -            }
> -        }
> -
> -        if (!has_preferred_many && nnodes != 1) {
> -            virReportError(VIR_ERR_INTERNAL_ERROR,
> -                           "%s", _("NUMA memory tuning in 'preferred' mode "
> -                                   "only supports single node"));
> -            return -1;
> -        }
> -
> -        /* The following automatically sets MPOL_PREFERRED_MANY
> -         * whenever possible, so no need to special case it. */
> -        numa_set_bind_policy(0);
> -
> -# ifdef WITH_NUMACTL_SET_PREFERRED_MANY
> -        bitmask = numa_bitmask_alloc(maxnode + 1);
> -        copy_nodemask_to_bitmask(&mask, bitmask);
> -        numa_set_preferred_many(bitmask);
> -        numa_bitmask_free(bitmask);
> -# else
> -        numa_set_preferred(node);
> -# endif
> -    }
> -    break;
> -
> -    case VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE:
> -        numa_set_interleave_mask(&mask);
> -        break;
> -
> -    case VIR_DOMAIN_NUMATUNE_MEM_RESTRICTIVE:
> -        break;
> -
> -    case VIR_DOMAIN_NUMATUNE_MEM_LAST:
> -        break;
> -    }
> -
> -    return 0;
> -}
> -
> -bool
> -virNumaIsAvailable(void)
> -{
> -    return numa_available() != -1;
> -}
> -
> -
> -/**
> - * virNumaGetMaxNode:
> - * Get the highest node number available on the current system.
> - * (See the node numbers in /sys/devices/system/node/ ).
> - *
> - * Returns the highest NUMA node id on success, -1 on error.
> - */
> -int
> -virNumaGetMaxNode(void)
> -{
> -    int ret;
> -
> -    if (!virNumaIsAvailable()) {
> -        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> -                       _("NUMA isn't available on this host"));
> -        return -1;
> -    }
> -
> -    if ((ret = numa_max_node()) < 0) {
> -        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> -                       _("Failed to request maximum NUMA node id"));
> -        return -1;
> -    }
> -
> -    return ret;
> -}
> -
> -
> -/**
> - * virNumaGetNodeMemory:
> - * @node: identifier of the requested NUMA node
> - * @memsize: returns the total size of memory in the NUMA node
> - * @memfree: returns the total free memory in a NUMA node
> - *
> - * Returns the size of the memory in one NUMA node in bytes via the @size
> - * argument and free memory of a node in the @free argument.  The caller has to
> - * guarantee that @node is in range (see virNumaGetMaxNode).
> - *
> - * Returns 0 on success, -1 on error. Does not report errors.
> - */
> -int
> -virNumaGetNodeMemory(int node,
> -                     unsigned long long *memsize,
> -                     unsigned long long *memfree)
> -{
> -    long long node_size;
> -    long long node_free;
> -
> -    if (memsize)
> -        *memsize = 0;
> -
> -    if (memfree)
> -        *memfree = 0;
> -
> -    if ((node_size = numa_node_size64(node, &node_free)) < 0)
> -        return -1;
> -
> -    if (memsize)
> -        *memsize = node_size;
> -
> -    if (memfree)
> -        *memfree = node_free;
> -
> -    return 0;
> -}
> -
> -
> -/**
> - * virNumaGetNodeCPUs:
> - * @node: identifier of the requested NUMA node
> - * @cpus: returns a bitmap of CPUs in @node
> - *
> - * Returns count of CPUs in the selected node and sets the map of the cpus to
> - * @cpus. On error if the @node doesn't exist in the system this function
> - * returns -2 and sets @cpus to NULL. On other errors -1 is returned, @cpus
> - * is set to NULL and an error is reported.
> - */
> -
> -# define n_bits(var) (8 * sizeof(var))
> -# define MASK_CPU_ISSET(mask, cpu) \
> -  (((mask)[((cpu) / n_bits(*(mask)))] >> ((cpu) % n_bits(*(mask)))) & 1)
> -int
> -virNumaGetNodeCPUs(int node,
> -                   virBitmap **cpus)
> -{
> -    int ncpus = 0;
> -    int max_n_cpus = virNumaGetMaxCPUs();
> -    int mask_n_bytes = max_n_cpus / 8;
> -    size_t i;
> -    g_autofree unsigned long *mask = NULL;
> -    g_autoptr(virBitmap) cpumap = NULL;
> -
> -    *cpus = NULL;
> -
> -    if (!virNumaNodeIsAvailable(node)) {
> -        VIR_DEBUG("NUMA topology for cell %d is not available, ignoring", node);
> -        return -2;
> -    }
> -
> -    mask = g_new0(unsigned long, mask_n_bytes / sizeof(*mask));
> -
> -    if (numa_node_to_cpus(node, mask, mask_n_bytes) < 0) {
> -        VIR_WARN("NUMA topology for cell %d is not available, ignoring", node);
> -        return -2;
> -    }
> -
> -    cpumap = virBitmapNew(max_n_cpus);
> -
> -    for (i = 0; i < max_n_cpus; i++) {
> -        if (MASK_CPU_ISSET(mask, i)) {
> -            ignore_value(virBitmapSetBit(cpumap, i));
> -            ncpus++;
> -        }
> -    }
> -
> -    *cpus = g_steal_pointer(&cpumap);
> -    return ncpus;
> -}
> -# undef MASK_CPU_ISSET
> -# undef n_bits
> -
> -
> -/**
> - * virNumaGetNodeOfCPU:
> - * @cpu: CPU ID
> - *
> - * For given @cpu, return NUMA node which it belongs to.
> - *
> - * Returns: NUMA node # on success,
> - *          -1 on failure (with errno set).
> - */
> -int
> -virNumaGetNodeOfCPU(int cpu)
> -{
> -    return numa_node_of_cpu(cpu);
> -}
> -
> -#else /* !WITH_NUMACTL */
> -
> -int
> -virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode G_GNUC_UNUSED,
> -                         virBitmap *nodeset)
> -{
> -    if (!virNumaNodesetIsAvailable(nodeset))
> -        return -1;
> -
> -    return 0;
> -}
> -
> -bool
> -virNumaIsAvailable(void)
> -{
> -    return false;
> -}
> -
> -
> -int
> -virNumaGetMaxNode(void)
> -{
> -    virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> -                   _("NUMA isn't available on this host"));
> -    return -1;
> -}
> -
> -
> -int
> -virNumaGetNodeMemory(int node G_GNUC_UNUSED,
> -                     unsigned long long *memsize,
> -                     unsigned long long *memfree)
> -{
> -    if (memsize)
> -        *memsize = 0;
> -
> -    if (memfree)
> -        *memfree = 0;
> -
> -    VIR_DEBUG("NUMA isn't available on this host");
> -    return -1;
> -}
> -
> -
> -int
> -virNumaGetNodeCPUs(int node G_GNUC_UNUSED,
> -                   virBitmap **cpus)
> -{
> -    *cpus = NULL;
> -
> -    virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> -                   _("NUMA isn't available on this host"));
> -    return -1;
> -}
> -
> -int
> -virNumaGetNodeOfCPU(int cpu G_GNUC_UNUSED)
> -{
> -    errno = ENOSYS;
> -    return -1;
> -}
> -
> -
> -#endif /* !WITH_NUMACTL */
> -
> -/**
> - * virNumaGetMaxCPUs:
> - *
> - * Get the maximum count of CPUs supportable in the host.
> - *
> - * Returns the count of CPUs supported.
> - */
> -unsigned int
> -virNumaGetMaxCPUs(void)
> -{
> -    return NUMA_MAX_N_CPUS;
> -}
> -
> -
> -#if WITH_NUMACTL
> -/**
> - * virNumaNodeIsAvailable:
> - * @node: node to check
> - *
> - * On some hosts the set of NUMA nodes isn't continuous.
> - * Use this function to test if the @node is available.
> - *
> - * Returns: true if @node is available,
> - *          false if @node doesn't exist
> - */
> -bool
> -virNumaNodeIsAvailable(int node)
> -{
> -    return numa_bitmask_isbitset(numa_nodes_ptr, node);
> -}
> -
> -
> -/**
> - * virNumaGetDistances:
> - * @node: identifier of the requested NUMA node
> - * @distances: array of distances to sibling nodes
> - * @ndistances: size of @distances
> - *
> - * Get array of distances to sibling nodes from @node. If a
> - * distances[x] equals to zero, the node x is not enabled or
> - * doesn't exist. As a special case, if @node itself refers to
> - * disabled or nonexistent NUMA node, then @distances and
> - * @ndistances are set to NULL and zero respectively.
> - *
> - * The distances are a bit of magic. For a local node the value
> - * is 10, for remote it's typically 20 meaning that time penalty
> - * for accessing a remote node is two time bigger than when
> - * accessing a local node.
> - *
> - * Returns 0 on success, -1 otherwise.
> - */
> -int
> -virNumaGetDistances(int node,
> -                    int **distances,
> -                    int *ndistances)
> -{
> -    int max_node;
> -    size_t i;
> -
> -    if (!virNumaNodeIsAvailable(node)) {
> -        VIR_DEBUG("Node %d does not exist", node);
> -        *distances = NULL;
> -        *ndistances = 0;
> -        return 0;
> -    }
> -
> -    if ((max_node = virNumaGetMaxNode()) < 0)
> -        return -1;
> -
> -    *distances = g_new0(int, max_node + 1);
> -    *ndistances = max_node + 1;
> -
> -    for (i = 0; i <= max_node; i++) {
> -        if (!virNumaNodeIsAvailable(node))
> -            continue;
> -
> -        (*distances)[i] = numa_distance(node, i);
> -    }
> -
> -    return 0;
> -}
> -
> -#else /* !WITH_NUMACTL */
> -
> -bool
> -virNumaNodeIsAvailable(int node)
> -{
> -    int max_node = virNumaGetMaxNode();
> -
> -    if (max_node < 0)
> -        return false;
> -
> -    /* Do we have anything better? */
> -    return (node >= 0) && (node <= max_node);
> -}
> -
> -
> -int
> -virNumaGetDistances(int node G_GNUC_UNUSED,
> -                    int **distances,
> -                    int *ndistances)
> -{
> -    *distances = NULL;
> -    *ndistances = 0;
> -    VIR_DEBUG("NUMA distance information isn't available on this host");
> -    return 0;
> -}
> -#endif /* !WITH_NUMACTL */
>  
>  
>  /* currently all the huge page stuff below is linux only */
> diff --git a/src/util/virnuma_common.h b/src/util/virnuma_common.h
> new file mode 100644
> index 000000000000..2ba1a9344ca4
> --- /dev/null
> +++ b/src/util/virnuma_common.h
> @@ -0,0 +1,57 @@
> +/*
> + * virnuma_common.h: helper APIs for managing NUMA with varying backends
> + *
> + * Copyright (C) 2011-2014 Red Hat, Inc.
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library.  If not, see
> + * <http://www.gnu.org/licenses/>.
> + *
> + */
> +
> +#pragma once
> +
> +#include "internal.h"
> +#include "virbitmap.h"
> +
> +#define NUMA_MAX_N_CPUS 4096
> +
> +int
> +virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode,
> +                         virBitmap *nodeset);
> +
> +bool
> +virNumaIsAvailable(void);
> +
> +int
> +virNumaGetMaxNode(void);
> +
> +int
> +virNumaGetNodeMemory(int node,
> +                     unsigned long long *memsize,
> +                     unsigned long long *memfree);
> +
> +int
> +virNumaGetNodeCPUs(int node,
> +                   virBitmap **cpus);
> +
> +int
> +virNumaGetNodeOfCPU(int cpu);
> +
> +bool
> +virNumaNodeIsAvailable(int node);
> +
> +int
> +virNumaGetDistances(int node,
> +                    int **distances,
> +                    int *ndistances);
> diff --git a/src/util/virnuma_numactl.c b/src/util/virnuma_numactl.c
> new file mode 100644
> index 000000000000..223c2e3e56db
> --- /dev/null
> +++ b/src/util/virnuma_numactl.c
> @@ -0,0 +1,357 @@
> +/*
> + * virnuma_numactl.c: NUMA functions for builds with numactl
> + *
> + * Copyright (C) 2011-2014 Red Hat, Inc.
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library.  If not, see
> + * <http://www.gnu.org/licenses/>.
> + *
> + */
> +
> +#include <config.h>
> +
> +#define NUMA_VERSION1_COMPATIBILITY 1
> +#include <numa.h>
> +
> +#include "virnuma_common.h"
> +#include "virnuma.h"
> +#include "virerror.h"
> +#include "virlog.h"
> +
> +#define VIR_FROM_THIS VIR_FROM_NONE
> +
> +VIR_LOG_INIT("util.numa");
> +
> +
> +int
> +virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode,
> +                         virBitmap *nodeset)
> +{
> +    nodemask_t mask;
> +    int bit = 0;
> +    size_t i;
> +    int maxnode = 0;
> +
> +    if (!nodeset)
> +        return 0;
> +
> +    if (!virNumaNodesetIsAvailable(nodeset))
> +        return -1;
> +
> +    maxnode = numa_max_node();
> +    maxnode = maxnode < NUMA_NUM_NODES ? maxnode : NUMA_NUM_NODES;
> +
> +    /* Convert nodemask to NUMA bitmask. */
> +    nodemask_zero(&mask);
> +    bit = -1;
> +    while ((bit = virBitmapNextSetBit(nodeset, bit)) >= 0) {
> +        if (bit > maxnode) {
> +            virReportError(VIR_ERR_INTERNAL_ERROR,
> +                           _("NUMA node %d is out of range"), bit);
> +            return -1;
> +        }
> +        nodemask_set(&mask, bit);
> +    }
> +
> +    switch (mode) {
> +    case VIR_DOMAIN_NUMATUNE_MEM_STRICT:
> +        numa_set_bind_policy(1);
> +        numa_set_membind(&mask);
> +        numa_set_bind_policy(0);
> +        break;
> +
> +    case VIR_DOMAIN_NUMATUNE_MEM_PREFERRED:
> +    {
> +#ifdef WITH_NUMACTL_SET_PREFERRED_MANY
> +        struct bitmask *bitmask = NULL;
> +#endif
> +        int G_GNUC_UNUSED node = -1;
> +        int nnodes = 0;
> +        bool has_preferred_many = false;
> +
> +#ifdef WITH_NUMACTL_SET_PREFERRED_MANY
> +        if (numa_has_preferred_many() > 0) {
> +            has_preferred_many = true;
> +        }
> +#endif
> +
> +        for (i = 0; i < NUMA_NUM_NODES; i++) {
> +            if (nodemask_isset(&mask, i)) {
> +                node = i;
> +                nnodes++;
> +            }
> +        }
> +
> +        if (!has_preferred_many && nnodes != 1) {
> +            virReportError(VIR_ERR_INTERNAL_ERROR,
> +                           "%s", _("NUMA memory tuning in 'preferred' mode "
> +                                   "only supports single node"));
> +            return -1;
> +        }
> +
> +        /* The following automatically sets MPOL_PREFERRED_MANY
> +         * whenever possible, so no need to special case it. */
> +        numa_set_bind_policy(0);
> +
> +#ifdef WITH_NUMACTL_SET_PREFERRED_MANY
> +        bitmask = numa_bitmask_alloc(maxnode + 1);
> +        copy_nodemask_to_bitmask(&mask, bitmask);
> +        numa_set_preferred_many(bitmask);
> +        numa_bitmask_free(bitmask);
> +#else
> +        numa_set_preferred(node);
> +#endif
> +    }
> +    break;
> +
> +    case VIR_DOMAIN_NUMATUNE_MEM_INTERLEAVE:
> +        numa_set_interleave_mask(&mask);
> +        break;
> +
> +    case VIR_DOMAIN_NUMATUNE_MEM_RESTRICTIVE:
> +        break;
> +
> +    case VIR_DOMAIN_NUMATUNE_MEM_LAST:
> +        break;
> +    }
> +
> +    return 0;
> +}
> +
> +bool
> +virNumaIsAvailable(void)
> +{
> +    return numa_available() != -1;
> +}
> +
> +
> +/**
> + * virNumaGetMaxNode:
> + * Get the highest node number available on the current system.
> + * (See the node numbers in /sys/devices/system/node/ ).
> + *
> + * Returns the highest NUMA node id on success, -1 on error.
> + */
> +int
> +virNumaGetMaxNode(void)
> +{
> +    int ret;
> +
> +    if (!virNumaIsAvailable()) {
> +        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> +                       _("NUMA isn't available on this host"));
> +        return -1;
> +    }
> +
> +    if ((ret = numa_max_node()) < 0) {
> +        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> +                       _("Failed to request maximum NUMA node id"));
> +        return -1;
> +    }
> +
> +    return ret;
> +}
> +
> +
> +/**
> + * virNumaGetNodeMemory:
> + * @node: identifier of the requested NUMA node
> + * @memsize: returns the total size of memory in the NUMA node
> + * @memfree: returns the total free memory in a NUMA node
> + *
> + * Returns the size of the memory in one NUMA node in bytes via the @size
> + * argument and free memory of a node in the @free argument.  The caller has to
> + * guarantee that @node is in range (see virNumaGetMaxNode).
> + *
> + * Returns 0 on success, -1 on error. Does not report errors.
> + */
> +int
> +virNumaGetNodeMemory(int node,
> +                     unsigned long long *memsize,
> +                     unsigned long long *memfree)
> +{
> +    long long node_size;
> +    long long node_free;
> +
> +    if (memsize)
> +        *memsize = 0;
> +
> +    if (memfree)
> +        *memfree = 0;
> +
> +    if ((node_size = numa_node_size64(node, &node_free)) < 0)
> +        return -1;
> +
> +    if (memsize)
> +        *memsize = node_size;
> +
> +    if (memfree)
> +        *memfree = node_free;
> +
> +    return 0;
> +}
> +
> +
> +/**
> + * virNumaGetNodeCPUs:
> + * @node: identifier of the requested NUMA node
> + * @cpus: returns a bitmap of CPUs in @node
> + *
> + * Returns count of CPUs in the selected node and sets the map of the cpus to
> + * @cpus. On error if the @node doesn't exist in the system this function
> + * returns -2 and sets @cpus to NULL. On other errors -1 is returned, @cpus
> + * is set to NULL and an error is reported.
> + */
> +
> +#define n_bits(var) (8 * sizeof(var))
> +#define MASK_CPU_ISSET(mask, cpu) \
> +  (((mask)[((cpu) / n_bits(*(mask)))] >> ((cpu) % n_bits(*(mask)))) & 1)
> +int
> +virNumaGetNodeCPUs(int node,
> +                   virBitmap **cpus)
> +{
> +    int ncpus = 0;
> +    int max_n_cpus = virNumaGetMaxCPUs();
> +    int mask_n_bytes = max_n_cpus / 8;
> +    size_t i;
> +    g_autofree unsigned long *mask = NULL;
> +    g_autoptr(virBitmap) cpumap = NULL;
> +
> +    *cpus = NULL;
> +
> +    if (!virNumaNodeIsAvailable(node)) {
> +        VIR_DEBUG("NUMA topology for cell %d is not available, ignoring", node);
> +        return -2;
> +    }
> +
> +    mask = g_new0(unsigned long, mask_n_bytes / sizeof(*mask));
> +
> +    if (numa_node_to_cpus(node, mask, mask_n_bytes) < 0) {
> +        VIR_WARN("NUMA topology for cell %d is not available, ignoring", node);
> +        return -2;
> +    }
> +
> +    cpumap = virBitmapNew(max_n_cpus);
> +
> +    for (i = 0; i < max_n_cpus; i++) {
> +        if (MASK_CPU_ISSET(mask, i)) {
> +            ignore_value(virBitmapSetBit(cpumap, i));
> +            ncpus++;
> +        }
> +    }
> +
> +    *cpus = g_steal_pointer(&cpumap);
> +    return ncpus;
> +}
> +#undef MASK_CPU_ISSET
> +#undef n_bits
> +
> +
> +/**
> + * virNumaGetNodeOfCPU:
> + * @cpu: CPU ID
> + *
> + * For given @cpu, return NUMA node which it belongs to.
> + *
> + * Returns: NUMA node # on success,
> + *          -1 on failure (with errno set).
> + */
> +int
> +virNumaGetNodeOfCPU(int cpu)
> +{
> +    return numa_node_of_cpu(cpu);
> +}
> +
> +/**
> + * virNumaGetMaxCPUs:
> + *
> + * Get the maximum count of CPUs supportable in the host.
> + *
> + * Returns the count of CPUs supported.
> + */
> +unsigned int
> +virNumaGetMaxCPUs(void)
> +{
> +#if LIBNUMA_API_VERSION > 1
> +    return numa_all_cpus_ptr->size;
> +#else
> +    return NUMA_MAX_N_CPUS;
> +#endif
> +}
> +
> +/**
> + * virNumaNodeIsAvailable:
> + * @node: node to check
> + *
> + * On some hosts the set of NUMA nodes isn't continuous.
> + * Use this function to test if the @node is available.
> + *
> + * Returns: true if @node is available,
> + *          false if @node doesn't exist
> + */
> +bool
> +virNumaNodeIsAvailable(int node)
> +{
> +    return numa_bitmask_isbitset(numa_nodes_ptr, node);
> +}
> +
> +/**
> + * virNumaGetDistances:
> + * @node: identifier of the requested NUMA node
> + * @distances: array of distances to sibling nodes
> + * @ndistances: size of @distances
> + *
> + * Get array of distances to sibling nodes from @node. If a
> + * distances[x] equals to zero, the node x is not enabled or
> + * doesn't exist. As a special case, if @node itself refers to
> + * disabled or nonexistent NUMA node, then @distances and
> + * @ndistances are set to NULL and zero respectively.
> + *
> + * The distances are a bit of magic. For a local node the value
> + * is 10, for remote it's typically 20 meaning that time penalty
> + * for accessing a remote node is two time bigger than when
> + * accessing a local node.
> + *
> + * Returns 0 on success, -1 otherwise.
> + */
> +int
> +virNumaGetDistances(int node,
> +                    int **distances,
> +                    int *ndistances)
> +{
> +    int max_node;
> +    size_t i;
> +
> +    if (!virNumaNodeIsAvailable(node)) {
> +        VIR_DEBUG("Node %d does not exist", node);
> +        *distances = NULL;
> +        *ndistances = 0;
> +        return 0;
> +    }
> +
> +    if ((max_node = virNumaGetMaxNode()) < 0)
> +        return -1;
> +
> +    *distances = g_new0(int, max_node + 1);
> +    *ndistances = max_node + 1;
> +
> +    for (i = 0; i <= max_node; i++) {
> +        if (!virNumaNodeIsAvailable(node))
> +            continue;
> +
> +        (*distances)[i] = numa_distance(node, i);
> +    }
> +
> +    return 0;
> +}
> diff --git a/src/util/virnuma_stubs.c b/src/util/virnuma_stubs.c
> new file mode 100644
> index 000000000000..384216bb6bf9
> --- /dev/null
> +++ b/src/util/virnuma_stubs.c
> @@ -0,0 +1,124 @@
> +/*
> + * virnuma_stubs.c: NUMA stub functions for builds without numactl
> + *
> + * Copyright (C) 2011-2014 Red Hat, Inc.
> + *
> + * This library is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library.  If not, see
> + * <http://www.gnu.org/licenses/>.
> + *
> + */
> +
> +#include <config.h>
> +
> +#include "virnuma_common.h"
> +#include "virnuma.h"
> +#include "virerror.h"
> +#include "virlog.h"
> +
> +#define VIR_FROM_THIS VIR_FROM_NONE
> +
> +VIR_LOG_INIT("util.numa");
> +
> +
> +int
> +virNumaSetupMemoryPolicy(virDomainNumatuneMemMode mode G_GNUC_UNUSED,
> +                         virBitmap *nodeset)
> +{
> +    if (!virNumaNodesetIsAvailable(nodeset))
> +        return -1;
> +
> +    return 0;
> +}
> +
> +bool
> +virNumaIsAvailable(void)
> +{
> +    return false;
> +}
> +
> +
> +int
> +virNumaGetMaxNode(void)
> +{
> +    virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> +                   _("NUMA isn't available on this host"));
> +    return -1;
> +}
> +
> +
> +int
> +virNumaGetNodeMemory(int node G_GNUC_UNUSED,
> +                     unsigned long long *memsize,
> +                     unsigned long long *memfree)
> +{
> +    if (memsize)
> +        *memsize = 0;
> +
> +    if (memfree)
> +        *memfree = 0;
> +
> +    VIR_DEBUG("NUMA isn't available on this host");
> +    return -1;
> +}
> +
> +
> +int
> +virNumaGetNodeCPUs(int node G_GNUC_UNUSED,
> +                   virBitmap **cpus)
> +{
> +    *cpus = NULL;
> +
> +    virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
> +                   _("NUMA isn't available on this host"));
> +    return -1;
> +}
> +
> +int
> +virNumaGetNodeOfCPU(int cpu G_GNUC_UNUSED)
> +{
> +    errno = ENOSYS;
> +    return -1;
> +}
> +
> +
> +bool
> +virNumaNodeIsAvailable(int node G_GNUC_UNUSED)
> +{
> +    return false;
> +}
> +
> +
> +int
> +virNumaGetDistances(int node G_GNUC_UNUSED,
> +                    int **distances,
> +                    int *ndistances)
> +{
> +    *distances = NULL;
> +    *ndistances = 0;
> +    VIR_DEBUG("NUMA distance information isn't available on this host");
> +    return 0;
> +}
> +
> +/**
> + * virNumaGetMaxCPUs:
> + *
> + * Get the maximum count of CPUs supportable in the host.
> + *
> + * Returns the count of CPUs supported.
> + */
> +unsigned int
> +virNumaGetMaxCPUs(void)
> +{
> +    return NUMA_MAX_N_CPUS;
> +}
> -- 
> 2.40.0
> 

With regards,
Daniel
-- 
|: https://berrange.com      -o-    https://www.flickr.com/photos/dberrange :|
|: https://libvirt.org         -o-            https://fstop138.berrange.com :|
|: https://entangle-photo.org    -o-    https://www.instagram.com/dberrange :|


More information about the libvir-list mailing list