[libvirt] [PATCH 1/2] Resctrl: Add some utils functions

Eli Qiao liyong.qiao at intel.com
Wed Jan 18 07:26:19 UTC 2017


This patch adds some utils struct and functions to expose resctrl
information.

virResCtrlAvailable: If resctrl interface exist on host
virResCtrlGet: get specify type resource contral information
virResCtrlInit: initialize resctrl struct from the host's sys fs.
ResCtrlAll[]: an array to maintain resource contral information.

Signed-off-by: Eli Qiao <liyong.qiao at intel.com>
---
 include/libvirt/virterror.h |   1 +
 src/Makefile.am             |   1 +
 src/libvirt_private.syms    |   4 +
 src/util/virerror.c         |   1 +
 src/util/virresctrl.c       | 500 ++++++++++++++++++++++++++++++++++++++++++++
 src/util/virresctrl.h       | 121 +++++++++++
 tests/Makefile.am           |   4 +
 7 files changed, 632 insertions(+)
 create mode 100644 src/util/virresctrl.c
 create mode 100644 src/util/virresctrl.h

diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h
index 2efee8f..3dd2d08 100644
--- a/include/libvirt/virterror.h
+++ b/include/libvirt/virterror.h
@@ -132,6 +132,7 @@ typedef enum {
 
     VIR_FROM_PERF = 65,         /* Error from perf */
     VIR_FROM_LIBSSH = 66,       /* Error from libssh connection transport */
+    VIR_FROM_RESCTRL = 67,      /* Error from resource control */
 
 # ifdef VIR_ENUM_SENTINELS
     VIR_ERR_DOMAIN_LAST
diff --git a/src/Makefile.am b/src/Makefile.am
index d84c984..4026f8d 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -162,6 +162,7 @@ UTIL_SOURCES =							\
 		util/virprocess.c util/virprocess.h		\
 		util/virqemu.c util/virqemu.h			\
 		util/virrandom.h util/virrandom.c		\
+		util/virresctrl.h util/virresctrl.c		\
 		util/virrotatingfile.h util/virrotatingfile.c   \
 		util/virscsi.c util/virscsi.h			\
 		util/virscsivhost.c util/virscsivhost.h		\
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index 2d23e46..fe1334d 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -2295,6 +2295,10 @@ virRandomGenerateWWN;
 virRandomInt;
 
 
+# util/virresctrl.h
+virResCtrlAvailable;
+virResCtrlInit;
+
 # util/virrotatingfile.h
 virRotatingFileReaderConsume;
 virRotatingFileReaderFree;
diff --git a/src/util/virerror.c b/src/util/virerror.c
index ef17fb5..93dfd4f 100644
--- a/src/util/virerror.c
+++ b/src/util/virerror.c
@@ -139,6 +139,7 @@ VIR_ENUM_IMPL(virErrorDomain, VIR_ERR_DOMAIN_LAST,
 
               "Perf", /* 65 */
               "Libssh transport layer",
+              "Rescouce Control",
     )
 
 
diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c
new file mode 100644
index 0000000..bcb47f1
--- /dev/null
+++ b/src/util/virresctrl.c
@@ -0,0 +1,500 @@
+/*
+ * virresctrl.c: methods for managing resource contral
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ *  Eli Qiao <liyong.qiao at intel.com>
+ */
+#include <config.h>
+
+#include <sys/ioctl.h>
+#if defined HAVE_SYS_SYSCALL_H
+# include <sys/syscall.h>
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "virresctrl.h"
+#include "viralloc.h"
+#include "virerror.h"
+#include "virfile.h"
+#include "virhostcpu.h"
+#include "virlog.h"
+#include "virstring.h"
+#include "nodeinfo.h"
+
+VIR_LOG_INIT("util.resctrl");
+
+#define VIR_FROM_THIS VIR_FROM_RESCTRL
+
+#define CONSTRUCT_RESCTRL_PATH(domain_name, item_name) \
+do { \
+    if (NULL == domain_name) { \
+        if (asprintf(&path, "%s/%s", RESCTRL_DIR, item_name) < 0)\
+            return -1; \
+    } \
+    else { \
+        if (asprintf(&path, "%s/%s/%s", RESCTRL_DIR, \
+                                        domain_name, \
+                                        item_name) < 0) \
+            return -1;  \
+    } \
+} while(0)
+
+static virResCtrl ResCtrlAll[] = {
+    {
+        .name = "L3",
+        .domains = NULL,
+        .cache_level = "l3",
+    },
+    {
+        .name = "L3DATA",
+        .domains = NULL,
+        .cache_level = "l3",
+    },
+    {
+        .name = "L3CODE",
+        .domains = NULL,
+        .cache_level = "l3",
+    },
+    {
+        .name = "L2",
+        .domains = NULL,
+        .cache_level = "l2",
+    },
+};
+
+/* Return pointer of  and ncount of schemata*/
+static virResSchemataPtr virParseSchemata(const char* schemata, int *ncount)
+{
+    char type[MAX_SCHEMATA_LEN];
+    const char *p, *q;
+    int pos;
+    int ischemata;
+    virResSchemataPtr tmpschemata, pschemata;
+
+    unsigned int socket_no = 0;
+    p = q = schemata;
+    pos = strchr(schemata, ':') - p;
+
+    if(virStrncpy(type, schemata, pos, strlen(schemata)) == NULL) {
+        return NULL;
+    }
+
+    *ncount = 1;
+    while((q = strchr(p, ';')) != 0) {
+        p = q + 1;
+        (*ncount) ++;
+    }
+
+    /* allocat an arrry to keep schemata */
+    if(VIR_ALLOC_N_QUIET(tmpschemata, *ncount) < 0) {
+        return NULL;
+    }
+
+    pschemata = tmpschemata;
+
+    p = q = schemata + pos + 1;
+
+    char *tmp;
+
+    while(*p != '\0'){
+        if (*p == '='){
+            q = p + 1;
+
+            if (VIR_STRDUP(tmpschemata->name, type) < 0)
+                goto cleanup;
+
+            tmpschemata->socket_no = socket_no++;
+
+            while(*p != ';' && *p != '\0') p++;
+
+            if (VIR_STRNDUP(tmp, q, p-q) < 0)
+                goto cleanup;
+
+            if (virStrToLong_i(tmp, NULL, 16, &ischemata) < 0)
+                goto cleanup;
+
+            VIR_FREE(tmp);
+            tmpschemata->schemata = ischemata;
+            tmpschemata ++;
+        }
+        p++;
+    }
+
+    return pschemata;
+
+cleanup:
+    VIR_FREE(pschemata);
+    return NULL;
+}
+
+static int virResCtrlGetStr(const char *domain_name, const char *item_name, char **ret)
+{
+    char *path;
+    int rc = 0;
+
+    CONSTRUCT_RESCTRL_PATH(domain_name, item_name);
+
+    if (virFileReadAll(path, MAX_FILE_LEN, ret) < 0) {
+        rc = -1;
+        goto cleanup;
+    }
+
+cleanup:
+    VIR_FREE(path);
+    return rc;
+}
+
+static int virResCtrlGetTasks(const char *domain_name, char **pids)
+{
+    return virResCtrlGetStr(domain_name, "tasks", pids);
+}
+
+static int virResCtrlGetSchemata(const int type, const char *name, char **schemata)
+{
+    int rc;
+    char *tmp, *end;
+    char *buf;
+
+    if ((rc = virResCtrlGetStr(name, "schemata", &buf)) < 0)
+        return rc;
+
+    tmp = strstr(buf, ResCtrlAll[type].name);
+    end = strchr(tmp, '\n');
+    *end = '\0';
+    if(VIR_STRDUP(*schemata, tmp) < 0)
+        rc = -1;
+
+    VIR_FREE(buf);
+    return rc;
+}
+
+static int virResCtrlGetInfoStr(const int type, const char *item, char **str)
+{
+    int ret = 0;
+    char *tmp;
+    char *path;
+
+    if (asprintf(&path, "%s/%s/%s", RESCTRL_INFO_DIR, ResCtrlAll[type].name, item) < 0)
+        return -1;
+    if (virFileReadAll(path, 10, str) < 0) {
+        ret = -1;
+        goto cleanup;
+    }
+
+    if ((tmp = strchr(*str, '\n'))) {
+        *tmp = '\0';
+    }
+
+cleanup:
+    VIR_FREE(path);
+    return ret;
+}
+
+static virResDomainPtr virResCtrlGetAllDomains(int type, int *len)
+{
+    struct dirent *ent;
+    DIR *dp = NULL;
+    int direrr;
+    char *schematas, *tasks;
+
+    *len = 0;
+    virResDomainPtr header, tmp, tmp_pre;
+    header = tmp = tmp_pre = NULL;
+    if (virDirOpenQuiet(&dp, RESCTRL_DIR) < 0) {
+        if (errno == ENOENT)
+            return NULL;
+        VIR_ERROR(_("Unable to open %s (%d)"), RESCTRL_DIR, errno);
+        goto cleanup;
+    }
+
+    /* read default domain */
+    if(VIR_ALLOC(header) < 0)
+        return NULL;
+    header->pre = NULL;
+    header->next = NULL;
+    if(VIR_STRDUP(header->name, "default") < 0)
+        goto cleanup;
+
+    if (virResCtrlGetSchemata(type, NULL, &schematas) < 0) {
+        goto cleanup;
+    }
+    header->schematas = virParseSchemata(schematas, &(header->n_sockets));
+    VIR_FREE(schematas);
+    *len = 1;
+
+    while ((direrr = virDirRead(dp, &ent, NULL)) > 0) {
+        if ((ent->d_type != DT_DIR) || STREQ(ent->d_name, "info"))
+            continue;
+
+        if(virResCtrlGetTasks(ent->d_name, &tasks) < 0)
+            continue;
+
+        if(VIR_ALLOC(tmp) < 0)
+            return NULL;
+
+        tmp->next = NULL;
+
+        if(header->next == NULL)
+            header->next = tmp;
+
+        if(tmp_pre == NULL)
+            tmp->pre = header;
+        else {
+            tmp->pre = tmp_pre;
+            tmp_pre->next = tmp;
+        }
+
+        if(VIR_STRDUP(tmp->name, ent->d_name) < 0)
+            goto cleanup;
+
+        tmp_pre = tmp;
+        if (virResCtrlGetSchemata(type, tmp->name, &schematas) < 0) {
+            goto cleanup;
+        }
+
+        (*len) ++;
+        tmp->schematas = virParseSchemata(schematas, &(tmp->n_sockets));
+        tmp->tasks = tasks;
+        VIR_FREE(schematas);
+    }
+    return header;
+
+cleanup:
+
+    VIR_DIR_CLOSE(dp);
+    return NULL;
+}
+
+
+static int virResCtrlGetCPUValue(const char* path, char** value)
+{
+    int ret = -1;
+    char* tmp;
+
+    if(virFileReadAll(path, 10, value) < 0) {
+        goto cleanup;
+    }
+    if ((tmp = strchr(*value, '\n'))) {
+        *tmp = '\0';
+    }
+    ret = 0;
+cleanup:
+    return ret;
+}
+
+static int virResctrlGetCPUSocketID(const size_t cpu, int* socket_id)
+{
+    int ret = -1;
+    char* physical_package_path = NULL;
+    char* physical_package = NULL;
+    if (virAsprintf(&physical_package_path,
+                    "%s/cpu/cpu%zu/topology/physical_package_id",
+                    SYSFS_SYSTEM_PATH, cpu) < 0) {
+        return -1;
+    }
+
+    if(virResCtrlGetCPUValue(physical_package_path,
+                             &physical_package) < 0)
+        goto cleanup;
+
+    if (virStrToLong_i(physical_package, NULL, 0, socket_id) < 0)
+        goto cleanup;
+
+    ret = 0;
+cleanup:
+    VIR_FREE(physical_package);
+    VIR_FREE(physical_package_path);
+    return ret;
+}
+
+static int virResCtrlGetCPUCache(const size_t cpu, int type, int *cache)
+{
+    int ret = -1;
+    char* cache_dir = NULL;
+    char* cache_str = NULL;
+    char* tmp;
+    int carry = -1;
+
+    if (virAsprintf(&cache_dir,
+                    "%s/cpu/cpu%zu/cache/index%d/size",
+                    SYSFS_SYSTEM_PATH, cpu, type) < 0)
+        return -1;
+
+    if(virResCtrlGetCPUValue(cache_dir, &cache_str) < 0)
+        goto cleanup;
+
+    tmp = cache_str;
+
+    while (*tmp != '\0')
+        tmp++;
+    if (*(tmp - 1) == 'K') {
+        *(tmp - 1) = '\0';
+        carry = 1;
+    }
+    else if (*(tmp - 1) == 'M') {
+        *(tmp - 1) = '\0';
+        carry = 1024;
+    }
+
+    if (virStrToLong_i(cache_str, NULL, 0, cache) < 0)
+        goto cleanup;
+
+    *cache = (*cache) * carry;
+
+    if (*cache < 0)
+        goto cleanup;
+
+    ret = 0;
+cleanup:
+    VIR_FREE(cache_dir);
+    VIR_FREE(cache_str);
+    return ret;
+}
+
+/*
+ * Fill cache informations for specify cache type
+*/
+static int virResCtrlParseCPUCache(int type)
+{
+    int index = -1;
+    int npresent_cpus;
+
+    if ((npresent_cpus = virHostCPUGetCount()) < 0)
+        return -1;
+
+    if (type == RDT_RESOURCE_L3
+            || type == RDT_RESOURCE_L3DATA
+            || type == RDT_RESOURCE_L3CODE)
+        index = 3;
+    else if (type == RDT_RESOURCE_L2) {
+        index = 2;
+    }
+
+    if (index == -1)
+        return -1;
+
+    for(size_t i = 0; i < npresent_cpus ; i ++) {
+        int s_id;
+        int cache_size;
+
+        if (virResctrlGetCPUSocketID(i, &s_id) < 0) {
+            return -1;
+        }
+        if (ResCtrlAll[type].cpu_mask[s_id] == NULL) {
+            if (!(ResCtrlAll[type].cpu_mask[s_id] = virBitmapNew(npresent_cpus)))
+                return -1;
+        }
+
+        ignore_value(virBitmapSetBit(ResCtrlAll[type].cpu_mask[s_id], i));
+
+        if (ResCtrlAll[type].cache_size[s_id] == 0) {
+            if (virResCtrlGetCPUCache(i, index, &cache_size) < 0) {
+                return -1;
+            }
+            ResCtrlAll[type].cache_size[s_id] = cache_size;
+            ResCtrlAll[type].cache_min[s_id] = cache_size / ResCtrlAll[type].cbm_len;
+        }
+    }
+    return 0;
+}
+
+static int virResCtrlGetConfig(int type)
+{
+    int ret;
+    int i;
+    char *str;
+
+    /* Read min_cbm_bits from resctrl.
+       eg: /sys/fs/resctrl/info/L3/num_closids
+    */
+    if ((ret = virResCtrlGetInfoStr(type, "num_closids", &str)) < 0) {
+        return ret;
+    }
+    if (virStrToLong_i(str, NULL, 10, &ResCtrlAll[type].num_closid) < 0) {
+        return -1;
+    }
+    VIR_FREE(str);
+
+    /* Read min_cbm_bits from resctrl.
+       eg: /sys/fs/resctrl/info/L3/cbm_mask
+    */
+    if ((ret = virResCtrlGetInfoStr(type, "min_cbm_bits", &str)) < 0) {
+        return ret;
+    }
+    if (virStrToLong_i(str, NULL, 10, &ResCtrlAll[type].min_cbm_bits) < 0) {
+        return -1;
+    }
+    VIR_FREE(str);
+
+    /* Read cbm_mask string from resctrl.
+       eg: /sys/fs/resctrl/info/L3/cbm_mask
+    */
+    if ((ret = virResCtrlGetInfoStr(type, "cbm_mask", &str)) < 0) {
+        return ret;
+    }
+
+    /* Calculate cbm length from the default cbm_mask. */
+    ResCtrlAll[type].cbm_len = strlen(str) * 4;
+    VIR_FREE(str);
+
+    /* Get all resctrl comains from /sys/fs/resctrl */
+    ResCtrlAll[type].domains = virResCtrlGetAllDomains(type, &ResCtrlAll[type].num_domains);
+    ResCtrlAll[type].num_sockets = ResCtrlAll[type].domains->n_sockets;
+
+    /* Get cache_left information */
+    for(i = 0; i < ResCtrlAll[type].num_sockets; i++) {
+        ResCtrlAll[type].cpu_mask[i] = NULL;
+    }
+
+    if((ret = virResCtrlParseCPUCache(type)) < 0)
+        return ret;
+
+    ResCtrlAll[type].enabled = true;
+
+    return ret;
+}
+
+int virResCtrlInit(void) {
+    int i = 0;
+    char *tmp;
+    int rc = 0;
+
+    for(i = 0; i <RDT_NUM_RESOURCES; i ++) {
+        if ((rc = asprintf(&tmp, "%s/%s", RESCTRL_INFO_DIR, ResCtrlAll[i].name)) < 0) {
+            continue;
+        }
+        if (virFileExists(tmp)) {
+            if ((rc = virResCtrlGetConfig(i)) < 0 )
+                VIR_WARN("Ignor error while get config for %d", i);
+        }
+
+        VIR_FREE(tmp);
+    }
+    return rc;
+}
+
+bool virResCtrlAvailable(void) {
+    if (!virFileExists(RESCTRL_INFO_DIR))
+        return false;
+    return true;
+}
+
+virResCtrlPtr virResCtrlGet(int type) {
+    return &ResCtrlAll[type];
+}
diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h
new file mode 100644
index 0000000..432f191
--- /dev/null
+++ b/src/util/virresctrl.h
@@ -0,0 +1,121 @@
+/*
+ *  * virrscctrl.h: methods for managing rscctrl
+ *  *
+ *  * Copyright (C) 2016 Intel, Inc.
+ *  *
+ *  * This library is free software; you can redistribute it and/or
+ *  * modify it under the terms of the GNU Lesser General Public
+ *  * License as published by the Free Software Foundation; either
+ *  * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library.  If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ * Eli Qiao <liyong.qiao at intel.com>
+ */
+
+#ifndef __VIR_RESCTRL_H__
+# define __VIR_RESCTRL_H__
+
+# include "virutil.h"
+# include "virbitmap.h"
+# include "domain_conf.h"
+
+#define RESCTRL_DIR "/sys/fs/resctrl"
+#define RESCTRL_INFO_DIR "/sys/fs/resctrl/info"
+#define SYSFS_SYSTEM_PATH "/sys/devices/system"
+
+#define MAX_CPU_SOCKET_NUM 8
+#define MAX_CBM_BIT_LEN 32
+#define MAX_SCHEMATA_LEN 1024
+#define MAX_FILE_LEN ( 10 * 1024 * 1024)
+
+enum {
+    RDT_RESOURCE_L3,
+    RDT_RESOURCE_L3DATA,
+    RDT_RESOURCE_L3CODE,
+    RDT_RESOURCE_L2,
+    /* Must be the last */
+    RDT_NUM_RESOURCES,
+};
+
+/**
+ * a virResSchemata represents a schemata object under a resource control
+ * domain.
+ */
+typedef struct _virResSchemata virResSchemata;
+typedef virResSchemata *virResSchemataPtr;
+
+struct _virResSchemata {
+    char *name;
+    unsigned int socket_no;
+    int schemata;
+};
+
+/**
+ * a virResDomain represents a resource control domain. It's a double linked
+ * list.
+ */
+
+typedef struct _virResDomain virResDomain;
+typedef virResDomain *virResDomainPtr;
+
+struct _virResDomain {
+    char* name;
+    virResSchemataPtr schematas;
+    char* tasks;
+    int n_sockets;
+    virResDomainPtr pre;
+    virResDomainPtr next;
+};
+
+/**
+ * struct rdt_resource - attributes of an RDT resource
+ * @enabled:                    Is this feature enabled on this machine
+ * @name:                       Name to use in "schemata" file
+ * @num_closid:                 Number of CLOSIDs available
+ * @max_cbm:                    Largest Cache Bit Mask allowed
+ * @min_cbm_bits:               Minimum number of consecutive bits to be set
+ *                              in a cache bit mask
+ * @domains:                    All resource domains for this resource
+ * @num_domains:                Number of domains active
+ * @num_tmp_cbms:               Number of CBMs in tmp_cbms
+ * @cache_level:                Which cache level defines scope of this domain
+ * @num_sockets:                Number of sockets on this machine.
+ * @cache_size:                 Cache size on each cpu socket in KiB.
+ * @cache_left:                 Cache left on each cpu socket in KiB.
+ * @cache_min:                  The minimum cache of allocatioin.
+ * @cpu_mask:                   cpu mask on each socket.
+ */
+typedef struct _virResCtrl virResCtrl;
+typedef virResCtrl *virResCtrlPtr;
+
+struct _virResCtrl {
+        bool                    enabled;
+        const char              *name;
+        int                     num_closid;
+        int                     cbm_len;
+        int                     min_cbm_bits;
+        virResDomainPtr         domains;
+        int                     num_domains;
+        const char*             cache_level;
+        int                     num_sockets;
+        int                     cache_size[MAX_CPU_SOCKET_NUM];
+        int                     cache_left[MAX_CPU_SOCKET_NUM];
+        int                     cache_min[MAX_CPU_SOCKET_NUM];
+        virBitmapPtr            cpu_mask[MAX_CPU_SOCKET_NUM];
+};
+
+
+bool virResCtrlAvailable(void);
+int virResCtrlInit(void);
+virResCtrlPtr virResCtrlGet(int);
+
+#endif
diff --git a/tests/Makefile.am b/tests/Makefile.am
index ecd04e8..e9c9abc 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -1185,6 +1185,10 @@ virrotatingfiletest_SOURCES = \
 virrotatingfiletest_CFLAGS = $(AM_CFLAGS)
 virrotatingfiletest_LDADD = $(LDADDS)
 
+virresctrltest_SOURCES = \
+	virresctrltest.c testutils.h testutils.c
+virresctrltest_LDADD = $(LDADDS)
+
 if WITH_LINUX
 virusbtest_SOURCES = \
 	virusbtest.c testutils.h testutils.c
-- 
1.9.1




More information about the libvir-list mailing list