[libvirt] [PATCH RESEND RFC V2 2/2] Resctrl: Do cache allocation while boot a qemu VM
Eli Qiao
liyong.qiao at intel.com
Wed Jul 5 02:39:04 UTC 2017
Set cachetune if user define cachetune in domain xml when using qemu
driver.
This patch adds 3 major private interface.
virResctrlGetFreeCache: return free cache, default cache substract cache
allocated.
virResctrlSetCachetunes: set cache banks which defined in a domain.
virResctrlRemoveCachetunes: remove cache allocation group from the
host.
Signed-off-by: Eli Qiao <liyong.qiao at intel.com>
---
include/libvirt/virterror.h | 1 +
src/Makefile.am | 1 +
src/libvirt_private.syms | 9 +
src/qemu/qemu_process.c | 81 +++
src/util/virerror.c | 1 +
src/util/virresctrl.c | 822 ++++++++++++++++++++++++++++++
src/util/virresctrl.h | 88 ++++
tests/Makefile.am | 8 +-
tests/virresctrldata/L3-free.schemata | 1 +
tests/virresctrldata/L3CODE-free.schemata | 1 +
tests/virresctrldata/L3DATA-free.schemata | 1 +
tests/virresctrldata/linux-resctrl | 1 +
tests/virresctrldata/linux-resctrl-cdp | 1 +
tests/virresctrltest.c | 119 +++++
14 files changed, 1134 insertions(+), 1 deletion(-)
create mode 100644 src/util/virresctrl.c
create mode 100644 src/util/virresctrl.h
create mode 100644 tests/virresctrldata/L3-free.schemata
create mode 100644 tests/virresctrldata/L3CODE-free.schemata
create mode 100644 tests/virresctrldata/L3DATA-free.schemata
create mode 120000 tests/virresctrldata/linux-resctrl
create mode 120000 tests/virresctrldata/linux-resctrl-cdp
create mode 100644 tests/virresctrltest.c
diff --git a/include/libvirt/virterror.h b/include/libvirt/virterror.h
index 2efee8f..4bc0c74 100644
--- a/include/libvirt/virterror.h
+++ b/include/libvirt/virterror.h
@@ -132,6 +132,7 @@ typedef enum {
VIR_FROM_PERF = 65, /* Error from perf */
VIR_FROM_LIBSSH = 66, /* Error from libssh connection transport */
+ VIR_FROM_RESCTRL = 67, /* Error from resctrl */
# ifdef VIR_ENUM_SENTINELS
VIR_ERR_DOMAIN_LAST
diff --git a/src/Makefile.am b/src/Makefile.am
index eae32dc..8dbb778 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -167,6 +167,7 @@ UTIL_SOURCES = \
util/virprocess.c util/virprocess.h \
util/virqemu.c util/virqemu.h \
util/virrandom.h util/virrandom.c \
+ util/virresctrl.h util/virresctrl.c \
util/virrotatingfile.h util/virrotatingfile.c \
util/virscsi.c util/virscsi.h \
util/virscsihost.c util/virscsihost.h \
diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms
index c1e9471..72e5096 100644
--- a/src/libvirt_private.syms
+++ b/src/libvirt_private.syms
@@ -2439,6 +2439,15 @@ virRandomGenerateWWN;
virRandomInt;
+# util/virresctrl.h
+virResctrlBitmap2String;
+virResctrlFreeSchemata;
+virResctrlGetFreeCache;
+virResctrlRemoveCachetunes;
+virResctrlSetCachetunes;
+virResctrlTypeToString;
+
+
# util/virrotatingfile.h
virRotatingFileReaderConsume;
virRotatingFileReaderFree;
diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index fa9990e..3b11caa 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -70,6 +70,7 @@
#include "virbitmap.h"
#include "viratomic.h"
#include "virnuma.h"
+#include "virresctrl.h"
#include "virstring.h"
#include "virhostdev.h"
#include "secret_util.h"
@@ -5130,6 +5131,77 @@ qemuProcessSetupVcpus(virDomainObjPtr vm)
return 0;
}
+static int
+qemuProcessSetCacheBanks(virCapsHostPtr caps, virDomainObjPtr vm)
+{
+ size_t i, j;
+ virDomainCachetunePtr cachetune;
+ virResctrlCachetunePtr resctrl_cachetune;
+ unsigned int max_vcpus = virDomainDefGetVcpusMax(vm->def);
+ pid_t *pids = NULL;
+ virDomainVcpuDefPtr vcpu;
+ size_t npids = 0;
+ size_t count = 0;
+ int ret = -1;
+
+ cachetune = &(vm->def->cachetune);
+
+ if (VIR_ALLOC_N(resctrl_cachetune, cachetune->n_banks) < 0)
+ goto cleanup;
+
+ /* construct resctrl_cachetune array */
+ for (i = 0; i < cachetune->n_banks; i++) {
+
+ resctrl_cachetune[i].cache_id = cachetune->cache_banks[i].cache_id;
+ resctrl_cachetune[i].type = cachetune->cache_banks[i].type;
+ resctrl_cachetune[i].size = cachetune->cache_banks[i].size;
+
+ /* get granularity from host's capabilities */
+ for (j = 0; j < caps->ncaches; j++) {
+ /* even enable CDP, granularity for code and data are same */
+ if (caps->caches[j]->id == resctrl_cachetune[i].cache_id &&
+ caps->caches[j]->controls) {
+ resctrl_cachetune[i].granularity = caps->caches[j]->controls[0]->granularity;
+ break;
+ }
+ }
+
+ /* create pids of vcpus array */
+ if (cachetune->cache_banks[i].vcpus) {
+ for (j = 0; j < max_vcpus; j++) {
+ if (virBitmapIsBitSet(cachetune->cache_banks[i].vcpus, j)) {
+
+ vcpu = virDomainDefGetVcpu(vm->def, j);
+ if (!vcpu->online)
+ continue;
+
+ if (VIR_RESIZE_N(pids, npids, count, 1) < 0)
+ goto cleanup;
+ pids[count ++] = qemuDomainGetVcpuPid(vm, j);
+ }
+ }
+ }
+ }
+
+ /* If not specify vcpus in cachetune, add vm->pid */
+ if (pids == NULL) {
+ if (VIR_ALLOC_N(pids, 1) < 0)
+ goto cleanup;
+ pids[0] = vm->pid;
+ count = 1;
+ }
+
+ ret = virResctrlSetCachetunes(vm->def->uuid,
+ resctrl_cachetune,
+ cachetune->n_banks,
+ pids,
+ count);
+
+ cleanup:
+ VIR_FREE(resctrl_cachetune);
+ VIR_FREE(pids);
+ return ret;
+}
int
qemuProcessSetupIOThread(virDomainObjPtr vm,
@@ -5966,6 +6038,12 @@ qemuProcessLaunch(virConnectPtr conn,
qemuProcessAutoDestroyAdd(driver, vm, conn) < 0)
goto cleanup;
+ VIR_DEBUG("Cache allocation");
+ if (vm->def->cachetune.n_banks > 0 &&
+ qemuProcessSetCacheBanks(&driver->caps->host,
+ vm) < 0)
+ goto cleanup;
+
ret = 0;
cleanup:
@@ -6472,6 +6550,9 @@ void qemuProcessStop(virQEMUDriverPtr driver,
virPerfFree(priv->perf);
priv->perf = NULL;
+ if (&(vm->def->cachetune) != NULL)
+ virResctrlRemoveCachetunes(vm->def->uuid);
+
qemuProcessRemoveDomainStatus(driver, vm);
/* Remove VNC and Spice ports from port reservation bitmap, but only if
diff --git a/src/util/virerror.c b/src/util/virerror.c
index ef17fb5..02fabcc 100644
--- a/src/util/virerror.c
+++ b/src/util/virerror.c
@@ -139,6 +139,7 @@ VIR_ENUM_IMPL(virErrorDomain, VIR_ERR_DOMAIN_LAST,
"Perf", /* 65 */
"Libssh transport layer",
+ "Resource Control",
)
diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c
new file mode 100644
index 0000000..541c69c
--- /dev/null
+++ b/src/util/virresctrl.c
@@ -0,0 +1,822 @@
+/*
+ * virresctrl.c: methods for managing resource control
+ *
+ * Copyright (C) 2017 Intel, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ * Eli Qiao <liyong.qiao at intel.com>
+ */
+
+#include <config.h>
+#include <fcntl.h>
+#include <sys/file.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include "viralloc.h"
+#include "virerror.h"
+#include "virfile.h"
+#include "virlog.h"
+#include "virresctrl.h"
+#include "virstring.h"
+#include "viruuid.h"
+
+VIR_LOG_INIT("util.resctrl");
+
+#define VIR_FROM_THIS VIR_FROM_RESCTRL
+#define SYSFS_RESCTRL_PATH "/sys/fs/resctrl"
+#define MAX_CBM_LEN 20
+#define VIR_RESCTRL_LOCK(fd, op) flock(fd, op)
+#define VIR_RESCTRL_UNLOCK(fd) flock(fd, LOCK_UN)
+#define CONSTRUCT_RESCTRL_PATH(domain_name, item_name) \
+do { \
+ if (NULL == domain_name) { \
+ if (virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, item_name) < 0) \
+ return -1; \
+ } else { \
+ if (virAsprintf(&path, "%s/%s/%s", SYSFS_RESCTRL_PATH, domain_name, \
+ item_name) < 0) \
+ return -1; \
+ } \
+} while (0)
+
+VIR_ENUM_IMPL(virResctrl, VIR_RESCTRL_TYPE_LAST,
+ "L3",
+ "L3CODE",
+ "L3DATA")
+
+/**
+ * a virResctrlGroup represents a resource control group, it's a directory
+ * under /sys/fs/resctrl.
+ * e.g. /sys/fs/resctrl/CG1
+ * |-- cpus
+ * |-- schemata
+ * `-- tasks
+ * # cat schemata
+ * L3DATA:0=fffff;1=fffff
+ * L3CODE:0=fffff;1=fffff
+ *
+ * Besides, it can also represent the default resource control group of the
+ * host.
+ */
+
+typedef struct _virResctrlGroup virResctrlGroup;
+typedef virResctrlGroup *virResctrlGroupPtr;
+struct _virResctrlGroup {
+ char *name; /* resource group name, NULL for default host group */
+ size_t n_tasks; /* number of tasks assigned to the resource group */
+ char **tasks; /* task id list */
+ virResctrlSchemataPtr schemata[VIR_RESCTRL_TYPE_LAST]; /* Array for schemata */
+};
+
+/* All resource control groups on this host, including default resource group */
+typedef struct _virResctrlHost virResctrlHost;
+typedef virResctrlHost *virResctrlHostPtr;
+struct _virResctrlHost {
+ size_t n_groups; /* number of resource control group */
+ virResctrlGroupPtr *groups; /* list of resource control group */
+};
+
+void
+virResctrlFreeSchemata(virResctrlSchemataPtr ptr)
+{
+ size_t i;
+
+ if (!ptr)
+ return;
+
+ for (i = 0; i < ptr->n_masks; i++) {
+ virBitmapFree(ptr->masks[i]->mask);
+ VIR_FREE(ptr->masks[i]);
+ }
+
+ VIR_FREE(ptr);
+ ptr = NULL;
+}
+
+static void
+virResctrlFreeGroup(virResctrlGroupPtr ptr)
+{
+ size_t i;
+
+ if (!ptr)
+ return;
+
+ for (i = 0; i < ptr->n_tasks; i++)
+ VIR_FREE(ptr->tasks[i]);
+ VIR_FREE(ptr->name);
+
+ for (i = 0; i < VIR_RESCTRL_TYPE_LAST; i++)
+ virResctrlFreeSchemata(ptr->schemata[i]);
+
+ VIR_FREE(ptr);
+ ptr = NULL;
+}
+
+/* Return specify type of schemata string from schematalval.
+ e.g., 0=f;1=f */
+static int
+virResctrlGetSchemataString(virResctrlType type,
+ const char *schemataval,
+ char **schematastr)
+{
+ int rc = -1;
+ char *prefix = NULL;
+ char **lines = NULL;
+
+ if (virAsprintf(&prefix,
+ "%s:",
+ virResctrlTypeToString(type)) < 0)
+ return -1;
+
+ lines = virStringSplit(schemataval, "\n", 0);
+
+ if (VIR_STRDUP(*schematastr,
+ virStringListGetFirstWithPrefix(lines, prefix)) < 0)
+ goto cleanup;
+
+ if (*schematastr == NULL)
+ rc = -1;
+ else
+ rc = 0;
+
+ cleanup:
+ VIR_FREE(prefix);
+ virStringListFree(lines);
+ return rc;
+}
+
+static int
+virResctrlRemoveSysGroup(const char* name)
+{
+ char *path = NULL;
+ int ret = -1;
+
+ if ((ret = virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, name)) < 0)
+ return ret;
+
+ ret = rmdir(path);
+
+ VIR_FREE(path);
+ return ret;
+}
+
+static int
+virResctrlNewSysGroup(const char *name)
+{
+ char *path = NULL;
+ int ret = -1;
+ mode_t mode = 0755;
+
+ if (virAsprintf(&path, "%s/%s", SYSFS_RESCTRL_PATH, name) < 0)
+ return -1;
+
+ if (virDirCreate(path, mode, 0, 0, 0) < 0)
+ goto cleanup;
+
+ ret = 0;
+
+ cleanup:
+ VIR_FREE(path);
+ return ret;
+}
+
+static int
+virResctrlWrite(const char *name, const char *item, const char *content)
+{
+ char *path;
+ int writefd;
+ int rc = -1;
+
+ CONSTRUCT_RESCTRL_PATH(name, item);
+
+ if (!virFileExists(path))
+ goto cleanup;
+
+ if ((writefd = open(path, O_WRONLY | O_APPEND, S_IRUSR | S_IWUSR)) < 0)
+ goto cleanup;
+
+ if (safewrite(writefd, content, strlen(content)) < 0)
+ goto cleanup;
+ rc = 0;
+
+ cleanup:
+ VIR_FREE(path);
+ VIR_FORCE_CLOSE(writefd);
+ return rc;
+}
+
+static
+virBitmapPtr virResctrlMask2Bitmap(const char *mask)
+{
+ virBitmapPtr bitmap;
+ unsigned int tmp;
+ size_t i;
+
+ if (virStrToLong_ui(mask, NULL, 16, &tmp) < 0)
+ return NULL;
+
+ bitmap = virBitmapNewEmpty();
+
+ for (i = 0; i < MAX_CBM_LEN; i++) {
+ if (((tmp & 0x1) == 0x1) &&
+ (virBitmapSetBitExpand(bitmap, i) < 0))
+ goto error;
+ tmp = tmp >> 1;
+ }
+ return bitmap;
+
+ error:
+ virBitmapFree(bitmap);
+ return NULL;
+}
+
+char *virResctrlBitmap2String(virBitmapPtr bitmap)
+{
+ char *tmp;
+ char *ret = NULL;
+ char *p;
+ tmp = virBitmapString(bitmap);
+ /* skip "0x" */
+ p = tmp + 2;
+
+ /* first non-0 position */
+ while (*++p == '0');
+
+ if (VIR_STRDUP(ret, p) < 0)
+ ret = NULL;
+
+ VIR_FREE(tmp);
+ return ret;
+}
+
+static int
+virResctrlParseSchemata(const char* schemata_str,
+ virResctrlSchemataPtr schemata)
+{
+ VIR_DEBUG("schemata_str=%s, schemata=%p", schemata_str, schemata);
+
+ int ret = -1;
+ size_t i;
+ virResctrlMaskPtr mask;
+ char **schemata_list;
+ char *mask_str;
+
+ /* parse 0=fffff;1=f */
+ schemata_list = virStringSplit(schemata_str, ";", 0);
+
+ if (!schemata_list)
+ goto cleanup;
+
+ for (i = 0; schemata_list[i] != NULL; i++) {
+ /* parse 0=fffff */
+ mask_str = strchr(schemata_list[i], '=');
+
+ if (!mask_str)
+ goto cleanup;
+
+ if (VIR_ALLOC(mask) < 0)
+ goto cleanup;
+
+ mask->cache_id = i;
+ mask->mask = virResctrlMask2Bitmap(mask_str + 1);
+ schemata->n_masks += 1;
+ schemata->masks[i] = mask;
+
+ }
+ ret = 0;
+
+ cleanup:
+ virStringListFree(schemata_list);
+ return ret;
+}
+
+static int
+virResctrlLoadGroup(const char *name,
+ virResctrlHostPtr host)
+{
+ VIR_DEBUG("name=%s, host=%p\n", name, host);
+
+ int ret = -1;
+ char *schemataval = NULL;
+ char *schemata_str = NULL;
+ virResctrlType i;
+ int rv;
+ virResctrlGroupPtr grp;
+ virResctrlSchemataPtr schemata;
+
+ rv = virFileReadValueString(&schemataval,
+ SYSFS_RESCTRL_PATH "/%s/schemata",
+ name ? name : "");
+
+ if (rv < 0)
+ return -1;
+
+ if (VIR_ALLOC(grp) < 0)
+ goto cleanup;
+
+ if (VIR_STRDUP(grp->name, name) < 0)
+ goto cleanup;
+
+ for (i = 0; i < VIR_RESCTRL_TYPE_LAST; i++) {
+ rv = virResctrlGetSchemataString(i, schemataval, &schemata_str);
+
+ if (rv < 0)
+ continue;
+
+ if (VIR_ALLOC(schemata) < 0)
+ goto cleanup;
+
+ schemata->type = i;
+
+ if (virResctrlParseSchemata(schemata_str, schemata) < 0) {
+ VIR_FREE(schemata);
+ VIR_FREE(schemata_str);
+ goto cleanup;
+ }
+
+ grp->schemata[i] = schemata;
+ VIR_FREE(schemata_str);
+ }
+
+ if (VIR_APPEND_ELEMENT(host->groups,
+ host->n_groups,
+ grp) < 0) {
+ virResctrlFreeGroup(grp);
+ goto cleanup;
+ }
+
+ ret = 0;
+
+ cleanup:
+ VIR_FREE(schemataval);
+ return ret;
+}
+
+static int
+virResctrlLoadHost(virResctrlHostPtr host)
+{
+ int rv = -1;
+ DIR *dirp = NULL;
+ char *path = NULL;
+ struct dirent *ent;
+
+ rv = virDirOpenIfExists(&dirp, SYSFS_RESCTRL_PATH);
+ if (rv < 0)
+ return -1;
+
+ /* load default group first */
+ if (virResctrlLoadGroup(NULL, host) < 0)
+ return -1;
+
+ while ((rv = virDirRead(dirp, &ent, path)) > 0) {
+ /* resctrl is not hierarchical, only read directory under
+ /sys/fs/resctrl */
+ if ((ent->d_type != DT_DIR) || STREQ(ent->d_name, "info"))
+ continue;
+
+ if (virResctrlLoadGroup(ent->d_name, host) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static void
+virResctrlRefreshHost(virResctrlHostPtr host)
+{
+ virResctrlGroupPtr default_grp = NULL;
+ virResctrlSchemataPtr schemata = NULL;
+ size_t i, j;
+ virResctrlType t;
+
+ default_grp = host->groups[0];
+
+ for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+ if (default_grp->schemata[t] != NULL) {
+ for (i = 0; i < default_grp->schemata[t]->n_masks; i++) {
+ /* Reset default group's mask */
+ virBitmapSetAll(default_grp->schemata[t]->masks[i]->mask);
+ /* Loop each other resource group except default group */
+ for (j = 1; j < host->n_groups; j++) {
+ schemata = host->groups[j]->schemata[t];
+ virBitmapSubtract(default_grp->schemata[t]->masks[i]->mask,
+ schemata->masks[i]->mask);
+ }
+ }
+ }
+ }
+}
+
+static virResctrlGroupPtr
+virResctrlGetFreeGroup(void)
+{
+ size_t i;
+ virResctrlHostPtr host = NULL;
+ virResctrlGroupPtr grp = NULL;
+
+ if (VIR_ALLOC(host) < 0)
+ return NULL;
+
+ if (virResctrlLoadHost(host) < 0)
+ goto error;
+
+ virResctrlRefreshHost(host);
+
+ for (i = 1; i < host->n_groups; i++)
+ virResctrlFreeGroup(host->groups[i]);
+
+ grp = host->groups[0];
+ VIR_FREE(host);
+
+ return grp;
+
+ error:
+ virResctrlFreeGroup(grp);
+ return NULL;
+}
+
+virResctrlSchemataPtr
+virResctrlGetFreeCache(virResctrlType type)
+{
+ VIR_DEBUG("type=%d", type);
+
+ virResctrlType t;
+ virResctrlGroupPtr grp = NULL;
+ virResctrlSchemataPtr schemata = NULL;
+ int lockfd = -1;
+
+ lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY);
+ if (lockfd < 0)
+ return NULL;
+
+ VIR_RESCTRL_LOCK(lockfd, LOCK_SH);
+
+ if ((grp = virResctrlGetFreeGroup()) == NULL)
+ goto cleanup;
+
+ for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+ if (t == type)
+ schemata = grp->schemata[t];
+ else
+ virResctrlFreeSchemata(grp->schemata[t]);
+ }
+
+ cleanup:
+ VIR_RESCTRL_UNLOCK(lockfd);
+ VIR_FORCE_CLOSE(lockfd);
+ return schemata;
+}
+
+static int
+virResctrlCalculateCbm(int cbm_len,
+ virBitmapPtr defaultcbm,
+ virBitmapPtr newcbm)
+{
+ VIR_DEBUG("cbm_len=%d, defaultcbm=%p, newcbm=%p",
+ cbm_len, defaultcbm, newcbm);
+
+ ssize_t pos = -1;
+ size_t i;
+
+ /* not enough cache way to be allocated */
+ if (virBitmapCountBits(defaultcbm) < cbm_len + 1)
+ return -1;
+
+ while ((pos = virBitmapNextSetBit(defaultcbm, pos)) >= 0) {
+ for (i = 0; i < cbm_len; i++)
+ ignore_value(virBitmapSetBitExpand(newcbm, i + pos));
+ /* Test if newcbm is sub set of defaultcbm */
+ if (virBitmapNextClearBit(defaultcbm, pos) > i + pos) {
+ break;
+ } else {
+ pos = pos + i - 1;
+ virBitmapClearAll(newcbm);
+ }
+ }
+
+ if (virBitmapCountBits(newcbm) != cbm_len)
+ return -1;
+
+ /* consume default cbm after allocation */
+ virBitmapSubtract(defaultcbm, newcbm);
+
+ return 0;
+}
+
+/* Fill mask value for newly created resource group base on hostcachebank
+ * and domcachebank */
+static int
+virResctrlFillMask(virResctrlGroupPtr grp,
+ virResctrlGroupPtr free_grp,
+ virResctrlCachetunePtr cachetune)
+{
+ VIR_DEBUG("grp=%p, free_grp=%p, cachetune=%p",
+ grp, free_grp, cachetune);
+
+ unsigned int cache_id;
+ unsigned int cache_type;
+ int cbm_candidate_len;
+ virResctrlSchemataPtr schemata;
+ virResctrlMaskPtr mask;
+
+ cache_type = cachetune->type;
+ cache_id = cachetune->cache_id;
+ schemata = grp->schemata[cache_type];
+
+ if ((schemata == NULL) && (VIR_ALLOC(schemata) < 0))
+ return -1;
+
+ if (VIR_ALLOC(mask) < 0)
+ return -1;
+
+ mask->cache_id = cache_id;
+ mask->mask = virBitmapNewEmpty();
+
+ /* here should be control->granularity and control->min
+ also domcachebank size should be checked while define domain xml */
+ cbm_candidate_len = cachetune->size / cachetune->granularity;
+ if (virResctrlCalculateCbm(cbm_candidate_len,
+ free_grp->schemata[cache_type]->masks[cache_id]->mask,
+ mask->mask) < 0)
+ goto error;
+
+ schemata->type = cache_type;
+ schemata->n_masks += 1;
+ schemata->masks[cache_id] = mask;
+ grp->schemata[cache_type] = schemata;
+
+ return 0;
+
+ error:
+ VIR_FREE(schemata);
+ return -1;
+}
+
+/* only keep the highest consecutive bits */
+static void
+virResctrlTrimMask(virResctrlMaskPtr mask)
+{
+ size_t i;
+ ssize_t setbit = -1;
+ ssize_t clearbit = -1;
+
+ clearbit = virBitmapNextClearBit(mask->mask, -1);
+ setbit = virBitmapNextSetBit(mask->mask, -1);
+ for (i = setbit; i < clearbit; i++)
+ ignore_value(virBitmapClearBit(mask->mask, i));
+}
+
+static int
+virResctrlCompleteMask(virResctrlSchemataPtr schemata,
+ virResctrlSchemataPtr defaultschemata)
+{
+ size_t i;
+ virResctrlMaskPtr mask;
+
+ if (schemata == NULL && VIR_ALLOC(schemata) < 0)
+ return -1;
+
+ if (schemata->n_masks == defaultschemata->n_masks)
+ return 0;
+
+ for (i = 0; i < defaultschemata->n_masks; i++) {
+ if (schemata->masks[i] == NULL) {
+ if (VIR_ALLOC(mask) < 0)
+ goto error;
+
+ mask->cache_id = i;
+ mask->mask = virBitmapNewEmpty();
+ schemata->n_masks += 1;
+ schemata->masks[i] = mask;
+ /* resctrl doesn't allow mask to be zero
+ use higher bits to fill up the cbm which
+ domaincache bank doens't provide */
+ ignore_value(virBitmapSetBitExpand(mask->mask,
+ virBitmapLastSetBit(defaultschemata->masks[i]->mask)));
+ }
+ /* only keep the highest consecutive bits for default group */
+ virResctrlTrimMask(defaultschemata->masks[i]);
+ }
+
+ return 0;
+
+ error:
+ VIR_FREE(schemata);
+ return -1;
+}
+
+/* complete the schemata in the resrouce group before it can be write back
+ to resctrl */
+static int
+virResctrlCompleteGroup(virResctrlGroupPtr grp,
+ virResctrlGroupPtr default_grp)
+{
+ virResctrlType t;
+ virResctrlSchemataPtr schemata;
+ virResctrlSchemataPtr defaultschemata;
+
+ /* NOTES: resctrl system require we need provide all cache's cbm mask */
+ for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+ defaultschemata = default_grp->schemata[t];
+ if (defaultschemata != NULL) {
+ schemata = grp->schemata[t];
+ if (virResctrlCompleteMask(schemata, defaultschemata) < 0)
+ return -1;
+ }
+ }
+ return 0;
+}
+
+static
+char *virResctrlGetSchemataStr(virResctrlSchemataPtr schemata)
+{
+ virBuffer buf = VIR_BUFFER_INITIALIZER;
+ size_t i;
+
+ virBufferAsprintf(&buf, "%s:%u=%s",
+ virResctrlTypeToString(schemata->type),
+ schemata->masks[0]->cache_id,
+ virResctrlBitmap2String(schemata->masks[0]->mask));
+
+ for (i = 1; i < schemata->n_masks; i ++)
+ virBufferAsprintf(&buf, ";%u=%s",
+ schemata->masks[i]->cache_id,
+ virResctrlBitmap2String(schemata->masks[i]->mask));
+
+ return virBufferContentAndReset(&buf);
+}
+
+static int
+virResctrlFlushGroup(virResctrlGroupPtr grp)
+{
+ int ret = -1;
+ size_t i;
+ char *schemata_str = NULL;
+ virResctrlType t;
+ virBuffer buf = VIR_BUFFER_INITIALIZER;
+
+ VIR_DEBUG("grp=%p", grp);
+
+ if (grp->name != NULL && virResctrlNewSysGroup(grp->name) < 0)
+ return -1;
+
+ for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+ if (grp->schemata[t] != NULL) {
+ schemata_str = virResctrlGetSchemataStr(grp->schemata[t]);
+ virBufferAsprintf(&buf, "%s\n", schemata_str);
+ VIR_FREE(schemata_str);
+ }
+ }
+
+ schemata_str = virBufferContentAndReset(&buf);
+
+ if (virResctrlWrite(grp->name, "schemata", schemata_str) < 0)
+ goto cleanup;
+
+ for (i = 0; i < grp->n_tasks; i++) {
+ if (virResctrlWrite(grp->name, "tasks", grp->tasks[i]) < 0)
+ goto cleanup;
+ }
+
+ ret = 0;
+
+ cleanup:
+ VIR_FREE(schemata_str);
+ return ret;
+}
+
+int virResctrlSetCachetunes(unsigned char* uuid,
+ virResctrlCachetunePtr cachetunes, size_t ncachetune,
+ pid_t *pids, size_t npid)
+{
+ size_t i;
+ int ret = -1;
+ char name[VIR_UUID_STRING_BUFLEN];
+ char *tmp;
+ int lockfd = -1;
+ virResctrlGroupPtr grp = NULL;
+ virResctrlGroupPtr default_grp = NULL;
+
+ virUUIDFormat(uuid, name);
+
+ if (ncachetune < 1)
+ return 0;
+
+ /* create new resource group */
+ if (VIR_ALLOC(grp) < 0)
+ goto error;
+
+ if (VIR_STRDUP(grp->name, name) < 0)
+ goto error;
+
+ /* allocate file lock */
+ lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY);
+ if (lockfd < 0)
+ goto error;
+
+ VIR_RESCTRL_LOCK(lockfd, LOCK_EX);
+
+ if ((default_grp = virResctrlGetFreeGroup()) == NULL)
+ goto error;
+
+ /* Allocate cache for each cache bank defined in cache tune */
+ for (i = 0; i < ncachetune; i++) {
+ /* fill up newly crated grp and consume from default_grp */
+ if (virResctrlFillMask(grp, default_grp, &cachetunes[i]) < 0)
+ goto error;
+ }
+
+ /* Add tasks to grp */
+ for (i = 0; i < npid; i++) {
+ if (virAsprintf(&tmp, "%llu", (long long)pids[i]) < 0)
+ goto error;
+
+ if (VIR_APPEND_ELEMENT(grp->tasks,
+ grp->n_tasks,
+ tmp) < 0) {
+ VIR_FREE(tmp);
+ goto error;
+ }
+ }
+
+ if (virResctrlCompleteGroup(grp, default_grp) < 0) {
+ VIR_WARN("Failed to complete group");
+ goto error;
+ }
+
+ if (virResctrlFlushGroup(grp) < 0)
+ goto error;
+
+ if (virResctrlFlushGroup(default_grp) < 0) {
+ virResctrlRemoveSysGroup(grp->name);
+ goto error;
+ }
+
+ ret = 0;
+
+ error:
+ VIR_RESCTRL_UNLOCK(lockfd);
+ VIR_FORCE_CLOSE(lockfd);
+ virResctrlFreeGroup(grp);
+ virResctrlFreeGroup(default_grp);
+ return ret;
+}
+
+int virResctrlRemoveCachetunes(unsigned char* uuid)
+{
+ int ret = -1;
+ int lockfd = -1;
+ size_t i;
+ virResctrlType t;
+ virResctrlSchemataPtr schemata;
+ char name[VIR_UUID_STRING_BUFLEN];
+ virResctrlGroupPtr default_grp = NULL;
+
+ virUUIDFormat(uuid, name);
+
+ VIR_DEBUG("name=%s", name);
+
+ lockfd = open(SYSFS_RESCTRL_PATH, O_DIRECTORY);
+ if (lockfd < 0)
+ return -1;
+
+ VIR_RESCTRL_LOCK(lockfd, LOCK_SH);
+
+ if (virResctrlRemoveSysGroup(name) < 0)
+ goto cleanup;
+
+ if ((default_grp = virResctrlGetFreeGroup()) == NULL)
+ goto cleanup;
+
+ for (t = 0; t < VIR_RESCTRL_TYPE_LAST; t++) {
+ schemata = default_grp->schemata[t];
+ if (schemata != NULL) {
+ for (i = 0; i < schemata->n_masks; i++)
+ virResctrlTrimMask(schemata->masks[i]);
+ }
+ }
+
+ if (virResctrlFlushGroup(default_grp) < 0)
+ goto cleanup;
+
+ ret = 0;
+
+ cleanup:
+ VIR_RESCTRL_UNLOCK(lockfd);
+ VIR_FORCE_CLOSE(lockfd);
+ return ret;
+}
diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h
new file mode 100644
index 0000000..49781ad
--- /dev/null
+++ b/src/util/virresctrl.h
@@ -0,0 +1,88 @@
+/*
+ * virresctrl.h: header for managing resctrl control
+ *
+ * Copyright (C) 2017 Intel, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ * Eli Qiao <liyong.qiao at intel.com>
+ */
+
+#ifndef __VIR_RESCTRL_H__
+# define __VIR_RESCTRL_H__
+
+#include "virutil.h"
+#include "virbitmap.h"
+
+#define MAX_CACHE_ID 16
+
+typedef enum {
+ VIR_RESCTRL_TYPE_L3,
+ VIR_RESCTRL_TYPE_L3_CODE,
+ VIR_RESCTRL_TYPE_L3_DATA,
+
+ VIR_RESCTRL_TYPE_LAST
+} virResctrlType;
+
+VIR_ENUM_DECL(virResctrl);
+
+typedef struct _virResctrlCachetune virResctrlCachetune;
+typedef virResctrlCachetune *virResctrlCachetunePtr;
+struct _virResctrlCachetune {
+ unsigned int cache_id;
+ virResctrlType type;
+ unsigned long long size; /* in B */
+ unsigned long long granularity; /* min size of the cache */
+};
+
+/*
+ * a virResctrlMask represents one of mask object in a
+ * resource control group.
+ * e.g., 0=f
+ */
+typedef struct _virResctrlMask virResctrlMask;
+typedef virResctrlMask *virResctrlMaskPtr;
+struct _virResctrlMask {
+ unsigned int cache_id; /* cache resource id */
+ virBitmapPtr mask; /* the cbm mask */
+};
+
+/*
+ * a virResctrlSchemata represents schemata objects of specific type of
+ * resource in a resource control group.
+ * eg: L3:0=f,1=ff
+ */
+typedef struct _virResctrlSchemata virResctrlSchemata;
+typedef virResctrlSchemata *virResctrlSchemataPtr;
+struct _virResctrlSchemata {
+ virResctrlType type; /* resource control type, e.g., L3 */
+ size_t n_masks; /* number of masks */
+ virResctrlMaskPtr masks[MAX_CACHE_ID]; /* array of mask, use array for easy index */
+};
+
+/* Get free cache of the host, result saved in schemata */
+virResctrlSchemataPtr virResctrlGetFreeCache(virResctrlType type);
+
+/* Get mask string from Bitmap */
+char *virResctrlBitmap2String(virBitmapPtr bitmap);
+
+void virResctrlFreeSchemata(virResctrlSchemataPtr ptr);
+
+int virResctrlSetCachetunes(unsigned char* uuid,
+ virResctrlCachetunePtr cachetunes, size_t ncachetune,
+ pid_t *pids, size_t npid);
+int virResctrlRemoveCachetunes(unsigned char* uuid);
+#endif
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 19986dc..e0b9923 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -229,6 +229,7 @@ if WITH_LINUX
test_programs += fchosttest
test_programs += scsihosttest
test_programs += vircaps2xmltest
+test_programs += virresctrltest
test_libraries += virusbmock.la \
virnetdevbandwidthmock.la \
virnumamock.la \
@@ -1149,8 +1150,13 @@ virnumamock_la_CFLAGS = $(AM_CFLAGS)
virnumamock_la_LDFLAGS = $(MOCKLIBS_LDFLAGS)
virnumamock_la_LIBADD = $(MOCKLIBS_LIBS)
+virresctrltest_SOURCES = \
+ virresctrltest.c testutils.h testutils.c virfilewrapper.h virfilewrapper.c
+virresctrltest_LDADD = $(LDADDS)
+
else ! WITH_LINUX
-EXTRA_DIST += vircaps2xmltest.c virnumamock.c virfilewrapper.c virfilewrapper.h
+EXTRA_DIST += vircaps2xmltest.c virnumamock.c virfilewrapper.c \
+ virfilewrapper.h virresctrltest.c
endif ! WITH_LINUX
if WITH_NSS
diff --git a/tests/virresctrldata/L3-free.schemata b/tests/virresctrldata/L3-free.schemata
new file mode 100644
index 0000000..9b47d25
--- /dev/null
+++ b/tests/virresctrldata/L3-free.schemata
@@ -0,0 +1 @@
+L3:0=1ffff;1=1ffff
diff --git a/tests/virresctrldata/L3CODE-free.schemata b/tests/virresctrldata/L3CODE-free.schemata
new file mode 100644
index 0000000..7039c45
--- /dev/null
+++ b/tests/virresctrldata/L3CODE-free.schemata
@@ -0,0 +1 @@
+L3CODE:0=cffff;1=cffff
diff --git a/tests/virresctrldata/L3DATA-free.schemata b/tests/virresctrldata/L3DATA-free.schemata
new file mode 100644
index 0000000..30f1cbd
--- /dev/null
+++ b/tests/virresctrldata/L3DATA-free.schemata
@@ -0,0 +1 @@
+L3DATA:0=3ffff;1=3ffff
diff --git a/tests/virresctrldata/linux-resctrl b/tests/virresctrldata/linux-resctrl
new file mode 120000
index 0000000..069dfb2
--- /dev/null
+++ b/tests/virresctrldata/linux-resctrl
@@ -0,0 +1 @@
+../vircaps2xmldata/linux-resctrl
\ No newline at end of file
diff --git a/tests/virresctrldata/linux-resctrl-cdp b/tests/virresctrldata/linux-resctrl-cdp
new file mode 120000
index 0000000..c5a973f
--- /dev/null
+++ b/tests/virresctrldata/linux-resctrl-cdp
@@ -0,0 +1 @@
+../vircaps2xmldata/linux-resctrl-cdp
\ No newline at end of file
diff --git a/tests/virresctrltest.c b/tests/virresctrltest.c
new file mode 100644
index 0000000..a9d75f1
--- /dev/null
+++ b/tests/virresctrltest.c
@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) Intel, Inc. 2017
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library. If not, see
+ * <http://www.gnu.org/licenses/>.
+ *
+ * Authors:
+ * Eli Qiao <liyong.qiao at intel.com>
+ */
+
+#include <config.h>
+#include <stdlib.h>
+
+#include "testutils.h"
+#include "virbitmap.h"
+#include "virfilewrapper.h"
+#include "virresctrl.h"
+
+
+#define VIR_FROM_THIS VIR_FROM_NONE
+
+struct virResctrlData {
+ const char *filename;
+ virResctrlType type;
+};
+
+static void
+GetSchemataStr(virResctrlSchemataPtr schemata, char **str)
+{
+ size_t i;
+
+ virBuffer buf = VIR_BUFFER_INITIALIZER;
+ virBufferAsprintf(&buf, "%s:%u=%s",
+ virResctrlTypeToString(schemata->type),
+ schemata->masks[0]->cache_id,
+ virResctrlBitmap2String(schemata->masks[0]->mask));
+
+ for (i = 1; i < schemata->n_masks; i ++)
+ virBufferAsprintf(&buf, ";%u=%s",
+ schemata->masks[i]->cache_id,
+ virResctrlBitmap2String(schemata->masks[i]->mask));
+
+ *str = virBufferContentAndReset(&buf);
+}
+
+static int
+test_virResctrl(const void *opaque)
+{
+ struct virResctrlData *data = (struct virResctrlData *) opaque;
+ char *dir = NULL;
+ char *resctrl = NULL;
+ int ret = -1;
+ virResctrlSchemataPtr schemata = NULL;
+ char *schemata_str = NULL;
+ char *schemata_file;
+
+ if (virAsprintf(&resctrl, "%s/virresctrldata/linux-%s/resctrl",
+ abs_srcdir, data->filename) < 0)
+ goto cleanup;
+
+ if (virAsprintf(&schemata_file, "%s/virresctrldata/%s-free.schemata",
+ abs_srcdir, virResctrlTypeToString(data->type)) < 0)
+ goto cleanup;
+
+ if (virFileWrapperAddPrefix("/sys/fs/resctrl", resctrl) < 0)
+ goto cleanup;
+
+ if ((schemata = virResctrlGetFreeCache(data->type)) == NULL)
+ goto cleanup;
+
+ virFileWrapperClearPrefixes();
+
+ GetSchemataStr(schemata, &schemata_str);
+
+ if (virTestCompareToFile(schemata_str, schemata_file) < 0)
+ goto cleanup;
+
+ ret = 0;
+
+ cleanup:
+ VIR_FREE(dir);
+ VIR_FREE(resctrl);
+ VIR_FREE(schemata_str);
+ virResctrlFreeSchemata(schemata);
+ return ret;
+}
+
+static int
+mymain(void)
+{
+ int ret = 0;
+
+#define DO_TEST_FULL(filename, type) \
+ do { \
+ struct virResctrlData data = {filename, \
+ type}; \
+ if (virTestRun(filename, test_virResctrl, &data) < 0) \
+ ret = -1; \
+ } while (0)
+
+ DO_TEST_FULL("resctrl", VIR_RESCTRL_TYPE_L3);
+ DO_TEST_FULL("resctrl-cdp", VIR_RESCTRL_TYPE_L3_CODE);
+ DO_TEST_FULL("resctrl-cdp", VIR_RESCTRL_TYPE_L3_DATA);
+
+ return ret;
+}
+
+VIR_TEST_MAIN(mymain)
--
1.9.1
More information about the libvir-list
mailing list