[libvirt] [PATH V10 09/12] Resctrl: concurrence support

Eli Qiao liyong.qiao at intel.com
Mon Mar 6 09:50:40 UTC 2017


The internal struct list domainall is a list which are resctral domain
status shared by all VMs, especiall the default domain, each VM should
access it concomitantly. Ues a mutex to control it.

Each bank's cache_left field is also a global shared resource we need
to be care, add a mutex for each bank.

We need also to add lock to access /sys/fs/resctrl, use flock.

Signed-off-by: Eli Qiao <liyong.qiao at intel.com>
---
 src/util/virresctrl.c | 93 ++++++++++++++++++++++++++++++++++++++++++++-------
 src/util/virresctrl.h |  3 ++
 2 files changed, 83 insertions(+), 13 deletions(-)

diff --git a/src/util/virresctrl.c b/src/util/virresctrl.c
index 9f0d05f..3c54e44 100644
--- a/src/util/virresctrl.c
+++ b/src/util/virresctrl.c
@@ -20,7 +20,9 @@
  */
 #include <config.h>
 #include <fcntl.h>
+#include <sys/file.h>
 #include <sys/stat.h>
+#include <sys/types.h>
 
 #include "virresctrl.h"
 #include "viralloc.h"
@@ -62,6 +64,9 @@ do { \
 
 #define VIR_RESCTRL_GET_SCHEMATA(count) ((1 << count) - 1)
 
+#define VIR_RESCTRL_LOCK(fd, op) flock(fd, op)
+#define VIR_RESCTRL_UNLOCK(fd) flock(fd, LOCK_UN)
+
 /**
  * a virResSchemata represents a schemata object under a resource control
  * domain.
@@ -105,6 +110,8 @@ typedef virResCtrlDomain *virResCtrlDomainPtr;
 struct _virResCtrlDomain {
     unsigned int num_domains;
     virResDomainPtr domains;
+
+    virMutex lock;
 };
 
 static unsigned int host_id;
@@ -176,11 +183,16 @@ static int virResCtrlGetStr(const char *domain_name, const char *item_name, char
 
     CONSTRUCT_RESCTRL_PATH(domain_name, item_name);
 
+    if (!virFileExists(path))
+        goto cleanup;
+
     if (virFileReadAll(path, MAX_FILE_LEN, ret) < 0) {
         rc = -1;
         goto cleanup;
     }
 
+    rc = 0;
+
  cleanup:
     VIR_FREE(path);
     return rc;
@@ -687,10 +699,15 @@ static int
 virResCtrlAppendDomain(virResDomainPtr dom)
 {
     virResDomainPtr p = domainall.domains;
+
+    virMutexLock(&domainall.lock);
+
     while (p->next != NULL) p = p->next;
     p->next = dom;
     dom->pre = p;
     domainall.num_domains += 1;
+
+    virMutexUnlock(&domainall.lock);
     return 0;
 }
 
@@ -713,18 +730,22 @@ virResCtrlCalculateSchemata(int type,
 {
     size_t i;
     int count;
+    int rc = -1;
     virResDomainPtr p;
     unsigned int tmp_schemata;
     unsigned int schemata_sum = 0;
     int pair_type = 0;
 
+    virMutexLock(&resctrlall[type].cache_banks[sid].lock);
+
     if (resctrlall[type].cache_banks[sid].cache_left < size) {
         VIR_ERROR(_("Not enough cache left on bank %u"), hostid);
-        return -1;
+        goto cleanup;
     }
+
     if ((count = size / resctrlall[type].cache_banks[sid].cache_min) <= 0) {
         VIR_ERROR(_("Error cache size %llu"), size);
-        return -1;
+        goto cleanup;
     }
 
     tmp_schemata = VIR_RESCTRL_GET_SCHEMATA(count);
@@ -739,7 +760,7 @@ virResCtrlCalculateSchemata(int type,
     if (type == VIR_RDT_RESOURCE_L3CODE)
         pair_type = VIR_RDT_RESOURCE_L3DATA;
 
-    for (i = 1; i < domainall.num_domains; i ++) {
+    for (i = 1; i < domainall.num_domains; i++) {
         schemata_sum |= p->schematas[type]->schemata_items[sid].schemata;
         if (pair_type > 0)
             schemata_sum |= p->schematas[pair_type]->schemata_items[sid].schemata;
@@ -750,7 +771,16 @@ virResCtrlCalculateSchemata(int type,
 
     while ((tmp_schemata & schemata_sum) != 0)
         tmp_schemata = tmp_schemata >> 1;
-    return tmp_schemata;
+
+    resctrlall[type].cache_banks[sid].cache_left -= size;
+    if (pair_type > 0)
+        resctrlall[pair_type].cache_banks[sid].cache_left = resctrlall[type].cache_banks[sid].cache_left;
+
+    rc = tmp_schemata;
+
+ cleanup:
+    virMutexUnlock(&resctrlall[type].cache_banks[sid].lock);
+    return rc;
 }
 
 int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
@@ -763,8 +793,8 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
     int pair_type = -1;
     int sid;
     int schemata;
-
-    virUUIDFormat(uuid, name);
+    int lockfd;
+    int rc = -1;
 
     for (i = 0; i < cachetune->n_banks; i++) {
         VIR_DEBUG("cache_banks %u, %u, %llu, %s",
@@ -777,12 +807,21 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
     if (cachetune->n_banks < 1)
         return 0;
 
+    virUUIDFormat(uuid, name);
+
+    if ((lockfd = open(RESCTRL_DIR, O_RDONLY)) < 0)
+        goto cleanup;
+
+    if (VIR_RESCTRL_LOCK(lockfd, LOCK_EX) < 0) {
+        virReportSystemError(errno, _("Unable to lock '%s'"), RESCTRL_DIR);
+        goto cleanup;
+    }
+
     p = virResCtrlGetDomain(name);
     if (p == NULL) {
         VIR_DEBUG("no domain name %s found, create new one!", name);
         p = virResCtrlCreateDomain(name);
     }
-
     if (p != NULL) {
 
         virResCtrlAppendDomain(p);
@@ -828,19 +867,25 @@ int virResCtrlSetCacheBanks(virDomainCachetunePtr cachetune,
         if (virResCtrlFlushDomainToSysfs(p) < 0) {
             VIR_ERROR(_("failed to flush domain %s to sysfs"), name);
             virResCtrlDestroyDomain(p);
-            return -1;
+            goto cleanup;
         }
     } else {
         VIR_ERROR(_("Failed to create a domain in sysfs"));
-        return -1;
+        goto cleanup;
     }
 
     virResCtrlRefreshSchemata();
     /* after refresh, flush header's schemata changes to sys fs */
-    if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0)
-        VIR_WARN("failed to flush domain to sysfs");
+    if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0) {
+        VIR_ERROR(_("failed to flush domain to sysfs"));
+        goto cleanup;
+    }
 
-    return 0;
+    rc = 0;
+
+ cleanup:
+    VIR_RESCTRL_UNLOCK(lockfd);
+    return rc;
 }
 
 /* Should be called after pid disappeared, we recalculate
@@ -856,6 +901,9 @@ int virResCtrlUpdate(unsigned char *uuid)
     del = virResCtrlGetDomain(name);
 
     if (del != NULL) {
+
+        virMutexLock(&domainall.lock);
+
         del->pre->next = del->next;
         if (del->next != NULL)
             del->next->pre = del->pre;
@@ -864,6 +912,7 @@ int virResCtrlUpdate(unsigned char *uuid)
         virResCtrlRefreshSchemata();
         if (virResCtrlFlushDomainToSysfs(domainall.domains) < 0)
             VIR_WARN("failed to flush domain to sysfs");
+        virMutexUnlock(&domainall.lock);
     }
     return 0;
 }
@@ -871,7 +920,7 @@ int virResCtrlUpdate(unsigned char *uuid)
 int
 virResCtrlInit(void)
 {
-    size_t i = 0;
+    size_t i, j;
     char *tmp;
     int rc = 0;
 
@@ -898,6 +947,24 @@ virResCtrlInit(void)
 
     domainall.domains = virResCtrlGetAllDomains(&(domainall.num_domains));
 
+    for (i = 0; i < VIR_RDT_RESOURCE_LAST; i++) {
+        if (VIR_RESCTRL_ENABLED(i)) {
+            for (j = 0; j < resctrlall[i].num_banks; j++) {
+                if (virMutexInit(&resctrlall[i].cache_banks[j].lock) < 0) {
+                    virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                            _("Unable to initialize mutex"));
+                    return -1;
+                }
+            }
+        }
+    }
+
+    if (virMutexInit(&domainall.lock) < 0) {
+        virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
+                _("Unable to initialize mutex"));
+        return -1;
+    }
+
     if ((rc = virResCtrlRefreshSchemata()) < 0)
         virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
                 _("Failed to refresh resource control"));
diff --git a/src/util/virresctrl.h b/src/util/virresctrl.h
index d639de1..968e0dc 100644
--- a/src/util/virresctrl.h
+++ b/src/util/virresctrl.h
@@ -27,6 +27,7 @@
 # include "virbitmap.h"
 # include "virutil.h"
 # include "conf/domain_conf.h"
+# include "virthread.h"
 
 #define MAX_CPU_SOCKET_NUM 8
 
@@ -49,6 +50,8 @@ struct _virResCacheBank {
     unsigned long long cache_left;
     unsigned long long cache_min;
     virBitmapPtr cpu_mask;
+
+    virMutex lock;
 };
 
 /**
-- 
1.9.1




More information about the libvir-list mailing list