[libvirt] [PATCH] set cpu bandwidth for the vm

[Wen Congyang]

The cpu bandwidth is applied at the vcpu group level. We should apply it
at the vm group level too, because the vm may do heavy I/O, and it will affect
the other vm.

We apply cpu bandwidth at the vcpu and the vm group level, so we must ensure
that max(child_quota) <= parent_quota when we modify cpu bandwidth.

---
 src/qemu/qemu_cgroup.c |   38 ++++++++++-------
 src/qemu/qemu_driver.c |  103 ++++++++++++++++++++++++++++++++++++-----------
 2 files changed, 101 insertions(+), 40 deletions(-)

diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
index d6e4cbc..2a10bd2 100644
--- a/src/qemu/qemu_cgroup.c
+++ b/src/qemu/qemu_cgroup.c
@@ -435,6 +435,7 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
     unsigned int i;
     unsigned long long period = vm->def->cputune.period;
     long long quota = vm->def->cputune.quota;
+    long long vm_quota = 0;
 
     if (driver->cgroup == NULL)
         return 0; /* Not supported, so claim success */
@@ -447,26 +448,31 @@ int qemuSetupCgroupForVcpu(struct qemud_driver *driver, virDomainObjPtr vm)
         goto cleanup;
     }
 
+    /* Set cpu bandwidth for the vm */
+    if (period || quota) {
+        if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPU)) {
+            /* Ensure that we can multiply by vcpus without overflowing. */
+            if (quota > LLONG_MAX / vm->def->vcpus) {
+                virReportSystemError(EINVAL,
+                                     _("%s"),
+                                     "Unable to set cpu bandwidth quota");
+                goto cleanup;
+            }
+
+            if (quota > 0)
+                vm_quota = quota * vm->def->vcpus;
+            else
+                vm_quota = quota;
+            if (qemuSetupCgroupVcpuBW(cgroup, period, vm_quota) < 0)
+                goto cleanup;
+        }
+    }
+
     if (priv->nvcpupids == 0 || priv->vcpupids[0] == vm->pid) {
         /* If we does not know VCPU<->PID mapping or all vcpu runs in the same
          * thread, we cannot control each vcpu.
          */
-        if (period || quota) {
-            if (qemuCgroupControllerActive(driver, VIR_CGROUP_CONTROLLER_CPU)) {
-                /* Ensure that we can multiply by vcpus without overflowing. */
-                if (quota > LLONG_MAX / vm->def->vcpus) {
-                    virReportSystemError(EINVAL,
-                                         _("%s"),
-                                         "Unable to set cpu bandwidth quota");
-                    goto cleanup;
-                }
-
-                if (quota > 0)
-                    quota *= vm->def->vcpus;
-                if (qemuSetupCgroupVcpuBW(cgroup, period, quota) < 0)
-                    goto cleanup;
-            }
-        }
+        virCgroupFree(&cgroup);
         return 0;
     }
 
diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c
index 5df58b1..52e5d69 100644
--- a/src/qemu/qemu_driver.c
+++ b/src/qemu/qemu_driver.c
@@ -6051,43 +6051,98 @@ qemuSetVcpusBWLive(virDomainObjPtr vm, virCgroupPtr cgroup,
     qemuDomainObjPrivatePtr priv = vm->privateData;
     virCgroupPtr cgroup_vcpu = NULL;
     int rc;
+    long long vm_quota = 0;
+    long long old_quota = 0;
+    unsigned long long old_period = 0;
 
     if (period == 0 && quota == 0)
         return 0;
 
-    if (priv->nvcpupids == 0 || priv->vcpupids[0] == vm->pid) {
-        /* If we does not know VCPU<->PID mapping or all vcpu runs in the same
-         * thread, we cannot control each vcpu.
-         */
-        /* Ensure that we can multiply by vcpus without overflowing. */
-        if (quota > LLONG_MAX / vm->def->vcpus) {
-            virReportSystemError(EINVAL,
-                                 _("%s"),
-                                 "Unable to set cpu bandwidth quota");
-            goto cleanup;
-        }
+    /* Ensure that we can multiply by vcpus without overflowing. */
+    if (quota > LLONG_MAX / vm->def->vcpus) {
+        virReportSystemError(EINVAL,
+                             _("%s"),
+                             "Unable to set cpu bandwidth quota");
+        goto cleanup;
+    }
+
+    if (quota > 0)
+        vm_quota = quota * vm->def->vcpus;
+    else
+        vm_quota = quota;
 
-        if (quota > 0)
-            quota *= vm->def->vcpus;
-        return qemuSetupCgroupVcpuBW(cgroup, period, quota);
+    rc = virCgroupGetCpuCfsQuota(cgroup, &old_quota);
+    if (rc < 0) {
+        virReportSystemError(-rc, "%s",
+                             _("unable to get cpu bandwidth tunable"));
+        goto cleanup;
     }
 
-    for (i = 0; i < priv->nvcpupids; i++) {
-        rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 0);
-        if (rc < 0) {
-            virReportSystemError(-rc,
-                                 _("Unable to find vcpu cgroup for %s(vcpu:"
-                                   " %d)"),
-                                 vm->def->name, i);
-            goto cleanup;
+    rc = virCgroupGetCpuCfsPeriod(cgroup, &old_period);
+    if (rc < 0) {
+        virReportSystemError(-rc, "%s",
+                             _("unable to get cpu bandwidth period tunable"));
+        goto cleanup;
+    }
+
+    /*
+     * If quota will be changed to a small value, we should modify vcpu's quota
+     * first. Otherwise, we should modify vm's quota first.
+     *
+     * If period will be changed to a small value, we should modify vm's period
+     * first. Otherwise, we should modify vcpu's period first.
+     *
+     * If both quota and period will be changed to a big/small value, we cannot
+     * modify period and quota together.
+     */
+    if ((quota != 0) && (period != 0)) {
+        if (((quota > old_quota) && (period > old_period)) ||
+            ((quota < old_quota) && (period < old_period))) {
+            /* modify period */
+            if (qemuSetVcpusBWLive(vm, cgroup, period, 0) < 0)
+                goto cleanup;
+
+            /* modify quota */
+            if (qemuSetVcpusBWLive(vm, cgroup, 0, quota) < 0)
+                goto cleanup;
+            return 0;
         }
+    }
 
-        if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
+    if (((vm_quota != 0) && (vm_quota > old_quota)) ||
+        ((period != 0) && (period < old_period)))
+        /* Set cpu bandwidth for the vm */
+        if (qemuSetupCgroupVcpuBW(cgroup, period, vm_quota) < 0)
             goto cleanup;
 
-        virCgroupFree(&cgroup_vcpu);
+    /* If we does not know VCPU<->PID mapping or all vcpu runs in the same
+     * thread, we cannot control each vcpu. So we only modify cpu bandwidth
+     * when each vcpu has a separated thread.
+     */
+    if (priv->nvcpupids != 0 && priv->vcpupids[0] != vm->pid) {
+        for (i = 0; i < priv->nvcpupids; i++) {
+            rc = virCgroupForVcpu(cgroup, i, &cgroup_vcpu, 0);
+            if (rc < 0) {
+                virReportSystemError(-rc,
+                                     _("Unable to find vcpu cgroup for %s(vcpu:"
+                                       " %d)"),
+                                     vm->def->name, i);
+                goto cleanup;
+            }
+
+            if (qemuSetupCgroupVcpuBW(cgroup_vcpu, period, quota) < 0)
+                goto cleanup;
+
+            virCgroupFree(&cgroup_vcpu);
+        }
     }
 
+    if (((vm_quota != 0) && (vm_quota <= old_quota)) ||
+        ((period != 0) && (period >= old_period)))
+        /* Set cpu bandwidth for the vm */
+        if (qemuSetupCgroupVcpuBW(cgroup, period, vm_quota) < 0)
+            goto cleanup;
+
     return 0;
 
 cleanup:
-- 
1.7.1