[libvirt] [PATCH 2 of 2] Use cgroup functions to set resource limits on LXC domains

[Dan Smith]

This patch adds code to the controller to set up a cgroup called
libvirt/lxc/$name, set the memory limit, and restrict devices.  It also
adds bits to lxc_driver to properly clean up the cgroup on domain death.

If virCgroupHaveSupport() says that no support is available, then we just
allow the domain creation to proceed as it did before without resource
controls in place.

diff -r 8e948eb88328 -r f452190ac168 src/Makefile.am
--- a/src/Makefile.am	Mon Sep 29 09:37:42 2008 -0700
+++ b/src/Makefile.am	Mon Sep 29 09:37:43 2008 -0700
@@ -90,7 +90,8 @@
 		lxc_conf.c lxc_conf.h				\
 		lxc_container.c lxc_container.h			\
 		lxc_driver.c lxc_driver.h			\
-		veth.c veth.h
+		veth.c veth.h                                   \
+		cgroup.c cgroup.h
 
 LXC_CONTROLLER_SOURCES =					\
 		lxc_conf.c lxc_conf.h				\
diff -r 8e948eb88328 -r f452190ac168 src/lxc_controller.c
--- a/src/lxc_controller.c	Mon Sep 29 09:37:42 2008 -0700
+++ b/src/lxc_controller.c	Mon Sep 29 09:37:43 2008 -0700
@@ -42,12 +42,125 @@
 #include "veth.h"
 #include "memory.h"
 #include "util.h"
-
+#include "cgroup.h"
 
 #define DEBUG(fmt,...) VIR_DEBUG(__FILE__, fmt, __VA_ARGS__)
 #define DEBUG0(msg) VIR_DEBUG(__FILE__, "%s", msg)
 
 int debugFlag = 0;
+
+struct cgroup_device_policy {
+    char type;
+    int major;
+    int minor;
+};
+
+static int lxcSetMemory(virDomainDefPtr def, virCgroupPtr cgroup)
+{
+    return virCgroupSetValueU64(cgroup,
+                                "memory.limit_in_bytes",
+                                (uint64_t)(def->maxmem << 10));
+}
+
+static int lxcSetDevices(virDomainDefPtr def, virCgroupPtr cgroup)
+{
+    int rc;
+    int i;
+    struct cgroup_device_policy devices[] = {
+        {'c', 1, 3},
+        {'c', 1, 5},
+        {'c', 1, 7},
+        {'c', 1, 8},
+        {'c', 1, 9},
+        {'c', 5, 1},
+        {0,   0, 0}};
+
+    rc = virCgroupSetValueStr(cgroup,
+                              "devices.deny",
+                              "a");
+    if (rc != 0)
+        return rc;
+
+    for (i = 0; devices[i].type != 0; i++) {
+        char *devstr = NULL;
+
+        rc = asprintf(&devstr,
+                      "%c %i:%i rwm",
+                      devices[i].type,
+                      devices[i].major,
+                      devices[i].minor);
+        if (rc == -1)
+            return -ENOMEM;
+
+        rc = virCgroupSetValueStr(cgroup, "devices.allow", devstr);
+        free(devstr);
+
+        if (rc != 0)
+            return rc;
+    }
+
+    return 0;
+}
+
+static int lxcCreateTypeGroup(virCgroupPtr *tgrp)
+{
+    int ret;
+
+    if (virCgroupOpen(NULL, "lxc", tgrp) != 0) {
+        ret = virCgroupCreate(NULL, "lxc", tgrp);
+        if (ret != 0)
+            fprintf(stderr, "Failed to create lxc: %s\n", strerror(-ret));
+    }
+
+    return ret;
+}
+
+/**
+ * lxcSetContainerResources
+ * @def: pointer to virtual machine structure
+ *
+ * Creates a cgroup for the container, moves the task inside,
+ * and sets resource limits
+ *
+ * Returns 0 on success or -1 in case of error
+ */
+static int lxcSetContainerResources(virDomainDefPtr def)
+{
+    virCgroupPtr lxcgroup;
+    virCgroupPtr cgroup;
+    int rc = -1;
+
+    if (virCgroupHaveSupport() != 0)
+        return 0; /* Not supported, so claim success */
+
+    rc = lxcCreateTypeGroup(&lxcgroup);
+    if (rc != 0)
+        goto out;
+
+    rc = virCgroupCreate(lxcgroup, def->name, &cgroup);
+    if (rc != 0) {
+        fprintf(stderr, "Unable to create cgroup for %s\n", def->name);
+        goto out;
+    }
+
+    rc = lxcSetMemory(def, cgroup);
+    if (rc != 0)
+        goto out;
+
+    rc = lxcSetDevices(def, cgroup);
+    if (rc != 0)
+        goto out;
+
+    rc = virCgroupAddTask(cgroup, getpid());
+out:
+    virCgroupFree(&lxcgroup);
+    virCgroupFree(&cgroup);
+
+    if (rc != 0)
+        fprintf(stderr, "Failed to set lxc resources: %s\n", strerror(-rc));
+
+    return rc;
+}
 
 static char*lxcMonitorPath(virDomainDefPtr def)
 {
@@ -394,6 +507,9 @@
     if (lxcControllerMoveInterfaces(nveths, veths, container) < 0)
         goto cleanup;
 
+    if (lxcSetContainerResources(def) < 0)
+        goto cleanup;
+
     if (lxcContainerSendContinue(control[0]) < 0)
         goto cleanup;
 
@@ -412,6 +528,7 @@
         kill(container, SIGTERM);
         waitpid(container, NULL, 0);
     }
+
     return rc;
 }
 
diff -r 8e948eb88328 -r f452190ac168 src/lxc_driver.c
--- a/src/lxc_driver.c	Mon Sep 29 09:37:42 2008 -0700
+++ b/src/lxc_driver.c	Mon Sep 29 09:37:43 2008 -0700
@@ -43,7 +43,7 @@
 #include "bridge.h"
 #include "veth.h"
 #include "event.h"
-
+#include "cgroup.h"
 
 /* debug macros */
 #define DEBUG(fmt,...) VIR_DEBUG(__FILE__, fmt, __VA_ARGS__)
@@ -376,6 +376,8 @@
     int waitRc;
     int childStatus = -1;
     virDomainNetDefPtr net;
+    virCgroupPtr lxcgroup;
+    virCgroupPtr cgroup;
 
     while (((waitRc = waitpid(vm->pid, &childStatus, 0)) == -1) &&
            errno == EINTR)
@@ -408,6 +410,19 @@
     for (net = vm->def->nets; net; net = net->next) {
         vethInterfaceUpOrDown(net->ifname, 0);
         vethDelete(net->ifname);
+    }
+
+    rc = virCgroupOpen(NULL, "lxc", &lxcgroup);
+    if (rc == 0) {
+        rc = virCgroupOpen(lxcgroup, vm->def->name, &cgroup);
+        if (rc == 0) {
+            virCgroupRemove(cgroup);
+            virCgroupFree(&cgroup);
+        } else {
+            DEBUG("Unable to find cgroup %s to remove", vm->def->name);
+        }
+    } else {
+        DEBUG0("Unable to find lxc cgroup to remove");
     }
 
     return rc;