[libvirt] [PATCH] qemu: Add support for overriding max threads per process limit
Jim Fehlig
jfehlig at suse.com
Wed Jun 12 17:14:47 UTC 2019
On 6/6/19 11:40 AM, Jim Fehlig wrote:
> Some VM configurations may result in a large number of threads created by
> the associated qemu process which can exceed the system default limit. The
> maximum number of threads allowed per process is controlled by the pids
> cgroup controller and is set to 16k when creating VMs with systemd's
> machined service. The maximum number of threads per process is recorded
> in the pids.max file under the machine's pids controller cgroup hierarchy,
> e.g.
>
> $cgrp-mnt/pids/machine.slice/machine-qemu\\x2d1\\x2dtest.scope/pids.max
>
> Maximum threads per process is controlled with the TasksMax property of
> the systemd scope for the machine. This patch adds an option to qemu.conf
> which can be used to override the maximum number of threads allowed per
> qemu process. If the value of option is greater than zero, it will be set
> in the TasksMax property of the machine's scope after creating the machine.
>
> Signed-off-by: Jim Fehlig <jfehlig at suse.com>
> ---
> src/lxc/lxc_cgroup.c | 1 +
> src/qemu/libvirtd_qemu.aug | 1 +
> src/qemu/qemu.conf | 10 ++++++++++
> src/qemu/qemu_cgroup.c | 1 +
> src/qemu/qemu_conf.c | 2 ++
> src/qemu/qemu_conf.h | 1 +
> src/qemu/test_libvirtd_qemu.aug.in | 1 +
> src/util/vircgroup.c | 6 +++++-
> src/util/vircgroup.h | 1 +
> src/util/virsystemd.c | 24 +++++++++++++++++++++++-
> src/util/virsystemd.h | 3 ++-
> tests/virsystemdtest.c | 12 ++++++------
> 12 files changed, 54 insertions(+), 9 deletions(-)
>
> diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c
> index d93a19d684..76014f3bfd 100644
> --- a/src/lxc/lxc_cgroup.c
> +++ b/src/lxc/lxc_cgroup.c
> @@ -455,6 +455,7 @@ virCgroupPtr virLXCCgroupCreate(virDomainDefPtr def,
> nnicindexes, nicindexes,
> def->resource->partition,
> -1,
> + 0,
> &cgroup) < 0)
> goto cleanup;
>
> diff --git a/src/qemu/libvirtd_qemu.aug b/src/qemu/libvirtd_qemu.aug
> index b311f02da6..c70b903fed 100644
> --- a/src/qemu/libvirtd_qemu.aug
> +++ b/src/qemu/libvirtd_qemu.aug
> @@ -94,6 +94,7 @@ module Libvirtd_qemu =
> | limits_entry "max_core"
> | bool_entry "dump_guest_core"
> | str_entry "stdio_handler"
> + | int_entry "max_threads_per_process"
>
> let device_entry = bool_entry "mac_filter"
> | bool_entry "relaxed_acs_check"
> diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf
> index 5a85789d81..ab044c9cf3 100644
> --- a/src/qemu/qemu.conf
> +++ b/src/qemu/qemu.conf
> @@ -608,6 +608,16 @@
> #max_processes = 0
> #max_files = 0
>
> +# If max_threads_per_process is set to a positive integer, libvirt
> +# will use it to set the maximum number of threads that can be
> +# created by a qemu process. Some VM configurations can result in
> +# qemu processes with tens of thousands of threads. systemd-based
> +# systems typically limit the number of threads per process to
> +# 16k. max_threads_per_process can be used to override default
> +# limits in the host OS.
> +#
> +#max_threads_per_process = 0
> +
> # If max_core is set to a non-zero integer, then QEMU will be
> # permitted to create core dumps when it crashes, provided its
> # RAM size is smaller than the limit set.
> diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c
> index ca76c4fdfa..9603f33e8a 100644
> --- a/src/qemu/qemu_cgroup.c
> +++ b/src/qemu/qemu_cgroup.c
> @@ -930,6 +930,7 @@ qemuInitCgroup(virDomainObjPtr vm,
> nnicindexes, nicindexes,
> vm->def->resource->partition,
> cfg->cgroupControllers,
> + cfg->maxThreadsPerProc,
> &priv->cgroup) < 0) {
> if (virCgroupNewIgnoreError())
> goto done;
> diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c
> index daea11dacb..8ac2dc92b5 100644
> --- a/src/qemu/qemu_conf.c
> +++ b/src/qemu/qemu_conf.c
> @@ -687,6 +687,8 @@ virQEMUDriverConfigLoadProcessEntry(virQEMUDriverConfigPtr cfg,
> return -1;
> if (virConfGetValueUInt(conf, "max_files", &cfg->maxFiles) < 0)
> return -1;
> + if (virConfGetValueUInt(conf, "max_threads_per_process", &cfg->maxThreadsPerProc) < 0)
> + return -1;
>
> if (virConfGetValueType(conf, "max_core") == VIR_CONF_STRING) {
> if (virConfGetValueString(conf, "max_core", &corestr) < 0)
> diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h
> index 983e74a3cf..48b8711cbd 100644
> --- a/src/qemu/qemu_conf.h
> +++ b/src/qemu/qemu_conf.h
> @@ -171,6 +171,7 @@ struct _virQEMUDriverConfig {
>
> unsigned int maxProcesses;
> unsigned int maxFiles;
> + unsigned int maxThreadsPerProc;
> unsigned long long maxCore;
> bool dumpGuestCore;
>
> diff --git a/src/qemu/test_libvirtd_qemu.aug.in b/src/qemu/test_libvirtd_qemu.aug.in
> index fea1d308b7..ac7ad59ba8 100644
> --- a/src/qemu/test_libvirtd_qemu.aug.in
> +++ b/src/qemu/test_libvirtd_qemu.aug.in
> @@ -75,6 +75,7 @@ module Test_libvirtd_qemu =
> { "set_process_name" = "1" }
> { "max_processes" = "0" }
> { "max_files" = "0" }
> +{ "max_threads_per_process" = "0" }
> { "max_core" = "unlimited" }
> { "dump_guest_core" = "1" }
> { "mac_filter" = "1" }
> diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c
> index f58e336404..c31c34e5f8 100644
> --- a/src/util/vircgroup.c
> +++ b/src/util/vircgroup.c
> @@ -1106,6 +1106,7 @@ virCgroupNewMachineSystemd(const char *name,
> int *nicindexes,
> const char *partition,
> int controllers,
> + unsigned int maxthreads,
> virCgroupPtr *group)
> {
> int rv;
> @@ -1122,7 +1123,8 @@ virCgroupNewMachineSystemd(const char *name,
> isContainer,
> nnicindexes,
> nicindexes,
> - partition)) < 0)
> + partition,
> + maxthreads)) < 0)
> return rv;
>
> if (controllers != -1)
> @@ -1234,6 +1236,7 @@ virCgroupNewMachine(const char *name,
> int *nicindexes,
> const char *partition,
> int controllers,
> + unsigned int maxthreads,
> virCgroupPtr *group)
> {
> int rv;
> @@ -1250,6 +1253,7 @@ virCgroupNewMachine(const char *name,
> nicindexes,
> partition,
> controllers,
> + maxthreads,
> group)) == 0)
> return 0;
>
> diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h
> index 377e0fd870..3fb99f70ef 100644
> --- a/src/util/vircgroup.h
> +++ b/src/util/vircgroup.h
> @@ -99,6 +99,7 @@ int virCgroupNewMachine(const char *name,
> int *nicindexes,
> const char *partition,
> int controllers,
> + unsigned int maxthreads,
> virCgroupPtr *group)
> ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2)
> ATTRIBUTE_NONNULL(3);
> diff --git a/src/util/virsystemd.c b/src/util/virsystemd.c
> index 3f03e3bd63..497d100a5c 100644
> --- a/src/util/virsystemd.c
> +++ b/src/util/virsystemd.c
> @@ -238,12 +238,14 @@ int virSystemdCreateMachine(const char *name,
> bool iscontainer,
> size_t nnicindexes,
> int *nicindexes,
> - const char *partition)
> + const char *partition,
> + unsigned int maxthreads)
> {
> int ret;
> DBusConnection *conn;
> char *creatorname = NULL;
> char *slicename = NULL;
> + char *scopename = NULL;
> static int hasCreateWithNetwork = 1;
>
> if ((ret = virSystemdHasMachined()) < 0)
> @@ -389,11 +391,31 @@ int virSystemdCreateMachine(const char *name,
> goto cleanup;
> }
>
> + if (maxthreads > 0) {
> + if (!(scopename = virSystemdMakeScopeName(name, drivername, false)))
> + goto cleanup;
> +
> + if (virDBusCallMethod(conn,
> + NULL,
> + NULL,
> + "org.freedesktop.systemd1",
> + "/org/freedesktop/systemd1",
> + "org.freedesktop.systemd1.Manager",
> + "SetUnitProperties",
> + "sba(sv)",
> + scopename,
> + true,
> + 1,
> + "TasksMax", "t", maxthreads) < 0)
While testing this on older systems (SLES12 SP2/3/4) I needed to cast maxthreads
to unit64_t, else I got some pretty large values passed to the SetUnitProperties
method. E.g. with a setting of 65536 in qemu.conf
method call sender=:1.18 -> dest=org.freedesktop.systemd1 serial=17
path=/org/freedesktop/systemd1; interface=org.freedesktop.systemd1.Manager;
member=SetUnitProperties
string "machine-qemu\x2d1\x2dsles12sp2.scope"
boolean true
array [
struct {
string "TasksMax"
variant uint64 6803228262400
}
]
I've squashed the following in my local branch
diff --git a/src/util/virsystemd.c b/src/util/virsystemd.c
index 497d100a5c..ba6b1edc6f 100644
--- a/src/util/virsystemd.c
+++ b/src/util/virsystemd.c
@@ -406,7 +406,7 @@ int virSystemdCreateMachine(const char *name,
scopename,
true,
1,
- "TasksMax", "t", maxthreads) < 0)
+ "TasksMax", "t", (uint64_t)maxthreads) < 0)
goto cleanup;
}
Regards,
Jim
> + goto cleanup;
> + }
> +
> ret = 0;
>
> cleanup:
> VIR_FREE(creatorname);
> VIR_FREE(slicename);
> + VIR_FREE(scopename);
> return ret;
> }
>
> diff --git a/src/util/virsystemd.h b/src/util/virsystemd.h
> index 7d9c0ebd62..bdce3b2e9d 100644
> --- a/src/util/virsystemd.h
> +++ b/src/util/virsystemd.h
> @@ -37,7 +37,8 @@ int virSystemdCreateMachine(const char *name,
> bool iscontainer,
> size_t nnicindexes,
> int *nicindexes,
> - const char *partition);
> + const char *partition,
> + unsigned int maxthreads);
>
> int virSystemdTerminateMachine(const char *name);
>
> diff --git a/tests/virsystemdtest.c b/tests/virsystemdtest.c
> index 82c02decd1..478fa844fa 100644
> --- a/tests/virsystemdtest.c
> +++ b/tests/virsystemdtest.c
> @@ -172,7 +172,7 @@ static int testCreateContainer(const void *opaque ATTRIBUTE_UNUSED)
> 123,
> true,
> 0, NULL,
> - "highpriority.slice") < 0) {
> + "highpriority.slice", 0) < 0) {
> fprintf(stderr, "%s", "Failed to create LXC machine\n");
> return -1;
> }
> @@ -205,7 +205,7 @@ static int testCreateMachine(const void *opaque ATTRIBUTE_UNUSED)
> 123,
> false,
> 0, NULL,
> - NULL) < 0) {
> + NULL, 0) < 0) {
> fprintf(stderr, "%s", "Failed to create KVM machine\n");
> return -1;
> }
> @@ -242,7 +242,7 @@ static int testCreateNoSystemd(const void *opaque ATTRIBUTE_UNUSED)
> 123,
> false,
> 0, NULL,
> - NULL)) == 0) {
> + NULL, 0)) == 0) {
> unsetenv("FAIL_NO_SERVICE");
> fprintf(stderr, "%s", "Unexpected create machine success\n");
> return -1;
> @@ -276,7 +276,7 @@ static int testCreateSystemdNotRunning(const void *opaque ATTRIBUTE_UNUSED)
> 123,
> false,
> 0, NULL,
> - NULL)) == 0) {
> + NULL, 0)) == 0) {
> unsetenv("FAIL_NOT_REGISTERED");
> fprintf(stderr, "%s", "Unexpected create machine success\n");
> return -1;
> @@ -310,7 +310,7 @@ static int testCreateBadSystemd(const void *opaque ATTRIBUTE_UNUSED)
> 123,
> false,
> 0, NULL,
> - NULL)) == 0) {
> + NULL, 0)) == 0) {
> unsetenv("FAIL_BAD_SERVICE");
> fprintf(stderr, "%s", "Unexpected create machine success\n");
> return -1;
> @@ -345,7 +345,7 @@ static int testCreateNetwork(const void *opaque ATTRIBUTE_UNUSED)
> 123,
> true,
> nnicindexes, nicindexes,
> - "highpriority.slice") < 0) {
> + "highpriority.slice", 0) < 0) {
> fprintf(stderr, "%s", "Failed to create LXC machine\n");
> return -1;
> }
>
More information about the libvir-list
mailing list