[Virtio-fs] [RFC] [PATCH][v2]: Auto switch between inline and thread pool processing
Vivek Goyal
vgoyal at redhat.com
Wed Apr 28 18:06:39 UTC 2021
For low queue depth workloads, inline processing works well. But for
high queue depth (and multiple process/thread) workloads, parallel
processing of thread pool can benefit.
This patch is an experiment which tries to switch to between inline and
thread-pool processing.
If number of requests received on queue is less than or equal to user
specified threshold, inline processing is done. Otherwise requests are
handed over to a thread pool for processing.
A user can specify the threshold using option
"--auto-switch-threshold=<threshold>".
I have got good results with "--thread-pool-size=16 --auto-switch-threshold=3"
on my system.
Signed-off-by: Vivek Goyal <vgoyal at redhat.com>
---
Changes since v1:
Added a knob to specify auto switch threshold. This will also enable
auto switching. By default it is turned off.
---
tools/virtiofsd/fuse_i.h | 1 +
tools/virtiofsd/fuse_lowlevel.c | 5 ++++-
tools/virtiofsd/fuse_virtio.c | 32 ++++++++++++++++++++++----------
3 files changed, 27 insertions(+), 11 deletions(-)
Index: rhvgoyal-qemu/tools/virtiofsd/fuse_virtio.c
===================================================================
--- rhvgoyal-qemu.orig/tools/virtiofsd/fuse_virtio.c 2021-04-28 13:42:34.855145073 -0400
+++ rhvgoyal-qemu/tools/virtiofsd/fuse_virtio.c 2021-04-28 13:48:21.980538754 -0400
@@ -446,6 +446,15 @@ err:
static __thread bool clone_fs_called;
/* Process one FVRequest in a thread pool */
+static void fv_queue_push_to_pool(gpointer data, gpointer user_data)
+{
+ FVRequest *req = data;
+ GThreadPool *pool = user_data;
+
+ g_thread_pool_push(pool, req, NULL);
+}
+
+/* Process one FVRequest in a thread pool */
static void fv_queue_worker(gpointer data, gpointer user_data)
{
struct fv_QueueInfo *qi = user_data;
@@ -605,10 +614,12 @@ static void *fv_queue_thread(void *opaqu
struct fuse_session *se = qi->virtio_dev->se;
GThreadPool *pool = NULL;
GList *req_list = NULL;
+ int nr_req = 0;
if (se->thread_pool_size) {
- fuse_log(FUSE_LOG_DEBUG, "%s: Creating thread pool for Queue %d\n",
- __func__, qi->qidx);
+ fuse_log(FUSE_LOG_DEBUG, "%s: Creating thread pool for Queue %d."
+ " auto_switch_threshold=%u\n", __func__, qi->qidx,
+ se->auto_switch_threshold);
pool = g_thread_pool_new(fv_queue_worker, qi, se->thread_pool_size,
FALSE, NULL);
if (!pool) {
@@ -686,22 +697,23 @@ static void *fv_queue_thread(void *opaqu
}
req->reply_sent = false;
-
- if (!se->thread_pool_size) {
- req_list = g_list_prepend(req_list, req);
- } else {
- g_thread_pool_push(pool, req, NULL);
- }
+ req_list = g_list_prepend(req_list, req);
+ nr_req++;
}
pthread_mutex_unlock(&qi->vq_lock);
vu_dispatch_unlock(qi->virtio_dev);
/* Process all the requests. */
- if (!se->thread_pool_size && req_list != NULL) {
- g_list_foreach(req_list, fv_queue_worker, qi);
+ if (req_list != NULL) {
+ if (!se->thread_pool_size || nr_req <= se->auto_switch_threshold) {
+ g_list_foreach(req_list, fv_queue_worker, qi);
+ } else {
+ g_list_foreach(req_list, fv_queue_push_to_pool, pool);
+ }
g_list_free(req_list);
req_list = NULL;
+ nr_req = 0;
}
}
Index: rhvgoyal-qemu/tools/virtiofsd/fuse_i.h
===================================================================
--- rhvgoyal-qemu.orig/tools/virtiofsd/fuse_i.h 2021-04-28 13:42:34.855145073 -0400
+++ rhvgoyal-qemu/tools/virtiofsd/fuse_i.h 2021-04-28 13:42:35.903188526 -0400
@@ -73,6 +73,7 @@ struct fuse_session {
int vu_socketfd;
struct fv_VuDev *virtio_dev;
int thread_pool_size;
+ unsigned auto_switch_threshold;
};
struct fuse_chan {
Index: rhvgoyal-qemu/tools/virtiofsd/fuse_lowlevel.c
===================================================================
--- rhvgoyal-qemu.orig/tools/virtiofsd/fuse_lowlevel.c 2021-04-28 13:42:34.855145073 -0400
+++ rhvgoyal-qemu/tools/virtiofsd/fuse_lowlevel.c 2021-04-28 13:42:35.904188567 -0400
@@ -2432,6 +2432,7 @@ static const struct fuse_opt fuse_ll_opt
LL_OPTION("--socket-group=%s", vu_socket_group, 0),
LL_OPTION("--fd=%d", vu_listen_fd, 0),
LL_OPTION("--thread-pool-size=%d", thread_pool_size, 0),
+ LL_OPTION("--auto-switch-threshold=%u", auto_switch_threshold, 0),
FUSE_OPT_END
};
@@ -2452,7 +2453,8 @@ void fuse_lowlevel_help(void)
" --socket-path=PATH path for the vhost-user socket\n"
" --socket-group=GRNAME name of group for the vhost-user socket\n"
" --fd=FDNUM fd number of vhost-user socket\n"
- " --thread-pool-size=NUM thread pool size limit (default %d)\n",
+ " --thread-pool-size=NUM thread pool size limit (default %d)\n"
+ " --auto-switch-threshold=NUM number of request threshold to switch between inline and thread pool processing of request\n",
THREAD_POOL_SIZE);
}
@@ -2508,6 +2510,7 @@ struct fuse_session *fuse_session_new(st
se->fd = -1;
se->vu_listen_fd = -1;
se->thread_pool_size = THREAD_POOL_SIZE;
+ se->auto_switch_threshold = 0;
se->conn.max_write = UINT_MAX;
se->conn.max_readahead = UINT_MAX;
More information about the Virtio-fs
mailing list