[Cluster-devel] [GFS2] Introduce workqueue into glock.c

[Steven Whitehouse]

>From f2186755db781995d7dbc4b2f1993e503a64bde7 Mon Sep 17 00:00:00 2001
From: Steven Whitehouse <swhiteho at redhat.com>
Date: Wed, 1 Aug 2007 13:42:51 +0100
Subject: [PATCH] [GFS2] Introduce workqueue into glock.c

The workqueue is used to run the callbacks from the lock
modules so that these no longer block. This also means that
we can either reduce the number of threads in lock_dlm to a single
thread or allow both threads to deliver both types of callback.

The introduction of a workqueue will also allow many future
optimisations as well, but those are for later patches.

Signed-off-by: Steven Whitehouse <swhiteho at redhat.com>

diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index 5e2c269..34c17e1 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -25,6 +25,7 @@
 #include <asm/uaccess.h>
 #include <linux/seq_file.h>
 #include <linux/debugfs.h>
+#include <linux/workqueue.h>
 
 #include "gfs2.h"
 #include "incore.h"
@@ -55,8 +56,11 @@ typedef void (*glock_examiner) (struct gfs2_glock * gl);
 static int gfs2_dump_lockstate(struct gfs2_sbd *sdp);
 static int dump_glock(struct glock_iter *gi, struct gfs2_glock *gl);
 static void gfs2_glock_xmote_th(struct gfs2_glock *gl, struct gfs2_holder *gh);
+static void run_queue(struct gfs2_glock *gl);
+
 static DECLARE_RWSEM(gfs2_umount_flush_sem);
 static struct dentry *gfs2_root;
+static struct workqueue_struct *glock_workqueue;
 
 #define GFS2_GL_HASH_SHIFT      15
 #define GFS2_GL_HASH_SIZE       (1 << GFS2_GL_HASH_SHIFT)
@@ -272,6 +276,16 @@ static struct gfs2_glock *gfs2_glock_find(const struct gfs2_sbd *sdp,
 	return gl;
 }
 
+static void glock_work_func(struct work_struct *work)
+{
+	struct gfs2_glock *gl = container_of(work, struct gfs2_glock, gl_work);
+
+	spin_lock(&gl->gl_spin);
+	run_queue(gl);
+	spin_unlock(&gl->gl_spin);
+	gfs2_glock_put(gl);
+}
+
 /**
  * gfs2_glock_get() - Get a glock, or create one if one doesn't exist
  * @sdp: The GFS2 superblock
@@ -322,6 +336,7 @@ int gfs2_glock_get(struct gfs2_sbd *sdp, u64 number,
 	gl->gl_sbd = sdp;
 	gl->gl_aspace = NULL;
 	lops_init_le(&gl->gl_le, &gfs2_glock_lops);
+	INIT_WORK(&gl->gl_work, glock_work_func);
 
 	/* If this glock protects actual on-disk data or metadata blocks,
 	   create a VFS inode to manage the pages/buffers holding them. */
@@ -783,7 +798,6 @@ out:
 		spin_lock(&gl->gl_spin);
 		gl->gl_req_gh = NULL;
 		clear_bit(GLF_LOCK, &gl->gl_flags);
-		run_queue(gl);
 		spin_unlock(&gl->gl_spin);
 	}
 
@@ -1329,12 +1343,8 @@ static void blocking_cb(struct gfs2_sbd *sdp, struct lm_lockname *name,
 		return;
 
 	handle_callback(gl, state, 1);
-
-	spin_lock(&gl->gl_spin);
-	run_queue(gl);
-	spin_unlock(&gl->gl_spin);
-
-	gfs2_glock_put(gl);
+	if (queue_work(glock_workqueue, &gl->gl_work) == 0)
+		gfs2_glock_put(gl);
 }
 
 /**
@@ -1374,7 +1384,8 @@ void gfs2_glock_cb(void *cb_data, unsigned int type, void *data)
 		if (gfs2_assert_warn(sdp, gl))
 			return;
 		xmote_bh(gl, async->lc_ret);
-		gfs2_glock_put(gl);
+		if (queue_work(glock_workqueue, &gl->gl_work) == 0)
+			gfs2_glock_put(gl);
 		up_read(&gfs2_umount_flush_sem);
 		return;
 	}
@@ -1852,9 +1863,19 @@ int __init gfs2_glock_init(void)
 		rwlock_init(&gl_hash_locks[i]);
 	}
 #endif
+
+	glock_workqueue = create_workqueue("glock_workqueue");
+	if (IS_ERR(glock_workqueue))
+		return PTR_ERR(glock_workqueue);
+
 	return 0;
 }
 
+void gfs2_glock_exit(void)
+{
+	destroy_workqueue(glock_workqueue);
+}
+
 static int gfs2_glock_iter_next(struct glock_iter *gi)
 {
 	struct gfs2_glock *gl;
diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h
index 7721ca3..5ae1102 100644
--- a/fs/gfs2/glock.h
+++ b/fs/gfs2/glock.h
@@ -137,6 +137,8 @@ void gfs2_scand_internal(struct gfs2_sbd *sdp);
 void gfs2_gl_hash_clear(struct gfs2_sbd *sdp, int wait);
 
 int __init gfs2_glock_init(void);
+void gfs2_glock_exit(void);
+
 int gfs2_create_debugfs_file(struct gfs2_sbd *sdp);
 void gfs2_delete_debugfs_file(struct gfs2_sbd *sdp);
 int gfs2_register_debugfs(void);
diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
index cc5ca58..73fcac9 100644
--- a/fs/gfs2/incore.h
+++ b/fs/gfs2/incore.h
@@ -11,6 +11,7 @@
 #define __INCORE_DOT_H__
 
 #include <linux/fs.h>
+#include <linux/workqueue.h>
 
 #define DIO_WAIT	0x00000010
 #define DIO_METADATA	0x00000020
@@ -196,6 +197,7 @@ struct gfs2_glock {
 	struct gfs2_log_element gl_le;
 	struct list_head gl_ail_list;
 	atomic_t gl_ail_count;
+	struct work_struct gl_work;
 };
 
 struct gfs2_alloc {
diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c
index d5d4e68..79c91fd 100644
--- a/fs/gfs2/main.c
+++ b/fs/gfs2/main.c
@@ -107,6 +107,8 @@ static int __init init_gfs2_fs(void)
 fail_unregister:
 	unregister_filesystem(&gfs2_fs_type);
 fail:
+	gfs2_glock_exit();
+
 	if (gfs2_bufdata_cachep)
 		kmem_cache_destroy(gfs2_bufdata_cachep);
 
@@ -127,6 +129,7 @@ fail:
 
 static void __exit exit_gfs2_fs(void)
 {
+	gfs2_glock_exit();
 	gfs2_unregister_debugfs();
 	unregister_filesystem(&gfs2_fs_type);
 	unregister_filesystem(&gfs2meta_fs_type);
-- 
1.5.1.2