[Cluster-devel] [GFS2] Fix bz 224480, pass on the baton for unlinked, but open inodes
Steven Whitehouse
swhiteho at redhat.com
Mon Mar 12 12:18:33 UTC 2007
Hi,
This is my current patch to fix bz 224480 which survives my torture test
of setting the drop_count too low and then running postmark. It needs
wider testing though as it makes a number of changes to the demote code.
This patch removes the memory allocation which was previously required
in order to queue a demotion request on a glock. This was mentioned in
bz 221152 as a future objective. Also, this then allows the use of the
GLF_DEMOTE flag in GFS2's new ->drop_inode() routine to clear the link
count on the inode (the callback only occurs if another node tries to
demote the iopen glock for the inode, and this only happens in the case
that the link count has hit zero). This ensures that inodes which are
open, but otherwise with zero link count cannot get "lost" unless a node
crashes (in which case we have another method of recovering the free
space).
Steve.
diff --git a/fs/gfs2/acl.c b/fs/gfs2/acl.c
diff --git a/fs/gfs2/bmap.c b/fs/gfs2/bmap.c
diff --git a/fs/gfs2/daemon.c b/fs/gfs2/daemon.c
diff --git a/fs/gfs2/dir.c b/fs/gfs2/dir.c
diff --git a/fs/gfs2/eaops.c b/fs/gfs2/eaops.c
diff --git a/fs/gfs2/eattr.c b/fs/gfs2/eattr.c
diff --git a/fs/gfs2/glock.c b/fs/gfs2/glock.c
index a3a24f2..e3f62b3 100644
--- a/fs/gfs2/glock.c
+++ b/fs/gfs2/glock.c
@@ -54,7 +54,7 @@ struct glock_iter {
typedef void (*glock_examiner) (struct gfs2_glock * gl);
static int gfs2_dump_lockstate(struct gfs2_sbd *sdp);
-static void gfs2_glock_xmote_th(struct gfs2_holder *gh);
+static void gfs2_glock_xmote_th(struct gfs2_glock *gl, struct gfs2_holder *gh);
static void gfs2_glock_drop_th(struct gfs2_glock *gl);
static DECLARE_RWSEM(gfs2_umount_flush_sem);
static struct dentry *gfs2_root;
@@ -212,7 +212,6 @@ int gfs2_glock_put(struct gfs2_glock *gl)
gfs2_assert(sdp, list_empty(&gl->gl_reclaim));
gfs2_assert(sdp, list_empty(&gl->gl_holders));
gfs2_assert(sdp, list_empty(&gl->gl_waiters1));
- gfs2_assert(sdp, list_empty(&gl->gl_waiters2));
gfs2_assert(sdp, list_empty(&gl->gl_waiters3));
glock_free(gl);
rv = 1;
@@ -399,7 +398,7 @@ void gfs2_holder_reinit(unsigned int state, unsigned flags, struct gfs2_holder *
{
gh->gh_state = state;
gh->gh_flags = flags;
- gh->gh_iflags &= 1 << HIF_ALLOCED;
+ gh->gh_iflags = 0;
gh->gh_ip = (unsigned long)__builtin_return_address(0);
}
@@ -416,54 +415,8 @@ void gfs2_holder_uninit(struct gfs2_holder *gh)
gh->gh_ip = 0;
}
-/**
- * gfs2_holder_get - get a struct gfs2_holder structure
- * @gl: the glock
- * @state: the state we're requesting
- * @flags: the modifier flags
- * @gfp_flags:
- *
- * Figure out how big an impact this function has. Either:
- * 1) Replace it with a cache of structures hanging off the struct gfs2_sbd
- * 2) Leave it like it is
- *
- * Returns: the holder structure, NULL on ENOMEM
- */
-
-static struct gfs2_holder *gfs2_holder_get(struct gfs2_glock *gl,
- unsigned int state,
- int flags, gfp_t gfp_flags)
-{
- struct gfs2_holder *gh;
-
- gh = kmalloc(sizeof(struct gfs2_holder), gfp_flags);
- if (!gh)
- return NULL;
-
- gfs2_holder_init(gl, state, flags, gh);
- set_bit(HIF_ALLOCED, &gh->gh_iflags);
- gh->gh_ip = (unsigned long)__builtin_return_address(0);
- return gh;
-}
-
-/**
- * gfs2_holder_put - get rid of a struct gfs2_holder structure
- * @gh: the holder structure
- *
- */
-
-static void gfs2_holder_put(struct gfs2_holder *gh)
-{
- gfs2_holder_uninit(gh);
- kfree(gh);
-}
-
-static void gfs2_holder_dispose_or_wake(struct gfs2_holder *gh)
+static void gfs2_holder_wake(struct gfs2_holder *gh)
{
- if (test_bit(HIF_DEALLOC, &gh->gh_iflags)) {
- gfs2_holder_put(gh);
- return;
- }
clear_bit(HIF_WAIT, &gh->gh_iflags);
smp_mb();
wake_up_bit(&gh->gh_iflags, HIF_WAIT);
@@ -529,7 +482,7 @@ static int rq_promote(struct gfs2_holder *gh)
gfs2_reclaim_glock(sdp);
}
- gfs2_glock_xmote_th(gh);
+ gfs2_glock_xmote_th(gh->gh_gl, gh);
spin_lock(&gl->gl_spin);
}
return 1;
@@ -552,7 +505,7 @@ static int rq_promote(struct gfs2_holder *gh)
gh->gh_error = 0;
set_bit(HIF_HOLDER, &gh->gh_iflags);
- gfs2_holder_dispose_or_wake(gh);
+ gfs2_holder_wake(gh);
return 0;
}
@@ -564,32 +517,24 @@ static int rq_promote(struct gfs2_holder *gh)
* Returns: 1 if the queue is blocked
*/
-static int rq_demote(struct gfs2_holder *gh)
+static int rq_demote(struct gfs2_glock *gl)
{
- struct gfs2_glock *gl = gh->gh_gl;
-
if (!list_empty(&gl->gl_holders))
return 1;
- if (gl->gl_state == gh->gh_state || gl->gl_state == LM_ST_UNLOCKED) {
- list_del_init(&gh->gh_list);
- gh->gh_error = 0;
- spin_unlock(&gl->gl_spin);
- gfs2_holder_dispose_or_wake(gh);
- spin_lock(&gl->gl_spin);
- } else {
- gl->gl_req_gh = gh;
- set_bit(GLF_LOCK, &gl->gl_flags);
- spin_unlock(&gl->gl_spin);
-
- if (gh->gh_state == LM_ST_UNLOCKED ||
- gl->gl_state != LM_ST_EXCLUSIVE)
- gfs2_glock_drop_th(gl);
- else
- gfs2_glock_xmote_th(gh);
-
- spin_lock(&gl->gl_spin);
+ if (gl->gl_state == gl->gl_demote_state ||
+ gl->gl_state == LM_ST_UNLOCKED) {
+ clear_bit(GLF_DEMOTE, &gl->gl_flags);
+ return 0;
}
+ set_bit(GLF_LOCK, &gl->gl_flags);
+ spin_unlock(&gl->gl_spin);
+ if (gl->gl_demote_state == LM_ST_UNLOCKED ||
+ gl->gl_state != LM_ST_EXCLUSIVE)
+ gfs2_glock_drop_th(gl);
+ else
+ gfs2_glock_xmote_th(gl, NULL);
+ spin_lock(&gl->gl_spin);
return 0;
}
@@ -617,16 +562,8 @@ static void run_queue(struct gfs2_glock *gl)
else
gfs2_assert_warn(gl->gl_sbd, 0);
- } else if (!list_empty(&gl->gl_waiters2) &&
- !test_bit(GLF_SKIP_WAITERS2, &gl->gl_flags)) {
- gh = list_entry(gl->gl_waiters2.next,
- struct gfs2_holder, gh_list);
-
- if (test_bit(HIF_DEMOTE, &gh->gh_iflags))
- blocked = rq_demote(gh);
- else
- gfs2_assert_warn(gl->gl_sbd, 0);
-
+ } else if (test_bit(GLF_DEMOTE, &gl->gl_flags)) {
+ blocked = rq_demote(gl);
} else if (!list_empty(&gl->gl_waiters3)) {
gh = list_entry(gl->gl_waiters3.next,
struct gfs2_holder, gh_list);
@@ -717,50 +654,24 @@ static void gfs2_glmutex_unlock(struct gfs2_glock *gl)
}
/**
- * handle_callback - add a demote request to a lock's queue
+ * handle_callback - process a demote request
* @gl: the glock
* @state: the state the caller wants us to change to
*
- * Note: This may fail sliently if we are out of memory.
+ * There are only two requests that we are going to see in actual
+ * practise: LM_ST_SHARED and LM_ST_UNLOCKED
*/
static void handle_callback(struct gfs2_glock *gl, unsigned int state)
{
- struct gfs2_holder *gh, *new_gh = NULL;
-
-restart:
spin_lock(&gl->gl_spin);
-
- list_for_each_entry(gh, &gl->gl_waiters2, gh_list) {
- if (test_bit(HIF_DEMOTE, &gh->gh_iflags) &&
- gl->gl_req_gh != gh) {
- if (gh->gh_state != state)
- gh->gh_state = LM_ST_UNLOCKED;
- goto out;
- }
- }
-
- if (new_gh) {
- list_add_tail(&new_gh->gh_list, &gl->gl_waiters2);
- new_gh = NULL;
- } else {
- spin_unlock(&gl->gl_spin);
-
- new_gh = gfs2_holder_get(gl, state, LM_FLAG_TRY, GFP_NOFS);
- if (!new_gh)
- return;
- set_bit(HIF_DEMOTE, &new_gh->gh_iflags);
- set_bit(HIF_DEALLOC, &new_gh->gh_iflags);
- set_bit(HIF_WAIT, &new_gh->gh_iflags);
-
- goto restart;
+ if (test_and_set_bit(GLF_DEMOTE, &gl->gl_flags) == 0) {
+ gl->gl_demote_state = state;
+ gl->gl_demote_time = jiffies;
+ } else if (gl->gl_demote_state != LM_ST_UNLOCKED) {
+ gl->gl_demote_state = state;
}
-
-out:
spin_unlock(&gl->gl_spin);
-
- if (new_gh)
- gfs2_holder_put(new_gh);
}
/**
@@ -820,56 +731,37 @@ static void xmote_bh(struct gfs2_glock *gl, unsigned int ret)
/* Deal with each possible exit condition */
- if (!gh)
+ if (!gh) {
gl->gl_stamp = jiffies;
- else if (unlikely(test_bit(SDF_SHUTDOWN, &sdp->sd_flags))) {
+ if (ret & LM_OUT_CANCELED)
+ op_done = 0;
+ else
+ clear_bit(GLF_DEMOTE, &gl->gl_flags);
+ } else {
spin_lock(&gl->gl_spin);
list_del_init(&gh->gh_list);
gh->gh_error = -EIO;
- spin_unlock(&gl->gl_spin);
- } else if (test_bit(HIF_DEMOTE, &gh->gh_iflags)) {
- spin_lock(&gl->gl_spin);
- list_del_init(&gh->gh_list);
- if (gl->gl_state == gh->gh_state ||
- gl->gl_state == LM_ST_UNLOCKED) {
+ if (unlikely(test_bit(SDF_SHUTDOWN, &sdp->sd_flags)))
+ goto out;
+ gh->gh_error = GLR_CANCELED;
+ if (ret & LM_OUT_CANCELED)
+ goto out;
+ if (relaxed_state_ok(gl->gl_state, gh->gh_state, gh->gh_flags)) {
+ list_add_tail(&gh->gh_list, &gl->gl_holders);
gh->gh_error = 0;
- } else {
- if (gfs2_assert_warn(sdp, gh->gh_flags &
- (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) == -1)
- fs_warn(sdp, "ret = 0x%.8X\n", ret);
- gh->gh_error = GLR_TRYFAILED;
+ set_bit(HIF_HOLDER, &gh->gh_iflags);
+ set_bit(HIF_FIRST, &gh->gh_iflags);
+ op_done = 0;
+ goto out;
}
- spin_unlock(&gl->gl_spin);
-
- if (ret & LM_OUT_CANCELED)
- handle_callback(gl, LM_ST_UNLOCKED);
-
- } else if (ret & LM_OUT_CANCELED) {
- spin_lock(&gl->gl_spin);
- list_del_init(&gh->gh_list);
- gh->gh_error = GLR_CANCELED;
- spin_unlock(&gl->gl_spin);
-
- } else if (relaxed_state_ok(gl->gl_state, gh->gh_state, gh->gh_flags)) {
- spin_lock(&gl->gl_spin);
- list_move_tail(&gh->gh_list, &gl->gl_holders);
- gh->gh_error = 0;
- set_bit(HIF_HOLDER, &gh->gh_iflags);
- spin_unlock(&gl->gl_spin);
-
- set_bit(HIF_FIRST, &gh->gh_iflags);
-
- op_done = 0;
-
- } else if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB)) {
- spin_lock(&gl->gl_spin);
- list_del_init(&gh->gh_list);
gh->gh_error = GLR_TRYFAILED;
- spin_unlock(&gl->gl_spin);
-
- } else {
+ if (gh->gh_flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB))
+ goto out;
+ gh->gh_error = -EINVAL;
if (gfs2_assert_withdraw(sdp, 0) == -1)
fs_err(sdp, "ret = 0x%.8X\n", ret);
+out:
+ spin_unlock(&gl->gl_spin);
}
if (glops->go_xmote_bh)
@@ -887,7 +779,7 @@ static void xmote_bh(struct gfs2_glock *gl, unsigned int ret)
gfs2_glock_put(gl);
if (gh)
- gfs2_holder_dispose_or_wake(gh);
+ gfs2_holder_wake(gh);
}
/**
@@ -898,12 +790,11 @@ static void xmote_bh(struct gfs2_glock *gl, unsigned int ret)
*
*/
-void gfs2_glock_xmote_th(struct gfs2_holder *gh)
+void gfs2_glock_xmote_th(struct gfs2_glock *gl, struct gfs2_holder *gh)
{
- struct gfs2_glock *gl = gh->gh_gl;
struct gfs2_sbd *sdp = gl->gl_sbd;
- int flags = gh->gh_flags;
- unsigned state = gh->gh_state;
+ int flags = gh ? gh->gh_flags : 0;
+ unsigned state = gh ? gh->gh_state : gl->gl_demote_state;
const struct gfs2_glock_operations *glops = gl->gl_ops;
int lck_flags = flags & (LM_FLAG_TRY | LM_FLAG_TRY_1CB |
LM_FLAG_NOEXP | LM_FLAG_ANY |
@@ -953,6 +844,7 @@ static void drop_bh(struct gfs2_glock *gl, unsigned int ret)
gfs2_assert_warn(sdp, !ret);
state_change(gl, LM_ST_UNLOCKED);
+ clear_bit(GLF_DEMOTE, &gl->gl_flags);
if (glops->go_inval)
glops->go_inval(gl, DIO_METADATA);
@@ -974,7 +866,7 @@ static void drop_bh(struct gfs2_glock *gl, unsigned int ret)
gfs2_glock_put(gl);
if (gh)
- gfs2_holder_dispose_or_wake(gh);
+ gfs2_holder_wake(gh);
}
/**
@@ -1291,8 +1183,6 @@ void gfs2_glock_dq(struct gfs2_holder *gh)
if (glops->go_unlock)
glops->go_unlock(gh);
- gl->gl_stamp = jiffies;
-
spin_lock(&gl->gl_spin);
}
@@ -1981,11 +1871,6 @@ static int dump_glock(struct glock_iter *gi, struct gfs2_glock *gl)
if (error)
goto out;
}
- list_for_each_entry(gh, &gl->gl_waiters2, gh_list) {
- error = dump_holder(gi, "Waiter2", gh);
- if (error)
- goto out;
- }
list_for_each_entry(gh, &gl->gl_waiters3, gh_list) {
error = dump_holder(gi, "Waiter3", gh);
if (error)
diff --git a/fs/gfs2/glock.h b/fs/gfs2/glock.h
index d7cef74..5e662ea 100644
--- a/fs/gfs2/glock.h
+++ b/fs/gfs2/glock.h
@@ -67,7 +67,7 @@ static inline int gfs2_glock_is_blocking(struct gfs2_glock *gl)
{
int ret;
spin_lock(&gl->gl_spin);
- ret = !list_empty(&gl->gl_waiters2) || !list_empty(&gl->gl_waiters3);
+ ret = test_bit(GLF_DEMOTE, &gl->gl_flags) || !list_empty(&gl->gl_waiters3);
spin_unlock(&gl->gl_spin);
return ret;
}
diff --git a/fs/gfs2/glops.c b/fs/gfs2/glops.c
diff --git a/fs/gfs2/incore.h b/fs/gfs2/incore.h
index 7555261..9c12582 100644
--- a/fs/gfs2/incore.h
+++ b/fs/gfs2/incore.h
@@ -115,11 +115,8 @@ enum {
/* Actions */
HIF_MUTEX = 0,
HIF_PROMOTE = 1,
- HIF_DEMOTE = 2,
/* States */
- HIF_ALLOCED = 4,
- HIF_DEALLOC = 5,
HIF_HOLDER = 6,
HIF_FIRST = 7,
HIF_ABORTED = 9,
@@ -142,8 +139,8 @@ struct gfs2_holder {
enum {
GLF_LOCK = 1,
GLF_STICKY = 2,
+ GLF_DEMOTE = 3,
GLF_DIRTY = 5,
- GLF_SKIP_WAITERS2 = 6,
};
struct gfs2_glock {
@@ -156,11 +153,12 @@ struct gfs2_glock {
unsigned int gl_state;
unsigned int gl_hash;
+ unsigned int gl_demote_state; /* state requested by remote node */
+ unsigned long gl_demote_time; /* time of first demote request */
struct task_struct *gl_owner;
unsigned long gl_ip;
struct list_head gl_holders;
struct list_head gl_waiters1; /* HIF_MUTEX */
- struct list_head gl_waiters2; /* HIF_DEMOTE */
struct list_head gl_waiters3; /* HIF_PROMOTE */
const struct gfs2_glock_operations *gl_ops;
diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
diff --git a/fs/gfs2/lm.c b/fs/gfs2/lm.c
diff --git a/fs/gfs2/locking.c b/fs/gfs2/locking.c
diff --git a/fs/gfs2/log.c b/fs/gfs2/log.c
diff --git a/fs/gfs2/lops.c b/fs/gfs2/lops.c
diff --git a/fs/gfs2/main.c b/fs/gfs2/main.c
index 2183953..c4bb374 100644
--- a/fs/gfs2/main.c
+++ b/fs/gfs2/main.c
@@ -45,7 +45,6 @@ static void gfs2_init_glock_once(void *foo, struct kmem_cache *cachep, unsigned
spin_lock_init(&gl->gl_spin);
INIT_LIST_HEAD(&gl->gl_holders);
INIT_LIST_HEAD(&gl->gl_waiters1);
- INIT_LIST_HEAD(&gl->gl_waiters2);
INIT_LIST_HEAD(&gl->gl_waiters3);
gl->gl_lvb = NULL;
atomic_set(&gl->gl_lvb_count, 0);
diff --git a/fs/gfs2/meta_io.c b/fs/gfs2/meta_io.c
diff --git a/fs/gfs2/mount.c b/fs/gfs2/mount.c
diff --git a/fs/gfs2/ondisk.c b/fs/gfs2/ondisk.c
diff --git a/fs/gfs2/ops_address.c b/fs/gfs2/ops_address.c
diff --git a/fs/gfs2/ops_dentry.c b/fs/gfs2/ops_dentry.c
diff --git a/fs/gfs2/ops_export.c b/fs/gfs2/ops_export.c
diff --git a/fs/gfs2/ops_file.c b/fs/gfs2/ops_file.c
diff --git a/fs/gfs2/ops_fstype.c b/fs/gfs2/ops_fstype.c
diff --git a/fs/gfs2/ops_inode.c b/fs/gfs2/ops_inode.c
diff --git a/fs/gfs2/ops_super.c b/fs/gfs2/ops_super.c
index b89999d..485ce3d 100644
--- a/fs/gfs2/ops_super.c
+++ b/fs/gfs2/ops_super.c
@@ -284,6 +284,31 @@ static int gfs2_remount_fs(struct super_block *sb, int *flags, char *data)
}
/**
+ * gfs2_drop_inode - Drop an inode (test for remote unlink)
+ * @inode: The inode to drop
+ *
+ * If we've received a callback on an iopen lock then its because a
+ * remote node tried to deallocate the inode but failed due to this node
+ * still having the inode open. Here we mark the link count zero
+ * since we know that it must have reached zero if the GLF_DEMOTE flag
+ * is set on the iopen glock. If we didn't do a disk read since the
+ * remote node removed the final link then we might otherwise miss
+ * this event. This check ensures that this node will deallocate the
+ * inode's blocks, or alternatively pass the baton on to another
+ * node for later deallocation.
+ */
+static void gfs2_drop_inode(struct inode *inode)
+{
+ if (inode->i_private && inode->i_nlink) {
+ struct gfs2_inode *ip = GFS2_I(inode);
+ struct gfs2_glock *gl = ip->i_iopen_gh.gh_gl;
+ if (gl && test_bit(GLF_DEMOTE, &gl->gl_flags))
+ clear_nlink(inode);
+ }
+ generic_drop_inode(inode);
+}
+
+/**
* gfs2_clear_inode - Deallocate an inode when VFS is done with it
* @inode: The VFS inode
*
@@ -441,7 +466,7 @@ out_unlock:
out_uninit:
gfs2_holder_uninit(&ip->i_iopen_gh);
gfs2_glock_dq_uninit(&gh);
- if (error)
+ if (error && error != GLR_TRYFAILED)
fs_warn(sdp, "gfs2_delete_inode: %d\n", error);
out:
truncate_inode_pages(&inode->i_data, 0);
@@ -481,6 +506,7 @@ const struct super_operations gfs2_super_ops = {
.statfs = gfs2_statfs,
.remount_fs = gfs2_remount_fs,
.clear_inode = gfs2_clear_inode,
+ .drop_inode = gfs2_drop_inode,
.show_options = gfs2_show_options,
};
diff --git a/fs/gfs2/ops_vm.c b/fs/gfs2/ops_vm.c
diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c
diff --git a/fs/gfs2/recovery.c b/fs/gfs2/recovery.c
diff --git a/fs/gfs2/rgrp.c b/fs/gfs2/rgrp.c
diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c
diff --git a/fs/gfs2/sys.c b/fs/gfs2/sys.c
diff --git a/fs/gfs2/trans.c b/fs/gfs2/trans.c
diff --git a/fs/gfs2/util.c b/fs/gfs2/util.c
More information about the Cluster-devel
mailing list