[Linux-cluster] GFS Mount Problem

[Leo Pleiman]

I have a 7 node virtualized RHEL5.3 cluster. All nodes mount mulitple gfs filesystems. One node only only of the file systems won't mount. I know (from a previous incident) that rebooting the node will clear the problem. Below is some information, I'd appreciate any help to resolve this without rebooting the node.

[root at rdccluster2-5 ~]# umount /decennial/
umount: /decennial/: not mounted
[root at rdccluster2-5 ~]# mount /decennial
/sbin/mount.gfs: fs is being unmounted
/sbin/mount.gfs: error mounting lockproto lock_dlm
[root at rdccluster2-5 ~]# strace /decennial
...snip
lstat("/etc/mtab", {st_mode=S_IFREG|0644, st_size=1817, ...}) = 0
readlink("/decennial", 0x7fff43151700, 4096) = -1 EINVAL (Invalid argument)
umask(077)                              = 022
open("/etc/fstab", O_RDONLY)            = 3
umask(022)                              = 077
fstat(3, {st_mode=S_IFREG|0644, st_size=1695, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2ab
read(3, "/dev/vgrp00/Root        /       "..., 4096) = 1695
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x2abd6af59000, 4096)            = 0
stat("/sbin/mount.gfs", {st_mode=S_IFREG|0755, st_size=41968, ...}) = 0
clone(child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, chil
wait4(-1, /sbin/mount.gfs: fs is being unmounted
/sbin/mount.gfs: error mounting lockproto lock_dlm
[{WIFEXITED(s) && WEXITSTATUS(s) == 1}], 0, NULL) = 22646
--- SIGCHLD (Child exited) @ 0 (0) ---
exit_group(1)                           = ?
[root at rdccluster2-5 ~]#

>From /var/log/messages
Feb  5 09:18:39 rdccluster2-5 gfs_controld[2862]: mount: reject mount due to unmount
Feb  5 09:19:00 rdccluster2-5 gfs_controld[2862]: mount: reject mount due to unmount

--Leo