[Linux-cluster] Rgmanager fails to restart
Janne Peltonen
janne.peltonen at helsinki.fi
Sun Jul 1 11:17:48 UTC 2007
Hi!
Sometimes, when I have cleanly shut down rgmanager on one node, and the
services have nicely migrated to other nodes, trying to start rgmanager
fails. Trying to access /dev/misc/dlm_rgmanager results in "No such
device". clurgmgrd concludes that locks are not working and exits.
(See strace output attached.)
--cut--
[jmmpelto at pcn1 ~]$ sudo service rgmanager start
Starting Cluster Service Manager: [ OK ]
[jmmpelto at pcn1 ~]$ sudo service rgmanager status
clurgmgrd dead but pid file exists
--cut--
Trying to stop cman fails:
--clip--
[jmmpelto at pcn1 ~]$ sudo service cman restart
Stopping cluster:
Stopping fencing... done
Stopping cman... failed
/usr/sbin/cman_tool: Error leaving cluster: Device or resource busy
[FAILED]
Starting cluster:
Loading modules... done
Mounting configfs... done
Starting ccsd... done
Starting cman... done
Starting daemons... done
Starting fencing... done
[ OK ]
--clip--
And indeed, the rgmanager that isn't there is there:
--clip--
[jmmpelto at pcn1 ~]$ sudo cman_tool services
type level name id state
fence 0 default 00010001 none
[1 2 3 4 100]
dlm 1 clvmd 00010002 none
[1 2 3 4 100]
dlm 1 rgmanager 00020002 none
[1 2 3 4]
--clip--
If I say 'cman_tool leave force', it succeeds. But if I then try starting the cluster:
--cut--
[jmmpelto at pcn1 ~]$ sudo service cman start
Starting cluster:
Loading modules... done
Mounting configfs... done
Starting ccsd... done
Starting cman... done
Starting daemons... done
Starting fencing... failed
[FAILED]
--cut--
Log (oops, I forgot to shut down clvmd there... it would have gone down cleanly):
--cut--
Jul 1 14:11:02 pcn1.mappi.helsinki.fi ccsd[4427]: Initial status:: Inquorate
Jul 1 14:11:28 pcn1.mappi.helsinki.fi groupd[557]: found uncontrolled kernel object rgmanager in /sys/kernel/dlm
Jul 1 14:11:28 pcn1.mappi.helsinki.fi groupd[557]: found uncontrolled kernel object clvmd in /sys/kernel/dlm
Jul 1 14:11:28 pcn1.mappi.helsinki.fi groupd[557]: local node must be reset to clear 2 uncontrolled instances of gfs and/or dlm
Jul 1 14:11:28 pcn1.mappi.helsinki.fi fenced[568]: cman_init error 0 111
Jul 1 14:11:28 pcn1.mappi.helsinki.fi dlm_controld[576]: cman_init error 0 111
Jul 1 14:11:28 pcn1.mappi.helsinki.fi gfs_controld[583]: cman_init error 111
--cut--
Thereafter, one of the other nodes fences this one:
--cut--
Jul 1 14:11:50 pcn1.mappi.helsinki.fi init: Switching to runlevel: 0
Jul 1 14:11:50 pcn1.mappi.helsinki.fi ccsd[4427]: Unable to connect to cluster infrastructure after 30 seconds.
Jul 1 14:11:52 pcn1.mappi.helsinki.fi rgmanager: [667]: <notice> Cluster Service Manager is stopped.
--cut--
(Now I wonder where that rgmanager log line came from? It isn't from any
clurgmgrd, I checked with ps that there were none running.)
Any ideas?
(version of relevant packages:
lvm2-2.02.16-3.el5
cman-2.0.60-1.el5
rgmanager-2.0.23-1.el5.centos
)
--Janne
-------------- next part --------------
execve("/usr/sbin/clurgmgrd", ["clurgmgrd"], [/* 17 vars */]) = 0
brk(0) = 0xc4f2000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaaab000
uname({sys="Linux", node="pcn1.mappi.helsinki.fi", ...}) = 0
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=45165, ...}) = 0
mmap(NULL, 45165, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2aaaaaaac000
close(3) = 0
open("/usr/lib64/libxml2.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \262B_<\0\0\0@\0\0\0\0\0\0\0\260\303\23\0\0\0\0\0\0\0\0\0@\0008\0\5\0@\0\35\0\34\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0 at _<\0\0\0\0\0 at _<\0\0\0\24\"\23\0\0\0\0\0\24\"\23\0\0\0\0\0\0\0 \0\0\0\0\0\1\0\0\0\6\0\0\0\0000\23\0\0\0\0\0\0000s_<\0\0\0\000"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1297136, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaab8000
mmap(0x3c5f400000, 3395256, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3c5f400000
mprotect(0x3c5f533000, 2097152, PROT_NONE) = 0
mmap(0x3c5f733000, 36864, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x133000) = 0x3c5f733000
mmap(0x3c5f73c000, 3768, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3c5f73c000
close(3) = 0
open("/lib64/libpthread.so.0", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20W\300Y<\0\0\0@\0\0\0\0\0\0\0\330\35\2\0\0\0\0\0\0\0\0\0@\0008\0\t\0@\0\'\0&\0\6\0\0\0\5\0\0\0@\0\0\0\0\0\0\0@\0\300Y<\0\0\0@\0\300Y<\0\0\0\370\1\0\0\0\0\0\0\370\1\0\0\0\0\0\0\10\0\0\0\0\0\0\0\3\0\0\0\4\0\0\0@\375\0\0\0\0\0\0@\375\300Y<\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=141208, ...}) = 0
mmap(0x3c59c00000, 2200432, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3c59c00000
mprotect(0x3c59c15000, 2093056, PROT_NONE) = 0
mmap(0x3c59e14000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x14000) = 0x3c59e14000
mmap(0x3c59e16000, 13168, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3c59e16000
close(3) = 0
open("/lib64/libdl.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0 \16\200Y<\0\0\0@\0\0\0\0\0\0\0\240R\0\0\0\0\0\0\0\0\0\0@\0008\0\t\0@\0%\0$\0\6\0\0\0\5\0\0\0@\0\0\0\0\0\0\0@\0\200Y<\0\0\0@\0\200Y<\0\0\0\370\1\0\0\0\0\0\0\370\1\0\0\0\0\0\0\10\0\0\0\0\0\0\0\3\0\0\0\4\0\0\0\240\32\0\0\0\0\0\0\240\32\200Y<\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=23520, ...}) = 0
mmap(0x3c59800000, 2109728, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3c59800000
mprotect(0x3c59802000, 2097152, PROT_NONE) = 0
mmap(0x3c59a02000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x3c59a02000
close(3) = 0
open("/usr/lib64/libcman.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0p\20\200Z<\0\0\0@\0\0\0\0\0\0\0\240L\0\0\0\0\0\0\0\0\0\0@\0008\0\5\0@\0\35\0\34\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\200Z<\0\0\0\0\0\200Z<\0\0\0\34A\0\0\0\0\0\0\34A\0\0\0\0\0\0\0\0 \0\0\0\0\0\1\0\0\0\6\0\0\0 A\0\0\0\0\0\0 A\240Z<\0\0\0 A\240Z<"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=21472, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaab9000
mmap(0x3c5a800000, 2114456, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3c5a800000
mprotect(0x3c5a805000, 2093056, PROT_NONE) = 0
mmap(0x3c5aa04000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x4000) = 0x3c5aa04000
close(3) = 0
open("/usr/lib64/libdlm.so.2", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\360\27\300Z<\0\0\0@\0\0\0\0\0\0\0\20H\0\0\0\0\0\0\0\0\0\0@\0008\0\5\0@\0\35\0\34\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\300Z<\0\0\0\0\0\300Z<\0\0\0L;\0\0\0\0\0\0L;\0\0\0\0\0\0\0\0 \0\0\0\0\0\1\0\0\0\6\0\0\0P;\0\0\0\0\0\0P;\340Z<\0\0\0P;\340Z<\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=20304, ...}) = 0
mmap(0x3c5ac00000, 2113272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3c5ac00000
mprotect(0x3c5ac04000, 2093056, PROT_NONE) = 0
mmap(0x3c5ae03000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x3000) = 0x3c5ae03000
close(3) = 0
open("/lib64/libc.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240\331AY<\0\0\0@\0\0\0\0\0\0\0P\211\31\0\0\0\0\0\0\0\0\0@\0008\0\n\0@\0M\0L\0\6\0\0\0\5\0\0\0@\0\0\0\0\0\0\0@\0 at Y<\0\0\0@\0 at Y<\0\0\0000\2\0\0\0\0\0\0000\2\0\0\0\0\0\0\10\0\0\0\0\0\0\0\3\0\0\0\4\0\0\0\240\257\21\0\0\0\0\0\240\257QY<\0\0\0\240\257"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1678480, ...}) = 0
mmap(0x3c59400000, 3461272, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3c59400000
mprotect(0x3c59544000, 2097152, PROT_NONE) = 0
mmap(0x3c59744000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x144000) = 0x3c59744000
mmap(0x3c59749000, 16536, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x3c59749000
close(3) = 0
open("/usr/lib64/libz.so.1", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\260\36 at Z<\0\0\0@\0\0\0\0\0\0\0(G\1\0\0\0\0\0\0\0\0\0@\0008\0\5\0@\0\35\0\34\0\1\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0 at Z<\0\0\0\0\0 at Z<\0\0\0\3648\1\0\0\0\0\0\3648\1\0\0\0\0\0\0\0 \0\0\0\0\0\1\0\0\0\6\0\0\0\3708\1\0\0\0\0\0\3708aZ<\0\0\0\3708aZ<\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=85608, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaaba000
mmap(0x3c5a400000, 2178600, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3c5a400000
mprotect(0x3c5a414000, 2093056, PROT_NONE) = 0
mmap(0x3c5a613000, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x13000) = 0x3c5a613000
close(3) = 0
open("/lib64/libm.so.6", O_RDONLY) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\200>\0Z<\0\0\0@\0\0\0\0\0\0\0\240X\t\0\0\0\0\0\0\0\0\0@\0008\0\t\0@\0)\0(\0\6\0\0\0\5\0\0\0@\0\0\0\0\0\0\0@\0\0Z<\0\0\0@\0\0Z<\0\0\0\370\1\0\0\0\0\0\0\370\1\0\0\0\0\0\0\10\0\0\0\0\0\0\0\3\0\0\0\4\0\0\0\260\304\7\0\0\0\0\0\260\304\7Z<\0\0\0\260"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=615136, ...}) = 0
mmap(0x3c5a000000, 2629848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x3c5a000000
mprotect(0x3c5a082000, 2093056, PROT_NONE) = 0
mmap(0x3c5a281000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x81000) = 0x3c5a281000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaabb000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaabc000
arch_prctl(ARCH_SET_FS, 0x2aaaaaabba00) = 0
mprotect(0x3c59e14000, 4096, PROT_READ) = 0
mprotect(0x3c59a02000, 4096, PROT_READ) = 0
mprotect(0x3c59744000, 16384, PROT_READ) = 0
mprotect(0x3c5a281000, 4096, PROT_READ) = 0
mprotect(0x3c59219000, 4096, PROT_READ) = 0
munmap(0x2aaaaaaac000, 45165) = 0
set_tid_address(0x2aaaaaabba90) = 393
set_robust_list(0x2aaaaaabbaa0, 0x18) = 0
rt_sigaction(SIGRTMIN, {0x3c59c05350, [], SA_RESTORER|SA_SIGINFO, 0x3c59c0dd40}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x3c59c052a0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x3c59c0dd40}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=10240*1024, rlim_max=RLIM_INFINITY}) = 0
geteuid() = 0
getuid() = 0
stat("/var/run/clurgmgrd.pid", {st_mode=S_IFREG|0644, st_size=3, ...}) = 0
brk(0) = 0xc4f2000
brk(0xc513000) = 0xc513000
open("/var/run/clurgmgrd.pid", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=3, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaaac000
read(3, "369", 4096) = 3
read(3, "", 4096) = 0
close(3) = 0
munmap(0x2aaaaaaac000, 4096) = 0
open("/proc/369", O_RDONLY|O_NONBLOCK|O_DIRECTORY) = -1 ENOENT (No such file or directory)
rt_sigprocmask(SIG_BLOCK, ~[QUIT ILL TRAP ABRT BUS FPE SEGV RTMIN RT_1], NULL, 8) = 0
clone(Process 394 attached (waiting for parent)
Process 394 resumed (parent 393 ready)
child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x2aaaaaabba90) = 394
[pid 393] exit_group(0) = ?
Process 393 detached
setsid() = 394
chdir("/") = 0
open("/dev/null", O_RDWR) = 3
fstat(3, {st_mode=S_IFCHR|0666, st_rdev=makedev(1, 3), ...}) = 0
dup2(3, 0) = 0
dup2(3, 1) = 1
dup2(3, 2) = 2
close(3) = 0
open("/var/run/clurgmgrd.pid", O_WRONLY|O_CREAT|O_TRUNC, 0666) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=0, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaaac000
write(3, "394", 3) = 3
close(3) = 0
munmap(0x2aaaaaaac000, 4096) = 0
getpriority(PRIO_PROCESS, 0) = 20
setpriority(PRIO_PROCESS, 0, 4294967295) = 0
getpriority(PRIO_PROCESS, 0) = 21
clone(Process 395 attached (waiting for parent)
Process 395 resumed (parent 394 ready)
child_stack=0, flags=CLONE_CHILD_CLEARTID|CLONE_CHILD_SETTID|SIGCHLD, child_tidptr=0x2aaaaaabba90) = 395
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [], NULL, 8) = 0
[pid 394] rt_sigaction(SIG_0, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = -1 EINVAL (Invalid argument)
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [HUP], NULL, 8) = 0
[pid 394] rt_sigaction(SIGHUP, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [INT], NULL, 8) = 0
[pid 394] rt_sigaction(SIGINT, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [QUIT], NULL, 8) = 0
[pid 394] rt_sigaction(SIGQUIT, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [ILL], NULL, 8) = 0
[pid 394] rt_sigaction(SIGILL, {SIG_DFL}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [TRAP], NULL, 8) = 0
[pid 394] rt_sigaction(SIGTRAP, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [ABRT], NULL, 8) = 0
[pid 394] rt_sigaction(SIGABRT, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [BUS], NULL, 8) = 0
[pid 394] rt_sigaction(SIGBUS, {SIG_DFL}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [FPE], NULL, 8) = 0
[pid 394] rt_sigaction(SIGFPE, {SIG_DFL}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [KILL], NULL, 8) = 0
[pid 394] rt_sigaction(SIGKILL, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = -1 EINVAL (Invalid argument)
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [USR1], NULL, 8) = 0
[pid 394] rt_sigaction(SIGUSR1, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [SEGV], NULL, 8) = 0
[pid 394] rt_sigaction(SIGSEGV, {SIG_DFL}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [USR2], NULL, 8) = 0
[pid 394] rt_sigaction(SIGUSR2, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [PIPE], NULL, 8) = 0
[pid 394] rt_sigaction(SIGPIPE, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [ALRM], NULL, 8) = 0
[pid 394] rt_sigaction(SIGALRM, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [TERM], NULL, 8) = 0
[pid 394] rt_sigaction(SIGTERM, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [STKFLT], NULL, 8) = 0
[pid 394] rt_sigaction(SIGSTKFLT, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 395] socket(PF_FILE, SOCK_STREAM, 0 <unfinished ...>
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [CHLD], <unfinished ...>
[pid 395] <... socket resumed> ) = 3
[pid 394] <... rt_sigprocmask resumed> NULL, 8) = 0
[pid 395] fcntl(3, F_SETFD, FD_CLOEXEC) = 0
[pid 394] rt_sigaction(SIGCHLD, {SIG_DFL}, <unfinished ...>
[pid 395] connect(3, {sa_family=AF_FILE, path="/var/run/cman_client"}, 110 <unfinished ...>
[pid 394] <... rt_sigaction resumed> {SIG_DFL}, 8) = 0
[pid 395] <... connect resumed> ) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [CONT], <unfinished ...>
[pid 395] open("/dev/zero", O_RDONLY <unfinished ...>
[pid 394] <... rt_sigprocmask resumed> NULL, 8) = 0
[pid 395] <... open resumed> ) = 4
[pid 394] rt_sigaction(SIGCONT, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, <unfinished ...>
[pid 395] writev(3, [{"NAMC\3\0\0\20\24\0\0\0\5\0\0\0\0\0\0\0", 20}], 1 <unfinished ...>
[pid 394] <... rt_sigaction resumed> {SIG_DFL}, 8) = 0
[pid 395] <... writev resumed> ) = 20
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [STOP], <unfinished ...>
[pid 395] recvfrom(3, <unfinished ...>
[pid 394] <... rt_sigprocmask resumed> NULL, 8) = 0
[pid 395] <... recvfrom resumed> "NAMC\0\0\0\0\30\0\0\0\5\0\0@\0\0\0\0", 20, 0, NULL, NULL) = 20
[pid 395] read(3, <unfinished ...>
[pid 394] rt_sigaction(SIGSTOP, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, <unfinished ...>
[pid 395] <... read resumed> "\1\0\0\0", 4) = 4
[pid 394] <... rt_sigaction resumed> {SIG_DFL}, 8) = -1 EINVAL (Invalid argument)
[pid 395] pipe( <unfinished ...>
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [TSTP], <unfinished ...>
[pid 395] <... pipe resumed> [5, 6]) = 0
[pid 395] fcntl(5, F_GETFL <unfinished ...>
[pid 394] <... rt_sigprocmask resumed> NULL, 8) = 0
[pid 395] <... fcntl resumed> ) = 0 (flags O_RDONLY)
[pid 395] fcntl(5, F_SETFL, O_RDONLY|O_NONBLOCK <unfinished ...>
[pid 394] rt_sigaction(SIGTSTP, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, <unfinished ...>
[pid 395] <... fcntl resumed> ) = 0
[pid 394] <... rt_sigaction resumed> {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [TTIN], <unfinished ...>
[pid 395] open("/dev/misc/dlm_rgmanager", O_RDWR <unfinished ...>
[pid 394] <... rt_sigprocmask resumed> NULL, 8) = 0
[pid 394] rt_sigaction(SIGTTIN, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [TTOU], NULL, 8) = 0
[pid 394] rt_sigaction(SIGTTOU, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [URG], NULL, 8) = 0
[pid 394] rt_sigaction(SIGURG, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [XCPU], NULL, 8) = 0
[pid 394] rt_sigaction(SIGXCPU, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [XFSZ], NULL, 8) = 0
[pid 394] rt_sigaction(SIGXFSZ, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [VTALRM], NULL, 8) = 0
[pid 394] rt_sigaction(SIGVTALRM, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [PROF], NULL, 8) = 0
[pid 394] rt_sigaction(SIGPROF, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [WINCH], NULL, 8) = 0
[pid 394] rt_sigaction(SIGWINCH, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [IO], NULL, 8) = 0
[pid 394] rt_sigaction(SIGIO, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [PWR], NULL, 8) = 0
[pid 394] rt_sigaction(SIGPWR, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [SYS], NULL, 8) = 0
[pid 394] rt_sigaction(SIGSYS, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RTMIN], NULL, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_1], NULL, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_2], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_2, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_3], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_3, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_4], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_4, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_5], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_5, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_6], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_6, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_7], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_7, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_8], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_8, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_9], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_9, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_10], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_10, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_11], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_11, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_12], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_12, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_13], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_13, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_14], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_14, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_15], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_15, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_16], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_16, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_17], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_17, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_18], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_18, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_19], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_19, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_20], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_20, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_21], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_21, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_22], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_22, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_23], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_23, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_24], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_24, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_25], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_25, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_26], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_26, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_27], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_27, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_28], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_28, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_29], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_29, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_30], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_30, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [RT_31], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_31, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] rt_sigprocmask(SIG_UNBLOCK, [], NULL, 8) = 0
[pid 394] rt_sigaction(SIGRT_32, {0x411210, [], SA_RESTORER, 0x3c59c0dd40}, {SIG_DFL}, 8) = 0
[pid 394] wait4(395, Process 394 suspended
<unfinished ...>
[pid 395] <... open resumed> ) = -1 ENODEV (No such device)
[pid 395] stat("/dev/misc/dlm-control", {st_mode=S_IFCHR|0600, st_rdev=makedev(10, 62), ...}) = 0
[pid 395] open("/proc/misc", O_RDONLY) = 7
[pid 395] fstat(7, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
[pid 395] mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaaac000
[pid 395] read(7, "209 cpqci\n 60 dlm_clvmd\n 61 lock_dlm_plock\n 62 dlm-control\n 63 device-mapper\n144 nvram\n228 hpet\n135 rtc\n231 snapshot\n227 mcelog\n", 4096) = 128
[pid 395] close(7) = 0
[pid 395] munmap(0x2aaaaaaac000, 4096) = 0
[pid 395] open("/dev/misc/dlm-control", O_RDWR) = 7
[pid 395] fcntl(7, F_SETFD, FD_CLOEXEC) = 0
[pid 395] write(7, "\5\0\0\0\0\0\0\0\0\0\0\0\4\1\0\0\0\0\0\0\0\0\0\0rgmanager\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\300\233!Y<\0\0\0\200 O\f\0\0\0\0\330\333A\0\0\0\0\0\320*\0\203\377\177\0\0\0\0\0\0\0\0\0\0\263\270DY<\0\0\0 ", 113) = -1 EEXIST (File exists)
[pid 395] open("/proc/misc", O_RDONLY) = 8
[pid 395] fstat(8, {st_mode=S_IFREG|0444, st_size=0, ...}) = 0
[pid 395] mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaaac000
[pid 395] read(8, "209 cpqci\n 60 dlm_clvmd\n 61 lock_dlm_plock\n 62 dlm-control\n 63 device-mapper\n144 nvram\n228 hpet\n135 rtc\n231 snapshot\n227 mcelog\n", 4096) = 128
[pid 395] read(8, "", 4096) = 0
[pid 395] close(8) = 0
[pid 395] munmap(0x2aaaaaaac000, 4096) = 0
[pid 395] stat("/dev/misc/dlm_rgmanager", {st_mode=S_IFCHR|0644, st_rdev=makedev(10, 0), ...}) = 0
[pid 395] stat("/dev/misc/dlm_rgmanager", {st_mode=S_IFCHR|0644, st_rdev=makedev(10, 0), ...}) = 0
[pid 395] open("/dev/misc/dlm_rgmanager", O_RDWR) = -1 ENODEV (No such device)
[pid 395] write(2, "failed acquiring lockspace: No such device\n", 43) = 43
[pid 395] fstat(1, {st_mode=S_IFCHR|0666, st_rdev=makedev(1, 3), ...}) = 0
[pid 395] ioctl(1, SNDCTL_TMR_TIMEBASE or TCGETS, 0x7fff83002690) = -1 ENOTTY (Inappropriate ioctl for device)
[pid 395] mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaaaac000
[pid 395] write(1, "Locks not working!\n", 19) = 19
[pid 395] exit_group(-1) = ?
Process 394 resumed
Process 395 detached
<... wait4 resumed> [{WIFEXITED(s) && WEXITSTATUS(s) == 255}], 0, NULL) = 395
--- SIGCHLD (Child exited) @ 0 (0) ---
exit_group(255) = ?
Process 394 detached
More information about the Linux-cluster
mailing list