[Cluster-devel] cluster/group/daemon app.c cman.c cpg.c gd_int ...
teigland at sourceware.org
teigland at sourceware.org
Fri Dec 1 20:27:37 UTC 2006
CVSROOT: /cvs/cluster
Module name: cluster
Branch: RHEL50
Changes by: teigland at sourceware.org 2006-12-01 20:27:36
Modified files:
group/daemon : app.c cman.c cpg.c gd_internal.h
Log message:
Be more intelligent about handling recovery sets so we can deal with
cases where a node fails, rejoins, then fails again before recovery has
completed for the first failure. Also handles case where the groupd
process exits without the node going down. If that happens, we want
to kill the node (via cman) if the node was in any groups and ignore
it otherwise.
bz 218103
Patches:
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/app.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.52&r2=1.52.4.1
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cman.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.27&r2=1.27.4.1
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/cpg.c.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.36&r2=1.36.4.1
http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/group/daemon/gd_internal.h.diff?cvsroot=cluster&only_with_tag=RHEL50&r1=1.44&r2=1.44.4.1
--- cluster/group/daemon/app.c 2006/10/16 15:52:05 1.52
+++ cluster/group/daemon/app.c 2006/12/01 20:27:36 1.52.4.1
@@ -209,20 +209,61 @@
level must complete recovery (on all nodes) before recovery can begin for
the next level. */
-void add_recovery_set(int nodeid)
+/* FIXME: do we need to worry about the case where we get an
+ add_recovery_set_cman() that finds an old rs, the old rs completes
+ and goes away, and then we get the add_recovery_set_cpg() matching
+ the _cman() variant that we ignored? */
+
+void add_recovery_set_cman(int nodeid)
+{
+ struct recovery_set *rs;
+
+ log_debug("add_recovery_set_cman nodeid %d", nodeid);
+
+ rs = get_recovery_set(nodeid);
+ if (rs->cman_update) {
+ log_debug("old recovery for %d still in progress", nodeid);
+ return;
+ }
+ rs->cman_update = 1;
+
+ if (!rs->cpg_update && !in_groupd_cpg(nodeid)) {
+ log_debug("free recovery set %d not running groupd", nodeid);
+ list_del(&rs->list);
+ free(rs);
+ return;
+ }
+
+ if (list_empty(&rs->entries) && rs->cpg_update) {
+ log_debug("free unused recovery set %d cman", nodeid);
+ list_del(&rs->list);
+ free(rs);
+ }
+}
+
+/* procdown of 1 means the groupd daemon process exited, but the node didn't
+ fail. when only the process fails, we won't get a cman callback which is
+ only for nodedown. if the node wasn't in any groups we don't add a recovery
+ set and don't care about the exited groupd; if the node with the failed
+ groupd _was_ in any groups, we add a rs and process_groupd_confchg() will do
+ cman_kill_node() to make the node really fail (and we'll get an
+ add_recovery_set_cman()). */
+
+struct recovery_set *add_recovery_set_cpg(int nodeid, int procdown)
{
struct recovery_set *rs;
struct recovery_entry *re;
group_t *g;
node_t *node;
- log_debug("add_recovery_set for nodeid %d", nodeid);
+ log_debug("add_recovery_set_cpg nodeid %d procdown %d",
+ nodeid, procdown);
rs = get_recovery_set(nodeid);
-
- ASSERT(list_empty(&rs->entries));
- ASSERT(!rs->cpg_update);
-
+ if (rs->cpg_update) {
+ log_debug("old recovery for %d still in progress", nodeid);
+ return rs;
+ }
rs->cpg_update = 1;
list_for_each_entry(g, &gd_groups, list) {
@@ -238,11 +279,16 @@
}
}
- if (list_empty(&rs->entries) && rs->cman_update) {
- log_debug("free unused recovery set %d cpg", nodeid);
- list_del(&rs->list);
- free(rs);
+ if (list_empty(&rs->entries)) {
+ if (rs->cman_update || procdown) {
+ log_debug("free unused recovery set %d cpg", nodeid);
+ list_del(&rs->list);
+ free(rs);
+ return NULL;
+ }
}
+
+ return rs;
}
void _del_recovery_set(group_t *g, int nodeid, int purge)
--- cluster/group/daemon/cman.c 2006/06/30 21:00:33 1.27
+++ cluster/group/daemon/cman.c 2006/12/01 20:27:36 1.27.4.1
@@ -53,7 +53,6 @@
static void statechange(void)
{
- struct recovery_set *rs;
int i, rv;
old_quorate = cman_quorate;
@@ -95,24 +94,7 @@
log_debug("cman: node %d removed",
old_nodes[i].cn_nodeid);
-
- rs = get_recovery_set(old_nodes[i].cn_nodeid);
- rs->cman_update = 1;
-
- if (!rs->cpg_update && !in_groupd_cpg(rs->nodeid)) {
- log_debug("free recovery set %d not in cpg",
- rs->nodeid);
- list_del(&rs->list);
- free(rs);
- continue;
- }
-
- if (rs->cpg_update && list_empty(&rs->entries)) {
- log_debug("free unused recovery set %d cman",
- rs->nodeid);
- list_del(&rs->list);
- free(rs);
- }
+ add_recovery_set_cman(old_nodes[i].cn_nodeid);
}
}
--- cluster/group/daemon/cpg.c 2006/10/13 15:57:23 1.36
+++ cluster/group/daemon/cpg.c 2006/12/01 20:27:36 1.36.4.1
@@ -151,6 +151,7 @@
void process_groupd_confchg(void)
{
+ struct recovery_set *rs;
int i, found = 0;
log_debug("groupd confchg total %d left %d joined %d",
@@ -172,14 +173,26 @@
log_print("we are not in groupd confchg: %u %u",
our_nodeid, (uint32_t) getpid());
- /* FIXME: we probably want to do a cman_kill_node() on a node
- where groupd exits but cman is still running. */
-
for (i = 0; i < saved_left_count; i++) {
- if (saved_left[i].reason != CPG_REASON_LEAVE) {
- add_recovery_set(saved_left[i].nodeid);
- groupd_down(saved_left[i].nodeid);
+ if (saved_left[i].reason == CPG_REASON_LEAVE)
+ continue;
+
+ if (saved_left[i].reason == CPG_REASON_NODEDOWN) {
+ /* a nice clean failure */
+ add_recovery_set_cpg(saved_left[i].nodeid, 0);
+ } else if (saved_left[i].reason == CPG_REASON_PROCDOWN) {
+ /* groupd failed, but the node is still up; if
+ the node was in any groups (non-NULL rs is
+ returned), then kill the node so it'll be a
+ real nodedown */
+ rs = add_recovery_set_cpg(saved_left[i].nodeid, 1);
+ if (rs) {
+ log_print("kill node %d - groupd PROCDOWN",
+ saved_left[i].nodeid);
+ kill_cman(saved_left[i].nodeid);
+ }
}
+ groupd_down(saved_left[i].nodeid);
}
}
--- cluster/group/daemon/gd_internal.h 2006/10/06 16:55:19 1.44
+++ cluster/group/daemon/gd_internal.h 2006/12/01 20:27:36 1.44.4.1
@@ -236,7 +236,8 @@
/* app.c */
-void add_recovery_set(int nodeid);
+void add_recovery_set_cman(int nodeid);
+struct recovery_set *add_recovery_set_cpg(int nodeid, int procdown);
int queue_app_recover(group_t *g, int nodeid);
int queue_app_join(group_t *g, int nodeid);
int queue_app_leave(group_t *g, int nodeid);
More information about the Cluster-devel
mailing list