[Cluster-devel] [PATCH] qdiskd: backport dual socket connection to cman
Fabio M. Di Nitto
fdinitto at redhat.com
Mon Aug 13 12:39:51 UTC 2012
From: "Fabio M. Di Nitto" <fdinitto at redhat.com>
Patch 76741bb2a94ae94e493c609d50f570d02e2f3029 had a not so obvious
dependency on 08ae3ce147b2771c5ee6e1d364a5e48c88384427.
Backport portion of 08ae3ce147b2771c5ee6e1d364a5e48c88384427 to handle
dual cman socket (admin and user) and use the correct socket (user)
for send/receive data.
Move cman_alive check and heartbeat (for dispatch) to ch_user.
Resolves: rhbz#782900
Signed-off-by: Fabio M. Di Nitto <fdinitto at redhat.com>
---
cman/qdisk/disk.h | 5 ++-
cman/qdisk/disk_util.c | 7 +++--
cman/qdisk/iostate.c | 8 +++---
cman/qdisk/main.c | 69 ++++++++++++++++++++++-------------------------
4 files changed, 43 insertions(+), 46 deletions(-)
diff --git a/cman/qdisk/disk.h b/cman/qdisk/disk.h
index d491de1..83167ea 100644
--- a/cman/qdisk/disk.h
+++ b/cman/qdisk/disk.h
@@ -270,7 +270,8 @@ typedef struct {
int qc_master; /* Master?! */
int qc_status_sock;
run_flag_t qc_flags;
- cman_handle_t qc_ch;
+ cman_handle_t qc_ch_admin;
+ cman_handle_t qc_ch_user;
char *qc_device;
char *qc_label;
char *qc_status_file;
@@ -299,7 +300,7 @@ typedef struct {
int qd_write_status(qd_ctx *ctx, int nid, disk_node_state_t state,
disk_msg_t *msg, memb_mask_t mask, memb_mask_t master);
int qd_read_print_status(target_info_t *disk, int nid);
-int qd_init(qd_ctx *ctx, cman_handle_t ch, int me);
+int qd_init(qd_ctx *ctx, cman_handle_t ch_admin, cman_handle_t ch_user, int me);
void qd_destroy(qd_ctx *ctx);
/* proc.c */
diff --git a/cman/qdisk/disk_util.c b/cman/qdisk/disk_util.c
index f5539c0..25f4013 100644
--- a/cman/qdisk/disk_util.c
+++ b/cman/qdisk/disk_util.c
@@ -312,16 +312,17 @@ generate_token(void)
Initialize a quorum disk context, given a CMAN handle and a nodeid.
*/
int
-qd_init(qd_ctx *ctx, cman_handle_t ch, int me)
+qd_init(qd_ctx *ctx, cman_handle_t ch_admin, cman_handle_t ch_user, int me)
{
- if (!ctx || !ch || !me) {
+ if (!ctx || !ch_admin || !ch_user || !me) {
errno = EINVAL;
return -1;
}
memset(ctx, 0, sizeof(*ctx));
ctx->qc_incarnation = generate_token();
- ctx->qc_ch = ch;
+ ctx->qc_ch_admin = ch_admin;
+ ctx->qc_ch_user = ch_user;
ctx->qc_my_id = me;
ctx->qc_status_sock = -1;
diff --git a/cman/qdisk/iostate.c b/cman/qdisk/iostate.c
index eb74ad2..ba7ad12 100644
--- a/cman/qdisk/iostate.c
+++ b/cman/qdisk/iostate.c
@@ -69,7 +69,7 @@ io_nanny_thread(void *arg)
iostate_t last_main_state = 0, current_main_state = 0;
int last_main_incarnation = 0, current_main_incarnation = 0;
int logged_incarnation = 0;
- cman_handle_t ch = (cman_handle_t)arg;
+ cman_handle_t ch_user = (cman_handle_t)arg;
int32_t whine_state;
/* Start with wherever we're at now */
@@ -105,7 +105,7 @@ io_nanny_thread(void *arg)
/* Whine on CMAN api */
whine_state = (int32_t)current_main_state;
swab32(whine_state);
- cman_send_data(ch, &whine_state, sizeof(int32_t), 0, CLUSTER_PORT_QDISKD, 0);
+ cman_send_data(ch_user, &whine_state, sizeof(int32_t), 0, CLUSTER_PORT_QDISKD, 0);
/* Don't log things twice */
if (logged_incarnation == current_main_incarnation)
@@ -125,7 +125,7 @@ io_nanny_thread(void *arg)
int
-io_nanny_start(cman_handle_t ch, int timeout)
+io_nanny_start(cman_handle_t ch_user, int timeout)
{
int ret;
@@ -135,7 +135,7 @@ io_nanny_start(cman_handle_t ch, int timeout)
qdisk_timeout = timeout;
thread_active = 1;
- ret = pthread_create(&io_nanny_tid, NULL, io_nanny_thread, ch);
+ ret = pthread_create(&io_nanny_tid, NULL, io_nanny_thread, ch_user);
pthread_mutex_unlock(&state_mutex);
return ret;
diff --git a/cman/qdisk/main.c b/cman/qdisk/main.c
index 90d00ab..72a3c07 100644
--- a/cman/qdisk/main.c
+++ b/cman/qdisk/main.c
@@ -287,7 +287,7 @@ check_transitions(qd_ctx *ctx, node_info_t *ni, int max, memb_mask_t mask)
if (ctx->qc_flags & RF_ALLOW_KILL) {
clulog(LOG_DEBUG, "Telling CMAN to "
"kill the node\n");
- cman_kill_node(ctx->qc_ch,
+ cman_kill_node(ctx->qc_ch_admin,
ni[x].ni_status.ps_nodeid);
}
}
@@ -325,7 +325,7 @@ check_transitions(qd_ctx *ctx, node_info_t *ni, int max, memb_mask_t mask)
if (ctx->qc_flags & RF_ALLOW_KILL) {
clulog(LOG_DEBUG, "Telling CMAN to "
"kill the node\n");
- cman_kill_node(ctx->qc_ch,
+ cman_kill_node(ctx->qc_ch_admin,
ni[x].ni_status.ps_nodeid);
}
continue;
@@ -580,7 +580,7 @@ check_cman(qd_ctx *ctx, memb_mask_t mask, memb_mask_t master_mask)
cman_node_t nodes[MAX_NODES_DISK];
int retnodes, x;
- if (cman_get_nodes(ctx->qc_ch, MAX_NODES_DISK,
+ if (cman_get_nodes(ctx->qc_ch_admin, MAX_NODES_DISK,
&retnodes, nodes) <0 )
return;
@@ -894,7 +894,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
_running = 1;
while (_running) {
/* perform a forceful cman dispatch */
- if (cman_alive(ctx->qc_ch) < 0) {
+ if (cman_alive(ctx->qc_ch_user) < 0) {
clulog(LOG_ERR, "cman: %s\n", strerror(errno));
}
@@ -931,11 +931,11 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
msg.m_msg = M_NONE;
++msg.m_seq;
bid_pending = 0;
- if (cman_alive(ctx->qc_ch) < 0) {
+ if (cman_alive(ctx->qc_ch_user) < 0) {
clulog(LOG_ERR, "cman: %s\n",
strerror(errno));
} else {
- cman_poll_quorum_device(ctx->qc_ch, 0);
+ cman_poll_quorum_device(ctx->qc_ch_admin, 0);
}
if (ctx->qc_flags & RF_REBOOT)
reboot(RB_AUTOBOOT);
@@ -1039,7 +1039,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
/* We are the master. Poll the quorum device.
We can't be the master unless we score high
enough on our heuristics. */
- if (cman_alive(ctx->qc_ch) < 0) {
+ if (cman_alive(ctx->qc_ch_user) < 0) {
clulog(LOG_ERR, "cman_dispatch: %s\n",
strerror(errno));
clulog(LOG_ERR,
@@ -1048,7 +1048,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
}
check_cman(ctx, mask, master_mask);
if (!errors)
- cman_poll_quorum_device(ctx->qc_ch, 1);
+ cman_poll_quorum_device(ctx->qc_ch_admin, 1);
} else if (ctx->qc_status == S_RUN && ctx->qc_master &&
ctx->qc_master != ctx->qc_my_id) {
@@ -1060,7 +1060,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
ni[ctx->qc_master-1].ni_status.ps_master_mask,
ctx->qc_my_id-1,
sizeof(memb_mask_t))) {
- if (cman_alive(ctx->qc_ch) < 0) {
+ if (cman_alive(ctx->qc_ch_user) < 0) {
clulog(LOG_ERR, "cman_dispatch: %s\n",
strerror(errno));
clulog(LOG_ERR,
@@ -1069,7 +1069,7 @@ quorum_loop(qd_ctx *ctx, node_info_t *ni, int max)
}
if (!errors &&
(!(ctx->qc_flags & RF_MASTER_WINS)))
- cman_poll_quorum_device(ctx->qc_ch, 1);
+ cman_poll_quorum_device(ctx->qc_ch_admin, 1);
}
}
@@ -1506,12 +1506,7 @@ check_stop_cman(qd_ctx *ctx)
clulog(LOG_WARNING, "Telling CMAN to leave the cluster; qdisk is not"
" available\n");
-#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2)
- if (cman_shutdown(ctx->qc_ch, 0) < 0) {
-#else
- int x = 0;
- if (ioctl(cman_get_fd(ctx->qc_ch), SIOCCLUSTER_LEAVE_CLUSTER, &x) < 0) {
-#endif
+ if (cman_shutdown(ctx->qc_ch_admin, 0) < 0) {
clulog(LOG_CRIT, "Could not leave the cluster - rebooting\n");
sleep(5);
if (ctx->qc_flags & RF_DEBUG)
@@ -1552,7 +1547,8 @@ main(int argc, char **argv)
cman_node_t me;
int cfh, rv, forked = 0, nfd = -1, ret = -1;
qd_ctx ctx;
- cman_handle_t ch = NULL;
+ cman_handle_t ch_admin = NULL;
+ cman_handle_t ch_user = NULL;
node_info_t ni[MAX_NODES_DISK];
struct h_data h[10];
char debug = 0, foreground = 0;
@@ -1597,12 +1593,8 @@ main(int argc, char **argv)
qp.ni = &ni[0];
qp.ni_len = MAX_NODES_DISK;
-#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2)
- ch = cman_admin_init(&qp);
-#else
- ch = cman_init(&qp);
-#endif
- if (!ch) {
+ ch_admin = cman_admin_init(&qp);
+ if (!ch_admin) {
if (!foreground && !forked) {
if (daemon_init(argv[0]) < 0)
goto out;
@@ -1614,22 +1606,24 @@ main(int argc, char **argv)
do {
sleep(5);
-#if (defined(LIBCMAN_VERSION) && LIBCMAN_VERSION >= 2)
- ch = cman_admin_init(&qp);
-#else
- ch = cman_init(&qp);
-#endif
- } while (!ch);
+ ch_admin = cman_admin_init(&qp);
+ } while (!ch_admin);
+ }
+
+ ch_user = cman_init(&qp);
+ if (!ch_user) {
+ clulog_and_print(LOG_CRIT, "Could not register with CMAN: %s\n", strerror(errno));
+ goto out;
}
- if (cman_start_recv_data(ch, qdisk_whine, CLUSTER_PORT_QDISKD) != 0) {
+ if (cman_start_recv_data(ch_user, qdisk_whine, CLUSTER_PORT_QDISKD) != 0) {
clulog_and_print(LOG_CRIT, "Could not register with CMAN: %s\n",
strerror(errno));
goto out;
}
memset(&me, 0, sizeof(me));
- while (cman_get_node(ch, CMAN_NODEID_US, &me) < 0) {
+ while (cman_get_node(ch_admin, CMAN_NODEID_US, &me) < 0) {
if (!foreground && !forked) {
if (daemon_init(argv[0]) < 0)
goto out;
@@ -1639,7 +1633,7 @@ main(int argc, char **argv)
sleep(5);
}
- qd_init(&ctx, ch, me.cn_nodeid);
+ qd_init(&ctx, ch_admin, ch_user, me.cn_nodeid);
signal(SIGINT, int_handler);
signal(SIGTERM, int_handler);
@@ -1715,13 +1709,13 @@ main(int argc, char **argv)
if (!_running)
goto out;
- if ((rv = cman_register_quorum_device(ctx.qc_ch,
+ if ((rv = cman_register_quorum_device(ctx.qc_ch_admin,
(ctx.qc_flags&RF_CMAN_LABEL)?
ctx.qc_cman_label:
ctx.qc_device,
ctx.qc_votes)) < 0) {
if (errno == EBUSY) {
- if ((rv = cman_update_quorum_device(ctx.qc_ch,
+ if ((rv = cman_update_quorum_device(ctx.qc_ch_admin,
(ctx.qc_flags&RF_CMAN_LABEL)?
ctx.qc_cman_label:
ctx.qc_device,
@@ -1740,17 +1734,18 @@ main(int argc, char **argv)
}
}
- io_nanny_start(ch, ctx.qc_tko * ctx.qc_interval);
+ io_nanny_start(ch_user, ctx.qc_tko * ctx.qc_interval);
if (quorum_loop(&ctx, ni, MAX_NODES_DISK) == 0)
- cman_unregister_quorum_device(ctx.qc_ch);
+ cman_unregister_quorum_device(ctx.qc_ch_admin);
io_nanny_stop();
quorum_logout(&ctx);
/* free cman handle to avoid leak in cman */
out:
- cman_finish(ctx.qc_ch);
+ cman_finish(ctx.qc_ch_admin);
+ cman_finish(ctx.qc_ch_user);
qd_destroy(&ctx);
return ret;
--
1.7.7.6
More information about the Cluster-devel
mailing list