merge from peter

(This used to be ctdb commit ddf390da2bceb5b3f431433aec424d99d98c05f4)

9 files changed, 121 insertions, 83 deletions

diff --git a/ctdb/common/ctdb.c b/ctdb/common/ctdb.c
index d7e4241250d..56c621fd7e7 100644
--- a/ctdb/common/ctdb.c
+++ b/ctdb/common/ctdb.c
@@ -31,24 +31,11 @@
 */
 int ctdb_set_transport(struct ctdb_context *ctdb, const char *transport)
 {
-	int ctdb_tcp_init(struct ctdb_context *ctdb);
-#ifdef USE_INFINIBAND
-	int ctdb_ibw_init(struct ctdb_context *ctdb);
-#endif /* USE_INFINIBAND */
-
-	if (strcmp(transport, "tcp") == 0) {
-		return ctdb_tcp_init(ctdb);
-	}
-#ifdef USE_INFINIBAND
-	if (strcmp(transport, "ib") == 0) {
-		return ctdb_ibw_init(ctdb);
-	}
-#endif /* USE_INFINIBAND */
-
-	ctdb_set_error(ctdb, "Unknown transport '%s'\n", transport);
-	return -1;
+	ctdb->transport = talloc_strdup(ctdb, transport);
+	return 0;
 }
 
+
 /*
   set some ctdb flags
 */
@@ -116,11 +103,6 @@ static int ctdb_add_node(struct ctdb_context *ctdb, char *nstr)
 	   will change! */
 	node->vnn = ctdb->num_nodes;
 
-	if (ctdb->methods->add_node(node) != 0) {
-		talloc_free(node);
-		return -1;
-	}
-
 	if (ctdb_same_address(&ctdb->address, &node->address)) {
 		ctdb->vnn = node->vnn;
 	}
diff --git a/ctdb/common/ctdb_daemon.c b/ctdb/common/ctdb_daemon.c
index efb3f5431ec..053c5af6dd1 100644
--- a/ctdb/common/ctdb_daemon.c
+++ b/ctdb/common/ctdb_daemon.c
@@ -44,6 +44,24 @@ static void daemon_incoming_packet(void *, uint8_t *, uint32_t );
 
 static void ctdb_main_loop(struct ctdb_context *ctdb)
 {
+	int ret = -1;
+
+	if (strcmp(ctdb->transport, "tcp") == 0) {
+		int ctdb_tcp_init(struct ctdb_context *);
+		ret = ctdb_tcp_init(ctdb);
+	}
+#ifdef USE_INFINBAND
+	if (strcmp(ctdb->transport, "ib") == 0) {
+		int ctdb_ibw_init(struct ctdb_context *);
+		ret = ctdb_ibw_init(ctdb);
+	}
+#endif
+	if (ret != 0) {
+		DEBUG(0,("Failed to initialise transport '%s'\n", ctdb->transport));
+		return;
+	}
+
+	/* start the transport running */
 	ctdb->methods->start(ctdb);
 
 	/* go into a wait loop to allow other nodes to complete */
diff --git a/ctdb/ib/ibw_ctdb_init.c b/ctdb/ib/ibw_ctdb_init.c
index b4adfe6f124..c8f9e97feb7 100644
--- a/ctdb/ib/ibw_ctdb_init.c
+++ b/ctdb/ib/ibw_ctdb_init.c
@@ -54,11 +54,38 @@ static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog)
 }
 
 /*
+ * initialise ibw portion of a ctdb node 
+ */
+static int ctdb_ibw_add_node(struct ctdb_node *node)
+{
+	struct ibw_ctx *ictx = talloc_get_type(node->ctdb->private_data, struct ibw_ctx);
+	struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node);
+
+	assert(cn!=NULL);
+	cn->conn = ibw_conn_new(ictx, node);
+	node->private_data = (void *)cn;
+
+	return (cn->conn!=NULL ? 0 : -1);
+}
+
+/*
  * Start infiniband
  */
 static int ctdb_ibw_start(struct ctdb_context *ctdb)
 {
-	int i;
+	int i, ret;
+
+	ret = ctdb_ibw_init(ctdb);
+	if (ret != 0) {
+		return ret;
+	}
+
+	for (i=0; i<ctdb->num_nodes; i++) {
+		if (ctdb_ibw_add_node(ctdb->nodes[i]) != 0) {
+			DEBUG(0, ("methods->add_node failed at %d\n", i));
+			return -1;
+		}
+	}
 
 	/* listen on our own address */
 	if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */
@@ -76,21 +103,6 @@ static int ctdb_ibw_start(struct ctdb_context *ctdb)
 	return 0;
 }
 
-/*
- * initialise ibw portion of a ctdb node 
- */
-static int ctdb_ibw_add_node(struct ctdb_node *node)
-{
-	struct ibw_ctx *ictx = talloc_get_type(node->ctdb->private_data, struct ibw_ctx);
-	struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node);
-
-	assert(cn!=NULL);
-	cn->conn = ibw_conn_new(ictx, node);
-	node->private_data = (void *)cn;
-
-	return (cn->conn!=NULL ? 0 : -1);
-}
-
 static int ctdb_ibw_send_pkt(struct ibw_conn *conn, uint8_t *data, uint32_t length)
 {
 	void	*buf, *key;
@@ -178,8 +190,8 @@ static int ctdb_ibw_stop(struct ctdb_context *cctx)
 
 static const struct ctdb_methods ctdb_ibw_methods = {
 	.start     = ctdb_ibw_start,
-	.add_node  = ctdb_ibw_add_node,
 	.queue_pkt = ctdb_ibw_queue_pkt,
+	.add_node = ctdb_ibw_add_node,
 	.allocate_pkt = ctdb_ibw_allocate_pkt,
 
 //	.stop = ctdb_ibw_stop
diff --git a/ctdb/ib/ibwrapper.c b/ctdb/ib/ibwrapper.c
index f7b233954d2..a558ab44b8f 100644
--- a/ctdb/ib/ibwrapper.c
+++ b/ctdb/ib/ibwrapper.c
@@ -38,8 +38,9 @@
 #include "lib/events/events.h"
 #include "ibwrapper.h"
 
+#include <infiniband/kern-abi.h>
+#include <rdma/rdma_cma_abi.h>
 #include <rdma/rdma_cma.h>
-#include "infiniband/sa-kern-abi.h"
 
 #include "ibwrapper_internal.h"
 #include "lib/util/dlinklist.h"
@@ -83,7 +84,7 @@ static void *ibw_alloc_mr(struct ibw_ctx_priv *pctx, struct ibw_conn_priv *pconn
 
 static void ibw_free_mr(char **ppbuf, struct ibv_mr **ppmr)
 {
-	DEBUG(10, ("ibw_free_mr(%u %u)\n", (uint32_t)*ppbuf, (uint32_t)*ppmr));
+	DEBUG(10, ("ibw_free_mr(%p %p)\n", *ppbuf, *ppmr));
 	if (*ppmr!=NULL) {
 		ibv_dereg_mr(*ppmr);
 		*ppmr = NULL;
@@ -133,7 +134,7 @@ static int ibw_init_memory(struct ibw_conn *conn)
 
 static int ibw_ctx_priv_destruct(struct ibw_ctx_priv *pctx)
 {
-	DEBUG(10, ("ibw_ctx_priv_destruct(%u)\n", (uint32_t)pctx));
+	DEBUG(10, ("ibw_ctx_priv_destruct(%p)\n", pctx));
 
 	/* destroy cm */
 	if (pctx->cm_channel) {
@@ -155,7 +156,7 @@ static int ibw_ctx_priv_destruct(struct ibw_ctx_priv *pctx)
 
 static int ibw_ctx_destruct(struct ibw_ctx *ctx)
 {
-	DEBUG(10, ("ibw_ctx_destruct(%u)\n", (uint32_t)ctx));
+	DEBUG(10, ("ibw_ctx_destruct(%p)\n", ctx));
 	return 0;
 }
 
@@ -217,7 +218,7 @@ static int ibw_wr_destruct(struct ibw_wr *wr)
 
 static int ibw_conn_destruct(struct ibw_conn *conn)
 {
-	DEBUG(10, ("ibw_conn_destruct(%u)\n", (uint32_t)conn));
+	DEBUG(10, ("ibw_conn_destruct(%p)\n", conn));
 	
 	/* important here: ctx is a talloc _parent_ */
 	DLIST_REMOVE(conn->ctx->conn_list, conn);
@@ -428,6 +429,7 @@ static void ibw_event_handler_cm(struct event_context *ev,
 	rc = rdma_get_cm_event(pctx->cm_channel, &event);
 	if (rc) {
 		ctx->state = IBWS_ERROR;
+		event = NULL;
 		sprintf(ibw_lasterr, "rdma_get_cm_event error %d\n", rc);
 		goto error;
 	}
@@ -520,9 +522,10 @@ static void ibw_event_handler_cm(struct event_context *ev,
 		if (conn) {
 			if ((rc=rdma_ack_cm_event(event)))
 				DEBUG(0, ("reject/rdma_ack_cm_event failed with %d\n", rc));
-			event = NULL;
-			pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
-			ibw_conn_priv_destruct(pconn);
+			event = NULL; /* not to touch cma_id or conn */
+			conn->state = IBWC_ERROR;
+			/* it should free the conn */
+			pctx->connstate_func(NULL, conn);
 		}
 		goto error;
 
@@ -556,22 +559,26 @@ static void ibw_event_handler_cm(struct event_context *ev,
 
 	return;
 error:
-	if (event!=NULL && (rc=rdma_ack_cm_event(event))) {
-		sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n", rc);
-		goto error;
-	}
-
 	DEBUG(0, ("cm event handler: %s", ibw_lasterr));
 
-	if (cma_id!=pctx->cm_id) {
-		conn = talloc_get_type(cma_id->context, struct ibw_conn);
-		if (conn)
-			conn->state = IBWC_ERROR;
-		pctx->connstate_func(NULL, conn);
-	} else {
-		ctx->state = IBWS_ERROR;
-		pctx->connstate_func(ctx, NULL);
+	if (event!=NULL) {
+		if (cma_id!=NULL && cma_id!=pctx->cm_id) {
+			conn = talloc_get_type(cma_id->context, struct ibw_conn);
+			if (conn) {
+				conn->state = IBWC_ERROR;
+				pctx->connstate_func(NULL, conn);
+			}
+		} else {
+			ctx->state = IBWS_ERROR;
+			pctx->connstate_func(ctx, NULL);
+		}
+
+		if ((rc=rdma_ack_cm_event(event))!=0) {
+			DEBUG(0, ("rdma_ack_cm_event failed with %d\n", rc));
+		}
 	}
+
+	return;
 }
 
 static void ibw_event_handler_verbs(struct event_context *ev,
@@ -967,7 +974,11 @@ struct ibw_ctx *ibw_init(struct ibw_initattr *attr, int nattr,
 	pctx->cm_channel_event = event_add_fd(pctx->ectx, pctx,
 		pctx->cm_channel->fd, EVENT_FD_READ, ibw_event_handler_cm, ctx);
 
+#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
 	rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx, RDMA_PS_TCP);
+#else
+	rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx);
+#endif
 	if (rc) {
 		rc = errno;
 		sprintf(ibw_lasterr, "rdma_create_id error %d\n", rc);
@@ -1088,7 +1099,11 @@ int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn
 	}
 
 	/* init cm */
+#if RDMA_USER_CM_MAX_ABI_VERSION >= 2
 	rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP);
+#else
+	rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn);
+#endif
 	if (rc) {
 		rc = errno;
 		sprintf(ibw_lasterr, "ibw_connect/rdma_create_id error %d\n", rc);
diff --git a/ctdb/ib/ibwrapper_test.c b/ctdb/ib/ibwrapper_test.c
index fa506500e46..6ef5aa1e679 100644
--- a/ctdb/ib/ibwrapper_test.c
+++ b/ctdb/ib/ibwrapper_test.c
@@ -336,7 +336,7 @@ int ibwtest_receive_handler(struct ibw_conn *conn, void *buf, int n)
 				ibw_stop(tcx->ibwctx);
 				goto error;
 			}
-		} else {
+		} else if (op!=TESTOP_SEND_ID) {
 			char *buf2;
 			void *key2;
 
diff --git a/ctdb/include/ctdb_private.h b/ctdb/include/ctdb_private.h
index ee8118a750e..88aa209d9ad 100644
--- a/ctdb/include/ctdb_private.h
+++ b/ctdb/include/ctdb_private.h
@@ -162,6 +162,7 @@ struct ctdb_context {
 	struct ctdb_address address;
 	const char *name;
 	const char *db_directory;
+	const char *transport;
 	uint32_t vnn; /* our own vnn */
 	uint32_t num_nodes;
 	uint32_t num_connected;
diff --git a/ctdb/lib/util/debug.c b/ctdb/lib/util/debug.c
index a3229015ca8..a048ed807ed 100644
--- a/ctdb/lib/util/debug.c
+++ b/ctdb/lib/util/debug.c
@@ -22,6 +22,7 @@
 #include "system/time.h"
 #include <unistd.h>
 
+
 void do_debug(const char *format, ...)
 {
 	struct timeval tm;
diff --git a/ctdb/tcp/tcp_init.c b/ctdb/tcp/tcp_init.c
index 9b54bb75baf..35b43bd5e29 100644
--- a/ctdb/tcp/tcp_init.c
+++ b/ctdb/tcp/tcp_init.c
@@ -26,6 +26,25 @@
 #include "../include/ctdb_private.h"
 #include "ctdb_tcp.h"
 
+
+/*
+  initialise tcp portion of a ctdb node 
+*/
+static int ctdb_tcp_add_node(struct ctdb_node *node)
+{
+	struct ctdb_tcp_node *tnode;
+	tnode = talloc_zero(node, struct ctdb_tcp_node);
+	CTDB_NO_MEMORY(node->ctdb, tnode);
+
+	tnode->fd = -1;
+	node->private_data = tnode;
+
+	tnode->queue = ctdb_queue_setup(node->ctdb, node, tnode->fd, CTDB_TCP_ALIGNMENT,
+					ctdb_tcp_tnode_cb, node);
+	
+	return 0;
+}
+
 /*
   start the protocol going
 */
@@ -33,6 +52,13 @@ static int ctdb_tcp_start(struct ctdb_context *ctdb)
 {
 	int i;
 
+	for (i=0; i<ctdb->num_nodes; i++) {
+		if (ctdb_tcp_add_node(ctdb->nodes[i]) != 0) {
+			DEBUG(0, ("methods->add_node failed at %d\n", i));
+			return -1;
+		}
+	}
+	
 	/* listen on our own address */
 	if (ctdb_tcp_listen(ctdb) != 0) return -1;
 
@@ -51,25 +77,6 @@ static int ctdb_tcp_start(struct ctdb_context *ctdb)
 
 
 /*
-  initialise tcp portion of a ctdb node 
-*/
-static int ctdb_tcp_add_node(struct ctdb_node *node)
-{
-	struct ctdb_tcp_node *tnode;
-	tnode = talloc_zero(node, struct ctdb_tcp_node);
-	CTDB_NO_MEMORY(node->ctdb, tnode);
-
-	tnode->fd = -1;
-	node->private_data = tnode;
-
-	tnode->queue = ctdb_queue_setup(node->ctdb, node, tnode->fd, CTDB_TCP_ALIGNMENT,
-					ctdb_tcp_tnode_cb, node);
-	
-	return 0;
-}
-
-
-/*
   transport packet allocator - allows transport to control memory for packets
 */
 static void *ctdb_tcp_allocate_pkt(TALLOC_CTX *mem_ctx, size_t size)
@@ -83,9 +90,9 @@ static void *ctdb_tcp_allocate_pkt(TALLOC_CTX *mem_ctx, size_t size)
 
 
 static const struct ctdb_methods ctdb_tcp_methods = {
-	.start     = ctdb_tcp_start,
-	.add_node  = ctdb_tcp_add_node,
-	.queue_pkt = ctdb_tcp_queue_pkt,
+	.start        = ctdb_tcp_start,
+	.queue_pkt    = ctdb_tcp_queue_pkt,
+	.add_node     = ctdb_tcp_add_node,
 	.allocate_pkt = ctdb_tcp_allocate_pkt
 };
 
diff --git a/ctdb/tests/ctdb_bench.c b/ctdb/tests/ctdb_bench.c
index 02fcc1f2d45..ead3184157e 100644
--- a/ctdb/tests/ctdb_bench.c
+++ b/ctdb/tests/ctdb_bench.c
@@ -204,7 +204,8 @@ int main(int argc, const char *argv[])
 	/* start the protocol running */
 	ret = ctdb_start(ctdb);
 
-	ctdb_set_message_handler(ctdb, 0, ring_message_handler,&msg_count);
+	if (ctdb_set_message_handler(ctdb, 0, ring_message_handler,&msg_count))
+		goto error;
 
 	/* wait until all nodes are connected (should not be needed
 	   outside of test code) */
@@ -212,6 +213,7 @@ int main(int argc, const char *argv[])
 
 	bench_ring(ctdb, ev);
        
+error:
 	/* shut it down */
 	ctdb_shutdown(ctdb);