diff options
author | Andrew Tridgell <tridge@samba.org> | 2007-04-26 15:28:13 +0200 |
---|---|---|
committer | Andrew Tridgell <tridge@samba.org> | 2007-04-26 15:28:13 +0200 |
commit | c23d1694db4e19a7349300ed475bf5f7688c522b (patch) | |
tree | 3ba6fc2460f8267b700f58eb68f4361a22724b86 /ctdb/ib | |
parent | 8ae14b4052bc565bee5ff8556121f613155dddd3 (diff) | |
parent | 9d521cf16a601dcce2dea2ba52c33dbeac738150 (diff) | |
download | samba-c23d1694db4e19a7349300ed475bf5f7688c522b.tar.gz |
merge from peter
(This used to be ctdb commit ddf390da2bceb5b3f431433aec424d99d98c05f4)
Diffstat (limited to 'ctdb/ib')
-rw-r--r-- | ctdb/ib/ibw_ctdb_init.c | 46 | ||||
-rw-r--r-- | ctdb/ib/ibwrapper.c | 57 | ||||
-rw-r--r-- | ctdb/ib/ibwrapper_test.c | 2 |
3 files changed, 66 insertions, 39 deletions
diff --git a/ctdb/ib/ibw_ctdb_init.c b/ctdb/ib/ibw_ctdb_init.c index b4adfe6f124..c8f9e97feb7 100644 --- a/ctdb/ib/ibw_ctdb_init.c +++ b/ctdb/ib/ibw_ctdb_init.c @@ -54,11 +54,38 @@ static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog) } /* + * initialise ibw portion of a ctdb node + */ +static int ctdb_ibw_add_node(struct ctdb_node *node) +{ + struct ibw_ctx *ictx = talloc_get_type(node->ctdb->private_data, struct ibw_ctx); + struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node); + + assert(cn!=NULL); + cn->conn = ibw_conn_new(ictx, node); + node->private_data = (void *)cn; + + return (cn->conn!=NULL ? 0 : -1); +} + +/* * Start infiniband */ static int ctdb_ibw_start(struct ctdb_context *ctdb) { - int i; + int i, ret; + + ret = ctdb_ibw_init(ctdb); + if (ret != 0) { + return ret; + } + + for (i=0; i<ctdb->num_nodes; i++) { + if (ctdb_ibw_add_node(ctdb->nodes[i]) != 0) { + DEBUG(0, ("methods->add_node failed at %d\n", i)); + return -1; + } + } /* listen on our own address */ if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */ @@ -76,21 +103,6 @@ static int ctdb_ibw_start(struct ctdb_context *ctdb) return 0; } -/* - * initialise ibw portion of a ctdb node - */ -static int ctdb_ibw_add_node(struct ctdb_node *node) -{ - struct ibw_ctx *ictx = talloc_get_type(node->ctdb->private_data, struct ibw_ctx); - struct ctdb_ibw_node *cn = talloc_zero(node, struct ctdb_ibw_node); - - assert(cn!=NULL); - cn->conn = ibw_conn_new(ictx, node); - node->private_data = (void *)cn; - - return (cn->conn!=NULL ? 0 : -1); -} - static int ctdb_ibw_send_pkt(struct ibw_conn *conn, uint8_t *data, uint32_t length) { void *buf, *key; @@ -178,8 +190,8 @@ static int ctdb_ibw_stop(struct ctdb_context *cctx) static const struct ctdb_methods ctdb_ibw_methods = { .start = ctdb_ibw_start, - .add_node = ctdb_ibw_add_node, .queue_pkt = ctdb_ibw_queue_pkt, + .add_node = ctdb_ibw_add_node, .allocate_pkt = ctdb_ibw_allocate_pkt, // .stop = ctdb_ibw_stop diff --git a/ctdb/ib/ibwrapper.c b/ctdb/ib/ibwrapper.c index f7b233954d2..a558ab44b8f 100644 --- a/ctdb/ib/ibwrapper.c +++ b/ctdb/ib/ibwrapper.c @@ -38,8 +38,9 @@ #include "lib/events/events.h" #include "ibwrapper.h" +#include <infiniband/kern-abi.h> +#include <rdma/rdma_cma_abi.h> #include <rdma/rdma_cma.h> -#include "infiniband/sa-kern-abi.h" #include "ibwrapper_internal.h" #include "lib/util/dlinklist.h" @@ -83,7 +84,7 @@ static void *ibw_alloc_mr(struct ibw_ctx_priv *pctx, struct ibw_conn_priv *pconn static void ibw_free_mr(char **ppbuf, struct ibv_mr **ppmr) { - DEBUG(10, ("ibw_free_mr(%u %u)\n", (uint32_t)*ppbuf, (uint32_t)*ppmr)); + DEBUG(10, ("ibw_free_mr(%p %p)\n", *ppbuf, *ppmr)); if (*ppmr!=NULL) { ibv_dereg_mr(*ppmr); *ppmr = NULL; @@ -133,7 +134,7 @@ static int ibw_init_memory(struct ibw_conn *conn) static int ibw_ctx_priv_destruct(struct ibw_ctx_priv *pctx) { - DEBUG(10, ("ibw_ctx_priv_destruct(%u)\n", (uint32_t)pctx)); + DEBUG(10, ("ibw_ctx_priv_destruct(%p)\n", pctx)); /* destroy cm */ if (pctx->cm_channel) { @@ -155,7 +156,7 @@ static int ibw_ctx_priv_destruct(struct ibw_ctx_priv *pctx) static int ibw_ctx_destruct(struct ibw_ctx *ctx) { - DEBUG(10, ("ibw_ctx_destruct(%u)\n", (uint32_t)ctx)); + DEBUG(10, ("ibw_ctx_destruct(%p)\n", ctx)); return 0; } @@ -217,7 +218,7 @@ static int ibw_wr_destruct(struct ibw_wr *wr) static int ibw_conn_destruct(struct ibw_conn *conn) { - DEBUG(10, ("ibw_conn_destruct(%u)\n", (uint32_t)conn)); + DEBUG(10, ("ibw_conn_destruct(%p)\n", conn)); /* important here: ctx is a talloc _parent_ */ DLIST_REMOVE(conn->ctx->conn_list, conn); @@ -428,6 +429,7 @@ static void ibw_event_handler_cm(struct event_context *ev, rc = rdma_get_cm_event(pctx->cm_channel, &event); if (rc) { ctx->state = IBWS_ERROR; + event = NULL; sprintf(ibw_lasterr, "rdma_get_cm_event error %d\n", rc); goto error; } @@ -520,9 +522,10 @@ static void ibw_event_handler_cm(struct event_context *ev, if (conn) { if ((rc=rdma_ack_cm_event(event))) DEBUG(0, ("reject/rdma_ack_cm_event failed with %d\n", rc)); - event = NULL; - pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); - ibw_conn_priv_destruct(pconn); + event = NULL; /* not to touch cma_id or conn */ + conn->state = IBWC_ERROR; + /* it should free the conn */ + pctx->connstate_func(NULL, conn); } goto error; @@ -556,22 +559,26 @@ static void ibw_event_handler_cm(struct event_context *ev, return; error: - if (event!=NULL && (rc=rdma_ack_cm_event(event))) { - sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n", rc); - goto error; - } - DEBUG(0, ("cm event handler: %s", ibw_lasterr)); - if (cma_id!=pctx->cm_id) { - conn = talloc_get_type(cma_id->context, struct ibw_conn); - if (conn) - conn->state = IBWC_ERROR; - pctx->connstate_func(NULL, conn); - } else { - ctx->state = IBWS_ERROR; - pctx->connstate_func(ctx, NULL); + if (event!=NULL) { + if (cma_id!=NULL && cma_id!=pctx->cm_id) { + conn = talloc_get_type(cma_id->context, struct ibw_conn); + if (conn) { + conn->state = IBWC_ERROR; + pctx->connstate_func(NULL, conn); + } + } else { + ctx->state = IBWS_ERROR; + pctx->connstate_func(ctx, NULL); + } + + if ((rc=rdma_ack_cm_event(event))!=0) { + DEBUG(0, ("rdma_ack_cm_event failed with %d\n", rc)); + } } + + return; } static void ibw_event_handler_verbs(struct event_context *ev, @@ -967,7 +974,11 @@ struct ibw_ctx *ibw_init(struct ibw_initattr *attr, int nattr, pctx->cm_channel_event = event_add_fd(pctx->ectx, pctx, pctx->cm_channel->fd, EVENT_FD_READ, ibw_event_handler_cm, ctx); +#if RDMA_USER_CM_MAX_ABI_VERSION >= 2 rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx, RDMA_PS_TCP); +#else + rc = rdma_create_id(pctx->cm_channel, &pctx->cm_id, ctx); +#endif if (rc) { rc = errno; sprintf(ibw_lasterr, "rdma_create_id error %d\n", rc); @@ -1088,7 +1099,11 @@ int ibw_connect(struct ibw_conn *conn, struct sockaddr_in *serv_addr, void *conn } /* init cm */ +#if RDMA_USER_CM_MAX_ABI_VERSION >= 2 rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn, RDMA_PS_TCP); +#else + rc = rdma_create_id(pctx->cm_channel, &pconn->cm_id, conn); +#endif if (rc) { rc = errno; sprintf(ibw_lasterr, "ibw_connect/rdma_create_id error %d\n", rc); diff --git a/ctdb/ib/ibwrapper_test.c b/ctdb/ib/ibwrapper_test.c index fa506500e46..6ef5aa1e679 100644 --- a/ctdb/ib/ibwrapper_test.c +++ b/ctdb/ib/ibwrapper_test.c @@ -336,7 +336,7 @@ int ibwtest_receive_handler(struct ibw_conn *conn, void *buf, int n) ibw_stop(tcx->ibwctx); goto error; } - } else { + } else if (op!=TESTOP_SEND_ID) { char *buf2; void *key2; |