diff options
author | Peter Somogyi <psomogyi@gamax.hu> | 2007-01-02 18:16:39 +0100 |
---|---|---|
committer | Peter Somogyi <psomogyi@gamax.hu> | 2007-01-02 18:16:39 +0100 |
commit | 11fb3ef3b552556adb440774b24e5689cbe198bb (patch) | |
tree | e2132385f4e88342a301aa93c572b5b035d4f541 /ctdb/ib | |
parent | 96f10d7ff429470e72648ecbbcad217d0b8c088c (diff) | |
parent | 0d9ec11a5099c00603d9e97901cc14653179f40f (diff) | |
download | samba-11fb3ef3b552556adb440774b24e5689cbe198bb.tar.gz |
Joining ctdb and ibwrapper (incomplete).
It compiles.
TODO: modify ctdb as well.
(This used to be ctdb commit a080b85a5d8e46946f8308bb6c76aa62b5a254db)
Diffstat (limited to 'ctdb/ib')
-rw-r--r-- | ctdb/ib/config.m4 | 2 | ||||
-rw-r--r-- | ctdb/ib/ibw_ctdb.c | 105 | ||||
-rw-r--r-- | ctdb/ib/ibw_ctdb.h | 25 | ||||
-rw-r--r-- | ctdb/ib/ibw_ctdb_init.c | 164 |
4 files changed, 295 insertions, 1 deletions
diff --git a/ctdb/ib/config.m4 b/ctdb/ib/config.m4 index e84b7448a76..9d95ea7a5ab 100644 --- a/ctdb/ib/config.m4 +++ b/ctdb/ib/config.m4 @@ -7,7 +7,7 @@ if eval "test x$enable_infiniband = xyes"; then AC_DEFINE(USE_INFINIBAND,1,[Use infiniband]) HAVE_INFINIBAND=yes - INFINIBAND_WRAPPER_OBJ="ib/ibwrapper.o" + INFINIBAND_WRAPPER_OBJ="ib/ibwrapper.o ib/ibw_ctdb.o ib/ibw_ctdb_init.o" INFINIBAND_LIBS="-lrdmacm -libverbs" INFINIBAND_BINS="bin/ibwrapper_test" diff --git a/ctdb/ib/ibw_ctdb.c b/ctdb/ib/ibw_ctdb.c new file mode 100644 index 00000000000..4379cb75086 --- /dev/null +++ b/ctdb/ib/ibw_ctdb.c @@ -0,0 +1,105 @@ +/* + * Unix SMB/CIFS implementation. + * Join infiniband wrapper and ctdb. + * + * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006 + * + * Major code contributions by Peter Somogyi <psomogyi@gamax.hu> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include "includes.h" +#include "lib/events/events.h" +#include <system/network.h> +#include <assert.h> +#include "ctdb_private.h" +#include "ibwrapper.h" +#include "ibw_ctdb.h" + +int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn) +{ + if (ctx!=NULL) { + /* ctx->state changed */ + switch(ctx->state) { + case IBWS_INIT: /* ctx start - after ibw_init */ + break; + case IBWS_READY: /* after ibw_bind & ibw_listen */ + break; + case IBWS_CONNECT_REQUEST: /* after [IBWS_READY + incoming request] */ + /* => [(ibw_accept)IBWS_READY | (ibw_disconnect)STOPPED | ERROR] */ + if (ibw_accept(ctx, conn, NULL)) { + DEBUG(0, ("connstate_handler/ibw_accept failed\n")); + return -1; + } /* else continue in IBWC_CONNECTED */ + break; + case IBWS_STOPPED: /* normal stop <= ibw_disconnect+(IBWS_READY | IBWS_CONNECT_REQUEST) */ + /* TODO: have a CTDB upcall for which CTDB should wait in a (final) loop */ + break; + case IBWS_ERROR: /* abnormal state; ibw_stop must be called after this */ + break; + default: + assert(0); + break; + } + } + + if (conn!=NULL) { + /* conn->state changed */ + switch(conn->state) { + case IBWC_INIT: /* conn start - internal state */ + break; + case IBWC_CONNECTED: { /* after ibw_accept or ibw_connect */ + struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node); + if (node!=NULL) /* after ibw_connect */ + node->ctdb->upcalls->node_connected(node); + else { /* after ibw_accept */ + /* NOP in CTDB case */ + } + } break; + case IBWC_DISCONNECTED: /* after ibw_disconnect */ + /* TODO: have a CTDB upcall */ + break; + case IBWC_ERROR: { + struct ctdb_node *node = talloc_get_type(conn->conn_userdata, struct ctdb_node); + if (node!=NULL) + node->ctdb->upcalls->node_connected(node); + } break; + default: + assert(0); + break; + } + } + + return 0; +} + +int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n) +{ + struct ctdb_context *ctdb = talloc_get_type(conn->ctx->ctx_userdata, struct ctdb_context); + + assert(ctdb!=NULL); + assert(conn->state==IBWC_CONNECTED); + + /* TODO: shall I short-circuit this in ibwrapper? */ + /* maybe when everything go fine... */ + + /* TODO2: !!! here I can provide conn->conn_userdata (with no perf. penalty) - + * as struct ctdb_node in case the connection + * has been built up by ibw_connect !!! */ + ctdb->upcalls->recv_pkt(ctdb, (uint8_t *)buf, (uint32_t)n); + + return 0; +} diff --git a/ctdb/ib/ibw_ctdb.h b/ctdb/ib/ibw_ctdb.h new file mode 100644 index 00000000000..c43aca7df15 --- /dev/null +++ b/ctdb/ib/ibw_ctdb.h @@ -0,0 +1,25 @@ +/* + * Unix SMB/CIFS implementation. + * Join infiniband wrapper and ctdb. + * + * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006 + * + * Major code contributions by Peter Somogyi <psomogyi@gamax.hu> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +int ctdb_ibw_connstate_handler(struct ibw_ctx *ctx, struct ibw_conn *conn); +int ctdb_ibw_receive_handler(struct ibw_conn *conn, void *buf, int n); diff --git a/ctdb/ib/ibw_ctdb_init.c b/ctdb/ib/ibw_ctdb_init.c new file mode 100644 index 00000000000..a6a3a8beffa --- /dev/null +++ b/ctdb/ib/ibw_ctdb_init.c @@ -0,0 +1,164 @@ +/* + * Unix SMB/CIFS implementation. + * Join infiniband wrapper and ctdb. + * + * Copyright (C) Sven Oehme <oehmes@de.ibm.com> 2006 + * + * Major code contributions by Peter Somogyi <psomogyi@gamax.hu> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + */ + +#include "includes.h" +#include "lib/events/events.h" +#include <system/network.h> +#include <assert.h> +#include "ctdb_private.h" +#include "ibwrapper.h" +#include "ibw_ctdb.h" + +static int ctdb_ibw_listen(struct ctdb_context *ctdb, int backlog) +{ + struct ibw_ctx *ictx = talloc_get_type(ctdb->private, struct ibw_ctx); + struct sockaddr_in my_addr; + + assert(ictx!=NULL); + memset(&my_addr, 0, sizeof(struct sockaddr_in)); + my_addr.sin_port = htons(ctdb->address.port); + my_addr.sin_family = PF_INET; + inet_pton(AF_INET, ctdb->address.address, &my_addr.sin_addr); + + if (ibw_bind(ictx, &my_addr)) { + DEBUG(0, ("ctdb_ibw_listen: ibw_bind failed\n")); + return -1; + } + + if (ibw_listen(ictx, backlog)) { + DEBUG(0, ("ctdb_ibw_listen: ibw_listen failed\n")); + return -1; + } + + return 0; +} + +int ctdb_ibw_node_connect(struct ibw_ctx *ictx, struct ctdb_node *node) +{ + struct sockaddr_in sock_out; + + inet_pton(AF_INET, node->address.address, &sock_out.sin_addr); + sock_out.sin_port = htons(node->address.port); + sock_out.sin_family = PF_INET; + + if (ibw_connect(ictx, &sock_out, node)) { + DEBUG(0, ("ctdb_ibw_node_connect: ibw_connect failed\n")); + return -1; + } + + /* continues at ibw_ctdb.c/IBWC_CONNECTED in good case */ + return 0; +} + +/* + * Start infiniband + */ +static int ctdb_ibw_start(struct ctdb_context *ctdb) +{ + struct ibw_ctx *ictx = talloc_get_type(ctdb->private, struct ibw_ctx); + int i; + + /* listen on our own address */ + if (ctdb_ibw_listen(ctdb, 10)) /* TODO: backlog as param */ + return -1; + + /* everything async here */ + for (i=0;i<ctdb->num_nodes;i++) { + struct ctdb_node *node = ctdb->nodes[i]; + if (!(ctdb->flags & CTDB_FLAG_SELF_CONNECT) && + ctdb_same_address(&ctdb->address, &node->address)) + continue; + ctdb_ibw_node_connect(ictx, node); + } + + return 0; +} + + +/* + * initialise ibw portion of a ctdb node + */ +static int ctdb_ibw_add_node(struct ctdb_node *node) +{ + /* TODO: clarify whether is this necessary for us ? + - why not enough doing such thing internally at connect time ? */ + return 0; +} + +/* + * transport packet allocator - allows transport to control memory for packets + */ +static void *ctdb_ibw_allocate_pkt(struct ctdb_context *ctdb, size_t size) +{ + struct ibw_conn *conn = NULL; + void *buf = NULL; + void *key; /* TODO: expand the param list with this */ + + /* TODO2: !!! I need "node" or ibw_conn here */ + if (ibw_alloc_send_buf(conn, &buf, &key, (int)size)) + return NULL; + + return buf; +} + +static int ctdb_ibw_queue_pkt(struct ctdb_node *node, uint8_t *data, uint32_t length) +{ + struct ibw_conn *conn = talloc_get_type(node->private, struct ibw_conn); + void *key = NULL; /* TODO: expand the param list with this */ + + assert(conn!=NULL); + return ibw_send(conn, data, key, length); +} + +static const struct ctdb_methods ctdb_ibw_methods = { + .start = ctdb_ibw_start, + .add_node = ctdb_ibw_add_node, + .queue_pkt = ctdb_ibw_queue_pkt, + .allocate_pkt = ctdb_ibw_allocate_pkt + /* TODO: missing node_disconnect & final_stop upcalls */ +}; + +/* + * initialise ibw portion of ctdb + */ +int ctdb_ibw_init(struct ctdb_context *ctdb) +{ + struct ibw_ctx *ictx; + + ictx = ibw_init( + NULL, //struct ibw_initattr *attr, /* TODO */ + 0, //int nattr, /* TODO */ + ctdb, + ctdb_ibw_connstate_handler, + ctdb_ibw_receive_handler, + ctdb->ev); + + if (ictx==NULL) { + DEBUG(0, ("ctdb_ibw_init: ibw_init failed\n")); + return -1; + } + + ctdb->methods = &ctdb_ibw_methods; + ctdb->private = ictx; + return 0; +} |