Added overload test scenario + fixed 1 send queue bug.

TODO: externize debug level

(This used to be ctdb commit 98442ce3e2b3f09ca6736abfa0a8314cbdafaac7)

3 files changed, 50 insertions, 13 deletions

diff --git a/ctdb/ib/ibwrapper.c b/ctdb/ib/ibwrapper.c
index 2981e8dc6a7..62a9c19eada 100644
--- a/ctdb/ib/ibwrapper.c
+++ b/ctdb/ib/ibwrapper.c
@@ -455,7 +455,7 @@ static void ibw_event_handler_cm(struct event_context *ev,
 
 	case RDMA_CM_EVENT_ESTABLISHED:
 		/* expected after ibw_accept and ibw_connect[not directly] */
-		DEBUG(0, ("ESTABLISHED (conn: %u)\n", (unsigned int)cma_id->context));
+		DEBUG(0, ("ESTABLISHED (conn: %p)\n", cma_id->context));
 		conn = talloc_get_type(cma_id->context, struct ibw_conn);
 		assert(conn!=NULL); /* important assumption */
 
@@ -623,7 +623,7 @@ static inline int ibw_wc_send(struct ibw_conn *conn, struct ibv_wc *wc)
 	} else { /* "extra" request - not optimized */
 		DEBUG(10, ("ibw_wc_send#2 %u\n", (int)wc->wr_id));
 		for(p=pconn->extra_sent; p!=NULL; p=p->next)
-			if (p->wr_id==(int)wc->wr_id)
+			if ((p->wr_id + pctx->opts.max_recv_wr)==(int)wc->wr_id)
 				break;
 		if (p==NULL) {
 			sprintf(ibw_lasterr, "failed to find wr_id %d\n", (int)wc->wr_id);
@@ -1042,6 +1042,7 @@ int ibw_alloc_send_buf(struct ibw_conn *conn, void **buf, void **key, uint32_t l
 			}
 			*buf = (void *)p->msg_large;
 		}
+		/* p->wr_id is already filled in ibw_init_memory */
 	} else {
 		DEBUG(10, ("ibw_alloc_send_buf#2: cmid=%p, len=%d\n", pconn->cm_id, len));
 		/* not optimized */
diff --git a/ctdb/include/includes.h b/ctdb/include/includes.h
index 2bd527d23df..199f416b20a 100644
--- a/ctdb/include/includes.h
+++ b/ctdb/include/includes.h
@@ -12,7 +12,7 @@ typedef bool BOOL;
 #define True 1
 #define False 0
 
-#define LogLevel 10
+#define LogLevel 0
 
 #define DEBUG(lvl, x) if ((lvl) <= LogLevel) (printf x)
 
diff --git a/ctdb/tests/ibwrapper_test.c b/ctdb/tests/ibwrapper_test.c
index 79324f5c4bd..f2ab6a96bcf 100644
--- a/ctdb/tests/ibwrapper_test.c
+++ b/ctdb/tests/ibwrapper_test.c
@@ -51,13 +51,15 @@ struct ibwtest_ctx {
 	struct sockaddr_in *addrs; /* dynamic array of dest addrs */
 	int	naddrs;
 
-	unsigned int	nsec; /* nanosleep between messages */
+	unsigned int	nsec; /* delta times between messages in nanosec */
+	unsigned int	sleep_nsec; /* nanosleep in the main loop to emulate overloading */
 
 	int	cnt;
 
 	int	nmsg; /* number of messages to send (client) */
 
 	int	kill_me;
+	int	error;
 	struct ibw_ctx	*ibwctx;
 
 	struct timeval	start_time, end_time;
@@ -72,6 +74,26 @@ enum testopcode {
 	TESTOP_SEND_DATA = 2
 };
 
+int ibwtest_nanosleep(unsigned int sleep_nsec)
+{
+	int nanosleep(const struct timespec *req, struct timespec *rem);
+	struct timespec req, rem;
+
+	memset(&req, 0, sizeof(struct timespec));
+	memset(&rem, 0, sizeof(struct timespec));
+	req.tv_sec = 0;
+	req.tv_nsec = sleep_nsec;
+
+	while (nanosleep(&req, &rem) < 0) {
+		if (errno != EINTR) {
+			DEBUG(0, ("nanosleep ERROR: %d\n", errno));
+			return -1;
+		}
+		memcpy(&req, &rem, sizeof(struct timespec));
+	}
+	return 0;
+}
+
 int ibwtest_connect_everybody(struct ibwtest_ctx *tcx)
 {
 	struct ibwtest_conn	*pconn = talloc_zero(tcx, struct ibwtest_conn);
@@ -90,9 +112,9 @@ int ibwtest_connect_everybody(struct ibwtest_ctx *tcx)
 
 int ibwtest_send_id(struct ibw_conn *conn)
 {
+	struct ibwtest_ctx *tcx = talloc_get_type(conn->ctx->ctx_userdata, struct ibwtest_ctx);
 	char *buf;
 	void *key;
-	struct ibwtest_ctx *tcx = talloc_get_type(conn->ctx->ctx_userdata, struct ibwtest_ctx);
 	uint32_t	len;
 
 	DEBUG(10, ("test IBWC_CONNECTED\n"));
@@ -240,7 +262,7 @@ int ibwtest_receive_handler(struct ibw_conn *conn, void *buf, int n)
 		if (tcx->nmsg) {
 			char	msg[26];
 			sprintf(msg, "hello world %d", tcx->nmsg--);
-			ibwtest_send_test_msg(tcx, conn, msg);
+			tcx->error = ibwtest_send_test_msg(tcx, conn, msg);
 			if (tcx->nmsg==0)
 				tcx->kill_me = 1;
 		}
@@ -263,7 +285,9 @@ void ibwtest_timeout_handler(struct event_context *ev, struct timed_event *te,
 
 		/* send something to everybody... */
 		for(p=tcx->ibwctx->conn_list; p!=NULL; p=p->next) {
-			ibwtest_send_test_msg(tcx, p, msg);
+			if (p->state==IBWC_CONNECTED) {
+				tcx->error = ibwtest_send_test_msg(tcx, p, msg);
+			}
 		}
 	} /* else allow main loop run */
 }
@@ -375,6 +399,7 @@ void ibwtest_usage(struct ibwtest_ctx *tcx, char *name)
 	printf("\t\t send message periodically and endless when nsec is non-zero\n");
 	printf("\t-s server mode (you have to give exactly one -d address:port in this case)\n");
 	printf("\t-n number of messages to send [default %d]\n", tcx->nmsg);
+	printf("\t-l nsec time to sleep in the main loop [default %d]\n", tcx->sleep_nsec);
 	printf("Press ctrl+C to stop the program.\n");
 }
 
@@ -395,7 +420,7 @@ int main(int argc, char *argv[])
 	testctx = tcx;
 	signal(SIGQUIT, ibwtest_sigquit_handler);
 
-	while ((op=getopt(argc, argv, "i:o:d:m:st:n:")) != -1) {
+	while ((op=getopt(argc, argv, "i:o:d:m:st:n:l:")) != -1) {
 		switch (op) {
 		case 'i':
 			tcx->id = talloc_strdup(tcx, optarg);
@@ -419,6 +444,9 @@ int main(int argc, char *argv[])
 		case 'n':
 			tcx->nmsg = atoi(optarg);
 			break;
+		case 'l':
+			tcx->sleep_nsec = (unsigned int)atoi(optarg);
+			break;
 		default:
 			fprintf(stderr, "ERROR: unknown option -%c\n", (char)op);
 			ibwtest_usage(tcx, argv[0]);
@@ -450,16 +478,23 @@ int main(int argc, char *argv[])
 	if (rc)
 		goto cleanup;
 
-	while(!tcx->kill_me) {
-		if (tcx->nsec)
+	while(!tcx->kill_me && !tcx->error) {
+		if (tcx->nsec) {
 			event_add_timed(ev, tcx, timeval_current_ofs(0, tcx->nsec),
 				ibwtest_timeout_handler, tcx);
+		}
+
 		event_loop_once(ev);
+
+		if (tcx->sleep_nsec) {
+			if (ibwtest_nanosleep(tcx->sleep_nsec))
+				goto cleanup;
+		}
 	}
 
-	if (!tcx->is_server && nmsg!=0) {
+	if (!tcx->is_server && nmsg!=0 && !tcx->error) {
 		if (gettimeofday(&tcx->end_time, NULL)) {
-			DEBUG(0, ("gettimeofday error %d", errno));
+			DEBUG(0, ("gettimeofday error %d\n", errno));
 			goto cleanup;
 		}
 		usec = (tcx->end_time.tv_sec - tcx->start_time.tv_sec) * 1000000 +
@@ -468,7 +503,8 @@ int main(int argc, char *argv[])
 			usec, nmsg, usec/(float)nmsg);
 	}
 
-	result = 0; /* everything OK */
+	if (!tcx->error)
+		result = 0; /* everything OK */
 
 cleanup:
 	if (tcx)