ctdb-tcp: Move incoming fd and queue into struct ctdb_tcp_node

This makes it easy to track both incoming and outgoing connectivity
states.

BUG: https://bugzilla.samba.org/show_bug.cgi?id=14084

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
(cherry picked from commit c68b6f96f26664459187ab2fbd56767fb31767e0)

4 files changed, 61 insertions, 34 deletions

diff --git a/ctdb/tcp/ctdb_tcp.h b/ctdb/tcp/ctdb_tcp.h
index acd343fb8f3..9a615fc6393 100644
--- a/ctdb/tcp/ctdb_tcp.h
+++ b/ctdb/tcp/ctdb_tcp.h
@@ -27,22 +27,18 @@ struct ctdb_tcp {
 };
 
 /*
-  state associated with an incoming connection
-*/
-struct ctdb_incoming {
-	struct ctdb_context *ctdb;
-	int fd;
-	struct ctdb_queue *queue;
-};
-
-/*
   state associated with one tcp node
 */
 struct ctdb_tcp_node {
 	int out_fd;
 	struct ctdb_queue *out_queue;
+
 	struct tevent_fd *connect_fde;
 	struct tevent_timer *connect_te;
+
+	struct ctdb_context *ctdb;
+	int in_fd;
+	struct ctdb_queue *in_queue;
 };
 
 
diff --git a/ctdb/tcp/tcp_connect.c b/ctdb/tcp/tcp_connect.c
index 4253f3bef7c..866454f3d29 100644
--- a/ctdb/tcp/tcp_connect.c
+++ b/ctdb/tcp/tcp_connect.c
@@ -262,8 +262,8 @@ static void ctdb_listen_event(struct tevent_context *ev, struct tevent_fd *fde,
 	ctdb_sock_addr addr;
 	socklen_t len;
 	int fd;
-	uint32_t pnn;
-	struct ctdb_incoming *in;
+	struct ctdb_node *node;
+	struct ctdb_tcp_node *tnode;
 	int one = 1;
 	int ret;
 
@@ -273,41 +273,61 @@ static void ctdb_listen_event(struct tevent_context *ev, struct tevent_fd *fde,
 	if (fd == -1) return;
 	smb_set_close_on_exec(fd);
 
-	pnn = ctdb_ip_to_pnn(ctdb, &addr);
-
-	if (pnn == CTDB_UNKNOWN_PNN) {
+	node = ctdb_ip_to_node(ctdb, &addr);
+	if (node == NULL) {
 		D_ERR("Refused connection from unknown node %s\n",
 		      ctdb_addr_to_str(&addr));
 		close(fd);
 		return;
 	}
 
-	in = talloc_zero(ctcp, struct ctdb_incoming);
-	in->fd = fd;
-	in->ctdb = ctdb;
+	tnode = talloc_get_type_abort(node->private_data,
+				      struct ctdb_tcp_node);
+	if (tnode == NULL) {
+		/* This can't happen - see ctdb_tcp_initialise() */
+		DBG_ERR("INTERNAL ERROR setting up connection from node %s\n",
+			ctdb_addr_to_str(&addr));
+		close(fd);
+		return;
+	}
 
-	ret = set_blocking(in->fd, false);
+	ret = set_blocking(fd, false);
 	if (ret != 0) {
-		DEBUG(DEBUG_ERR,
-		      (__location__
-		       " failed to set socket non-blocking (%s)\n",
-		       strerror(errno)));
-		close(in->fd);
-		in->fd = -1;
+		DBG_ERR("Failed to set socket non-blocking (%s)\n",
+			strerror(errno));
+		close(fd);
 		return;
 	}
 
-	set_close_on_exec(in->fd);
+	set_close_on_exec(fd);
 
-	DEBUG(DEBUG_DEBUG, (__location__ " Created SOCKET FD:%d to incoming ctdb connection\n", fd));
+	DBG_DEBUG("Created SOCKET FD:%d to incoming ctdb connection\n", fd);
 
-        if (setsockopt(in->fd,SOL_SOCKET,SO_KEEPALIVE,(char *)&one,sizeof(one)) == -1) {
-		DEBUG(DEBUG_WARNING, ("Failed to set KEEPALIVE on fd - %s\n",
-				      strerror(errno)));
+	ret = setsockopt(fd,
+			 SOL_SOCKET,
+			 SO_KEEPALIVE,
+			 (char *)&one,
+			 sizeof(one));
+	if (ret == -1) {
+		DBG_WARNING("Failed to set KEEPALIVE on fd - %s\n",
+			    strerror(errno));
+	}
+
+	tnode->in_queue = ctdb_queue_setup(ctdb,
+					   tnode,
+					   fd,
+					   CTDB_TCP_ALIGNMENT,
+					   ctdb_tcp_read_cb,
+					   tnode,
+					   "ctdbd-%s",
+					   ctdb_addr_to_str(&addr));
+	if (tnode->in_queue == NULL) {
+		DBG_ERR("Failed to set up incoming queue\n");
+		close(fd);
+		return;
 	}
 
-	in->queue = ctdb_queue_setup(ctdb, in, in->fd, CTDB_TCP_ALIGNMENT,
-				     ctdb_tcp_read_cb, in, "ctdbd-%s", ctdb_addr_to_str(&addr));
+	tnode->in_fd = fd;
 }
 
 
diff --git a/ctdb/tcp/tcp_init.c b/ctdb/tcp/tcp_init.c
index 6b1299f32b5..d4b42b0d0f2 100644
--- a/ctdb/tcp/tcp_init.c
+++ b/ctdb/tcp/tcp_init.c
@@ -43,6 +43,11 @@ static int tnode_destructor(struct ctdb_tcp_node *tnode)
 		tnode->out_fd = -1;
 	}
 
+	if (tnode->in_fd != -1) {
+		close(tnode->in_fd);
+		tnode->in_fd = -1;
+	}
+
 	return 0;
 }
 
@@ -56,6 +61,9 @@ static int ctdb_tcp_add_node(struct ctdb_node *node)
 	CTDB_NO_MEMORY(node->ctdb, tnode);
 
 	tnode->out_fd = -1;
+	tnode->in_fd = -1;
+	tnode->ctdb = node->ctdb;
+
 	node->private_data = tnode;
 	talloc_set_destructor(tnode, tnode_destructor);
 
diff --git a/ctdb/tcp/tcp_io.c b/ctdb/tcp/tcp_io.c
index 0eb8e25eea3..be4558b16ea 100644
--- a/ctdb/tcp/tcp_io.c
+++ b/ctdb/tcp/tcp_io.c
@@ -37,7 +37,8 @@
  */
 void ctdb_tcp_read_cb(uint8_t *data, size_t cnt, void *args)
 {
-	struct ctdb_incoming *in = talloc_get_type(args, struct ctdb_incoming);
+	struct ctdb_tcp_node *tnode = talloc_get_type_abort(
+		args, struct ctdb_tcp_node);
 	struct ctdb_req_header *hdr = (struct ctdb_req_header *)data;
 
 	if (data == NULL) {
@@ -69,11 +70,13 @@ void ctdb_tcp_read_cb(uint8_t *data, size_t cnt, void *args)
 	}
 
 	/* tell the ctdb layer above that we have a packet */
-	in->ctdb->upcalls->recv_pkt(in->ctdb, data, cnt);
+	tnode->ctdb->upcalls->recv_pkt(tnode->ctdb, data, cnt);
 	return;
 
 failed:
-	TALLOC_FREE(in);
+	TALLOC_FREE(tnode->in_queue);
+	close(tnode->in_fd);
+	tnode->in_fd = -1;
 	TALLOC_FREE(data);
 }