lib/tsocket: workaround sockets not supporting FIONREAD

Netlink sockets don't support querying pending bytes with ioctl(fd,
FIONREAD, ...) and would return EOPNOTSUPP, so use recvmsg() with
MSG_PEEK|MSG_TRUNC as a fallback.

The MSG_TRUNC flag to recvmsg() is Linux only, but netlink is as well,
so we're safe for now.

Bug: https://bugzilla.samba.org/show_bug.cgi?id=11714

Signed-off-by: Ralph Boehme <slow@samba.org>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>
Reviewed-by: Martin Schwenke <martin@meltin.net>

Autobuild-User(master): Ralph Böhme <slow@samba.org>
Autobuild-Date(master): Wed Feb 10 10:30:24 CET 2016 on sn-devel-144

1 files changed, 61 insertions, 1 deletions

diff --git a/lib/tsocket/tsocket_bsd.c b/lib/tsocket/tsocket_bsd.c
index ac0617d6578..9608dded535 100644
--- a/lib/tsocket/tsocket_bsd.c
+++ b/lib/tsocket/tsocket_bsd.c
@@ -132,6 +132,43 @@ static int tsocket_bsd_common_prepare_fd(int fd, bool high_fd)
 	return -1;
 }
 
+#ifdef HAVE_LINUX_RTNETLINK_H
+/**
+ * Get the amount of pending bytes from a netlink socket
+ *
+ * For some reason netlink sockets don't support querying the amount of pending
+ * data via ioctl with FIONREAD, which is what we use in tsocket_bsd_pending()
+ * below.
+ *
+ * We know we are on Linux as we're using netlink, which means we have a working
+ * MSG_TRUNC flag to recvmsg() as well, so we use that together with MSG_PEEK.
+ **/
+static ssize_t tsocket_bsd_netlink_pending(int fd)
+{
+	struct iovec iov;
+	struct msghdr msg;
+	char buf[1];
+
+	iov = (struct iovec) {
+		.iov_base = buf,
+		.iov_len = sizeof(buf)
+	};
+
+	msg = (struct msghdr) {
+		.msg_iov = &iov,
+		.msg_iovlen = 1
+	};
+
+	return recvmsg(fd, &msg, MSG_PEEK | MSG_TRUNC);
+}
+#else
+static ssize_t tsocket_bsd_netlink_pending(int fd)
+{
+	errno = ENOSYS;
+	return -1;
+}
+#endif
+
 static ssize_t tsocket_bsd_pending(int fd)
 {
 	int ret, error;
@@ -640,6 +677,7 @@ struct tdgram_bsd {
 	void *event_ptr;
 	struct tevent_fd *fde;
 	bool optimize_recvfrom;
+	bool netlink;
 
 	void *readable_private;
 	void (*readable_handler)(void *private_data);
@@ -892,7 +930,12 @@ static void tdgram_bsd_recvfrom_handler(void *private_data)
 	int err;
 	bool retry;
 
-	ret = tsocket_bsd_pending(bsds->fd);
+	if (bsds->netlink) {
+		ret = tsocket_bsd_netlink_pending(bsds->fd);
+	} else {
+		ret = tsocket_bsd_pending(bsds->fd);
+	}
+
 	if (state->first_try && ret == 0) {
 		state->first_try = false;
 		/* retry later */
@@ -1395,6 +1438,11 @@ int _tdgram_bsd_existing_socket(TALLOC_CTX *mem_ctx,
 {
 	struct tdgram_context *dgram;
 	struct tdgram_bsd *bsds;
+#ifdef HAVE_LINUX_RTNETLINK_H
+	int result;
+	struct sockaddr sa;
+	socklen_t sa_len = sizeof(struct sockaddr);
+#endif
 
 	dgram = tdgram_context_create(mem_ctx,
 				      &tdgram_bsd_ops,
@@ -1409,6 +1457,18 @@ int _tdgram_bsd_existing_socket(TALLOC_CTX *mem_ctx,
 	talloc_set_destructor(bsds, tdgram_bsd_destructor);
 
 	*_dgram = dgram;
+
+#ifdef HAVE_LINUX_RTNETLINK_H
+	/*
+	 * Try to determine the protocol family and remember if it's
+	 * AF_NETLINK. We don't care if this fails.
+	 */
+	result = getsockname(fd, &sa, &sa_len);
+	if (result == 0 && sa.sa_family == AF_NETLINK) {
+		bsds->netlink = true;
+	}
+#endif
+
 	return 0;
 }