net,pkt: add chunked support

As we don't know the length of the message we want to send to the
other end, we send a chunk size before each message. In later
versions, sending the wants might benefit from batching the lines
together.

Signed-off-by: Carlos Martín Nieto <carlos@cmartin.tk>

5 files changed, 72 insertions, 26 deletions

diff --git a/src/netops.c b/src/netops.c
index 7d8a7b28c..da242795b 100644
--- a/src/netops.c
+++ b/src/netops.c
@@ -23,6 +23,7 @@
 
 #include "common.h"
 #include "netops.h"
+#include "posix.h"
 
 void gitno_buffer_setup(gitno_buffer *buf, char *data, unsigned int len, int fd)
 {
@@ -138,6 +139,7 @@ int gitno_send(GIT_SOCKET s, const char *msg, size_t len, int flags)
 	return off;
 }
 
+
 #ifdef GIT_WIN32
 int gitno_close(GIT_SOCKET s)
 {
@@ -150,6 +152,20 @@ int gitno_close(GIT_SOCKET s)
 }
 #endif
 
+int gitno_send_chunk_size(int s, size_t len)
+{
+	char str[8] = {0};
+	int ret;
+
+	ret = p_snprintf(str, sizeof(str), "%zx", len);
+	if (ret >= (int) sizeof(str)) {
+		return git__throw(GIT_ESHORTBUFFER, "Your number is too fucking big");
+	}
+
+	return gitno_send(s, str, ret, 0 /* TODO: MSG_MORE */);
+}
+
+
 int gitno_select_in(gitno_buffer *buf, long int sec, long int usec)
 {
 	fd_set fds;
diff --git a/src/netops.h b/src/netops.h
index 203df85af..f9a812747 100644
--- a/src/netops.h
+++ b/src/netops.h
@@ -28,6 +28,7 @@ void gitno_consume_n(gitno_buffer *buf, size_t cons);
 int gitno_connect(const char *host, const char *port);
 int gitno_send(GIT_SOCKET s, const char *msg, size_t len, int flags);
 int gitno_close(GIT_SOCKET s);
+int gitno_send_chunk_size(int s, size_t len);
 int gitno_select_in(gitno_buffer *buf, long int sec, long int usec);
 
 int gitno_extract_host_and_port(char **host, char **port, const char *url, const char *default_port);
diff --git a/src/pkt.c b/src/pkt.c
index 319a22e01..a9ac4ad18 100644
--- a/src/pkt.c
+++ b/src/pkt.c
@@ -16,6 +16,7 @@
 #include "util.h"
 #include "netops.h"
 #include "posix.h"
+#include "buffer.h"
 
 #include <ctype.h>
 
@@ -261,18 +262,25 @@ void git_pkt_free(git_pkt *pkt)
 	free(pkt);
 }
 
-int git_pkt_send_flush(int s)
+int git_pkt_send_flush(int s, int chunked)
 {
 	char flush[] = "0000";
+	int error;
 
+	if (chunked) {
+		error = gitno_send_chunk_size(s, strlen(flush));
+		if (error < GIT_SUCCESS)
+			return git__rethrow(error, "Failed to send chunk size");
+	}
 	return gitno_send(s, flush, strlen(flush), 0);
 }
 
-static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps, int fd)
+static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps, int fd, int chunked)
 {
 	char capstr[20]; /* Longer than we need */
 	char oid[GIT_OID_HEXSZ +1] = {0}, *cmd;
 	int error, len;
+	git_buf buf = GIT_BUF_INIT;
 
 	if (caps->ofs_delta)
 		strcpy(capstr, GIT_CAP_OFS_DELTA);
@@ -283,9 +291,13 @@ static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps,
 		return GIT_ENOMEM;
 
 	git_oid_fmt(oid, &head->oid);
-	memset(cmd, 0x0, len + 1);
-	p_snprintf(cmd, len + 1, "%04xwant %s%c%s\n", len, oid, 0, capstr);
-	error = gitno_send(fd, cmd, len, 0);
+	git_buf_printf(&buf, "%04xwant %s%c%s\n", len, oid, 0, capstr);
+	if (chunked) {
+		error = gitno_send_chunk_size(fd, len);
+		if (error < GIT_SUCCESS)
+			return git__rethrow(error, "Failed to send first want chunk size");
+	}
+	error = gitno_send(fd, git_buf_cstr(&buf), len, 0);
 	free(cmd);
 	return error;
 }
@@ -296,7 +308,7 @@ static int send_want_with_caps(git_remote_head *head, git_transport_caps *caps,
  */
 #define WANT_PREFIX "0032want "
 
-int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd)
+int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd, int chunked)
 {
 	unsigned int i = 0;
 	int error = GIT_SUCCESS;
@@ -317,7 +329,7 @@ int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd)
 				break;
 		}
 
-		error = send_want_with_caps(refs->heads[i], caps, fd);
+		error = send_want_with_caps(refs->heads[i], caps, fd, chunked);
 		if (error < GIT_SUCCESS)
 			return git__rethrow(error, "Failed to send want pkt with caps");
 		/* Increase it here so it's correct whether we run this or not */
@@ -331,13 +343,17 @@ int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd)
 			continue;
 
 		git_oid_fmt(buf + strlen(WANT_PREFIX), &head->oid);
+		if (chunked) {
+			error = gitno_send_chunk_size(fd, strlen(buf));
+			if (error < GIT_SUCCESS)
+				return git__rethrow(error, "Failed to send want chunk size");
+		}
 		error = gitno_send(fd, buf, strlen(buf), 0);
-		if (error < 0) {
+		if (error < GIT_SUCCESS)
 			return git__rethrow(error, "Failed to send want pkt");
-		}
 	}
 
-	return git_pkt_send_flush(fd);
+	return git_pkt_send_flush(fd, chunked);
 }
 
 /*
@@ -346,17 +362,29 @@ int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd)
  */
 #define HAVE_PREFIX "0032have "
 
-int git_pkt_send_have(git_oid *oid, int fd)
+int git_pkt_send_have(git_oid *oid, int fd, int chunked)
 {
 	char buf[] = "0032have 0000000000000000000000000000000000000000\n";
+	int error;
 
+	if (chunked) {
+		error = gitno_send_chunk_size(fd, strlen(buf));
+		if (error < GIT_SUCCESS)
+			return git__rethrow(error, "Failed to send chunk size");
+	}
 	git_oid_fmt(buf + strlen(HAVE_PREFIX), oid);
 	return gitno_send(fd, buf, strlen(buf), 0);
 }
 
-int git_pkt_send_done(int fd)
+int git_pkt_send_done(int fd, int chunked)
 {
 	char buf[] = "0009done\n";
+	int error;
 
+	if (chunked) {
+		error = gitno_send_chunk_size(fd, strlen(buf));
+		if (error < GIT_SUCCESS)
+			return git__rethrow(error, "Failed to send chunk size");
+	}
 	return gitno_send(fd, buf, strlen(buf), 0);
 }
diff --git a/src/pkt.h b/src/pkt.h
index f4ed81c67..0689e2983 100644
--- a/src/pkt.h
+++ b/src/pkt.h
@@ -63,10 +63,10 @@ typedef struct {
 } git_pkt_comment;
 
 int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_t len);
-int git_pkt_send_flush(int s);
-int git_pkt_send_done(int s);
-int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd);
-int git_pkt_send_have(git_oid *oid, int fd);
+int git_pkt_send_flush(int s, int chunked);
+int git_pkt_send_done(int s, int chunked);
+int git_pkt_send_wants(git_headarray *refs, git_transport_caps *caps, int fd, int chunked);
+int git_pkt_send_have(git_oid *oid, int fd, int chunked);
 void git_pkt_free(git_pkt *pkt);
 
 #endif
diff --git a/src/transport_git.c b/src/transport_git.c
index b8a41379a..d3d57471d 100644
--- a/src/transport_git.c
+++ b/src/transport_git.c
@@ -269,14 +269,14 @@ static int git_send_wants(git_transport *transport, git_headarray *array)
 {
 	transport_git *t = (transport_git *) transport;
 
-	return git_pkt_send_wants(array, &t->caps, t->socket);
+	return git_pkt_send_wants(array, &t->caps, t->socket, 0);
 }
 
 static int git_send_have(git_transport *transport, git_oid *oid)
 {
 	transport_git *t = (transport_git *) transport;
 
-	return git_pkt_send_have(oid, t->socket);
+	return git_pkt_send_have(oid, t->socket, 0);
 }
 
 static int git_negotiate_fetch(git_transport *transport, git_repository *repo, git_headarray *GIT_UNUSED(list))
@@ -333,12 +333,12 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, g
 	 */
 	i = 0;
 	while ((error = git_revwalk_next(&oid, walk)) == GIT_SUCCESS) {
-		error = git_pkt_send_have(&oid, t->socket);
+		error = git_pkt_send_have(&oid, t->socket, 1);
 		i++;
 		if (i % 20 == 0) {
 			const char *ptr = buf.data, *line_end;
 			git_pkt *pkt;
-			git_pkt_send_flush(t->socket);
+			git_pkt_send_flush(t->socket, 0);
 			while (1) {
 				/* Wait for max. 1 second */
 				error = gitno_select_in(&buf, 1, 0);
@@ -384,8 +384,8 @@ static int git_negotiate_fetch(git_transport *transport, git_repository *repo, g
 		error = GIT_SUCCESS;
 
 done:
-	git_pkt_send_flush(t->socket);
-	git_pkt_send_done(t->socket);
+	git_pkt_send_flush(t->socket, 0);
+	git_pkt_send_done(t->socket, 0);
 
 cleanup:
 	git_revwalk_free(walk);
@@ -396,14 +396,14 @@ static int git_send_flush(git_transport *transport)
 {
 	transport_git *t = (transport_git *) transport;
 
-	return git_pkt_send_flush(t->socket);
+	return git_pkt_send_flush(t->socket, 1);
 }
 
 static int git_send_done(git_transport *transport)
 {
 	transport_git *t = (transport_git *) transport;
 
-	return git_pkt_send_done(t->socket);
+	return git_pkt_send_done(t->socket, 1);
 }
 
 static int store_pack(char **out, gitno_buffer *buf, git_repository *repo)
@@ -502,9 +502,10 @@ static int git_close(git_transport *transport)
 	transport_git *t = (transport_git*) transport;
 	int error;
 
-	/* Can't do anything if there's an error, so don't bother checking */
-	git_pkt_send_flush(t->socket);
+	/* Can't do anything if there's an error, so don't bother checking  */
+	git_pkt_send_flush(t->socket, 0);
 	error = gitno_close(t->socket);
+
 	if (error < 0)
 		error = git__throw(GIT_EOSERR, "Failed to close socket");