Merge pull request #405 from carlosmn/http-ls

Implement ls-remote over HTTP

10 files changed, 483 insertions, 40 deletions

diff --git a/src/buffer.c b/src/buffer.c
index b1be29241..0eeeecf2f 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -99,3 +99,15 @@ void git_buf_free(git_buf *buf)
 {
 	free(buf->ptr);
 }
+
+void git_buf_clear(git_buf *buf)
+{
+	buf->size = 0;
+}
+
+void git_buf_consume(git_buf *buf, const char *end)
+{
+	size_t consumed = end - buf->ptr;
+	memmove(buf->ptr, end, buf->size - consumed);
+	buf->size -= consumed;
+}
diff --git a/src/buffer.h b/src/buffer.h
index 14233b82b..ad3b8930f 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -24,6 +24,8 @@ void git_buf_puts(git_buf *buf, const char *string);
 void git_buf_printf(git_buf *buf, const char *format, ...) GIT_FORMAT_PRINTF(2, 3);
 const char *git_buf_cstr(git_buf *buf);
 void git_buf_free(git_buf *buf);
+void git_buf_clear(git_buf *buf);
+void git_buf_consume(git_buf *buf, const char *end);
 
 #define git_buf_PUTS(buf, str) git_buf_put(buf, str, sizeof(str) - 1)
 
diff --git a/src/netops.c b/src/netops.c
index a237fae73..187397ec6 100644
--- a/src/netops.c
+++ b/src/netops.c
@@ -27,7 +27,7 @@ void gitno_buffer_setup(gitno_buffer *buf, char *data, unsigned int len, int fd)
 	memset(buf, 0x0, sizeof(gitno_buffer));
 	memset(data, 0x0, len);
 	buf->data = data;
-	buf->len = len - 1;
+	buf->len = len;
 	buf->offset = 0;
 	buf->fd = fd;
 }
@@ -84,6 +84,7 @@ int gitno_connect(const char *host, const char *port)
 	ret = getaddrinfo(host, port, &hints, &info);
 	if (ret != 0) {
 		error = GIT_EOSERR;
+		info = NULL;
 		goto cleanup;
 	}
 
@@ -121,7 +122,7 @@ int gitno_send(int s, const char *msg, size_t len, int flags)
 	while (off < len) {
 		ret = send(s, msg + off, len - off, flags);
 		if (ret < 0)
-			return GIT_EOSERR;
+			return git__throw(GIT_EOSERR, "Error sending data: %s", strerror(errno));
 
 		off += ret;
 	}
@@ -143,3 +144,33 @@ int gitno_select_in(gitno_buffer *buf, long int sec, long int usec)
 	/* The select(2) interface is silly */
 	return select(buf->fd + 1, &fds, NULL, NULL, &tv);
 }
+
+int gitno_extract_host_and_port(char **host, char **port, const char *url, const char *default_port)
+{
+	char *colon, *slash, *delim;
+	int error = GIT_SUCCESS;
+
+	colon = strchr(url, ':');
+	slash = strchr(url, '/');
+
+	if (slash == NULL)
+			return git__throw(GIT_EOBJCORRUPTED, "Malformed URL: missing /");
+
+	if (colon == NULL) {
+		*port = git__strdup(default_port);
+	} else {
+		*port = git__strndup(colon + 1, slash - colon - 1);
+	}
+	if (*port == NULL)
+		return GIT_ENOMEM;;
+
+
+	delim = colon == NULL ? slash : colon;
+	*host = git__strndup(url, delim - url);
+	if (*host == NULL) {
+		free(*port);
+		error = GIT_ENOMEM;
+	}
+
+	return error;
+}
diff --git a/src/netops.h b/src/netops.h
index 1aa7aebac..0d962ef61 100644
--- a/src/netops.h
+++ b/src/netops.h
@@ -29,4 +29,6 @@ int gitno_connect(const char *host, const char *port);
 int gitno_send(int s, const char *msg, size_t len, int flags);
 int gitno_select_in(gitno_buffer *buf, long int sec, long int usec);
 
+int gitno_extract_host_and_port(char **host, char **port, const char *url, const char *default_port);
+
 #endif
diff --git a/src/pkt.c b/src/pkt.c
index e0beb72e0..a167866bd 100644
--- a/src/pkt.c
+++ b/src/pkt.c
@@ -80,13 +80,30 @@ static int pack_pkt(git_pkt **out)
 	return GIT_SUCCESS;
 }
 
+static int comment_pkt(git_pkt **out, const char *line, size_t len)
+{
+	git_pkt_comment *pkt;
+
+	pkt = git__malloc(sizeof(git_pkt_comment) + len + 1);
+	if  (pkt == NULL)
+		return GIT_ENOMEM;
+
+	pkt->type = GIT_PKT_COMMENT;
+	memcpy(pkt->comment, line, len);
+	pkt->comment[len] = '\0';
+
+	*out = (git_pkt *) pkt;
+
+	return GIT_SUCCESS;
+}
+
 /*
  * Parse an other-ref line.
  */
 static int ref_pkt(git_pkt **out, const char *line, size_t len)
 {
 	git_pkt_ref *pkt;
-	int error, has_caps = 0;
+	int error;
 
 	pkt = git__malloc(sizeof(git_pkt_ref));
 	if (pkt == NULL)
@@ -110,9 +127,6 @@ static int ref_pkt(git_pkt **out, const char *line, size_t len)
 	line += GIT_OID_HEXSZ + 1;
 	len -= (GIT_OID_HEXSZ + 1);
 
-	if (strlen(line) < len)
-		has_caps = 1;
-
 	if (line[len - 1] == '\n')
 		--len;
 
@@ -124,7 +138,7 @@ static int ref_pkt(git_pkt **out, const char *line, size_t len)
 	memcpy(pkt->head.name, line, len);
 	pkt->head.name[len] = '\0';
 
-	if (has_caps) {
+	if (strlen(pkt->head.name) < len) {
 		pkt->capabilities = strchr(pkt->head.name, '\0') + 1;
 	}
 
@@ -227,6 +241,8 @@ int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_
 		error = ack_pkt(head, line, len);
 	else if (!git__prefixcmp(line, "NAK"))
 		error = nak_pkt(head);
+	else if (*line == '#')
+		error = comment_pkt(head, line, len);
 	else
 		error = ref_pkt(head, line, len);
 
diff --git a/src/pkt.h b/src/pkt.h
index 5a94024de..f4ed81c67 100644
--- a/src/pkt.h
+++ b/src/pkt.h
@@ -20,6 +20,7 @@ enum git_pkt_type {
 	GIT_PKT_ACK,
 	GIT_PKT_NAK,
 	GIT_PKT_PACK,
+	GIT_PKT_COMMENT,
 };
 
 /* Used for multi-ack */
@@ -56,6 +57,11 @@ typedef struct {
 	enum git_ack_status status;
 } git_pkt_ack;
 
+typedef struct {
+	enum git_pkt_type type;
+	char comment[GIT_FLEX_ARRAY];
+} git_pkt_comment;
+
 int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_t len);
 int git_pkt_send_flush(int s);
 int git_pkt_send_done(int s);
diff --git a/src/transport-http.c b/src/transport-http.c
new file mode 100644
index 000000000..d111d5c38
--- /dev/null
+++ b/src/transport-http.c
@@ -0,0 +1,403 @@
+/*
+ * This file is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2,
+ * as published by the Free Software Foundation.
+ *
+ * In addition to the permissions in the GNU General Public License,
+ * the authors give you unlimited permission to link the compiled
+ * version of this file into combinations with other programs,
+ * and to distribute those combinations without any restriction
+ * coming from the use of this file.  (The General Public License
+ * restrictions do apply in other respects; for example, they cover
+ * modification of the file, and distribution when not linked into
+ * a combined executable.)
+ *
+ * This file is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; see the file COPYING.  If not, write to
+ * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#include <stdlib.h>
+#include "git2.h"
+#include "http_parser.h"
+
+#include "transport.h"
+#include "common.h"
+#include "netops.h"
+#include "buffer.h"
+#include "pkt.h"
+
+typedef enum {
+	NONE,
+	FIELD,
+	VALUE
+} last_cb_type;
+
+typedef struct {
+	git_transport parent;
+	git_vector refs;
+	int socket;
+	git_buf buf;
+	git_remote_head **heads;
+	int error;
+	int transfer_finished :1,
+		ct_found :1,
+		ct_finished :1,
+		last_cb :3;
+	char *content_type;
+	char *service;
+} transport_http;
+
+static int gen_request(git_buf *buf, const char *url, const char *host, const char *service)
+{
+	const char *path = url;
+
+	path = strchr(path, '/');
+	if (path == NULL) /* Is 'git fetch http://host.com/' valid? */
+		path = "/";
+
+	git_buf_printf(buf, "GET %s/info/refs?service=git-%s HTTP/1.1\r\n", path, service);
+	git_buf_puts(buf, "User-Agent: git/1.0 (libgit2 " LIBGIT2_VERSION ")\r\n");
+	git_buf_printf(buf, "Host: %s\r\n", host);
+	git_buf_puts(buf, "Accept: */*\r\n" "Pragma: no-cache\r\n\r\n");
+
+	if (git_buf_oom(buf))
+		return GIT_ENOMEM;
+
+	return GIT_SUCCESS;
+}
+
+static int do_connect(transport_http *t, const char *service)
+{
+	int s = -1, error;;
+	const char *url = t->parent.url, *prefix = "http://";
+	char *host = NULL, *port = NULL;
+	git_buf request = GIT_BUF_INIT;
+
+	if (!git__prefixcmp(url, prefix))
+		url += strlen(prefix);
+
+	error = gitno_extract_host_and_port(&host, &port, url, "80");
+	if (error < GIT_SUCCESS)
+		goto cleanup;
+
+	t->service = git__strdup(service);
+	if (t->service == NULL) {
+		error = GIT_ENOMEM;
+		goto cleanup;
+	}
+
+	s = gitno_connect(host, port);
+	if (s < GIT_SUCCESS) {
+	    error = git__throw(error, "Failed to connect to host");
+	}
+	t->socket = s;
+
+	/* Generate and send the HTTP request */
+	error = gen_request(&request, url, host, service);
+	if (error < GIT_SUCCESS) {
+		error = git__throw(error, "Failed to generate request");
+		goto cleanup;
+	}
+	error = gitno_send(s, git_buf_cstr(&request), strlen(git_buf_cstr(&request)), 0);
+	if (error < GIT_SUCCESS)
+		error = git__rethrow(error, "Failed to send the HTTP request");
+
+cleanup:
+	git_buf_free(&request);
+	free(host);
+	free(port);
+
+	return error;
+}
+
+/*
+ * The HTTP parser is streaming, so we need to wait until we're in the
+ * field handler before we can be sure that we can store the previous
+ * value. Right now, we only care about the
+ * Content-Type. on_header_{field,value} should be kept generic enough
+ * to work for any request.
+ */
+
+static const char *typestr = "Content-Type";
+
+static int on_header_field(http_parser *parser, const char *str, size_t len)
+{
+	transport_http *t = (transport_http *) parser->data;
+	git_buf *buf = &t->buf;
+
+	if (t->last_cb == VALUE && t->ct_found) {
+		t->ct_finished = 1;
+		t->ct_found = 0;
+		t->content_type = git__strdup(git_buf_cstr(buf));
+		if (t->content_type == NULL)
+			return t->error = GIT_ENOMEM;
+		git_buf_clear(buf);
+	}
+
+	if (t->ct_found) {
+		t->last_cb = FIELD;
+		return 0;
+	}
+
+	if (t->last_cb != FIELD)
+		git_buf_clear(buf);
+
+	git_buf_put(buf, str, len);
+	t->last_cb = FIELD;
+
+	return git_buf_oom(buf);
+}
+
+static int on_header_value(http_parser *parser, const char *str, size_t len)
+{
+	transport_http *t = (transport_http *) parser->data;
+	git_buf *buf = &t->buf;
+
+	if (t->ct_finished) {
+		t->last_cb = VALUE;
+		return 0;
+	}
+
+	if (t->last_cb == VALUE)
+		git_buf_put(buf, str, len);
+
+	if (t->last_cb == FIELD && !strcmp(git_buf_cstr(buf), typestr)) {
+		t->ct_found = 1;
+		git_buf_clear(buf);
+		git_buf_put(buf, str, len);
+	}
+
+	t->last_cb = VALUE;
+
+	return git_buf_oom(buf);
+}
+
+static int on_headers_complete(http_parser *parser)
+{
+	transport_http *t = (transport_http *) parser->data;
+	git_buf *buf = &t->buf;
+
+	if (t->content_type == NULL) {
+		t->content_type = git__strdup(git_buf_cstr(buf));
+		if (t->content_type == NULL)
+			return t->error = GIT_ENOMEM;
+	}
+
+	git_buf_clear(buf);
+	git_buf_printf(buf, "application/x-git-%s-advertisement", t->service);
+	if (git_buf_oom(buf))
+		return GIT_ENOMEM;
+
+	if (strcmp(t->content_type, git_buf_cstr(buf)))
+		return t->error = git__throw(GIT_EOBJCORRUPTED, "Content-Type '%s' is wrong", t->content_type);
+
+	git_buf_clear(buf);
+	return 0;
+}
+
+static int on_body_store_refs(http_parser *parser, const char *str, size_t len)
+{
+	transport_http *t = (transport_http *) parser->data;
+	git_buf *buf = &t->buf;
+	git_vector *refs = &t->refs;
+	int error;
+	const char *line_end, *ptr;
+	static int first_pkt = 1;
+
+	if (len == 0) { /* EOF */
+		if (buf->size != 0)
+			return t->error = git__throw(GIT_ERROR, "EOF and unprocessed data");
+		else
+			return 0;
+	}
+
+	git_buf_put(buf, str, len);
+	ptr = buf->ptr;
+	while (1) {
+		git_pkt *pkt;
+
+		if (buf->size == 0)
+			return 0;
+
+		error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->size);
+		if (error == GIT_ESHORTBUFFER)
+			return 0; /* Ask for more */
+		if (error < GIT_SUCCESS)
+			return t->error = git__rethrow(error, "Failed to parse pkt-line");
+
+		git_buf_consume(buf, line_end);
+
+		if (first_pkt) {
+			first_pkt = 0;
+			if (pkt->type != GIT_PKT_COMMENT)
+				return t->error = git__throw(GIT_EOBJCORRUPTED, "Not a valid smart HTTP response");
+		}
+
+		error = git_vector_insert(refs, pkt);
+		if (error < GIT_SUCCESS)
+			return t->error = git__rethrow(error, "Failed to add pkt to list");
+	}
+
+	return error;
+}
+
+static int on_message_complete(http_parser *parser)
+{
+	transport_http *t = (transport_http *) parser->data;
+
+	t->transfer_finished = 1;
+	return 0;
+}
+
+static int store_refs(transport_http *t)
+{
+	int error = GIT_SUCCESS;
+	http_parser parser;
+	http_parser_settings settings;
+	char buffer[1024];
+	gitno_buffer buf;
+
+	http_parser_init(&parser, HTTP_RESPONSE);
+	parser.data = t;
+	memset(&settings, 0x0, sizeof(http_parser_settings));
+	settings.on_header_field = on_header_field;
+	settings.on_header_value = on_header_value;
+	settings.on_headers_complete = on_headers_complete;
+	settings.on_body = on_body_store_refs;
+	settings.on_message_complete = on_message_complete;
+
+	gitno_buffer_setup(&buf, buffer, sizeof(buffer), t->socket);
+
+	while(1) {
+		size_t parsed;
+
+		error = gitno_recv(&buf);
+		if (error < GIT_SUCCESS)
+			return git__rethrow(error, "Error receiving data from network");
+
+		parsed = http_parser_execute(&parser, &settings, buf.data, buf.offset);
+		/* Both should happen at the same time */
+		if (parsed != buf.offset || t->error < GIT_SUCCESS)
+			return git__rethrow(t->error, "Error parsing HTTP data");
+
+		gitno_consume_n(&buf, parsed);
+
+		if (error == 0 || t->transfer_finished)
+			return GIT_SUCCESS;
+	}
+
+	return error;
+}
+
+static int http_connect(git_transport *transport, int direction)
+{
+	transport_http *t = (transport_http *) transport;
+	int error;
+
+	if (direction == GIT_DIR_PUSH)
+		return git__throw(GIT_EINVALIDARGS, "Pushing over HTTP is not supported");
+
+	t->parent.direction = direction;
+	error = git_vector_init(&t->refs, 16, NULL);
+	if (error < GIT_SUCCESS)
+		return git__rethrow(error, "Failed to init refs vector");
+
+	error = do_connect(t, "upload-pack");
+	if (error < GIT_SUCCESS) {
+		error = git__rethrow(error, "Failed to connect to host");
+		goto cleanup;
+	}
+
+	error = store_refs(t);
+
+cleanup:
+	git_buf_clear(&t->buf);
+	git_buf_free(&t->buf);
+
+	return error;
+}
+
+static int http_ls(git_transport *transport, git_headarray *array)
+{
+	transport_http *t = (transport_http *) transport;
+	git_vector *refs = &t->refs;
+	unsigned int i;
+	int len = 0;
+	git_pkt_ref *p;
+
+	array->heads = git__calloc(refs->length, sizeof(git_remote_head*));
+	if (array->heads == NULL)
+		return GIT_ENOMEM;
+
+	git_vector_foreach(refs, i, p) {
+		if (p->type != GIT_PKT_REF)
+			continue;
+
+		array->heads[len] = &p->head;
+		len++;
+	}
+
+	array->len = len;
+	t->heads = array->heads;
+
+	return GIT_SUCCESS;
+}
+
+static int http_close(git_transport *transport)
+{
+	transport_http *t = (transport_http *) transport;
+	int error;
+
+	error = close(t->socket);
+	if (error < 0)
+		return git__throw(GIT_EOSERR, "Failed to close the socket: %s", strerror(errno));
+
+	return GIT_SUCCESS;
+}
+
+
+static void http_free(git_transport *transport)
+{
+	transport_http *t = (transport_http *) transport;
+	git_vector *refs = &t->refs;
+	unsigned int i;
+	git_pkt *p;
+
+	git_vector_foreach(refs, i, p) {
+		git_pkt_free(p);
+	}
+	git_vector_free(refs);
+	free(t->heads);
+	free(t->content_type);
+	free(t->service);
+	free(t->parent.url);
+	free(t);
+}
+
+
+int git_transport_http(git_transport **out)
+{
+	transport_http *t;
+
+	t = git__malloc(sizeof(transport_http));
+	if (t == NULL)
+		return GIT_ENOMEM;
+
+	memset(t, 0x0, sizeof(transport_http));
+
+	t->parent.connect = http_connect;
+	t->parent.ls = http_ls;
+	t->parent.close = http_close;
+	t->parent.free = http_free;
+
+	*out = (git_transport *) t;
+
+	return GIT_SUCCESS;
+}
diff --git a/src/transport.c b/src/transport.c
index 6be8f4058..8d69db53e 100644
--- a/src/transport.c
+++ b/src/transport.c
@@ -15,7 +15,7 @@ struct {
 	git_transport_cb fn;
 } transports[] = {
 	{"git://", git_transport_git},
-	{"http://", git_transport_dummy},
+	{"http://", git_transport_http},
 	{"https://", git_transport_dummy},
 	{"file://", git_transport_local},
 	{"git+ssh://", git_transport_dummy},
diff --git a/src/transport.h b/src/transport.h
index 233cfb1db..eaa50d629 100644
--- a/src/transport.h
+++ b/src/transport.h
@@ -107,6 +107,7 @@ struct git_transport {
 
 int git_transport_local(struct git_transport **transport);
 int git_transport_git(struct git_transport **transport);
+int git_transport_http(struct git_transport **transport);
 int git_transport_dummy(struct git_transport **transport);
 
 #endif
diff --git a/src/transport_git.c b/src/transport_git.c
index ac268cd43..bcc612b43 100644
--- a/src/transport_git.c
+++ b/src/transport_git.c
@@ -84,37 +84,6 @@ cleanup:
 	return error;
 }
 
-/* The URL should already have been stripped of the protocol */
-static int extract_host_and_port(char **host, char **port, const char *url)
-{
-	char *colon, *slash, *delim;
-	int error = GIT_SUCCESS;
-
-	colon = strchr(url, ':');
-	slash = strchr(url, '/');
-
-	if (slash == NULL)
-			return git__throw(GIT_EOBJCORRUPTED, "Malformed URL: missing /");
-
-	if (colon == NULL) {
-		*port = git__strdup(GIT_DEFAULT_PORT);
-	} else {
-		*port = git__strndup(colon + 1, slash - colon - 1);
-	}
-	if (*port == NULL)
-		return GIT_ENOMEM;;
-
-
-	delim = colon == NULL ? slash : colon;
-	*host = git__strndup(url, delim - url);
-	if (*host == NULL) {
-		free(*port);
-		error = GIT_ENOMEM;
-	}
-
-	return error;
-}
-
 /*
  * Parse the URL and connect to a server, storing the socket in
  * out. For convenience this also takes care of asking for the remote
@@ -130,9 +99,10 @@ static int do_connect(transport_git *t, const char *url)
 	if (!git__prefixcmp(url, prefix))
 		url += strlen(prefix);
 
-	error = extract_host_and_port(&host, &port, url);
+	error = gitno_extract_host_and_port(&host, &port, url, GIT_DEFAULT_PORT);
 	if (error < GIT_SUCCESS)
 		return error;
+
 	s = gitno_connect(host, port);
 	connected = 1;
 	error = send_request(s, NULL, url);