odb: make the writestream's size a git_off_tcmn/stream-size

Restricting files to size_t is a silly limitation. The loose backend
writes to a file directly, so there is no issue in using 63 bits for the
size.

We still assume that the header is going to fit in 64 bytes, which does
mean quite a bit smaller files due to the run-length encoding, but it's
still a much larger size than you would want Git to handle.

7 files changed, 19 insertions, 13 deletions

diff --git a/include/git2/odb.h b/include/git2/odb.h
index 114f6b317..4f1e18bc1 100644
--- a/include/git2/odb.h
+++ b/include/git2/odb.h
@@ -247,7 +247,7 @@ GIT_EXTERN(int) git_odb_write(git_oid *out, git_odb *odb, const void *data, size
  * @param type type of the object that will be written
  * @return 0 if the stream was created; error code otherwise
  */
-GIT_EXTERN(int) git_odb_open_wstream(git_odb_stream **out, git_odb *db, size_t size, git_otype type);
+GIT_EXTERN(int) git_odb_open_wstream(git_odb_stream **out, git_odb *db, git_off_t size, git_otype type);
 
 /**
  * Write to an odb stream
diff --git a/include/git2/odb_backend.h b/include/git2/odb_backend.h
index 4d772cab9..b17cfd8ba 100644
--- a/include/git2/odb_backend.h
+++ b/include/git2/odb_backend.h
@@ -86,8 +86,8 @@ struct git_odb_stream {
 	unsigned int mode;
 	void *hash_ctx;
 
-	size_t declared_size;
-	size_t received_bytes;
+	git_off_t declared_size;
+	git_off_t received_bytes;
 
 	/**
 	 * Write at most `len` bytes into `buffer` and advance the stream.
diff --git a/include/git2/sys/odb_backend.h b/include/git2/sys/odb_backend.h
index 1fc3c3159..0a51c6dba 100644
--- a/include/git2/sys/odb_backend.h
+++ b/include/git2/sys/odb_backend.h
@@ -53,7 +53,7 @@ struct git_odb_backend {
 		git_odb_backend *, const git_oid *, const void *, size_t, git_otype);
 
 	int (* writestream)(
-		git_odb_stream **, git_odb_backend *, size_t, git_otype);
+		git_odb_stream **, git_odb_backend *, git_off_t, git_otype);
 
 	int (* readstream)(
 		git_odb_stream **, git_odb_backend *, const git_oid *);
diff --git a/src/blob.c b/src/blob.c
index 47216507b..07c4d92c8 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -76,10 +76,11 @@ static int write_file_stream(
 	int fd, error;
 	char buffer[FILEIO_BUFSIZE];
 	git_odb_stream *stream = NULL;
-	ssize_t read_len = -1, written = 0;
+	ssize_t read_len = -1;
+	git_off_t written = 0;
 
 	if ((error = git_odb_open_wstream(
-			&stream, odb, (size_t)file_size, GIT_OBJ_BLOB)) < 0)
+			&stream, odb, file_size, GIT_OBJ_BLOB)) < 0)
 		return error;
 
 	if ((fd = git_futils_open_ro(path)) < 0) {
diff --git a/src/odb.c b/src/odb.c
index c3ae15a6a..d0bad5331 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -47,7 +47,7 @@ static git_cache *odb_cache(git_odb *odb)
 
 static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_depth);
 
-int git_odb__format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
+int git_odb__format_object_header(char *hdr, size_t n, git_off_t obj_len, git_otype obj_type)
 {
 	const char *type_str = git_object_type2string(obj_type);
 	int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
@@ -327,10 +327,15 @@ static void fake_wstream__free(git_odb_stream *_stream)
 	git__free(stream);
 }
 
-static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
+static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, git_off_t size, git_otype type)
 {
 	fake_wstream *stream;
 
+	if (!git__is_ssizet(size)) {
+		giterr_set(GITERR_ODB, "object size too large to keep in memory");
+		return -1;
+	}
+
 	stream = git__calloc(1, sizeof(fake_wstream));
 	GITERR_CHECK_ALLOC(stream);
 
@@ -937,7 +942,7 @@ int git_odb_write(
 	return error;
 }
 
-static void hash_header(git_hash_ctx *ctx, size_t size, git_otype type)
+static void hash_header(git_hash_ctx *ctx, git_off_t size, git_otype type)
 {
 	char header[64];
 	int hdrlen;
@@ -947,7 +952,7 @@ static void hash_header(git_hash_ctx *ctx, size_t size, git_otype type)
 }
 
 int git_odb_open_wstream(
-	git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
+	git_odb_stream **stream, git_odb *db, git_off_t size, git_otype type)
 {
 	size_t i, writes = 0;
 	int error = GIT_ERROR;
diff --git a/src/odb.h b/src/odb.h
index 61dd9a7fd..281bd3a4d 100644
--- a/src/odb.h
+++ b/src/odb.h
@@ -49,7 +49,7 @@ int git_odb__hashobj(git_oid *id, git_rawobj *obj);
 /*
  * Format the object header such as it would appear in the on-disk object
  */
-int git_odb__format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type);
+int git_odb__format_object_header(char *hdr, size_t n, git_off_t obj_len, git_otype obj_type);
 /*
  * Hash an open file descriptor.
  * This is a performance call when the contents of a fd need to be hashed,
diff --git a/src/odb_loose.c b/src/odb_loose.c
index bfd95588b..99b8f7c91 100644
--- a/src/odb_loose.c
+++ b/src/odb_loose.c
@@ -834,7 +834,7 @@ static void loose_backend__stream_free(git_odb_stream *_stream)
 	git__free(stream);
 }
 
-static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, size_t length, git_otype type)
+static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, git_off_t length, git_otype type)
 {
 	loose_backend *backend;
 	loose_writestream *stream = NULL;
@@ -842,7 +842,7 @@ static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_
 	git_buf tmp_path = GIT_BUF_INIT;
 	int hdrlen;
 
-	assert(_backend);
+	assert(_backend && length >= 0);
 
 	backend = (loose_backend *)_backend;
 	*stream_out = NULL;