zlib: zlib can only process 4GB at a time

The size of objects we read from the repository and data we try to put
into the repository are represented in "unsigned long", so that on larger
architectures we can handle objects that weigh more than 4GB.

But the interface defined in zlib.h to communicate with inflate/deflate
limits avail_in (how many bytes of input are we calling zlib with) and
avail_out (how many bytes of output from zlib are we ready to accept)
fields effectively to 4GB by defining their type to be uInt.

In many places in our code, we allocate a large buffer (e.g. mmap'ing a
large loose object file) and tell zlib its size by assigning the size to
avail_in field of the stream, but that will truncate the high octets of
the real size. The worst part of this story is that we often pass around
z_stream (the state object used by zlib) to keep track of the number of
used bytes in input/output buffer by inspecting these two fields, which
practically limits our callchain to the same 4GB limit.

Wrap z_stream in another structure git_zstream that can express avail_in
and avail_out in unsigned long. For now, just die() when the caller gives
a size that cannot be given to a single zlib call. In later patches in the
series, we would make git_inflate() and git_deflate() internally loop to
give callers an illusion that our "improved" version of zlib interface can
operate on a buffer larger than 4GB in one go.

Signed-off-by: Junio C Hamano <gitster@pobox.com>

1 files changed, 91 insertions, 28 deletions

diff --git a/zlib.c b/zlib.c
index 8f19d2fe38..fe537e3ff6 100644
--- a/zlib.c
+++ b/zlib.c
@@ -22,45 +22,90 @@ static const char *zerr_to_string(int status)
 	}
 }
 
-void git_inflate_init(z_streamp strm)
+/*
+ * avail_in and avail_out in zlib are counted in uInt, which typically
+ * limits the size of the buffer we can use to 4GB when interacting
+ * with zlib in a single call to inflate/deflate.
+ */
+#define ZLIB_BUF_MAX ((uInt)-1)
+static inline uInt zlib_buf_cap(unsigned long len)
+{
+	if (ZLIB_BUF_MAX < len)
+		die("working buffer for zlib too large");
+	return len;
+}
+
+static void zlib_pre_call(git_zstream *s)
+{
+	s->z.next_in = s->next_in;
+	s->z.next_out = s->next_out;
+	s->z.total_in = s->total_in;
+	s->z.total_out = s->total_out;
+	s->z.avail_in = zlib_buf_cap(s->avail_in);
+	s->z.avail_out = zlib_buf_cap(s->avail_out);
+}
+
+static void zlib_post_call(git_zstream *s)
+{
+	s->next_in = s->z.next_in;
+	s->next_out = s->z.next_out;
+	s->total_in = s->z.total_in;
+	s->total_out = s->z.total_out;
+	s->avail_in = s->z.avail_in;
+	s->avail_out = s->z.avail_out;
+}
+
+void git_inflate_init(git_zstream *strm)
 {
-	int status = inflateInit(strm);
+	int status;
 
+	zlib_pre_call(strm);
+	status = inflateInit(&strm->z);
+	zlib_post_call(strm);
 	if (status == Z_OK)
 		return;
 	die("inflateInit: %s (%s)", zerr_to_string(status),
-	    strm->msg ? strm->msg : "no message");
+	    strm->z.msg ? strm->z.msg : "no message");
 }
 
-void git_inflate_init_gzip_only(z_streamp strm)
+void git_inflate_init_gzip_only(git_zstream *strm)
 {
 	/*
 	 * Use default 15 bits, +16 is to accept only gzip and to
 	 * yield Z_DATA_ERROR when fed zlib format.
 	 */
 	const int windowBits = 15 + 16;
-	int status = inflateInit2(strm, windowBits);
+	int status;
 
+	zlib_pre_call(strm);
+	status = inflateInit2(&strm->z, windowBits);
+	zlib_post_call(strm);
 	if (status == Z_OK)
 		return;
 	die("inflateInit2: %s (%s)", zerr_to_string(status),
-	    strm->msg ? strm->msg : "no message");
+	    strm->z.msg ? strm->z.msg : "no message");
 }
 
-void git_inflate_end(z_streamp strm)
+void git_inflate_end(git_zstream *strm)
 {
-	int status = inflateEnd(strm);
+	int status;
 
+	zlib_pre_call(strm);
+	status = inflateEnd(&strm->z);
+	zlib_post_call(strm);
 	if (status == Z_OK)
 		return;
 	error("inflateEnd: %s (%s)", zerr_to_string(status),
-	      strm->msg ? strm->msg : "no message");
+	      strm->z.msg ? strm->z.msg : "no message");
 }
 
-int git_inflate(z_streamp strm, int flush)
+int git_inflate(git_zstream *strm, int flush)
 {
-	int status = inflate(strm, flush);
+	int status;
 
+	zlib_pre_call(strm);
+	status = inflate(&strm->z, flush);
+	zlib_post_call(strm);
 	switch (status) {
 	/* Z_BUF_ERROR: normal, needs more space in the output buffer */
 	case Z_BUF_ERROR:
@@ -74,7 +119,7 @@ int git_inflate(z_streamp strm, int flush)
 		break;
 	}
 	error("inflate: %s (%s)", zerr_to_string(status),
-	      strm->msg ? strm->msg : "no message");
+	      strm->z.msg ? strm->z.msg : "no message");
 	return status;
 }
 
@@ -82,56 +127,74 @@ int git_inflate(z_streamp strm, int flush)
 #define deflateBound(c,s)  ((s) + (((s) + 7) >> 3) + (((s) + 63) >> 6) + 11)
 #endif
 
-unsigned long git_deflate_bound(z_streamp strm, unsigned long size)
+unsigned long git_deflate_bound(git_zstream *strm, unsigned long size)
 {
-	return deflateBound(strm, size);
+	return deflateBound(&strm->z, size);
 }
 
-void git_deflate_init(z_streamp strm, int level)
+void git_deflate_init(git_zstream *strm, int level)
 {
-	int status = deflateInit(strm, level);
+	int status;
 
+	zlib_pre_call(strm);
+	status = deflateInit(&strm->z, level);
+	zlib_post_call(strm);
 	if (status == Z_OK)
 		return;
 	die("deflateInit: %s (%s)", zerr_to_string(status),
-	    strm->msg ? strm->msg : "no message");
+	    strm->z.msg ? strm->z.msg : "no message");
 }
 
-void git_deflate_init_gzip(z_streamp strm, int level)
+void git_deflate_init_gzip(git_zstream *strm, int level)
 {
 	/*
 	 * Use default 15 bits, +16 is to generate gzip header/trailer
 	 * instead of the zlib wrapper.
 	 */
 	const int windowBits = 15 + 16;
-	int status = deflateInit2(strm, level,
+	int status;
+
+	zlib_pre_call(strm);
+	status = deflateInit2(&strm->z, level,
 				  Z_DEFLATED, windowBits,
 				  8, Z_DEFAULT_STRATEGY);
+	zlib_post_call(strm);
 	if (status == Z_OK)
 		return;
 	die("deflateInit2: %s (%s)", zerr_to_string(status),
-	    strm->msg ? strm->msg : "no message");
+	    strm->z.msg ? strm->z.msg : "no message");
 }
 
-void git_deflate_end(z_streamp strm)
+void git_deflate_end(git_zstream *strm)
 {
-	int status = deflateEnd(strm);
+	int status;
 
+	zlib_pre_call(strm);
+	status = deflateEnd(&strm->z);
+	zlib_post_call(strm);
 	if (status == Z_OK)
 		return;
 	error("deflateEnd: %s (%s)", zerr_to_string(status),
-	      strm->msg ? strm->msg : "no message");
+	      strm->z.msg ? strm->z.msg : "no message");
 }
 
-int git_deflate_end_gently(z_streamp strm)
+int git_deflate_end_gently(git_zstream *strm)
 {
-	return deflateEnd(strm);
+	int status;
+
+	zlib_pre_call(strm);
+	status = deflateEnd(&strm->z);
+	zlib_post_call(strm);
+	return status;
 }
 
-int git_deflate(z_streamp strm, int flush)
+int git_deflate(git_zstream *strm, int flush)
 {
-	int status = deflate(strm, flush);
+	int status;
 
+	zlib_pre_call(strm);
+	status = deflate(&strm->z, flush);
+	zlib_post_call(strm);
 	switch (status) {
 	/* Z_BUF_ERROR: normal, needs more space in the output buffer */
 	case Z_BUF_ERROR:
@@ -145,6 +208,6 @@ int git_deflate(z_streamp strm, int flush)
 		break;
 	}
 	error("deflate: %s (%s)", zerr_to_string(status),
-	      strm->msg ? strm->msg : "no message");
+	      strm->z.msg ? strm->z.msg : "no message");
 	return status;
 }