Implement filters for status/diff blobs

This adds support to diff and status for running filters (a la crlf)
on blobs in the workdir before computing SHAs and before generating
text diffs.  This ended up being a bit more code change than I had
thought since I had to reorganize some of the diff logic to minimize
peak memory use when filtering blobs in a diff.

This also adds a cap on the maximum size of data that will be loaded
to diff.  I set it at 512Mb which should match core git.  Right now
it is a #define in src/diff.h but it could be moved into the public
API if desired.

1 files changed, 40 insertions, 22 deletions

diff --git a/src/fileops.c b/src/fileops.c
index 95eacb5f1..d4def1a9a 100644
--- a/src/fileops.c
+++ b/src/fileops.c
@@ -115,10 +115,47 @@ mode_t git_futils_canonical_mode(mode_t raw_mode)
 		return 0;
 }
 
-int git_futils_readbuffer_updated(git_buf *buf, const char *path, time_t *mtime, int *updated)
+#define MAX_READ_STALLS 10
+
+int git_futils_readbuffer_fd(git_buf *buf, git_file fd, size_t len)
+{
+	int stalls = MAX_READ_STALLS;
+
+	git_buf_clear(buf);
+
+	if (git_buf_grow(buf, len + 1) < 0)
+		return -1;
+
+	buf->ptr[len] = '\0';
+
+	while (len > 0) {
+		ssize_t read_size = p_read(fd, buf->ptr + buf->size, len);
+
+		if (read_size < 0) {
+			giterr_set(GITERR_OS, "Failed to read descriptor");
+			return -1;
+		}
+
+		if (read_size == 0) {
+			stalls--;
+
+			if (!stalls) {
+				giterr_set(GITERR_OS, "Too many stalls reading descriptor");
+				return -1;
+			}
+		}
+
+		len -= read_size;
+		buf->size += read_size;
+	}
+
+	return 0;
+}
+
+int git_futils_readbuffer_updated(
+	git_buf *buf, const char *path, time_t *mtime, int *updated)
 {
 	git_file fd;
-	size_t len;
 	struct stat st;
 
 	assert(buf && path && *path);
@@ -147,30 +184,11 @@ int git_futils_readbuffer_updated(git_buf *buf, const char *path, time_t *mtime,
 	if (mtime != NULL)
 		*mtime = st.st_mtime;
 
-	len = (size_t) st.st_size;
-
-	git_buf_clear(buf);
-
-	if (git_buf_grow(buf, len + 1) < 0) {
+	if (git_futils_readbuffer_fd(buf, fd, (size_t)st.st_size) < 0) {
 		p_close(fd);
 		return -1;
 	}
 
-	buf->ptr[len] = '\0';
-
-	while (len > 0) {
-		ssize_t read_size = p_read(fd, buf->ptr, len);
-
-		if (read_size < 0) {
-			p_close(fd);
-			giterr_set(GITERR_OS, "Failed to read descriptor for '%s'", path);
-			return -1;
-		}
-
-		len -= read_size;
-		buf->size += read_size;
-	}
-
 	p_close(fd);
 
 	if (updated != NULL)