diff: determine DIFFABLE-ness for binaries

Always set `GIT_DIFF_PATCH_DIFFABLE` for all files, regardless of
binary-ness, so that the binary callback is invoked to either
show the binary contents, or just print the standard "Binary files
differ" message.  We may need to do deeper inspection for binary
files where we have avoided loading the contents into a file map.

2 files changed, 32 insertions, 12 deletions

diff --git a/src/diff_patch.c b/src/diff_patch.c
index 60648f8d7..ec4979a52 100644
--- a/src/diff_patch.c
+++ b/src/diff_patch.c
@@ -121,6 +121,35 @@ GIT_INLINE(bool) should_skip_binary(git_patch *patch, git_diff_file *file)
 	return (file->flags & GIT_DIFF_FLAG_BINARY) != 0;
 }
 
+static bool diff_patch_diffable(git_patch *patch)
+{
+	size_t olen, nlen;
+
+	if (patch->delta->status == GIT_DELTA_UNMODIFIED)
+		return false;
+
+	/* if we've determined this to be binary (and we are not showing binary
+	 * data) then we have skipped loading the map data.  instead, query the
+	 * file data itself.
+	 */
+	if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0 &&
+		(patch->diff_opts.flags & GIT_DIFF_SHOW_BINARY) == 0) {
+		olen = (size_t)patch->ofile.file->size;
+		nlen = (size_t)patch->nfile.file->size;
+	} else {
+		olen = patch->ofile.map.len;
+		nlen = patch->nfile.map.len;
+	}
+
+	/* if both sides are empty, files are identical */
+	if (!olen && !nlen)
+		return false;
+
+	/* otherwise, check the file sizes and the oid */
+	return (olen != nlen ||
+		!git_oid_equal(&patch->ofile.file->id, &patch->nfile.file->id));
+}
+
 static int diff_patch_load(git_patch *patch, git_diff_output *output)
 {
 	int error = 0;
@@ -186,18 +215,7 @@ cleanup:
 	diff_patch_update_binary(patch);
 
 	if (!error) {
-		bool skip_binary =
-			(patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0 &&
-			(patch->diff_opts.flags & GIT_DIFF_SHOW_BINARY) == 0;
-
-		/* patch is diffable only for non-binary, modified files where
-		* at least one side has data and the data actually changed
-		*/
-		if (!skip_binary &&
-			patch->delta->status != GIT_DELTA_UNMODIFIED &&
-			(patch->ofile.map.len || patch->nfile.map.len) &&
-			(patch->ofile.map.len != patch->nfile.map.len ||
-			 !git_oid_equal(&patch->ofile.file->id, &patch->nfile.file->id)))
+		if (diff_patch_diffable(patch))
 			patch->flags |= GIT_DIFF_PATCH_DIFFABLE;
 
 		patch->flags |= GIT_DIFF_PATCH_LOADED;
diff --git a/src/diff_patch.h b/src/diff_patch.h
index f6ce57ddd..7b4dacdde 100644
--- a/src/diff_patch.h
+++ b/src/diff_patch.h
@@ -24,7 +24,9 @@ enum {
 	GIT_DIFF_PATCH_ALLOCATED = (1 << 0),
 	GIT_DIFF_PATCH_INITIALIZED = (1 << 1),
 	GIT_DIFF_PATCH_LOADED = (1 << 2),
+	/* the two sides are different */
 	GIT_DIFF_PATCH_DIFFABLE = (1 << 3),
+	/* the difference between the two sides has been computed */
 	GIT_DIFF_PATCH_DIFFED = (1 << 4),
 	GIT_DIFF_PATCH_FLATTENED = (1 << 5),
 };