Merge branch 'development' into blame_rebased

Conflicts:
	include/git2.h

107 files changed, 6140 insertions, 2800 deletions

diff --git a/src/array.h b/src/array.h
index 2d77c71a0..b82079bd8 100644
--- a/src/array.h
+++ b/src/array.h
@@ -30,37 +30,45 @@
 #define git_array_init(a) \
 	do { (a).size = (a).asize = 0; (a).ptr = NULL; } while (0)
 
+#define git_array_init_to_size(a, desired) \
+	do { (a).size = 0; (a).asize = desired; (a).ptr = git__calloc(desired, sizeof(*(a).ptr)); } while (0)
+
 #define git_array_clear(a) \
 	do { git__free((a).ptr); git_array_init(a); } while (0)
 
 #define GITERR_CHECK_ARRAY(a) GITERR_CHECK_ALLOC((a).ptr)
 
 
-typedef git_array_t(void) git_array_generic_t;
+typedef git_array_t(char) git_array_generic_t;
 
 /* use a generic array for growth so this can return the new item */
-GIT_INLINE(void *) git_array_grow(git_array_generic_t *a, size_t item_size)
+GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size)
 {
+	git_array_generic_t *a = _a;
 	uint32_t new_size = (a->size < 8) ? 8 : a->asize * 3 / 2;
-	void *new_array = git__realloc(a->ptr, new_size * item_size);
+	char *new_array = git__realloc(a->ptr, new_size * item_size);
 	if (!new_array) {
 		git_array_clear(*a);
 		return NULL;
 	} else {
 		a->ptr = new_array; a->asize = new_size; a->size++;
-		return (((char *)a->ptr) + (a->size - 1) * item_size);
+		return a->ptr + (a->size - 1) * item_size;
 	}
 }
 
 #define git_array_alloc(a) \
 	((a).size >= (a).asize) ? \
-	git_array_grow((git_array_generic_t *)&(a), sizeof(*(a).ptr)) :	\
+	git_array_grow(&(a), sizeof(*(a).ptr)) : \
 	(a).ptr ? &(a).ptr[(a).size++] : NULL
 
 #define git_array_last(a) ((a).size ? &(a).ptr[(a).size - 1] : NULL)
 
+#define git_array_pop(a) ((a).size ? &(a).ptr[--(a).size] : NULL)
+
 #define git_array_get(a, i) (((i) < (a).size) ? &(a).ptr[(i)] : NULL)
 
 #define git_array_size(a) (a).size
 
+#define git_array_valid_index(a, i) ((i) < (a).size)
+
 #endif
diff --git a/src/attr_file.c b/src/attr_file.c
index d059cfec7..92702df98 100644
--- a/src/attr_file.c
+++ b/src/attr_file.c
@@ -79,9 +79,13 @@ int git_attr_file__parse_buffer(
 
 	while (!error && *scan) {
 		/* allocate rule if needed */
-		if (!rule && !(rule = git__calloc(1, sizeof(git_attr_rule)))) {
-			error = -1;
-			break;
+		if (!rule) {
+			if (!(rule = git__calloc(1, sizeof(git_attr_rule)))) {
+				error = -1;
+				break;
+			}
+			rule->match.flags = GIT_ATTR_FNMATCH_ALLOWNEG |
+				GIT_ATTR_FNMATCH_ALLOWMACRO;
 		}
 
 		/* parse the next "pattern attr attr attr" line */
@@ -351,8 +355,8 @@ int git_attr_fnmatch__parse(
 	if (parse_optimized_patterns(spec, pool, *base))
 		return 0;
 
-	spec->flags = (spec->flags & GIT_ATTR_FNMATCH_ALLOWSPACE);
-	allow_space = (spec->flags != 0);
+	spec->flags = (spec->flags & GIT_ATTR_FNMATCH__INCOMING);
+	allow_space = ((spec->flags & GIT_ATTR_FNMATCH_ALLOWSPACE) != 0);
 
 	pattern = *base;
 
@@ -362,7 +366,7 @@ int git_attr_fnmatch__parse(
 		return GIT_ENOTFOUND;
 	}
 
-	if (*pattern == '[') {
+	if (*pattern == '[' && (spec->flags & GIT_ATTR_FNMATCH_ALLOWMACRO) != 0) {
 		if (strncmp(pattern, "[attr]", 6) == 0) {
 			spec->flags = spec->flags | GIT_ATTR_FNMATCH_MACRO;
 			pattern += 6;
@@ -370,7 +374,7 @@ int git_attr_fnmatch__parse(
 		/* else a character range like [a-e]* which is accepted */
 	}
 
-	if (*pattern == '!') {
+	if (*pattern == '!' && (spec->flags & GIT_ATTR_FNMATCH_ALLOWNEG) != 0) {
 		spec->flags = spec->flags | GIT_ATTR_FNMATCH_NEGATIVE;
 		pattern++;
 	}
@@ -498,7 +502,7 @@ int git_attr_assignment__parse(
 
 	assert(assigns && !assigns->length);
 
-	assigns->_cmp = sort_by_hash_and_name;
+	git_vector_set_cmp(assigns, sort_by_hash_and_name);
 
 	while (*scan && *scan != '\n') {
 		const char *name_start, *value_start;
diff --git a/src/attr_file.h b/src/attr_file.h
index 15bba1c6a..3bc7c6cb8 100644
--- a/src/attr_file.h
+++ b/src/attr_file.h
@@ -28,6 +28,12 @@
 #define GIT_ATTR_FNMATCH_ALLOWSPACE	(1U << 6)
 #define GIT_ATTR_FNMATCH_ICASE		(1U << 7)
 #define GIT_ATTR_FNMATCH_MATCH_ALL	(1U << 8)
+#define GIT_ATTR_FNMATCH_ALLOWNEG   (1U << 9)
+#define GIT_ATTR_FNMATCH_ALLOWMACRO (1U << 10)
+
+#define GIT_ATTR_FNMATCH__INCOMING \
+	(GIT_ATTR_FNMATCH_ALLOWSPACE | \
+	 GIT_ATTR_FNMATCH_ALLOWNEG | GIT_ATTR_FNMATCH_ALLOWMACRO)
 
 extern const char *git_attr__true;
 extern const char *git_attr__false;
diff --git a/src/bitvec.h b/src/bitvec.h
new file mode 100644
index 000000000..fd6f0ccf8
--- /dev/null
+++ b/src/bitvec.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_bitvec_h__
+#define INCLUDE_bitvec_h__
+
+#include "util.h"
+
+/*
+ * This is a silly little fixed length bit vector type that will store
+ * vectors of 64 bits or less directly in the structure and allocate
+ * memory for vectors longer than 64 bits.  You can use the two versions
+ * transparently through the API and avoid heap allocation completely when
+ * using a short bit vector as a result.
+ */
+typedef struct {
+	size_t length;
+	union {
+		uint64_t *words;
+		uint64_t bits;
+	} u;
+} git_bitvec;
+
+GIT_INLINE(int) git_bitvec_init(git_bitvec *bv, size_t capacity)
+{
+	memset(bv, 0x0, sizeof(*bv));
+
+	if (capacity >= 64) {
+		bv->length = (capacity / 64) + 1;
+		bv->u.words = git__calloc(bv->length, sizeof(uint64_t));
+		if (!bv->u.words)
+			return -1;
+	}
+
+	return 0;
+}
+
+#define GIT_BITVEC_MASK(BIT) ((uint64_t)1 << (BIT % 64))
+#define GIT_BITVEC_WORD(BV, BIT) (BV->length ? &BV->u.words[BIT / 64] : &BV->u.bits)
+
+GIT_INLINE(void) git_bitvec_set(git_bitvec *bv, size_t bit, bool on)
+{
+	uint64_t *word = GIT_BITVEC_WORD(bv, bit);
+	uint64_t mask = GIT_BITVEC_MASK(bit);
+
+	if (on)
+		*word |= mask;
+	else
+		*word &= ~mask;
+}
+
+GIT_INLINE(bool) git_bitvec_get(git_bitvec *bv, size_t bit)
+{
+	uint64_t *word = GIT_BITVEC_WORD(bv, bit);
+	return (*word & GIT_BITVEC_MASK(bit)) != 0;
+}
+
+GIT_INLINE(void) git_bitvec_clear(git_bitvec *bv)
+{
+	if (!bv->length)
+		bv->u.bits = 0;
+	else
+		memset(bv->u.words, 0x0, bv->length * sizeof(uint64_t));
+}
+
+GIT_INLINE(void) git_bitvec_free(git_bitvec *bv)
+{
+	if (bv->length)
+		git__free(bv->u.words);
+}
+
+#endif
diff --git a/src/blob.c b/src/blob.c
index 2e4d5f479..6a289f43b 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -60,10 +60,10 @@ int git_blob_create_frombuffer(git_oid *oid, git_repository *repo, const void *b
 		(error = git_odb_open_wstream(&stream, odb, len, GIT_OBJ_BLOB)) < 0)
 		return error;
 
-	if ((error = stream->write(stream, buffer, len)) == 0)
-		error = stream->finalize_write(oid, stream);
+	if ((error = git_odb_stream_write(stream, buffer, len)) == 0)
+		error = git_odb_stream_finalize_write(oid, stream);
 
-	stream->free(stream);
+	git_odb_stream_free(stream);
 	return error;
 }
 
@@ -80,12 +80,12 @@ static int write_file_stream(
 		return error;
 
 	if ((fd = git_futils_open_ro(path)) < 0) {
-		stream->free(stream);
+		git_odb_stream_free(stream);
 		return -1;
 	}
 
 	while (!error && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
-		error = stream->write(stream, buffer, read_len);
+		error = git_odb_stream_write(stream, buffer, read_len);
 		written += read_len;
 	}
 
@@ -97,14 +97,15 @@ static int write_file_stream(
 	}
 
 	if (!error)
-		error = stream->finalize_write(oid, stream);
+		error = git_odb_stream_finalize_write(oid, stream);
 
-	stream->free(stream);
+	git_odb_stream_free(stream);
 	return error;
 }
 
 static int write_file_filtered(
 	git_oid *oid,
+	git_off_t *size,
 	git_odb *odb,
 	const char *full_path,
 	git_vector *filters)
@@ -123,8 +124,11 @@ static int write_file_filtered(
 	git_buf_free(&source);
 
 	/* Write the file to disk if it was properly filtered */
-	if (!error)
+	if (!error) {
+		*size = dest.size;
+
 		error = git_odb_write(oid, odb, dest.ptr, dest.size, GIT_OBJ_BLOB);
+	}
 
 	git_buf_free(&dest);
 	return error;
@@ -152,21 +156,46 @@ static int write_symlink(
 	return error;
 }
 
-static int blob_create_internal(git_oid *oid, git_repository *repo, const char *content_path, const char *hint_path, bool try_load_filters)
+int git_blob__create_from_paths(
+	git_oid *oid,
+	struct stat *out_st,
+	git_repository *repo,
+	const char *content_path,
+	const char *hint_path,
+	mode_t hint_mode,
+	bool try_load_filters)
 {
 	int error;
 	struct stat st;
 	git_odb *odb = NULL;
 	git_off_t size;
+	mode_t mode;
+	git_buf path = GIT_BUF_INIT;
 
 	assert(hint_path || !try_load_filters);
 
-	if ((error = git_path_lstat(content_path, &st)) < 0 || (error = git_repository_odb__weakptr(&odb, repo)) < 0)
-		return error;
+	if (!content_path) {
+		if (git_repository__ensure_not_bare(repo, "create blob from file") < 0)
+			return GIT_EBAREREPO;
+
+		if (git_buf_joinpath(
+				&path, git_repository_workdir(repo), hint_path) < 0)
+			return -1;
+
+		content_path = path.ptr;
+	}
+
+	if ((error = git_path_lstat(content_path, &st)) < 0 ||
+		(error = git_repository_odb(&odb, repo)) < 0)
+		goto done;
+
+	if (out_st)
+		memcpy(out_st, &st, sizeof(st));
 
 	size = st.st_size;
+	mode = hint_mode ? hint_mode : st.st_mode;
 
-	if (S_ISLNK(st.st_mode)) {
+	if (S_ISLNK(mode)) {
 		error = write_symlink(oid, odb, content_path, (size_t)size);
 	} else {
 		git_vector write_filters = GIT_VECTOR_INIT;
@@ -187,7 +216,8 @@ static int blob_create_internal(git_oid *oid, git_repository *repo, const char *
 			error = write_file_stream(oid, odb, content_path, size);
 		} else {
 			/* We need to apply one or more filters */
-			error = write_file_filtered(oid, odb, content_path, &write_filters);
+			error = write_file_filtered(
+				oid, &size, odb, content_path, &write_filters);
 		}
 
 		git_filters_free(&write_filters);
@@ -207,34 +237,21 @@ static int blob_create_internal(git_oid *oid, git_repository *repo, const char *
 		 */
 	}
 
+done:
+	git_odb_free(odb);
+	git_buf_free(&path);
+
 	return error;
 }
 
-int git_blob_create_fromworkdir(git_oid *oid, git_repository *repo, const char *path)
+int git_blob_create_fromworkdir(
+	git_oid *oid, git_repository *repo, const char *path)
 {
-	git_buf full_path = GIT_BUF_INIT;
-	const char *workdir;
-	int error;
-
-	if ((error = git_repository__ensure_not_bare(repo, "create blob from file")) < 0)
-		return error;
-
-	workdir = git_repository_workdir(repo);
-
-	if (git_buf_joinpath(&full_path, workdir, path) < 0) {
-		git_buf_free(&full_path);
-		return -1;
-	}
-
-	error = blob_create_internal(
-		oid, repo, git_buf_cstr(&full_path),
-		git_buf_cstr(&full_path) + strlen(workdir), true);
-
-	git_buf_free(&full_path);
-	return error;
+	return git_blob__create_from_paths(oid, NULL, repo, NULL, path, 0, true);
 }
 
-int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *path)
+int git_blob_create_fromdisk(
+	git_oid *oid, git_repository *repo, const char *path)
 {
 	int error;
 	git_buf full_path = GIT_BUF_INIT;
@@ -251,8 +268,8 @@ int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *pat
 	if (workdir && !git__prefixcmp(hintpath, workdir))
 		hintpath += strlen(workdir);
 
-	error = blob_create_internal(
-		oid, repo, git_buf_cstr(&full_path), hintpath, true);
+	error = git_blob__create_from_paths(
+		oid, NULL, repo, git_buf_cstr(&full_path), hintpath, 0, true);
 
 	git_buf_free(&full_path);
 	return error;
@@ -272,12 +289,9 @@ int git_blob_create_fromchunks(
 	git_filebuf file = GIT_FILEBUF_INIT;
 	git_buf path = GIT_BUF_INIT;
 
-	if (git_buf_join_n(
-		&path, '/', 3, 
-		git_repository_path(repo),
-		GIT_OBJECTS_DIR, 
-		"streamed") < 0)
-			goto cleanup;
+	if (git_buf_joinpath(
+			&path, git_repository_path(repo), GIT_OBJECTS_DIR "streamed") < 0)
+		goto cleanup;
 
 	content = git__malloc(BUFFER_SIZE);
 	GITERR_CHECK_ALLOC(content);
@@ -303,7 +317,8 @@ int git_blob_create_fromchunks(
 	if (git_filebuf_flush(&file) < 0)
 		goto cleanup;
 
-	error = blob_create_internal(oid, repo, file.path_lock, hintpath, hintpath != NULL);
+	error = git_blob__create_from_paths(
+		oid, NULL, repo, file.path_lock, hintpath, 0, hintpath != NULL);
 
 cleanup:
 	git_buf_free(&path);
diff --git a/src/blob.h b/src/blob.h
index 22e37cc3a..4cd9f1e0c 100644
--- a/src/blob.h
+++ b/src/blob.h
@@ -21,4 +21,13 @@ void git_blob__free(void *blob);
 int git_blob__parse(void *blob, git_odb_object *obj);
 int git_blob__getbuf(git_buf *buffer, git_blob *blob);
 
+extern int git_blob__create_from_paths(
+	git_oid *out_oid,
+	struct stat *out_st,
+	git_repository *repo,
+	const char *full_path,
+	const char *hint_path,
+	mode_t hint_mode,
+	bool apply_filters);
+
 #endif
diff --git a/src/branch.c b/src/branch.c
index de38e3355..7064fa7fc 100644
--- a/src/branch.c
+++ b/src/branch.c
@@ -132,7 +132,7 @@ int git_branch_foreach(
 {
 	git_reference_iterator *iter;
 	git_reference *ref;
-	int error;
+	int error = 0;
 
 	if (git_reference_iterator_new(&iter, repo) < 0)
 		return -1;
@@ -143,7 +143,6 @@ int git_branch_foreach(
 			if (callback(ref->name + strlen(GIT_REFS_HEADS_DIR),
 					GIT_BRANCH_LOCAL, payload)) {
 				error = GIT_EUSER;
-				break;
 			}
 		}
 
@@ -152,11 +151,14 @@ int git_branch_foreach(
 			if (callback(ref->name + strlen(GIT_REFS_REMOTES_DIR),
 					GIT_BRANCH_REMOTE, payload)) {
 				error = GIT_EUSER;
-				break;
 			}
 		}
 
 		git_reference_free(ref);
+
+		/* check if the callback has cancelled iteration */
+		if (error == GIT_EUSER)
+			break;
 	}
 
 	if (error == GIT_ITEROVER)
diff --git a/src/buf_text.c b/src/buf_text.c
index 443454b5f..ecf592b51 100644
--- a/src/buf_text.c
+++ b/src/buf_text.c
@@ -170,8 +170,14 @@ int git_buf_text_common_prefix(git_buf *buf, const git_strarray *strings)
 bool git_buf_text_is_binary(const git_buf *buf)
 {
 	const char *scan = buf->ptr, *end = buf->ptr + buf->size;
+	git_bom_t bom;
 	int printable = 0, nonprintable = 0;
 
+	scan += git_buf_text_detect_bom(&bom, buf, 0);
+
+	if (bom > GIT_BOM_UTF8)
+		return 1;
+
 	while (scan < end) {
 		unsigned char c = *scan++;
 
@@ -262,7 +268,7 @@ bool git_buf_text_gather_stats(
 	while (scan < end) {
 		unsigned char c = *scan++;
 
-		if ((c > 0x1F && c < 0x7F) || c > 0x9f)
+		if (c > 0x1F && c != 0x7F)
 			stats->printable++;
 		else switch (c) {
 			case '\0':
diff --git a/src/buffer.c b/src/buffer.c
index 6e3ffe560..b5b2fd678 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -259,6 +259,15 @@ void git_buf_truncate(git_buf *buf, size_t len)
 	}
 }
 
+void git_buf_shorten(git_buf *buf, size_t amount)
+{
+	if (amount > buf->size)
+		amount = buf->size;
+
+	buf->size = buf->size - amount;
+	buf->ptr[buf->size] = '\0';
+}
+
 void git_buf_rtruncate_at_char(git_buf *buf, char separator)
 {
 	ssize_t idx = git_buf_rfind_next(buf, separator);
diff --git a/src/buffer.h b/src/buffer.h
index 5402f3827..f3e1d506f 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -91,6 +91,7 @@ int git_buf_vprintf(git_buf *buf, const char *format, va_list ap);
 void git_buf_clear(git_buf *buf);
 void git_buf_consume(git_buf *buf, const char *end);
 void git_buf_truncate(git_buf *buf, size_t len);
+void git_buf_shorten(git_buf *buf, size_t amount);
 void git_buf_rtruncate_at_char(git_buf *path, char separator);
 
 int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...);
diff --git a/src/cc-compat.h b/src/cc-compat.h
index a5e4ce17e..37f1ea81e 100644
--- a/src/cc-compat.h
+++ b/src/cc-compat.h
@@ -54,8 +54,12 @@
 
 #if defined (_MSC_VER)
 	typedef unsigned char bool;
-#	define true 1
-#	define false 0
+#	ifndef true
+#		define true 1
+#	endif
+#	ifndef false
+#		define false 0
+#	endif
 #else
 #	include <stdbool.h>
 #endif
diff --git a/src/checkout.c b/src/checkout.c
index ede0be8e8..aae354ca6 100644
--- a/src/checkout.c
+++ b/src/checkout.c
@@ -220,9 +220,11 @@ static int checkout_action_no_wd(
 		action = CHECKOUT_ACTION_IF(SAFE_CREATE, UPDATE_BLOB, NONE);
 		break;
 	case GIT_DELTA_ADDED:    /* case 2 or 28 (and 5 but not really) */
-	case GIT_DELTA_MODIFIED: /* case 13 (and 35 but not really) */
 		action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
 		break;
+	case GIT_DELTA_MODIFIED: /* case 13 (and 35 but not really) */
+		action = CHECKOUT_ACTION_IF(SAFE_CREATE, UPDATE_BLOB, CONFLICT);
+		break;
 	case GIT_DELTA_TYPECHANGE: /* case 21 (B->T) and 28 (T->B)*/
 		if (delta->new_file.mode == GIT_FILEMODE_TREE)
 			action = CHECKOUT_ACTION_IF(SAFE, UPDATE_BLOB, NONE);
@@ -244,10 +246,10 @@ static int checkout_action_wd_only(
 	bool remove = false;
 	git_checkout_notify_t notify = GIT_CHECKOUT_NOTIFY_NONE;
 
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			pathspec, wd->path,
 			(data->strategy & GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH) != 0,
-			git_iterator_ignore_case(workdir), NULL))
+			git_iterator_ignore_case(workdir), NULL, NULL))
 		return 0;
 
 	/* check if item is tracked in the index but not in the checkout diff */
@@ -605,7 +607,7 @@ static int checkout_get_actions(
 	uint32_t *actions = NULL;
 
 	if (data->opts.paths.count > 0 &&
-		git_pathspec_init(&pathspec, &data->opts.paths, &pathpool) < 0)
+		git_pathspec__vinit(&pathspec, &data->opts.paths, &pathpool) < 0)
 		return -1;
 
 	if ((error = git_iterator_current(&wditem, workdir)) < 0 &&
@@ -657,7 +659,7 @@ static int checkout_get_actions(
 		goto fail;
 	}
 
-	git_pathspec_free(&pathspec);
+	git_pathspec__vfree(&pathspec);
 	git_pool_clear(&pathpool);
 
 	return 0;
@@ -668,7 +670,7 @@ fail:
 	*actions_ptr = NULL;
 	git__free(actions);
 
-	git_pathspec_free(&pathspec);
+	git_pathspec__vfree(&pathspec);
 	git_pool_clear(&pathpool);
 
 	return error;
@@ -691,17 +693,14 @@ static int buffer_to_file(
 			buffer, path, file_open_flags, file_mode)) < 0)
 		return error;
 
-	if (st != NULL && (error = p_stat(path, st)) < 0) {
-		giterr_set(GITERR_OS, "Error while statting '%s'", path);
-		return error;
-	}
+	if (st != NULL && (error = p_stat(path, st)) < 0)
+		giterr_set(GITERR_OS, "Error statting '%s'", path);
 
-	if ((file_mode & 0100) != 0 && (error = p_chmod(path, file_mode)) < 0) {
+	else if (GIT_PERMS_IS_EXEC(file_mode) &&
+			(error = p_chmod(path, file_mode)) < 0)
 		giterr_set(GITERR_OS, "Failed to set permissions on '%s'", path);
-		return error;
-	}
 
-	return 0;
+	return error;
 }
 
 static int blob_content_to_file(
@@ -856,7 +855,7 @@ static int checkout_submodule(
 		return 0;
 
 	if ((error = git_futils_mkdir(
-			file->path, git_repository_workdir(data->repo),
+			file->path, data->opts.target_directory,
 			data->opts.dir_mode, GIT_MKDIR_PATH)) < 0)
 		return error;
 
@@ -1028,7 +1027,7 @@ static int checkout_deferred_remove(git_repository *repo, const char *path)
 {
 #if 0
 	int error = git_futils_rmdir_r(
-		path, git_repository_workdir(repo), GIT_RMDIR_EMPTY_PARENTS);
+		path, data->opts.target_directory, GIT_RMDIR_EMPTY_PARENTS);
 
 	if (error == GIT_ENOTFOUND) {
 		error = 0;
@@ -1161,7 +1160,8 @@ static int checkout_data_init(
 		return -1;
 	}
 
-	if ((error = git_repository__ensure_not_bare(repo, "checkout")) < 0)
+	if ((!proposed || !proposed->target_directory) &&
+		(error = git_repository__ensure_not_bare(repo, "checkout")) < 0)
 		return error;
 
 	data->repo = repo;
@@ -1174,6 +1174,13 @@ static int checkout_data_init(
 	else
 		memmove(&data->opts, proposed, sizeof(git_checkout_opts));
 
+	if (!data->opts.target_directory)
+		data->opts.target_directory = git_repository_workdir(repo);
+	else if (!git_path_isdir(data->opts.target_directory) &&
+			 (error = git_futils_mkdir(data->opts.target_directory, NULL,
+					GIT_DIR_MODE, GIT_MKDIR_VERIFY_DIR)) < 0)
+		goto cleanup;
+
 	/* refresh config and index content unless NO_REFRESH is given */
 	if ((data->opts.checkout_strategy & GIT_CHECKOUT_NO_REFRESH) == 0) {
 		git_config *cfg;
@@ -1236,7 +1243,8 @@ static int checkout_data_init(
 
 	if ((error = git_vector_init(&data->removes, 0, git__strcmp_cb)) < 0 ||
 		(error = git_pool_init(&data->pool, 1, 0)) < 0 ||
-		(error = git_buf_puts(&data->path, git_repository_workdir(repo))) < 0)
+		(error = git_buf_puts(&data->path, data->opts.target_directory)) < 0 ||
+		(error = git_path_to_dir(&data->path)) < 0)
 		goto cleanup;
 
 	data->workdir_len = git_buf_len(&data->path);
@@ -1284,11 +1292,13 @@ int git_checkout_iterator(
 		GIT_ITERATOR_IGNORE_CASE : GIT_ITERATOR_DONT_IGNORE_CASE;
 
 	if ((error = git_iterator_reset(target, data.pfx, data.pfx)) < 0 ||
-		(error = git_iterator_for_workdir(
-			&workdir, data.repo, iterflags | GIT_ITERATOR_DONT_AUTOEXPAND,
+		(error = git_iterator_for_workdir_ext(
+			&workdir, data.repo, data.opts.target_directory,
+			iterflags | GIT_ITERATOR_DONT_AUTOEXPAND,
 			data.pfx, data.pfx)) < 0 ||
 		(error = git_iterator_for_tree(
-			&baseline, data.opts.baseline, iterflags, data.pfx, data.pfx)) < 0)
+			&baseline, data.opts.baseline,
+			iterflags, data.pfx, data.pfx)) < 0)
 		goto cleanup;
 
 	/* Should not have case insensitivity mismatch */
@@ -1356,8 +1366,19 @@ int git_checkout_index(
 	int error;
 	git_iterator *index_i;
 
-	if ((error = git_repository__ensure_not_bare(repo, "checkout index")) < 0)
-		return error;
+	if (!index && !repo) {
+		giterr_set(GITERR_CHECKOUT,
+			"Must provide either repository or index to checkout");
+		return -1;
+	}
+	if (index && repo && git_index_owner(index) != repo) {
+		giterr_set(GITERR_CHECKOUT,
+			"Index to checkout does not match repository");
+		return -1;
+	}
+
+	if (!repo)
+		repo = git_index_owner(index);
 
 	if (!index && (error = git_repository_index__weakptr(&index, repo)) < 0)
 		return error;
@@ -1381,8 +1402,19 @@ int git_checkout_tree(
 	git_tree *tree = NULL;
 	git_iterator *tree_i = NULL;
 
-	if ((error = git_repository__ensure_not_bare(repo, "checkout tree")) < 0)
-		return error;
+	if (!treeish && !repo) {
+		giterr_set(GITERR_CHECKOUT,
+			"Must provide either repository or tree to checkout");
+		return -1;
+	}
+	if (treeish && repo && git_object_owner(treeish) != repo) {
+		giterr_set(GITERR_CHECKOUT,
+			"Object to checkout does not match repository");
+		return -1;
+	}
+
+	if (!repo)
+		repo = git_object_owner(treeish);
 
 	if (git_object_peel((git_object **)&tree, treeish, GIT_OBJ_TREE) < 0) {
 		giterr_set(
@@ -1407,8 +1439,7 @@ int git_checkout_head(
 	git_tree *head = NULL;
 	git_iterator *head_i = NULL;
 
-	if ((error = git_repository__ensure_not_bare(repo, "checkout head")) < 0)
-		return error;
+	assert(repo);
 
 	if (!(error = checkout_lookup_head_tree(&head, repo)) &&
 		!(error = git_iterator_for_tree(&head_i, head, 0, NULL, NULL)))
diff --git a/src/clone.c b/src/clone.c
index 5b6c6f77d..5b8fc5e45 100644
--- a/src/clone.c
+++ b/src/clone.c
@@ -204,7 +204,7 @@ static int update_head_to_remote(git_repository *repo, git_remote *remote)
 
 	/* Get the remote's HEAD. This is always the first ref in remote->refs. */
 	remote_head = NULL;
-	
+
 	if (!remote->transport->ls(remote->transport, get_head_callback, &remote_head))
 		return -1;
 
@@ -220,7 +220,7 @@ static int update_head_to_remote(git_repository *repo, git_remote *remote)
 		memset(&dummy_spec, 0, sizeof(git_refspec));
 		head_info.refspec = &dummy_spec;
 	}
-	
+
 	/* Determine the remote tracking reference name from the local master */
 	if (git_refspec_transform_r(
 		&remote_master_name,
@@ -418,7 +418,7 @@ static bool should_checkout(
 	return !git_repository_head_orphan(repo);
 }
 
-static void normalize_options(git_clone_options *dst, const git_clone_options *src)
+static void normalize_options(git_clone_options *dst, const git_clone_options *src, git_repository_init_options *initOptions)
 {
 	git_clone_options default_options = GIT_CLONE_OPTIONS_INIT;
 	if (!src) src = &default_options;
@@ -427,6 +427,13 @@ static void normalize_options(git_clone_options *dst, const git_clone_options *s
 
 	/* Provide defaults for null pointers */
 	if (!dst->remote_name) dst->remote_name = "origin";
+	if (!dst->init_options)
+	{
+		dst->init_options = initOptions;
+		initOptions->flags = GIT_REPOSITORY_INIT_MKPATH;
+		if (dst->bare)
+			initOptions->flags |= GIT_REPOSITORY_INIT_BARE;
+	}
 }
 
 int git_clone(
@@ -439,10 +446,11 @@ int git_clone(
 	git_repository *repo = NULL;
 	git_clone_options normOptions;
 	int remove_directory_on_failure = 0;
+	git_repository_init_options initOptions = GIT_REPOSITORY_INIT_OPTIONS_INIT;
 
 	assert(out && url && local_path);
 
-	normalize_options(&normOptions, options);
+	normalize_options(&normOptions, options, &initOptions);
 	GITERR_CHECK_VERSION(&normOptions, GIT_CLONE_OPTIONS_VERSION, "git_clone_options");
 
 	/* Only clone to a new directory or an empty directory */
@@ -455,7 +463,7 @@ int git_clone(
 	/* Only remove the directory on failure if we create it */
 	remove_directory_on_failure = !git_path_exists(local_path);
 
-	if (!(retcode = git_repository_init(&repo, local_path, normOptions.bare))) {
+	if (!(retcode = git_repository_init_ext(&repo, local_path, normOptions.init_options))) {
 		if ((retcode = setup_remotes_and_fetch(repo, url, &normOptions)) < 0) {
 			/* Failed to fetch; clean up */
 			git_repository_free(repo);
diff --git a/src/commit.c b/src/commit.c
index 1ab9b34f7..15a195fe5 100644
--- a/src/commit.c
+++ b/src/commit.c
@@ -19,30 +19,19 @@
 
 #include <stdarg.h>
 
-static void clear_parents(git_commit *commit)
-{
-	size_t i;
-
-	for (i = 0; i < commit->parent_ids.length; ++i) {
-		git_oid *parent = git_vector_get(&commit->parent_ids, i);
-		git__free(parent);
-	}
-
-	git_vector_clear(&commit->parent_ids);
-}
-
 void git_commit__free(void *_commit)
 {
 	git_commit *commit = _commit;
 
-	clear_parents(commit);
-	git_vector_free(&commit->parent_ids);
+	git_array_clear(commit->parent_ids);
 
 	git_signature_free(commit->author);
 	git_signature_free(commit->committer);
 
+	git__free(commit->raw_header);
 	git__free(commit->message);
 	git__free(commit->message_encoding);
+
 	git__free(commit);
 }
 
@@ -171,12 +160,35 @@ int git_commit_create(
 int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 {
 	git_commit *commit = _commit;
-	const char *buffer = git_odb_object_data(odb_obj);
-	const char *buffer_end = buffer + git_odb_object_size(odb_obj);
+	const char *buffer_start = git_odb_object_data(odb_obj), *buffer;
+	const char *buffer_end = buffer_start + git_odb_object_size(odb_obj);
 	git_oid parent_id;
+	uint32_t parent_count = 0;
+	size_t header_len;
+
+	/* find end-of-header (counting parents as we go) */
+	for (buffer = buffer_start; buffer < buffer_end; ++buffer) {
+		if (!strncmp("\n\n", buffer, 2)) {
+			++buffer;
+			break;
+		}
+		if (!strncmp("\nparent ", buffer, strlen("\nparent ")))
+			++parent_count;
+	}
 
-	if (git_vector_init(&commit->parent_ids, 4, NULL) < 0)
-		return -1;
+	header_len = buffer - buffer_start;
+	commit->raw_header = git__strndup(buffer_start, header_len);
+	GITERR_CHECK_ALLOC(commit->raw_header);
+
+	/* point "buffer" to header data */
+	buffer = commit->raw_header;
+	buffer_end = commit->raw_header + header_len;
+
+	if (parent_count < 1)
+		parent_count = 1;
+
+	git_array_init_to_size(commit->parent_ids, parent_count);
+	GITERR_CHECK_ARRAY(commit->parent_ids);
 
 	if (git_oid__parse(&commit->tree_id, &buffer, buffer_end, "tree ") < 0)
 		goto bad_buffer;
@@ -186,13 +198,10 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 	 */
 
 	while (git_oid__parse(&parent_id, &buffer, buffer_end, "parent ") == 0) {
-		git_oid *new_id = git__malloc(sizeof(git_oid));
+		git_oid *new_id = git_array_alloc(commit->parent_ids);
 		GITERR_CHECK_ALLOC(new_id);
 
 		git_oid_cpy(new_id, &parent_id);
-
-		if (git_vector_insert(&commit->parent_ids, new_id) < 0)
-			return -1;
 	}
 
 	commit->author = git__malloc(sizeof(git_signature));
@@ -208,8 +217,8 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 	if (git_signature__parse(commit->committer, &buffer, buffer_end, "committer ", '\n') < 0)
 		return -1;
 
-	/* Parse add'l header entries until blank line found */
-	while (buffer < buffer_end && *buffer != '\n') {
+	/* Parse add'l header entries */
+	while (buffer < buffer_end) {
 		const char *eoln = buffer;
 		while (eoln < buffer_end && *eoln != '\n')
 			++eoln;
@@ -223,15 +232,18 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 
 		if (eoln < buffer_end && *eoln == '\n')
 			++eoln;
-
 		buffer = eoln;
 	}
 
-	/* buffer is now at the end of the header, double-check and move forward into the message */
-	if (buffer < buffer_end && *buffer == '\n')
-		buffer++;
+	/* point "buffer" to data after header */
+	buffer = git_odb_object_data(odb_obj);
+	buffer_end = buffer + git_odb_object_size(odb_obj);
+
+	buffer += header_len;
+	if (*buffer == '\n')
+		++buffer;
 
-	/* parse commit message */
+	/* extract commit message */
 	if (buffer <= buffer_end) {
 		commit->message = git__strndup(buffer, buffer_end - buffer);
 		GITERR_CHECK_ALLOC(commit->message);
@@ -255,9 +267,10 @@ GIT_COMMIT_GETTER(const git_signature *, author, commit->author)
 GIT_COMMIT_GETTER(const git_signature *, committer, commit->committer)
 GIT_COMMIT_GETTER(const char *, message, commit->message)
 GIT_COMMIT_GETTER(const char *, message_encoding, commit->message_encoding)
+GIT_COMMIT_GETTER(const char *, raw_header, commit->raw_header)
 GIT_COMMIT_GETTER(git_time_t, time, commit->committer->when.time)
 GIT_COMMIT_GETTER(int, time_offset, commit->committer->when.offset)
-GIT_COMMIT_GETTER(unsigned int, parentcount, (unsigned int)commit->parent_ids.length)
+GIT_COMMIT_GETTER(unsigned int, parentcount, (unsigned int)git_array_size(commit->parent_ids))
 GIT_COMMIT_GETTER(const git_oid *, tree_id, &commit->tree_id);
 
 int git_commit_tree(git_tree **tree_out, const git_commit *commit)
@@ -271,7 +284,7 @@ const git_oid *git_commit_parent_id(
 {
 	assert(commit);
 
-	return git_vector_get(&commit->parent_ids, n);
+	return git_array_get(commit->parent_ids, n);
 }
 
 int git_commit_parent(
diff --git a/src/commit.h b/src/commit.h
index d0981b125..22fc898a1 100644
--- a/src/commit.h
+++ b/src/commit.h
@@ -10,14 +10,14 @@
 #include "git2/commit.h"
 #include "tree.h"
 #include "repository.h"
-#include "vector.h"
+#include "array.h"
 
 #include <time.h>
 
 struct git_commit {
 	git_object object;
 
-	git_vector parent_ids;
+	git_array_t(git_oid) parent_ids;
 	git_oid tree_id;
 
 	git_signature *author;
@@ -25,6 +25,7 @@ struct git_commit {
 
 	char *message_encoding;
 	char *message;
+	char *raw_header;
 };
 
 void git_commit__free(void *commit);
diff --git a/src/commit_list.c b/src/commit_list.c
index bd5b5201a..64416e54d 100644
--- a/src/commit_list.c
+++ b/src/commit_list.c
@@ -36,7 +36,7 @@ git_commit_list *git_commit_list_insert_by_date(git_commit_list_node *item, git_
 	git_commit_list *p;
 
 	while ((p = *pp) != NULL) {
-		if (git_commit_list_time_cmp(p->item, item) < 0)
+		if (git_commit_list_time_cmp(p->item, item) > 0)
 			break;
 
 		pp = &p->next;
diff --git a/src/config.c b/src/config.c
index 068c40260..c98d6a52d 100644
--- a/src/config.c
+++ b/src/config.c
@@ -315,30 +315,241 @@ int git_config_refresh(git_config *cfg)
  * Loop over all the variables
  */
 
+typedef struct {
+	git_config_iterator parent;
+	git_config_iterator *current;
+	const git_config *cfg;
+	regex_t regex;
+	int has_regex;
+	size_t i;
+} all_iter;
+
+static int find_next_backend(size_t *out, const git_config *cfg, size_t i)
+{
+	file_internal *internal;
+
+	for (; i > 0; --i) {
+		internal = git_vector_get(&cfg->files, i - 1);
+		if (!internal || !internal->file)
+			continue;
+
+		*out = i;
+		return 0;
+	}
+
+	return -1;
+}
+
+static int all_iter_next(git_config_entry **entry, git_config_iterator *_iter)
+{
+	all_iter *iter = (all_iter *) _iter;
+	file_internal *internal;
+	git_config_backend *backend;
+	size_t i;
+	int error = 0;
+
+	if (iter->current != NULL &&
+	    (error = iter->current->next(entry, iter->current)) == 0) {
+		return 0;
+	}
+
+	if (error < 0 && error != GIT_ITEROVER)
+		return error;
+
+	do {
+		if (find_next_backend(&i, iter->cfg, iter->i) < 0)
+			return GIT_ITEROVER;
+
+		internal = git_vector_get(&iter->cfg->files, i - 1);
+		backend = internal->file;
+		iter->i = i - 1;
+
+		if (iter->current)
+			iter->current->free(iter->current);
+
+		iter->current = NULL;
+		error = backend->iterator(&iter->current, backend);
+		if (error == GIT_ENOTFOUND)
+			continue;
+
+		if (error < 0)
+			return error;
+
+		error = iter->current->next(entry, iter->current);
+		/* If this backend is empty, then keep going */
+		if (error == GIT_ITEROVER)
+			continue;
+
+		return error;
+
+	} while(1);
+
+	return GIT_ITEROVER;
+}
+
+static int all_iter_glob_next(git_config_entry **entry, git_config_iterator *_iter)
+{
+	int error;
+	all_iter *iter = (all_iter *) _iter;
+
+	/*
+	 * We use the "normal" function to grab the next one across
+	 * backends and then apply the regex
+	 */
+	while ((error = all_iter_next(entry, _iter)) == 0) {
+		/* skip non-matching keys if regexp was provided */
+		if (regexec(&iter->regex, (*entry)->name, 0, NULL, 0) != 0)
+			continue;
+
+		/* and simply return if we like the entry's name */
+		return 0;
+	}
+
+	return error;
+}
+
+static void all_iter_free(git_config_iterator *_iter)
+{
+	all_iter *iter = (all_iter *) _iter;
+
+	if (iter->current)
+		iter->current->free(iter->current);
+
+	git__free(iter);
+}
+
+static void all_iter_glob_free(git_config_iterator *_iter)
+{
+	all_iter *iter = (all_iter *) _iter;
+
+	regfree(&iter->regex);
+	all_iter_free(_iter);
+}
+
+int git_config_iterator_new(git_config_iterator **out, const git_config *cfg)
+{
+	all_iter *iter;
+
+	iter = git__calloc(1, sizeof(all_iter));
+	GITERR_CHECK_ALLOC(iter);
+
+	iter->parent.free = all_iter_free;
+	iter->parent.next = all_iter_next;
+
+	iter->i = cfg->files.length;
+	iter->cfg = cfg;
+
+	*out = (git_config_iterator *) iter;
+
+	return 0;
+}
+
+int git_config_iterator_glob_new(git_config_iterator **out, const git_config *cfg, const char *regexp)
+{
+	all_iter *iter;
+	int result;
+
+	if (regexp == NULL)
+		return git_config_iterator_new(out, cfg);
+
+	iter = git__calloc(1, sizeof(all_iter));
+	GITERR_CHECK_ALLOC(iter);
+
+	if ((result = regcomp(&iter->regex, regexp, REG_EXTENDED)) < 0) {
+		giterr_set_regex(&iter->regex, result);
+		regfree(&iter->regex);
+		return -1;
+	}
+
+	iter->parent.next = all_iter_glob_next;
+	iter->parent.free = all_iter_glob_free;
+	iter->i = cfg->files.length;
+	iter->cfg = cfg;
+
+	*out = (git_config_iterator *) iter;
+
+	return 0;
+}
+
 int git_config_foreach(
 	const git_config *cfg, git_config_foreach_cb cb, void *payload)
 {
 	return git_config_foreach_match(cfg, NULL, cb, payload);
 }
 
+int git_config_backend_foreach_match(
+	git_config_backend *backend,
+	const char *regexp,
+	int (*fn)(const git_config_entry *, void *),
+	void *data)
+{
+	git_config_entry *entry;
+	git_config_iterator* iter;
+	regex_t regex;
+	int result = 0;
+
+	if (regexp != NULL) {
+		if ((result = regcomp(&regex, regexp, REG_EXTENDED)) < 0) {
+			giterr_set_regex(&regex, result);
+			regfree(&regex);
+			return -1;
+		}
+	}
+
+	if ((result = backend->iterator(&iter, backend)) < 0) {
+		iter = NULL;
+		return -1;
+	}
+
+	while(!(iter->next(&entry, iter) < 0)) {
+		/* skip non-matching keys if regexp was provided */
+		if (regexp && regexec(&regex, entry->name, 0, NULL, 0) != 0)
+			continue;
+
+		/* abort iterator on non-zero return value */
+		if (fn(entry, data)) {
+			giterr_clear();
+			result = GIT_EUSER;
+			goto cleanup;
+		}
+	}
+
+cleanup:
+	if (regexp != NULL)
+		regfree(&regex);
+
+	iter->free(iter);
+
+	return result;
+}
+
 int git_config_foreach_match(
 	const git_config *cfg,
 	const char *regexp,
 	git_config_foreach_cb cb,
 	void *payload)
 {
-	int ret = 0;
-	size_t i;
-	file_internal *internal;
-	git_config_backend *file;
+	int error;
+	git_config_iterator *iter;
+	git_config_entry *entry;
 
-	for (i = 0; i < cfg->files.length && ret == 0; ++i) {
-		internal = git_vector_get(&cfg->files, i);
-		file = internal->file;
-		ret = file->foreach(file, regexp, cb, payload);
+	if ((error = git_config_iterator_glob_new(&iter, cfg, regexp)) < 0)
+		return error;
+
+	while ((error = git_config_next(&entry, iter)) == 0) {
+		if(cb(entry, payload)) {
+			giterr_clear();
+			error = GIT_EUSER;
+			break;
+		}
 	}
 
-	return ret;
+	git_config_iterator_free(iter);
+
+	if (error == GIT_ITEROVER)
+		error = 0;
+
+	return error;
 }
 
 /**************
@@ -528,31 +739,114 @@ int git_config_get_entry(const git_config_entry **out, const git_config *cfg, co
 	return config_error_notfound(name);
 }
 
-int git_config_get_multivar(
+int git_config_get_multivar_foreach(
 	const git_config *cfg, const char *name, const char *regexp,
 	git_config_foreach_cb cb, void *payload)
 {
-	file_internal *internal;
-	git_config_backend *file;
-	int ret = GIT_ENOTFOUND;
-	size_t i;
+	int err, found;
+	git_config_iterator *iter;
+	git_config_entry *entry;
+
+	if ((err = git_config_multivar_iterator_new(&iter, cfg, name, regexp)) < 0)
+		return err;
+
+	found = 0;
+	while ((err = iter->next(&entry, iter)) == 0) {
+		found = 1;
+		if(cb(entry, payload)) {
+			iter->free(iter);
+			return GIT_EUSER;
+		}
+	}
 
-	/*
-	 * This loop runs the "wrong" way 'round because we need to
-	 * look at every value from the most general to most specific
-	 */
-	for (i = cfg->files.length; i > 0; --i) {
-		internal = git_vector_get(&cfg->files, i - 1);
-		if (!internal || !internal->file)
+	iter->free(iter);
+	if (err == GIT_ITEROVER)
+		err = 0;
+
+	if (found == 0 && err == 0)
+		err = config_error_notfound(name);
+
+	return err;
+}
+
+typedef struct {
+	git_config_iterator parent;
+	git_config_iterator *iter;
+	char *name;
+	regex_t regex;
+	int have_regex;
+} multivar_iter;
+
+static int multivar_iter_next(git_config_entry **entry, git_config_iterator *_iter)
+{
+	multivar_iter *iter = (multivar_iter *) _iter;
+	int error = 0;
+
+	while ((error = iter->iter->next(entry, iter->iter)) == 0) {
+		if (git__strcmp(iter->name, (*entry)->name))
 			continue;
-		file = internal->file;
 
-		ret = file->get_multivar(file, name, regexp, cb, payload);
-		if (ret < 0 && ret != GIT_ENOTFOUND)
-			return ret;
+		if (!iter->have_regex)
+			return 0;
+
+		if (regexec(&iter->regex, (*entry)->value, 0, NULL, 0) == 0)
+			return 0;
+	}
+
+	return error;
+}
+
+void multivar_iter_free(git_config_iterator *_iter)
+{
+	multivar_iter *iter = (multivar_iter *) _iter;
+
+	iter->iter->free(iter->iter);
+
+	git__free(iter->name);
+	regfree(&iter->regex);
+	git__free(iter);
+}
+
+int git_config_multivar_iterator_new(git_config_iterator **out, const git_config *cfg, const char *name, const char *regexp)
+{
+	multivar_iter *iter = NULL;
+	git_config_iterator *inner = NULL;
+	int error;
+
+	if ((error = git_config_iterator_new(&inner, cfg)) < 0)
+		return error;
+
+	iter = git__calloc(1, sizeof(multivar_iter));
+	GITERR_CHECK_ALLOC(iter);
+
+	if ((error = git_config__normalize_name(name, &iter->name)) < 0)
+		goto on_error;
+
+	if (regexp != NULL) {
+		error = regcomp(&iter->regex, regexp, REG_EXTENDED);
+		if (error < 0) {
+			giterr_set_regex(&iter->regex, error);
+			error = -1;
+			regfree(&iter->regex);
+			goto on_error;
+		}
+
+		iter->have_regex = 1;
 	}
 
-	return (ret == GIT_ENOTFOUND) ? config_error_notfound(name) : 0;
+	iter->iter = inner;
+	iter->parent.free = multivar_iter_free;
+	iter->parent.next = multivar_iter_next;
+
+	*out = (git_config_iterator *) iter;
+
+	return 0;
+
+on_error:
+
+	inner->free(inner);
+	git__free(iter);
+	return error;
 }
 
 int git_config_set_multivar(git_config *cfg, const char *name, const char *regexp, const char *value)
@@ -568,6 +862,16 @@ int git_config_set_multivar(git_config *cfg, const char *name, const char *regex
 	return file->set_multivar(file, name, regexp, value);
 }
 
+int git_config_next(git_config_entry **entry, git_config_iterator *iter)
+{
+	return iter->next(entry, iter);
+}
+
+void git_config_iterator_free(git_config_iterator *iter)
+{
+	iter->free(iter);
+}
+
 static int git_config__find_file_to_path(
 	char *out, size_t outlen, int (*find)(git_buf *buf))
 {
@@ -811,6 +1115,41 @@ fail_parse:
 	return -1;
 }
 
+/* Take something the user gave us and make it nice for our hash function */
+int git_config__normalize_name(const char *in, char **out)
+{
+	char *name, *fdot, *ldot;
+
+	assert(in && out);
+
+	name = git__strdup(in);
+	GITERR_CHECK_ALLOC(name);
+
+	fdot = strchr(name, '.');
+	ldot = strrchr(name, '.');
+
+	if (fdot == NULL || fdot == name || ldot == NULL || !ldot[1])
+		goto invalid;
+
+	/* Validate and downcase up to first dot and after last dot */
+	if (git_config_file_normalize_section(name, fdot) < 0 ||
+		git_config_file_normalize_section(ldot + 1, NULL) < 0)
+		goto invalid;
+
+	/* If there is a middle range, make sure it doesn't have newlines */
+	while (fdot < ldot)
+		if (*fdot++ == '\n')
+			goto invalid;
+
+	*out = name;
+	return 0;
+
+invalid:
+	git__free(name);
+	giterr_set(GITERR_CONFIG, "Invalid config item name '%s'", in);
+	return GIT_EINVALIDSPEC;
+}
+
 struct rename_data {
 	git_config *config;
 	git_buf *name;
diff --git a/src/config.h b/src/config.h
index c5c11ae14..85db5e3e1 100644
--- a/src/config.h
+++ b/src/config.h
@@ -49,4 +49,7 @@ extern int git_config_rename_section(
  */
 extern int git_config_file__ondisk(struct git_config_backend **out, const char *path);
 
+extern int git_config__normalize_name(const char *in, char **out);
+
+
 #endif
diff --git a/src/config_file.c b/src/config_file.c
index dec952115..efc9df965 100644
--- a/src/config_file.c
+++ b/src/config_file.c
@@ -27,6 +27,13 @@ typedef struct cvar_t {
 	git_config_entry *entry;
 } cvar_t;
 
+typedef struct git_config_file_iter {
+	git_config_iterator parent;
+	git_strmap_iter iter;
+	cvar_t* next_var;
+} git_config_file_iter;
+
+
 #define CVAR_LIST_HEAD(list) ((list)->head)
 
 #define CVAR_LIST_TAIL(list) ((list)->tail)
@@ -129,41 +136,6 @@ int git_config_file_normalize_section(char *start, char *end)
 	return 0;
 }
 
-/* Take something the user gave us and make it nice for our hash function */
-static int normalize_name(const char *in, char **out)
-{
-	char *name, *fdot, *ldot;
-
-	assert(in && out);
-
-	name = git__strdup(in);
-	GITERR_CHECK_ALLOC(name);
-
-	fdot = strchr(name, '.');
-	ldot = strrchr(name, '.');
-
-	if (fdot == NULL || fdot == name || ldot == NULL || !ldot[1])
-		goto invalid;
-
-	/* Validate and downcase up to first dot and after last dot */
-	if (git_config_file_normalize_section(name, fdot) < 0 ||
-		git_config_file_normalize_section(ldot + 1, NULL) < 0)
-		goto invalid;
-
-	/* If there is a middle range, make sure it doesn't have newlines */
-	while (fdot < ldot)
-		if (*fdot++ == '\n')
-			goto invalid;
-
-	*out = name;
-	return 0;
-
-invalid:
-	git__free(name);
-	giterr_set(GITERR_CONFIG, "Invalid config item name '%s'", in);
-	return GIT_EINVALIDSPEC;
-}
-
 static void free_vars(git_strmap *values)
 {
 	cvar_t *var = NULL;
@@ -247,51 +219,56 @@ static void backend_free(git_config_backend *_backend)
 	git__free(backend);
 }
 
-static int file_foreach(
-	git_config_backend *backend,
-	const char *regexp,
-	int (*fn)(const git_config_entry *, void *),
-	void *data)
+static void config_iterator_free(
+	git_config_iterator* iter)
 {
-	diskfile_backend *b = (diskfile_backend *)backend;
-	cvar_t *var, *next_var;
-	const char *key;
-	regex_t regex;
-	int result = 0;
+	git__free(iter);
+}
 
-	if (!b->values)
-		return 0;
+static int config_iterator_next(
+	git_config_entry **entry,
+	git_config_iterator *iter)
+{
+	git_config_file_iter *it = (git_config_file_iter *) iter;
+	diskfile_backend *b = (diskfile_backend *) it->parent.backend;
+	int err = 0;
+	cvar_t * var;
 
-	if (regexp != NULL) {
-		if ((result = regcomp(&regex, regexp, REG_EXTENDED)) < 0) {
-			giterr_set_regex(&regex, result);
-			regfree(&regex);
-			return -1;
-		}
+	if (it->next_var == NULL) {
+		err = git_strmap_next((void**) &var, &(it->iter), b->values);
+	} else {
+		var = it->next_var;
 	}
 
-	git_strmap_foreach(b->values, key, var,
-		for (; var != NULL; var = next_var) {
-			next_var = CVAR_LIST_NEXT(var);
+	if (err < 0) {
+		it->next_var = NULL;
+		return err;
+	}
 
-			/* skip non-matching keys if regexp was provided */
-			if (regexp && regexec(&regex, key, 0, NULL, 0) != 0)
-				continue;
+	*entry = var->entry;
+	it->next_var = CVAR_LIST_NEXT(var);
 
-			/* abort iterator on non-zero return value */
-			if (fn(var->entry, data)) {
-				giterr_clear();
-				result = GIT_EUSER;
-				goto cleanup;
-			}
-		}
-	);
+	return 0;
+}
 
-cleanup:
-	if (regexp != NULL)
-		regfree(&regex);
+static int config_iterator_new(
+	git_config_iterator **iter,
+	struct git_config_backend* backend)
+{
+	diskfile_backend *b = (diskfile_backend *)backend;
+	git_config_file_iter *it = git__calloc(1, sizeof(git_config_file_iter));
 
-	return result;
+	GITERR_CHECK_ALLOC(it);
+
+	it->parent.backend = backend;
+	it->iter = git_strmap_begin(b->values);
+	it->next_var = NULL;
+
+	it->parent.next = config_iterator_next;
+	it->parent.free = config_iterator_free;
+	*iter = (git_config_iterator *) it;
+
+	return 0;
 }
 
 static int config_set(git_config_backend *cfg, const char *name, const char *value)
@@ -302,7 +279,7 @@ static int config_set(git_config_backend *cfg, const char *name, const char *val
 	khiter_t pos;
 	int rval, ret;
 
-	if ((rval = normalize_name(name, &key)) < 0)
+	if ((rval = git_config__normalize_name(name, &key)) < 0)
 		return rval;
 
 	/*
@@ -385,7 +362,7 @@ static int config_get(const git_config_backend *cfg, const char *name, const git
 	khiter_t pos;
 	int error;
 
-	if ((error = normalize_name(name, &key)) < 0)
+	if ((error = git_config__normalize_name(name, &key)) < 0)
 		return error;
 
 	pos = git_strmap_lookup_index(b->values, key);
@@ -400,70 +377,6 @@ static int config_get(const git_config_backend *cfg, const char *name, const git
 	return 0;
 }
 
-static int config_get_multivar(
-	git_config_backend *cfg,
-	const char *name,
-	const char *regex_str,
-	int (*fn)(const git_config_entry *, void *),
-	void *data)
-{
-	cvar_t *var;
-	diskfile_backend *b = (diskfile_backend *)cfg;
-	char *key;
-	khiter_t pos;
-	int error;
-
-	if ((error = normalize_name(name, &key)) < 0)
-		return error;
-
-	pos = git_strmap_lookup_index(b->values, key);
-	git__free(key);
-
-	if (!git_strmap_valid_index(b->values, pos))
-		return GIT_ENOTFOUND;
-
-	var = git_strmap_value_at(b->values, pos);
-
-	if (regex_str != NULL) {
-		regex_t regex;
-		int result;
-
-		/* regex matching; build the regex */
-		result = regcomp(&regex, regex_str, REG_EXTENDED);
-		if (result < 0) {
-			giterr_set_regex(&regex, result);
-			regfree(&regex);
-			return -1;
-		}
-
-		/* and throw the callback only on the variables that
-		 * match the regex */
-		do {
-			if (regexec(&regex, var->entry->value, 0, NULL, 0) == 0) {
-				/* early termination by the user is not an error;
-				 * just break and return successfully */
-				if (fn(var->entry, data) < 0)
-					break;
-			}
-
-			var = var->next;
-		} while (var != NULL);
-		regfree(&regex);
-	} else {
-		/* no regex; go through all the variables */
-		do {
-			/* early termination by the user is not an error;
-			 * just break and return successfully */
-			if (fn(var->entry, data) < 0)
-				break;
-
-			var = var->next;
-		} while (var != NULL);
-	}
-
-	return 0;
-}
-
 static int config_set_multivar(
 	git_config_backend *cfg, const char *name, const char *regexp, const char *value)
 {
@@ -477,7 +390,7 @@ static int config_set_multivar(
 
 	assert(regexp);
 
-	if ((result = normalize_name(name, &key)) < 0)
+	if ((result = git_config__normalize_name(name, &key)) < 0)
 		return result;
 
 	pos = git_strmap_lookup_index(b->values, key);
@@ -550,7 +463,7 @@ static int config_delete(git_config_backend *cfg, const char *name)
 	int result;
 	khiter_t pos;
 
-	if ((result = normalize_name(name, &key)) < 0)
+	if ((result = git_config__normalize_name(name, &key)) < 0)
 		return result;
 
 	pos = git_strmap_lookup_index(b->values, key);
@@ -590,11 +503,10 @@ int git_config_file__ondisk(git_config_backend **out, const char *path)
 
 	backend->parent.open = config_open;
 	backend->parent.get = config_get;
-	backend->parent.get_multivar = config_get_multivar;
 	backend->parent.set = config_set;
 	backend->parent.set_multivar = config_set_multivar;
 	backend->parent.del = config_delete;
-	backend->parent.foreach = file_foreach;
+	backend->parent.iterator = config_iterator_new;
 	backend->parent.refresh = config_refresh;
 	backend->parent.free = backend_free;
 
@@ -792,6 +704,11 @@ static int parse_section_header_ext(diskfile_backend *cfg, const char *line, con
 		}
 
 		switch (c) {
+		case 0:
+			set_parse_error(cfg, 0, "Unexpected end-of-line in section header");
+			git_buf_free(&buf);
+			return -1;
+
 		case '"':
 			++quote_marks;
 			continue;
@@ -801,6 +718,12 @@ static int parse_section_header_ext(diskfile_backend *cfg, const char *line, con
 
 			switch (c) {
 			case '"':
+				if (&line[rpos-1] == last_quote) {
+					set_parse_error(cfg, 0, "Missing closing quotation mark in section header");
+					git_buf_free(&buf);
+					return -1;
+				}
+
 			case '\\':
 				break;
 
@@ -1293,6 +1216,9 @@ static char *escape_value(const char *ptr)
 	assert(ptr);
 
 	len = strlen(ptr);
+	if (!len)
+		return git__calloc(1, sizeof(char));
+
 	git_buf_grow(&buf, len);
 
 	while (*ptr != '\0') {
@@ -1395,7 +1321,7 @@ static int parse_multiline_variable(diskfile_backend *cfg, git_buf *value, int i
 	 * standard, this character **has** to be last one in the buf, with
 	 * no whitespace after it */
 	assert(is_multiline_var(value->ptr));
-	git_buf_truncate(value, git_buf_len(value) - 1);
+	git_buf_shorten(value, 1);
 
 	proc_line = fixup_line(line, in_quotes);
 	if (proc_line == NULL) {
diff --git a/src/config_file.h b/src/config_file.h
index 7445859c4..d4a1a4061 100644
--- a/src/config_file.h
+++ b/src/config_file.h
@@ -42,7 +42,7 @@ GIT_INLINE(int) git_config_file_foreach(
 	int (*fn)(const git_config_entry *entry, void *data),
 	void *data)
 {
-	return cfg->foreach(cfg, NULL, fn, data);
+	return git_config_backend_foreach_match(cfg, NULL, fn, data);
 }
 
 GIT_INLINE(int) git_config_file_foreach_match(
@@ -51,7 +51,7 @@ GIT_INLINE(int) git_config_file_foreach_match(
 	int (*fn)(const git_config_entry *entry, void *data),
 	void *data)
 {
-	return cfg->foreach(cfg, regexp, fn, data);
+	return git_config_backend_foreach_match(cfg, regexp, fn, data);
 }
 
 extern int git_config_file_normalize_section(char *start, char *end);
diff --git a/src/date.c b/src/date.c
index 48841e4f9..7849c2f02 100644
--- a/src/date.c
+++ b/src/date.c
@@ -823,15 +823,13 @@ static void pending_number(struct tm *tm, int *num)
 }
 
 static git_time_t approxidate_str(const char *date,
-									const struct timeval *tv,
-									int *error_ret)
+	time_t time_sec,
+	int *error_ret)
 {
 	int number = 0;
 	int touched = 0;
 	struct tm tm = {0}, now;
-	time_t time_sec;
 
-	time_sec = tv->tv_sec;
 	p_localtime_r(&time_sec, &tm);
 	now = tm;
 
@@ -861,7 +859,7 @@ static git_time_t approxidate_str(const char *date,
 
 int git__date_parse(git_time_t *out, const char *date)
 {
-	struct timeval tv;
+	time_t time_sec;
 	git_time_t timestamp;
 	int offset, error_ret=0;
 
@@ -870,7 +868,9 @@ int git__date_parse(git_time_t *out, const char *date)
 		return 0;
 	}
 
-	p_gettimeofday(&tv, NULL);
-	*out = approxidate_str(date, &tv, &error_ret);
+	if (time(&time_sec) == -1)
+		return -1;
+
+	*out = approxidate_str(date, time_sec, &error_ret);
    return error_ret;
 }
diff --git a/src/diff.c b/src/diff.c
index 3bfe149e3..77dbbd8bc 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -13,6 +13,7 @@
 #include "pathspec.h"
 #include "index.h"
 #include "odb.h"
+#include "submodule.h"
 
 #define DIFF_FLAG_IS_SET(DIFF,FLAG) (((DIFF)->opts.flags & (FLAG)) != 0)
 #define DIFF_FLAG_ISNT_SET(DIFF,FLAG) (((DIFF)->opts.flags & (FLAG)) == 0)
@@ -77,15 +78,11 @@ static int diff_delta__from_one(
 		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_UNTRACKED))
 		return 0;
 
-	if (entry->mode == GIT_FILEMODE_COMMIT &&
-		DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES))
-		return 0;
-
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			&diff->pathspec, entry->path,
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE),
-			&matched_pathspec))
+			&matched_pathspec, NULL))
 		return 0;
 
 	delta = diff_delta__alloc(diff, status, entry->path);
@@ -134,16 +131,12 @@ static int diff_delta__from_two(
 {
 	git_diff_delta *delta;
 	int notify_res;
+	const char *canonical_path = old_entry->path;
 
 	if (status == GIT_DELTA_UNMODIFIED &&
 		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_UNMODIFIED))
 		return 0;
 
-	if (old_entry->mode == GIT_FILEMODE_COMMIT &&
-		new_entry->mode == GIT_FILEMODE_COMMIT &&
-		DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES))
-		return 0;
-
 	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE)) {
 		uint32_t temp_mode = old_mode;
 		const git_index_entry *temp_entry = old_entry;
@@ -153,7 +146,7 @@ static int diff_delta__from_two(
 		new_mode = temp_mode;
 	}
 
-	delta = diff_delta__alloc(diff, status, old_entry->path);
+	delta = diff_delta__alloc(diff, status, canonical_path);
 	GITERR_CHECK_ALLOC(delta);
 
 	git_oid_cpy(&delta->old_file.oid, &old_entry->oid);
@@ -246,6 +239,11 @@ GIT_INLINE(const char *) diff_delta__path(const git_diff_delta *delta)
 	return str;
 }
 
+const char *git_diff_delta__path(const git_diff_delta *delta)
+{
+	return diff_delta__path(delta);
+}
+
 int git_diff_delta__cmp(const void *a, const void *b)
 {
 	const git_diff_delta *da = a, *db = b;
@@ -253,6 +251,33 @@ int git_diff_delta__cmp(const void *a, const void *b)
 	return val ? val : ((int)da->status - (int)db->status);
 }
 
+int git_diff_delta__casecmp(const void *a, const void *b)
+{
+	const git_diff_delta *da = a, *db = b;
+	int val = strcasecmp(diff_delta__path(da), diff_delta__path(db));
+	return val ? val : ((int)da->status - (int)db->status);
+}
+
+GIT_INLINE(const char *) diff_delta__i2w_path(const git_diff_delta *delta)
+{
+	return delta->old_file.path ?
+		delta->old_file.path : delta->new_file.path;
+}
+
+int git_diff_delta__i2w_cmp(const void *a, const void *b)
+{
+	const git_diff_delta *da = a, *db = b;
+	int val = strcmp(diff_delta__i2w_path(da), diff_delta__i2w_path(db));
+	return val ? val : ((int)da->status - (int)db->status);
+}
+
+int git_diff_delta__i2w_casecmp(const void *a, const void *b)
+{
+	const git_diff_delta *da = a, *db = b;
+	int val = strcasecmp(diff_delta__i2w_path(da), diff_delta__i2w_path(db));
+	return val ? val : ((int)da->status - (int)db->status);
+}
+
 bool git_diff_delta__should_skip(
 	const git_diff_options *opts, const git_diff_delta *delta)
 {
@@ -356,6 +381,8 @@ static git_diff_list *diff_list_alloc(
 		diff->strncomp   = git__strncasecmp;
 		diff->pfxcomp    = git__prefixcmp_icase;
 		diff->entrycomp  = git_index_entry__cmp_icase;
+
+		git_vector_set_cmp(&diff->deltas, git_diff_delta__casecmp);
 	}
 
 	return diff;
@@ -377,7 +404,7 @@ static int diff_list_apply_options(
 		DIFF_FLAG_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE, icase);
 
 		/* initialize pathspec from options */
-		if (git_pathspec_init(&diff->pathspec, &opts->pathspec, pool) < 0)
+		if (git_pathspec__vinit(&diff->pathspec, &opts->pathspec, pool) < 0)
 			return -1;
 	}
 
@@ -415,8 +442,18 @@ static int diff_list_apply_options(
 	if (!opts) {
 		diff->opts.context_lines = config_int(cfg, "diff.context", 3);
 
-		if (config_bool(cfg, "diff.ignoreSubmodules", 0))
-			diff->opts.flags |= GIT_DIFF_IGNORE_SUBMODULES;
+		/* add other defaults here */
+	}
+
+	/* if ignore_submodules not explicitly set, check diff config */
+	if (diff->opts.ignore_submodules <= 0) {
+		const char *str;
+
+		if (git_config_get_string(&str , cfg, "diff.ignoreSubmodules") < 0)
+			giterr_clear();
+		else if (str != NULL &&
+			git_submodule_parse_ignore(&diff->opts.ignore_submodules, str) < 0)
+			giterr_clear();
 	}
 
 	/* if either prefix is not set, figure out appropriate value */
@@ -463,7 +500,7 @@ static void diff_list_free(git_diff_list *diff)
 	}
 	git_vector_free(&diff->deltas);
 
-	git_pathspec_free(&diff->pathspec);
+	git_pathspec__vfree(&diff->pathspec);
 	git_pool_clear(&diff->pool);
 
 	git__memzero(diff, sizeof(*diff));
@@ -580,35 +617,44 @@ static int maybe_modified_submodule(
 	int error = 0;
 	git_submodule *sub;
 	unsigned int sm_status = 0;
-	const git_oid *sm_oid;
+	git_submodule_ignore_t ign = diff->opts.ignore_submodules;
 
 	*status = GIT_DELTA_UNMODIFIED;
 
-	if (!DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES) &&
-		!(error = git_submodule_lookup(
-			  &sub, diff->repo, info->nitem->path)) &&
-		git_submodule_ignore(sub) != GIT_SUBMODULE_IGNORE_ALL &&
-		!(error = git_submodule_status(&sm_status, sub)))
-	{
-		/* check IS_WD_UNMODIFIED because this case is only used
-		 * when the new side of the diff is the working directory
-		 */
-		if (!GIT_SUBMODULE_STATUS_IS_WD_UNMODIFIED(sm_status))
-			*status = GIT_DELTA_MODIFIED;
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES) ||
+		ign == GIT_SUBMODULE_IGNORE_ALL)
+		return 0;
 
-		/* grab OID while we are here */
-		if (git_oid_iszero(&info->nitem->oid) &&
-			(sm_oid = git_submodule_wd_id(sub)) != NULL)
-			git_oid_cpy(found_oid, sm_oid);
-	}
+	if ((error = git_submodule_lookup(
+			&sub, diff->repo, info->nitem->path)) < 0) {
 
-	/* GIT_EEXISTS means a dir with .git in it was found - ignore it */
-	if (error == GIT_EEXISTS) {
-		giterr_clear();
-		error = 0;
+		/* GIT_EEXISTS means dir with .git in it was found - ignore it */
+		if (error == GIT_EEXISTS) {
+			giterr_clear();
+			error = 0;
+		}
+		return error;
 	}
 
-	return error;
+	if (ign <= 0 && git_submodule_ignore(sub) == GIT_SUBMODULE_IGNORE_ALL)
+		return 0;
+
+	if ((error = git_submodule__status(
+			&sm_status, NULL, NULL, found_oid, sub, ign)) < 0)
+		return error;
+
+	/* check IS_WD_UNMODIFIED because this case is only used
+	 * when the new side of the diff is the working directory
+	 */
+	if (!GIT_SUBMODULE_STATUS_IS_WD_UNMODIFIED(sm_status))
+		*status = GIT_DELTA_MODIFIED;
+
+	/* now that we have a HEAD OID, check if HEAD moved */
+	if ((sm_status & GIT_SUBMODULE_STATUS_IN_WD) != 0 &&
+		!git_oid_equal(&info->oitem->oid, found_oid))
+		*status = GIT_DELTA_MODIFIED;
+
+	return 0;
 }
 
 static int maybe_modified(
@@ -624,11 +670,11 @@ static int maybe_modified(
 	bool new_is_workdir = (info->new_iter->type == GIT_ITERATOR_TYPE_WORKDIR);
 	const char *matched_pathspec;
 
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			&diff->pathspec, oitem->path,
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE),
-			&matched_pathspec))
+			&matched_pathspec, NULL))
 		return 0;
 
 	memset(&noid, 0, sizeof(noid));
@@ -665,8 +711,10 @@ static int maybe_modified(
 		}
 	}
 
-	/* if oids and modes match, then file is unmodified */
-	else if (git_oid_equal(&oitem->oid, &nitem->oid) && omode == nmode)
+	/* if oids and modes match (and are valid), then file is unmodified */
+	else if (git_oid_equal(&oitem->oid, &nitem->oid) &&
+			 omode == nmode &&
+			 !git_oid_iszero(&oitem->oid))
 		status = GIT_DELTA_UNMODIFIED;
 
 	/* if we have an unknown OID and a workdir iterator, then check some
@@ -707,7 +755,7 @@ static int maybe_modified(
 	/* if we got here and decided that the files are modified, but we
 	 * haven't calculated the OID of the new item, then calculate it now
 	 */
-	if (status != GIT_DELTA_UNMODIFIED && git_oid_iszero(&nitem->oid)) {
+	if (status == GIT_DELTA_MODIFIED && git_oid_iszero(&nitem->oid)) {
 		if (git_oid_iszero(&noid)) {
 			if (git_diff__oid_for_file(diff->repo,
 					nitem->path, nitem->mode, nitem->file_size, &noid) < 0)
@@ -774,10 +822,15 @@ static int diff_scan_inside_untracked_dir(
 
 		/* need to recurse into non-ignored directories */
 		if (!is_ignored && S_ISDIR(info->nitem->mode)) {
-			if ((error = git_iterator_advance_into(
-					 &info->nitem, info->new_iter)) < 0)
-				break;
-			continue;
+			error = git_iterator_advance_into(&info->nitem, info->new_iter);
+
+			if (!error)
+				continue;
+			else if (error == GIT_ENOTFOUND) {
+				error = 0;
+				is_ignored = true; /* treat empty as ignored */
+			} else
+				break; /* real error, must stop */
 		}
 
 		/* found a non-ignored item - treat parent dir as untracked */
@@ -825,7 +878,7 @@ static int handle_unmatched_new_item(
 			git_buf_clear(&info->ignore_prefix);
 	}
 
-	if (S_ISDIR(nitem->mode)) {
+	if (nitem->mode == GIT_FILEMODE_TREE) {
 		bool recurse_into_dir = contains_oitem;
 
 		/* if not already inside an ignored dir, check if this is ignored */
@@ -929,6 +982,16 @@ static int handle_unmatched_new_item(
 	else if (info->new_iter->type != GIT_ITERATOR_TYPE_WORKDIR)
 		delta_type = GIT_DELTA_ADDED;
 
+	else if (nitem->mode == GIT_FILEMODE_COMMIT) {
+		git_submodule *sm;
+
+		/* ignore things that are not actual submodules */
+		if (git_submodule_lookup(&sm, info->repo, nitem->path) != 0) {
+			giterr_clear();
+			delta_type = GIT_DELTA_IGNORED;
+		}
+	}
+
 	/* Actually create the record for this item if necessary */
 	if ((error = diff_delta__from_one(diff, delta_type, nitem)) < 0)
 		return error;
@@ -1119,17 +1182,40 @@ int git_diff_tree_to_index(
 	const git_diff_options *opts)
 {
 	int error = 0;
+	bool reset_index_ignore_case = false;
 
 	assert(diff && repo);
 
 	if (!index && (error = git_repository_index__weakptr(&index, repo)) < 0)
 		return error;
 
+	if (index->ignore_case) {
+		git_index__set_ignore_case(index, false);
+		reset_index_ignore_case = true;
+	}
+
 	DIFF_FROM_ITERATORS(
 		git_iterator_for_tree(&a, old_tree, 0, pfx, pfx),
 		git_iterator_for_index(&b, index, 0, pfx, pfx)
 	);
 
+	if (reset_index_ignore_case) {
+		git_index__set_ignore_case(index, true);
+
+		if (!error) {
+			git_diff_list *d = *diff;
+
+			d->opts.flags |= GIT_DIFF_DELTAS_ARE_ICASE;
+			d->strcomp    = git__strcasecmp;
+			d->strncomp   = git__strncasecmp;
+			d->pfxcomp    = git__prefixcmp_icase;
+			d->entrycomp  = git_index_entry__cmp_icase;
+
+			git_vector_set_cmp(&d->deltas, git_diff_delta__casecmp);
+			git_vector_sort(&d->deltas);
+		}
+	}
+
 	return error;
 }
 
@@ -1195,52 +1281,99 @@ size_t git_diff_num_deltas_of_type(git_diff_list *diff, git_delta_t type)
 	return count;
 }
 
+int git_diff_is_sorted_icase(const git_diff_list *diff)
+{
+	return (diff->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0;
+}
+
 int git_diff__paired_foreach(
-	git_diff_list *idx2head,
-	git_diff_list *wd2idx,
-	int (*cb)(git_diff_delta *i2h, git_diff_delta *w2i, void *payload),
+	git_diff_list *head2idx,
+	git_diff_list *idx2wd,
+	int (*cb)(git_diff_delta *h2i, git_diff_delta *i2w, void *payload),
 	void *payload)
 {
 	int cmp;
-	git_diff_delta *i2h, *w2i;
+	git_diff_delta *h2i, *i2w;
 	size_t i, j, i_max, j_max;
-	int (*strcomp)(const char *, const char *);
+	int (*strcomp)(const char *, const char *) = git__strcmp;
+	bool h2i_icase, i2w_icase, icase_mismatch;
+
+	i_max = head2idx ? head2idx->deltas.length : 0;
+	j_max = idx2wd ? idx2wd->deltas.length : 0;
+	if (!i_max && !j_max)
+		return 0;
+
+	/* At some point, tree-to-index diffs will probably never ignore case,
+	 * even if that isn't true now.  Index-to-workdir diffs may or may not
+	 * ignore case, but the index filename for the idx2wd diff should
+	 * still be using the canonical case-preserving name.
+	 *
+	 * Therefore the main thing we need to do here is make sure the diffs
+	 * are traversed in a compatible order.  To do this, we temporarily
+	 * resort a mismatched diff to get the order correct.
+	 *
+	 * In order to traverse renames in the index->workdir, we need to
+	 * ensure that we compare the index name on both sides, so we
+	 * always sort by the old name in the i2w list.
+	 */
+	h2i_icase = head2idx != NULL &&
+		(head2idx->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0;
+
+	i2w_icase = idx2wd != NULL &&
+		(idx2wd->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0;
+
+	icase_mismatch =
+		(head2idx != NULL && idx2wd != NULL && h2i_icase != i2w_icase);
 
-	i_max = idx2head ? idx2head->deltas.length : 0;
-	j_max = wd2idx   ? wd2idx->deltas.length   : 0;
+	if (icase_mismatch && h2i_icase) {
+		git_vector_set_cmp(&head2idx->deltas, git_diff_delta__cmp);
+		git_vector_sort(&head2idx->deltas);
+	}
 
-	/* Get appropriate strcmp function */
-	strcomp = idx2head ? idx2head->strcomp : wd2idx ? wd2idx->strcomp : NULL;
+	if (i2w_icase && !icase_mismatch) {
+		strcomp = git__strcasecmp;
 
-	/* Assert both iterators use matching ignore-case. If this function ever
-	* supports merging diffs that are not sorted by the same function, then
-	* it will need to spool and sort on one of the results before merging
-	*/
-	if (idx2head && wd2idx) {
-		assert(idx2head->strcomp == wd2idx->strcomp);
+		git_vector_set_cmp(&idx2wd->deltas, git_diff_delta__i2w_casecmp);
+		git_vector_sort(&idx2wd->deltas);
+	} else if (idx2wd != NULL) {
+		git_vector_set_cmp(&idx2wd->deltas, git_diff_delta__i2w_cmp);
+		git_vector_sort(&idx2wd->deltas);
 	}
 
 	for (i = 0, j = 0; i < i_max || j < j_max; ) {
-		i2h = idx2head ? GIT_VECTOR_GET(&idx2head->deltas,i) : NULL;
-		w2i = wd2idx   ? GIT_VECTOR_GET(&wd2idx->deltas,j)   : NULL;
+		h2i = head2idx ? GIT_VECTOR_GET(&head2idx->deltas, i) : NULL;
+		i2w = idx2wd ? GIT_VECTOR_GET(&idx2wd->deltas, j) : NULL;
 
-		cmp = !w2i ? -1 : !i2h ? 1 :
-			strcomp(i2h->old_file.path, w2i->old_file.path);
+		cmp = !i2w ? -1 : !h2i ? 1 :
+			strcomp(h2i->new_file.path, i2w->old_file.path);
 
 		if (cmp < 0) {
-			if (cb(i2h, NULL, payload))
+			if (cb(h2i, NULL, payload))
 				return GIT_EUSER;
 			i++;
 		} else if (cmp > 0) {
-			if (cb(NULL, w2i, payload))
+			if (cb(NULL, i2w, payload))
 				return GIT_EUSER;
 			j++;
 		} else {
-			if (cb(i2h, w2i, payload))
+			if (cb(h2i, i2w, payload))
 				return GIT_EUSER;
 			i++; j++;
 		}
 	}
 
+	/* restore case-insensitive delta sort */
+	if (icase_mismatch && h2i_icase) {
+		git_vector_set_cmp(&head2idx->deltas, git_diff_delta__casecmp);
+		git_vector_sort(&head2idx->deltas);
+	}
+
+	/* restore idx2wd sort by new path */
+	if (idx2wd != NULL) {
+		git_vector_set_cmp(&idx2wd->deltas,
+			i2w_icase ? git_diff_delta__casecmp : git_diff_delta__cmp);
+		git_vector_sort(&idx2wd->deltas);
+	}
+
 	return 0;
 }
diff --git a/src/diff.h b/src/diff.h
index ad12e7731..bec7e27d7 100644
--- a/src/diff.h
+++ b/src/diff.h
@@ -16,6 +16,7 @@
 #include "iterator.h"
 #include "repository.h"
 #include "pool.h"
+#include "odb.h"
 
 #define DIFF_OLD_PREFIX_DEFAULT "a/"
 #define DIFF_NEW_PREFIX_DEFAULT "b/"
@@ -74,10 +75,20 @@ extern void git_diff__cleanup_modes(
 extern void git_diff_list_addref(git_diff_list *diff);
 
 extern int git_diff_delta__cmp(const void *a, const void *b);
+extern int git_diff_delta__casecmp(const void *a, const void *b);
+
+extern const char *git_diff_delta__path(const git_diff_delta *delta);
 
 extern bool git_diff_delta__should_skip(
 	const git_diff_options *opts, const git_diff_delta *delta);
 
+extern int git_diff_delta__format_file_header(
+	git_buf *out,
+	const git_diff_delta *delta,
+	const char *oldpfx,
+	const char *newpfx,
+	int oid_strlen);
+
 extern int git_diff__oid_for_file(
 	git_repository *, const char *, uint16_t, git_off_t, git_oid *);
 
@@ -94,17 +105,44 @@ extern int git_diff__paired_foreach(
 	int (*cb)(git_diff_delta *i2h, git_diff_delta *w2i, void *payload),
 	void *payload);
 
-int git_diff_find_similar__hashsig_for_file(
+extern int git_diff_find_similar__hashsig_for_file(
 	void **out, const git_diff_file *f, const char *path, void *p);
 
-int git_diff_find_similar__hashsig_for_buf(
+extern int git_diff_find_similar__hashsig_for_buf(
 	void **out, const git_diff_file *f, const char *buf, size_t len, void *p);
 
-void git_diff_find_similar__hashsig_free(void *sig, void *payload);
+extern void git_diff_find_similar__hashsig_free(void *sig, void *payload);
 
-int git_diff_find_similar__calc_similarity(
+extern int git_diff_find_similar__calc_similarity(
 	int *score, void *siga, void *sigb, void *payload);
 
+/*
+ * Sometimes a git_diff_file will have a zero size; this attempts to
+ * fill in the size without loading the blob if possible.  If that is
+ * not possible, then it will return the git_odb_object that had to be
+ * loaded and the caller can use it or dispose of it as needed.
+ */
+GIT_INLINE(int) git_diff_file__resolve_zero_size(
+	git_diff_file *file, git_odb_object **odb_obj, git_repository *repo)
+{
+	int error;
+	git_odb *odb;
+	size_t len;
+	git_otype type;
+
+	if ((error = git_repository_odb(&odb, repo)) < 0)
+		return error;
+
+	error = git_odb__read_header_or_object(
+		odb_obj, &len, &type, odb, &file->oid);
+
+	git_odb_free(odb);
+
+	if (!error)
+		file->size = (git_off_t)len;
+
+	return error;
+}
 
 #endif
 
diff --git a/src/diff_driver.c b/src/diff_driver.c
index 469be0d14..bd5a8fbd9 100644
--- a/src/diff_driver.c
+++ b/src/diff_driver.c
@@ -187,7 +187,7 @@ static int git_diff_driver_load(
 
 	git_buf_truncate(&name, namelen + strlen("diff.."));
 	git_buf_put(&name, "xfuncname", strlen("xfuncname"));
-	if ((error = git_config_get_multivar(
+	if ((error = git_config_get_multivar_foreach(
 			cfg, name.ptr, NULL, diff_driver_xfuncname, drv)) < 0) {
 		if (error != GIT_ENOTFOUND)
 			goto done;
@@ -196,7 +196,7 @@ static int git_diff_driver_load(
 
 	git_buf_truncate(&name, namelen + strlen("diff.."));
 	git_buf_put(&name, "funcname", strlen("funcname"));
-	if ((error = git_config_get_multivar(
+	if ((error = git_config_get_multivar_foreach(
 			cfg, name.ptr, NULL, diff_driver_funcname, drv)) < 0) {
 		if (error != GIT_ENOTFOUND)
 			goto done;
@@ -373,10 +373,11 @@ static long diff_context_find(
 		!ctxt->match_line(ctxt->driver, ctxt->line.ptr, ctxt->line.size))
 		return -1;
 
-	git_buf_truncate(&ctxt->line, (size_t)out_size);
-	git_buf_copy_cstr(out, (size_t)out_size, &ctxt->line);
+	if (out_size > (long)ctxt->line.size)
+		out_size = (long)ctxt->line.size;
+	memcpy(out, ctxt->line.ptr, (size_t)out_size);
 
-	return (long)ctxt->line.size;
+	return out_size;
 }
 
 void git_diff_find_context_init(
diff --git a/src/diff_file.c b/src/diff_file.c
index 4fd1177ae..bcfef13cd 100644
--- a/src/diff_file.c
+++ b/src/diff_file.c
@@ -18,23 +18,23 @@
 static bool diff_file_content_binary_by_size(git_diff_file_content *fc)
 {
 	/* if we have diff opts, check max_size vs file size */
-	if ((fc->file.flags & DIFF_FLAGS_KNOWN_BINARY) == 0 &&
+	if ((fc->file->flags & DIFF_FLAGS_KNOWN_BINARY) == 0 &&
 		fc->opts_max_size > 0 &&
-		fc->file.size > fc->opts_max_size)
-		fc->file.flags |= GIT_DIFF_FLAG_BINARY;
+		fc->file->size > fc->opts_max_size)
+		fc->file->flags |= GIT_DIFF_FLAG_BINARY;
 
-	return ((fc->file.flags & GIT_DIFF_FLAG_BINARY) != 0);
+	return ((fc->file->flags & GIT_DIFF_FLAG_BINARY) != 0);
 }
 
 static void diff_file_content_binary_by_content(git_diff_file_content *fc)
 {
-	if ((fc->file.flags & DIFF_FLAGS_KNOWN_BINARY) != 0)
+	if ((fc->file->flags & DIFF_FLAGS_KNOWN_BINARY) != 0)
 		return;
 
 	switch (git_diff_driver_content_is_binary(
 		fc->driver, fc->map.data, fc->map.len)) {
-	case 0: fc->file.flags |= GIT_DIFF_FLAG_NOT_BINARY; break;
-	case 1: fc->file.flags |= GIT_DIFF_FLAG_BINARY; break;
+	case 0: fc->file->flags |= GIT_DIFF_FLAG_NOT_BINARY; break;
+	case 1: fc->file->flags |= GIT_DIFF_FLAG_BINARY; break;
 	default: break;
 	}
 }
@@ -48,38 +48,39 @@ static int diff_file_content_init_common(
 		fc->opts_max_size = opts->max_size ?
 			opts->max_size : DIFF_MAX_FILESIZE;
 
-	if (!fc->driver) {
-		if (git_diff_driver_lookup(&fc->driver, fc->repo, "") < 0)
-			return -1;
+	if (fc->src == GIT_ITERATOR_TYPE_EMPTY)
 		fc->src = GIT_ITERATOR_TYPE_TREE;
-	}
+
+	if (!fc->driver &&
+		git_diff_driver_lookup(&fc->driver, fc->repo, fc->file->path) < 0)
+		return -1;
 
 	/* give driver a chance to modify options */
 	git_diff_driver_update_options(&fc->opts_flags, fc->driver);
 
 	/* make sure file is conceivable mmap-able */
-	if ((git_off_t)((size_t)fc->file.size) != fc->file.size)
-		fc->file.flags |= GIT_DIFF_FLAG_BINARY;
+	if ((git_off_t)((size_t)fc->file->size) != fc->file->size)
+		fc->file->flags |= GIT_DIFF_FLAG_BINARY;
 	/* check if user is forcing text diff the file */
 	else if (fc->opts_flags & GIT_DIFF_FORCE_TEXT) {
-		fc->file.flags &= ~GIT_DIFF_FLAG_BINARY;
-		fc->file.flags |= GIT_DIFF_FLAG_NOT_BINARY;
+		fc->file->flags &= ~GIT_DIFF_FLAG_BINARY;
+		fc->file->flags |= GIT_DIFF_FLAG_NOT_BINARY;
 	}
 	/* check if user is forcing binary diff the file */
 	else if (fc->opts_flags & GIT_DIFF_FORCE_BINARY) {
-		fc->file.flags &= ~GIT_DIFF_FLAG_NOT_BINARY;
-		fc->file.flags |= GIT_DIFF_FLAG_BINARY;
+		fc->file->flags &= ~GIT_DIFF_FLAG_NOT_BINARY;
+		fc->file->flags |= GIT_DIFF_FLAG_BINARY;
 	}
 
 	diff_file_content_binary_by_size(fc);
 
-	if ((fc->file.flags & GIT_DIFF_FLAG__NO_DATA) != 0) {
-		fc->file.flags |= GIT_DIFF_FLAG__LOADED;
+	if ((fc->flags & GIT_DIFF_FLAG__NO_DATA) != 0) {
+		fc->flags |= GIT_DIFF_FLAG__LOADED;
 		fc->map.len  = 0;
 		fc->map.data = "";
 	}
 
-	if ((fc->file.flags & GIT_DIFF_FLAG__LOADED) != 0)
+	if ((fc->flags & GIT_DIFF_FLAG__LOADED) != 0)
 		diff_file_content_binary_by_content(fc);
 
 	return 0;
@@ -92,15 +93,14 @@ int git_diff_file_content__init_from_diff(
 	bool use_old)
 {
 	git_diff_delta *delta = git_vector_get(&diff->deltas, delta_index);
-	git_diff_file *file = use_old ? &delta->old_file : &delta->new_file;
 	bool has_data = true;
 
 	memset(fc, 0, sizeof(*fc));
 	fc->repo = diff->repo;
+	fc->file = use_old ? &delta->old_file : &delta->new_file;
 	fc->src  = use_old ? diff->old_src : diff->new_src;
-	memcpy(&fc->file, file, sizeof(fc->file));
 
-	if (git_diff_driver_lookup(&fc->driver, fc->repo, file->path) < 0)
+	if (git_diff_driver_lookup(&fc->driver, fc->repo, fc->file->path) < 0)
 		return -1;
 
 	switch (delta->status) {
@@ -122,7 +122,7 @@ int git_diff_file_content__init_from_diff(
 	}
 
 	if (!has_data)
-		fc->file.flags |= GIT_DIFF_FLAG__NO_DATA;
+		fc->flags |= GIT_DIFF_FLAG__NO_DATA;
 
 	return diff_file_content_init_common(fc, &diff->opts);
 }
@@ -131,21 +131,24 @@ int git_diff_file_content__init_from_blob(
 	git_diff_file_content *fc,
 	git_repository *repo,
 	const git_diff_options *opts,
-	const git_blob *blob)
+	const git_blob *blob,
+	git_diff_file *as_file)
 {
 	memset(fc, 0, sizeof(*fc));
 	fc->repo = repo;
+	fc->file = as_file;
 	fc->blob = blob;
 
 	if (!blob) {
-		fc->file.flags |= GIT_DIFF_FLAG__NO_DATA;
+		fc->flags |= GIT_DIFF_FLAG__NO_DATA;
 	} else {
-		fc->file.flags |= GIT_DIFF_FLAG__LOADED | GIT_DIFF_FLAG_VALID_OID;
-		fc->file.size = git_blob_rawsize(blob);
-		fc->file.mode = 0644;
-		git_oid_cpy(&fc->file.oid, git_blob_id(blob));
+		fc->flags |= GIT_DIFF_FLAG__LOADED;
+		fc->file->flags |= GIT_DIFF_FLAG_VALID_OID;
+		fc->file->size = git_blob_rawsize(blob);
+		fc->file->mode = GIT_FILEMODE_BLOB;
+		git_oid_cpy(&fc->file->oid, git_blob_id(blob));
 
-		fc->map.len = (size_t)fc->file.size;
+		fc->map.len  = (size_t)fc->file->size;
 		fc->map.data = (char *)git_blob_rawcontent(blob);
 	}
 
@@ -157,18 +160,21 @@ int git_diff_file_content__init_from_raw(
 	git_repository *repo,
 	const git_diff_options *opts,
 	const char *buf,
-	size_t buflen)
+	size_t buflen,
+	git_diff_file *as_file)
 {
 	memset(fc, 0, sizeof(*fc));
 	fc->repo = repo;
+	fc->file = as_file;
 
 	if (!buf) {
-		fc->file.flags |= GIT_DIFF_FLAG__NO_DATA;
+		fc->flags |= GIT_DIFF_FLAG__NO_DATA;
 	} else {
-		fc->file.flags |= GIT_DIFF_FLAG__LOADED | GIT_DIFF_FLAG_VALID_OID;
-		fc->file.size = buflen;
-		fc->file.mode = 0644;
-		git_odb_hash(&fc->file.oid, buf, buflen, GIT_OBJ_BLOB);
+		fc->flags |= GIT_DIFF_FLAG__LOADED;
+		fc->file->flags |= GIT_DIFF_FLAG_VALID_OID;
+		fc->file->size = buflen;
+		fc->file->mode = GIT_FILEMODE_BLOB;
+		git_odb_hash(&fc->file->oid, buf, buflen, GIT_OBJ_BLOB);
 
 		fc->map.len  = buflen;
 		fc->map.data = (char *)buf;
@@ -190,7 +196,7 @@ static int diff_file_content_commit_to_str(
 		unsigned int sm_status = 0;
 		const git_oid *sm_head;
 
-		if ((error = git_submodule_lookup(&sm, fc->repo, fc->file.path)) < 0 ||
+		if ((error = git_submodule_lookup(&sm, fc->repo, fc->file->path)) < 0 ||
 			(error = git_submodule_status(&sm_status, sm)) < 0) {
 			/* GIT_EEXISTS means a "submodule" that has not been git added */
 			if (error == GIT_EEXISTS)
@@ -199,25 +205,25 @@ static int diff_file_content_commit_to_str(
 		}
 
 		/* update OID if we didn't have it previously */
-		if ((fc->file.flags & GIT_DIFF_FLAG_VALID_OID) == 0 &&
+		if ((fc->file->flags & GIT_DIFF_FLAG_VALID_OID) == 0 &&
 			((sm_head = git_submodule_wd_id(sm)) != NULL ||
 			 (sm_head = git_submodule_head_id(sm)) != NULL))
 		{
-			git_oid_cpy(&fc->file.oid, sm_head);
-			fc->file.flags |= GIT_DIFF_FLAG_VALID_OID;
+			git_oid_cpy(&fc->file->oid, sm_head);
+			fc->file->flags |= GIT_DIFF_FLAG_VALID_OID;
 		}
 
 		if (GIT_SUBMODULE_STATUS_IS_WD_DIRTY(sm_status))
 			status = "-dirty";
 	}
 
-	git_oid_tostr(oid, sizeof(oid), &fc->file.oid);
+	git_oid_tostr(oid, sizeof(oid), &fc->file->oid);
 	if (git_buf_printf(&content, "Subproject commit %s%s\n", oid, status) < 0)
 		return -1;
 
 	fc->map.len  = git_buf_len(&content);
 	fc->map.data = git_buf_detach(&content);
-	fc->file.flags |= GIT_DIFF_FLAG__FREE_DATA;
+	fc->flags |= GIT_DIFF_FLAG__FREE_DATA;
 
 	return 0;
 }
@@ -227,27 +233,17 @@ static int diff_file_content_load_blob(git_diff_file_content *fc)
 	int error = 0;
 	git_odb_object *odb_obj = NULL;
 
-	if (git_oid_iszero(&fc->file.oid))
+	if (git_oid_iszero(&fc->file->oid))
 		return 0;
 
-	if (fc->file.mode == GIT_FILEMODE_COMMIT)
+	if (fc->file->mode == GIT_FILEMODE_COMMIT)
 		return diff_file_content_commit_to_str(fc, false);
 
 	/* if we don't know size, try to peek at object header first */
-	if (!fc->file.size) {
-		git_odb *odb;
-		size_t len;
-		git_otype type;
-
-		if (!(error = git_repository_odb__weakptr(&odb, fc->repo))) {
-			error = git_odb__read_header_or_object(
-				&odb_obj, &len, &type, odb, &fc->file.oid);
-			git_odb_free(odb);
-		}
-		if (error)
+	if (!fc->file->size) {
+		if ((error = git_diff_file__resolve_zero_size(
+				fc->file, &odb_obj, fc->repo)) < 0)
 			return error;
-
-		fc->file.size = len;
 	}
 
 	if (diff_file_content_binary_by_size(fc))
@@ -259,11 +255,11 @@ static int diff_file_content_load_blob(git_diff_file_content *fc)
 		git_odb_object_free(odb_obj);
 	} else {
 		error = git_blob_lookup(
-			(git_blob **)&fc->blob, fc->repo, &fc->file.oid);
+			(git_blob **)&fc->blob, fc->repo, &fc->file->oid);
 	}
 
 	if (!error) {
-		fc->file.flags |= GIT_DIFF_FLAG__FREE_BLOB;
+		fc->flags |= GIT_DIFF_FLAG__FREE_BLOB;
 		fc->map.data = (void *)git_blob_rawcontent(fc->blob);
 		fc->map.len  = (size_t)git_blob_rawsize(fc->blob);
 	}
@@ -279,16 +275,16 @@ static int diff_file_content_load_workdir_symlink(
 	/* link path on disk could be UTF-16, so prepare a buffer that is
 	 * big enough to handle some UTF-8 data expansion
 	 */
-	alloc_len = (ssize_t)(fc->file.size * 2) + 1;
+	alloc_len = (ssize_t)(fc->file->size * 2) + 1;
 
 	fc->map.data = git__calloc(alloc_len, sizeof(char));
 	GITERR_CHECK_ALLOC(fc->map.data);
 
-	fc->file.flags |= GIT_DIFF_FLAG__FREE_DATA;
+	fc->flags |= GIT_DIFF_FLAG__FREE_DATA;
 
 	read_len = p_readlink(git_buf_cstr(path), fc->map.data, alloc_len);
 	if (read_len < 0) {
-		giterr_set(GITERR_OS, "Failed to read symlink '%s'", fc->file.path);
+		giterr_set(GITERR_OS, "Failed to read symlink '%s'", fc->file->path);
 		return -1;
 	}
 
@@ -307,28 +303,28 @@ static int diff_file_content_load_workdir_file(
 	if (fd < 0)
 		return fd;
 
-	if (!fc->file.size &&
-		!(fc->file.size = git_futils_filesize(fd)))
+	if (!fc->file->size &&
+		!(fc->file->size = git_futils_filesize(fd)))
 		goto cleanup;
 
 	if (diff_file_content_binary_by_size(fc))
 		goto cleanup;
 
 	if ((error = git_filters_load(
-			&filters, fc->repo, fc->file.path, GIT_FILTER_TO_ODB)) < 0)
+			&filters, fc->repo, fc->file->path, GIT_FILTER_TO_ODB)) < 0)
 		goto cleanup;
 	/* error >= is a filter count */
 
 	if (error == 0) {
 		if (!(error = git_futils_mmap_ro(
-				&fc->map, fd, 0, (size_t)fc->file.size)))
-			fc->file.flags |= GIT_DIFF_FLAG__UNMAP_DATA;
+				&fc->map, fd, 0, (size_t)fc->file->size)))
+			fc->flags |= GIT_DIFF_FLAG__UNMAP_DATA;
 		else /* fall through to try readbuffer below */
 			giterr_clear();
 	}
 
 	if (error != 0) {
-		error = git_futils_readbuffer_fd(&raw, fd, (size_t)fc->file.size);
+		error = git_futils_readbuffer_fd(&raw, fd, (size_t)fc->file->size);
 		if (error < 0)
 			goto cleanup;
 
@@ -340,7 +336,7 @@ static int diff_file_content_load_workdir_file(
 		if (!error) {
 			fc->map.len  = git_buf_len(&filtered);
 			fc->map.data = git_buf_detach(&filtered);
-			fc->file.flags |= GIT_DIFF_FLAG__FREE_DATA;
+			fc->flags |= GIT_DIFF_FLAG__FREE_DATA;
 		}
 
 		git_buf_free(&raw);
@@ -359,26 +355,26 @@ static int diff_file_content_load_workdir(git_diff_file_content *fc)
 	int error = 0;
 	git_buf path = GIT_BUF_INIT;
 
-	if (fc->file.mode == GIT_FILEMODE_COMMIT)
+	if (fc->file->mode == GIT_FILEMODE_COMMIT)
 		return diff_file_content_commit_to_str(fc, true);
 
-	if (fc->file.mode == GIT_FILEMODE_TREE)
+	if (fc->file->mode == GIT_FILEMODE_TREE)
 		return 0;
 
 	if (git_buf_joinpath(
-			&path, git_repository_workdir(fc->repo), fc->file.path) < 0)
+			&path, git_repository_workdir(fc->repo), fc->file->path) < 0)
 		return -1;
 
-	if (S_ISLNK(fc->file.mode))
+	if (S_ISLNK(fc->file->mode))
 		error = diff_file_content_load_workdir_symlink(fc, &path);
 	else
 		error = diff_file_content_load_workdir_file(fc, &path);
 
 	/* once data is loaded, update OID if we didn't have it previously */
-	if (!error && (fc->file.flags & GIT_DIFF_FLAG_VALID_OID) == 0) {
+	if (!error && (fc->file->flags & GIT_DIFF_FLAG_VALID_OID) == 0) {
 		error = git_odb_hash(
-			&fc->file.oid, fc->map.data, fc->map.len, GIT_OBJ_BLOB);
-		fc->file.flags |= GIT_DIFF_FLAG_VALID_OID;
+			&fc->file->oid, fc->map.data, fc->map.len, GIT_OBJ_BLOB);
+		fc->file->flags |= GIT_DIFF_FLAG_VALID_OID;
 	}
 
 	git_buf_free(&path);
@@ -389,10 +385,10 @@ int git_diff_file_content__load(git_diff_file_content *fc)
 {
 	int error = 0;
 
-	if ((fc->file.flags & GIT_DIFF_FLAG__LOADED) != 0)
+	if ((fc->flags & GIT_DIFF_FLAG__LOADED) != 0)
 		return 0;
 
-	if (fc->file.flags & GIT_DIFF_FLAG_BINARY)
+	if ((fc->file->flags & GIT_DIFF_FLAG_BINARY) != 0)
 		return 0;
 
 	if (fc->src == GIT_ITERATOR_TYPE_WORKDIR)
@@ -402,7 +398,7 @@ int git_diff_file_content__load(git_diff_file_content *fc)
 	if (error)
 		return error;
 
-	fc->file.flags |= GIT_DIFF_FLAG__LOADED;
+	fc->flags |= GIT_DIFF_FLAG__LOADED;
 
 	diff_file_content_binary_by_content(fc);
 
@@ -411,26 +407,29 @@ int git_diff_file_content__load(git_diff_file_content *fc)
 
 void git_diff_file_content__unload(git_diff_file_content *fc)
 {
-	if (fc->file.flags & GIT_DIFF_FLAG__FREE_DATA) {
+	if ((fc->flags & GIT_DIFF_FLAG__LOADED) == 0)
+		return;
+
+	if (fc->flags & GIT_DIFF_FLAG__FREE_DATA) {
 		git__free(fc->map.data);
 		fc->map.data = "";
 		fc->map.len  = 0;
-		fc->file.flags &= ~GIT_DIFF_FLAG__FREE_DATA;
+		fc->flags &= ~GIT_DIFF_FLAG__FREE_DATA;
 	}
-	else if (fc->file.flags & GIT_DIFF_FLAG__UNMAP_DATA) {
+	else if (fc->flags & GIT_DIFF_FLAG__UNMAP_DATA) {
 		git_futils_mmap_free(&fc->map);
 		fc->map.data = "";
 		fc->map.len  = 0;
-		fc->file.flags &= ~GIT_DIFF_FLAG__UNMAP_DATA;
+		fc->flags &= ~GIT_DIFF_FLAG__UNMAP_DATA;
 	}
 
-	if (fc->file.flags & GIT_DIFF_FLAG__FREE_BLOB) {
+	if (fc->flags & GIT_DIFF_FLAG__FREE_BLOB) {
 		git_blob_free((git_blob *)fc->blob);
 		fc->blob = NULL;
-		fc->file.flags &= ~GIT_DIFF_FLAG__FREE_BLOB;
+		fc->flags &= ~GIT_DIFF_FLAG__FREE_BLOB;
 	}
 
-	fc->file.flags &= ~GIT_DIFF_FLAG__LOADED;
+	fc->flags &= ~GIT_DIFF_FLAG__LOADED;
 }
 
 void git_diff_file_content__clear(git_diff_file_content *fc)
diff --git a/src/diff_file.h b/src/diff_file.h
index afad8510b..fb08cca6a 100644
--- a/src/diff_file.h
+++ b/src/diff_file.h
@@ -15,8 +15,9 @@
 /* expanded information for one side of a delta */
 typedef struct {
 	git_repository *repo;
-	git_diff_file file;
+	git_diff_file *file;
 	git_diff_driver *driver;
+	uint32_t flags;
 	uint32_t opts_flags;
 	git_off_t opts_max_size;
 	git_iterator_type_t src;
@@ -34,14 +35,16 @@ extern int git_diff_file_content__init_from_blob(
 	git_diff_file_content *fc,
 	git_repository *repo,
 	const git_diff_options *opts,
-	const git_blob *blob);
+	const git_blob *blob,
+	git_diff_file *as_file);
 
 extern int git_diff_file_content__init_from_raw(
 	git_diff_file_content *fc,
 	git_repository *repo,
 	const git_diff_options *opts,
 	const char *buf,
-	size_t buflen);
+	size_t buflen,
+	git_diff_file *as_file);
 
 /* this loads the blob/file-on-disk as needed */
 extern int git_diff_file_content__load(git_diff_file_content *fc);
diff --git a/src/diff_patch.c b/src/diff_patch.c
index a1e1fe84c..cc45b6ddb 100644
--- a/src/diff_patch.c
+++ b/src/diff_patch.c
@@ -10,6 +10,7 @@
 #include "diff_driver.h"
 #include "diff_patch.h"
 #include "diff_xdiff.h"
+#include "fileops.h"
 
 /* cached information about a single span in a diff */
 typedef struct diff_patch_line diff_patch_line;
@@ -41,7 +42,7 @@ struct git_diff_patch {
 	git_array_t(diff_patch_hunk) hunks;
 	git_array_t(diff_patch_line) lines;
 	size_t oldno, newno;
-	size_t content_size;
+	size_t content_size, context_size, header_size;
 	git_pool flattened;
 };
 
@@ -64,12 +65,12 @@ static void diff_patch_update_binary(git_diff_patch *patch)
 	if ((patch->delta->flags & DIFF_FLAGS_KNOWN_BINARY) != 0)
 		return;
 
-	if ((patch->ofile.file.flags & GIT_DIFF_FLAG_BINARY) != 0 ||
-		(patch->nfile.file.flags & GIT_DIFF_FLAG_BINARY) != 0)
+	if ((patch->ofile.file->flags & GIT_DIFF_FLAG_BINARY) != 0 ||
+		(patch->nfile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
 		patch->delta->flags |= GIT_DIFF_FLAG_BINARY;
 
-	else if ((patch->ofile.file.flags & DIFF_FLAGS_NOT_BINARY) != 0 &&
-			 (patch->nfile.file.flags & DIFF_FLAGS_NOT_BINARY) != 0)
+	else if ((patch->ofile.file->flags & DIFF_FLAGS_NOT_BINARY) != 0 &&
+			 (patch->nfile.file->flags & DIFF_FLAGS_NOT_BINARY) != 0)
 		patch->delta->flags |= GIT_DIFF_FLAG_NOT_BINARY;
 }
 
@@ -143,42 +144,46 @@ static int diff_patch_load(git_diff_patch *patch, git_diff_output *output)
 		output && !output->hunk_cb && !output->data_cb)
 		return 0;
 
-#define DIFF_FLAGS_KNOWN_DATA (GIT_DIFF_FLAG__NO_DATA|GIT_DIFF_FLAG_VALID_OID)
-
 	incomplete_data =
-		((patch->ofile.file.flags & DIFF_FLAGS_KNOWN_DATA) != 0 &&
-		 (patch->nfile.file.flags & DIFF_FLAGS_KNOWN_DATA) != 0);
+		(((patch->ofile.flags & GIT_DIFF_FLAG__NO_DATA) != 0 ||
+		  (patch->ofile.file->flags & GIT_DIFF_FLAG_VALID_OID) != 0) &&
+		 ((patch->nfile.flags & GIT_DIFF_FLAG__NO_DATA) != 0 ||
+		  (patch->nfile.file->flags & GIT_DIFF_FLAG_VALID_OID) != 0));
 
 	/* always try to load workdir content first because filtering may
 	 * need 2x data size and this minimizes peak memory footprint
 	 */
 	if (patch->ofile.src == GIT_ITERATOR_TYPE_WORKDIR) {
 		if ((error = git_diff_file_content__load(&patch->ofile)) < 0 ||
-			(patch->ofile.file.flags & GIT_DIFF_FLAG_BINARY) != 0)
+			(patch->ofile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
 			goto cleanup;
 	}
 	if (patch->nfile.src == GIT_ITERATOR_TYPE_WORKDIR) {
 		if ((error = git_diff_file_content__load(&patch->nfile)) < 0 ||
-			(patch->nfile.file.flags & GIT_DIFF_FLAG_BINARY) != 0)
+			(patch->nfile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
 			goto cleanup;
 	}
 
 	/* once workdir has been tried, load other data as needed */
 	if (patch->ofile.src != GIT_ITERATOR_TYPE_WORKDIR) {
 		if ((error = git_diff_file_content__load(&patch->ofile)) < 0 ||
-			(patch->ofile.file.flags & GIT_DIFF_FLAG_BINARY) != 0)
+			(patch->ofile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
 			goto cleanup;
 	}
 	if (patch->nfile.src != GIT_ITERATOR_TYPE_WORKDIR) {
 		if ((error = git_diff_file_content__load(&patch->nfile)) < 0 ||
-			(patch->nfile.file.flags & GIT_DIFF_FLAG_BINARY) != 0)
+			(patch->nfile.file->flags & GIT_DIFF_FLAG_BINARY) != 0)
 			goto cleanup;
 	}
 
-	/* if we were previously missing an oid, reassess UNMODIFIED state */
+	/* if previously missing an oid, and now that we have it the two sides
+	 * are the same (and not submodules), update MODIFIED -> UNMODIFIED
+	 */
 	if (incomplete_data &&
-		patch->ofile.file.mode == patch->nfile.file.mode &&
-		git_oid_equal(&patch->ofile.file.oid, &patch->nfile.file.oid))
+		patch->ofile.file->mode == patch->nfile.file->mode &&
+		patch->ofile.file->mode != GIT_FILEMODE_COMMIT &&
+		git_oid_equal(&patch->ofile.file->oid, &patch->nfile.file->oid) &&
+		patch->delta->status == GIT_DELTA_MODIFIED) /* not RENAMED/COPIED! */
 		patch->delta->status = GIT_DELTA_UNMODIFIED;
 
 cleanup:
@@ -192,7 +197,7 @@ cleanup:
 			patch->delta->status != GIT_DELTA_UNMODIFIED &&
 			(patch->ofile.map.len || patch->nfile.map.len) &&
 			(patch->ofile.map.len != patch->nfile.map.len ||
-			 !git_oid_equal(&patch->ofile.file.oid, &patch->nfile.file.oid)))
+			 !git_oid_equal(&patch->ofile.file->oid, &patch->nfile.file->oid)))
 			patch->flags |= GIT_DIFF_PATCH_DIFFABLE;
 
 		patch->flags |= GIT_DIFF_PATCH_LOADED;
@@ -225,6 +230,10 @@ static int diff_patch_generate(git_diff_patch *patch, git_diff_output *output)
 	if ((patch->flags & GIT_DIFF_PATCH_DIFFED) != 0)
 		return 0;
 
+	/* if we are not looking at the hunks and lines, don't do the diff */
+	if (!output->hunk_cb && !output->data_cb)
+		return 0;
+
 	if ((patch->flags & GIT_DIFF_PATCH_LOADED) == 0 &&
 		(error = diff_patch_load(patch, output)) < 0)
 		return error;
@@ -279,21 +288,22 @@ int git_diff_foreach(
 	if (diff_required(diff, "git_diff_foreach") < 0)
 		return -1;
 
-	diff_output_init((git_diff_output *)&xo,
-		&diff->opts, file_cb, hunk_cb, data_cb, payload);
+	diff_output_init(
+		&xo.output, &diff->opts, file_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, &diff->opts);
 
 	git_vector_foreach(&diff->deltas, idx, patch.delta) {
+
 		/* check flags against patch status */
 		if (git_diff_delta__should_skip(&diff->opts, patch.delta))
 			continue;
 
 		if (!(error = diff_patch_init_from_diff(&patch, diff, idx))) {
 
-			error = diff_patch_file_callback(&patch, (git_diff_output *)&xo);
+			error = diff_patch_file_callback(&patch, &xo.output);
 
 			if (!error)
-				error = diff_patch_generate(&patch, (git_diff_output *)&xo);
+				error = diff_patch_generate(&patch, &xo.output);
 
 			git_diff_patch_free(&patch);
 		}
@@ -310,26 +320,31 @@ int git_diff_foreach(
 typedef struct {
 	git_diff_patch patch;
 	git_diff_delta delta;
+	char paths[GIT_FLEX_ARRAY];
 } diff_patch_with_delta;
 
 static int diff_single_generate(diff_patch_with_delta *pd, git_xdiff_output *xo)
 {
 	int error = 0;
 	git_diff_patch *patch = &pd->patch;
-	bool has_old = ((patch->ofile.file.flags & GIT_DIFF_FLAG__NO_DATA) == 0);
-	bool has_new = ((patch->nfile.file.flags & GIT_DIFF_FLAG__NO_DATA) == 0);
+	bool has_old = ((patch->ofile.flags & GIT_DIFF_FLAG__NO_DATA) == 0);
+	bool has_new = ((patch->nfile.flags & GIT_DIFF_FLAG__NO_DATA) == 0);
 
 	pd->delta.status = has_new ?
 		(has_old ? GIT_DELTA_MODIFIED : GIT_DELTA_ADDED) :
 		(has_old ? GIT_DELTA_DELETED : GIT_DELTA_UNTRACKED);
 
-	if (git_oid_equal(&patch->nfile.file.oid, &patch->ofile.file.oid))
+	if (git_oid_equal(&patch->nfile.file->oid, &patch->ofile.file->oid))
 		pd->delta.status = GIT_DELTA_UNMODIFIED;
 
 	patch->delta = &pd->delta;
 
 	diff_patch_init_common(patch);
 
+	if (pd->delta.status == GIT_DELTA_UNMODIFIED &&
+		!(patch->ofile.opts_flags & GIT_DIFF_INCLUDE_UNMODIFIED))
+		return error;
+
 	error = diff_patch_file_callback(patch, (git_diff_output *)xo);
 
 	if (!error)
@@ -345,7 +360,9 @@ static int diff_patch_from_blobs(
 	diff_patch_with_delta *pd,
 	git_xdiff_output *xo,
 	const git_blob *old_blob,
+	const char *old_path,
 	const git_blob *new_blob,
+	const char *new_path,
 	const git_diff_options *opts)
 {
 	int error = 0;
@@ -355,29 +372,61 @@ static int diff_patch_from_blobs(
 
 	GITERR_CHECK_VERSION(opts, GIT_DIFF_OPTIONS_VERSION, "git_diff_options");
 
-	pd->patch.delta = &pd->delta;
-
-	if (!repo) /* return two NULL items as UNMODIFIED delta */
-		return 0;
-
 	if (opts && (opts->flags & GIT_DIFF_REVERSE) != 0) {
-		const git_blob *swap = old_blob;
-		old_blob = new_blob;
-		new_blob = swap;
+		const git_blob *tmp_blob;
+		const char *tmp_path;
+		tmp_blob = old_blob; old_blob = new_blob; new_blob = tmp_blob;
+		tmp_path = old_path; old_path = new_path; new_path = tmp_path;
 	}
 
+	pd->patch.delta = &pd->delta;
+
+	pd->delta.old_file.path = old_path;
+	pd->delta.new_file.path = new_path;
+
 	if ((error = git_diff_file_content__init_from_blob(
-			&pd->patch.ofile, repo, opts, old_blob)) < 0 ||
+			&pd->patch.ofile, repo, opts, old_blob, &pd->delta.old_file)) < 0 ||
 		(error = git_diff_file_content__init_from_blob(
-			&pd->patch.nfile, repo, opts, new_blob)) < 0)
+			&pd->patch.nfile, repo, opts, new_blob, &pd->delta.new_file)) < 0)
 		return error;
 
 	return diff_single_generate(pd, xo);
 }
 
+static int diff_patch_with_delta_alloc(
+	diff_patch_with_delta **out,
+	const char **old_path,
+	const char **new_path)
+{
+	diff_patch_with_delta *pd;
+	size_t old_len = *old_path ? strlen(*old_path) : 0;
+	size_t new_len = *new_path ? strlen(*new_path) : 0;
+
+	*out = pd = git__calloc(1, sizeof(*pd) + old_len + new_len + 2);
+	GITERR_CHECK_ALLOC(pd);
+
+	pd->patch.flags = GIT_DIFF_PATCH_ALLOCATED;
+
+	if (*old_path) {
+		memcpy(&pd->paths[0], *old_path, old_len);
+		*old_path = &pd->paths[0];
+	} else if (*new_path)
+		*old_path = &pd->paths[old_len + 1];
+
+	if (*new_path) {
+		memcpy(&pd->paths[old_len + 1], *new_path, new_len);
+		*new_path = &pd->paths[old_len + 1];
+	} else if (*old_path)
+		*new_path = &pd->paths[0];
+
+	return 0;
+}
+
 int git_diff_blobs(
 	const git_blob *old_blob,
+	const char *old_path,
 	const git_blob *new_blob,
+	const char *new_path,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
@@ -392,12 +441,18 @@ int git_diff_blobs(
 	memset(&xo, 0, sizeof(xo));
 
 	diff_output_init(
-		(git_diff_output *)&xo, opts, file_cb, hunk_cb, data_cb, payload);
+		&xo.output, opts, file_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, opts);
 
-	error = diff_patch_from_blobs(&pd, &xo, old_blob, new_blob, opts);
+	if (!old_path && new_path)
+		old_path = new_path;
+	else if (!new_path && old_path)
+		new_path = old_path;
 
-	git_diff_patch_free((git_diff_patch *)&pd);
+	error = diff_patch_from_blobs(
+		&pd, &xo, old_blob, old_path, new_blob, new_path, opts);
+
+	git_diff_patch_free(&pd.patch);
 
 	return error;
 }
@@ -405,7 +460,9 @@ int git_diff_blobs(
 int git_diff_patch_from_blobs(
 	git_diff_patch **out,
 	const git_blob *old_blob,
+	const char *old_path,
 	const git_blob *new_blob,
+	const char *new_path,
 	const git_diff_options *opts)
 {
 	int error = 0;
@@ -415,16 +472,18 @@ int git_diff_patch_from_blobs(
 	assert(out);
 	*out = NULL;
 
-	pd = git__calloc(1, sizeof(*pd));
-	GITERR_CHECK_ALLOC(pd);
-	pd->patch.flags = GIT_DIFF_PATCH_ALLOCATED;
+	if (diff_patch_with_delta_alloc(&pd, &old_path, &new_path) < 0)
+		return -1;
 
 	memset(&xo, 0, sizeof(xo));
 
-	diff_output_to_patch((git_diff_output *)&xo, &pd->patch);
+	diff_output_to_patch(&xo.output, &pd->patch);
 	git_xdiff_init(&xo, opts);
 
-	if (!(error = diff_patch_from_blobs(pd, &xo, old_blob, new_blob, opts)))
+	error = diff_patch_from_blobs(
+		pd, &xo, old_blob, old_path, new_blob, new_path, opts);
+
+	if (!error)
 		*out = (git_diff_patch *)pd;
 	else
 		git_diff_patch_free((git_diff_patch *)pd);
@@ -436,8 +495,10 @@ static int diff_patch_from_blob_and_buffer(
 	diff_patch_with_delta *pd,
 	git_xdiff_output *xo,
 	const git_blob *old_blob,
+	const char *old_path,
 	const char *buf,
 	size_t buflen,
+	const char *buf_path,
 	const git_diff_options *opts)
 {
 	int error = 0;
@@ -448,28 +509,36 @@ static int diff_patch_from_blob_and_buffer(
 
 	pd->patch.delta = &pd->delta;
 
-	if (!repo && !buf) /* return two NULL items as UNMODIFIED delta */
-		return 0;
-
 	if (opts && (opts->flags & GIT_DIFF_REVERSE) != 0) {
+		pd->delta.old_file.path = buf_path;
+		pd->delta.new_file.path = old_path;
+
 		if (!(error = git_diff_file_content__init_from_raw(
-				&pd->patch.ofile, repo, opts, buf, buflen)))
+				&pd->patch.ofile, repo, opts, buf, buflen, &pd->delta.old_file)))
 			error = git_diff_file_content__init_from_blob(
-				&pd->patch.nfile, repo, opts, old_blob);
+				&pd->patch.nfile, repo, opts, old_blob, &pd->delta.new_file);
 	} else {
+		pd->delta.old_file.path = old_path;
+		pd->delta.new_file.path = buf_path;
+
 		if (!(error = git_diff_file_content__init_from_blob(
-				&pd->patch.ofile, repo, opts, old_blob)))
+				&pd->patch.ofile, repo, opts, old_blob, &pd->delta.old_file)))
 			error = git_diff_file_content__init_from_raw(
-				&pd->patch.nfile, repo, opts, buf, buflen);
+				&pd->patch.nfile, repo, opts, buf, buflen, &pd->delta.new_file);
 	}
 
+	if (error < 0)
+		return error;
+
 	return diff_single_generate(pd, xo);
 }
 
 int git_diff_blob_to_buffer(
 	const git_blob *old_blob,
+	const char *old_path,
 	const char *buf,
 	size_t buflen,
+	const char *buf_path,
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
@@ -484,13 +553,18 @@ int git_diff_blob_to_buffer(
 	memset(&xo, 0, sizeof(xo));
 
 	diff_output_init(
-		(git_diff_output *)&xo, opts, file_cb, hunk_cb, data_cb, payload);
+		&xo.output, opts, file_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, opts);
 
+	if (!old_path && buf_path)
+		old_path = buf_path;
+	else if (!buf_path && old_path)
+		buf_path = old_path;
+
 	error = diff_patch_from_blob_and_buffer(
-		&pd, &xo, old_blob, buf, buflen, opts);
+		&pd, &xo, old_blob, old_path, buf, buflen, buf_path, opts);
 
-	git_diff_patch_free((git_diff_patch *)&pd);
+	git_diff_patch_free(&pd.patch);
 
 	return error;
 }
@@ -498,8 +572,10 @@ int git_diff_blob_to_buffer(
 int git_diff_patch_from_blob_and_buffer(
 	git_diff_patch **out,
 	const git_blob *old_blob,
+	const char *old_path,
 	const char *buf,
 	size_t buflen,
+	const char *buf_path,
 	const git_diff_options *opts)
 {
 	int error = 0;
@@ -509,17 +585,18 @@ int git_diff_patch_from_blob_and_buffer(
 	assert(out);
 	*out = NULL;
 
-	pd = git__calloc(1, sizeof(*pd));
-	GITERR_CHECK_ALLOC(pd);
-	pd->patch.flags = GIT_DIFF_PATCH_ALLOCATED;
+	if (diff_patch_with_delta_alloc(&pd, &old_path, &buf_path) < 0)
+		return -1;
 
 	memset(&xo, 0, sizeof(xo));
 
-	diff_output_to_patch((git_diff_output *)&xo, &pd->patch);
+	diff_output_to_patch(&xo.output, &pd->patch);
 	git_xdiff_init(&xo, opts);
 
-	if (!(error = diff_patch_from_blob_and_buffer(
-			pd, &xo, old_blob, buf, buflen, opts)))
+	error = diff_patch_from_blob_and_buffer(
+		pd, &xo, old_blob, old_path, buf, buflen, buf_path, opts);
+
+	if (!error)
 		*out = (git_diff_patch *)pd;
 	else
 		git_diff_patch_free((git_diff_patch *)pd);
@@ -565,13 +642,13 @@ int git_diff_get_patch(
 	if ((error = diff_patch_alloc_from_diff(&patch, diff, idx)) < 0)
 		return error;
 
-	diff_output_to_patch((git_diff_output *)&xo, patch);
+	diff_output_to_patch(&xo.output, patch);
 	git_xdiff_init(&xo, &diff->opts);
 
-	error = diff_patch_file_callback(patch, (git_diff_output *)&xo);
+	error = diff_patch_file_callback(patch, &xo.output);
 
 	if (!error)
-		error = diff_patch_generate(patch, (git_diff_output *)&xo);
+		error = diff_patch_generate(patch, &xo.output);
 
 	if (!error) {
 		/* if cumulative diff size is < 0.5 total size, flatten the patch */
@@ -733,6 +810,39 @@ notfound:
 	return diff_error_outofrange(thing);
 }
 
+size_t git_diff_patch_size(
+	git_diff_patch *patch,
+	int include_context,
+	int include_hunk_headers,
+	int include_file_headers)
+{
+	size_t out;
+
+	assert(patch);
+
+	out = patch->content_size;
+
+	if (!include_context)
+		out -= patch->context_size;
+
+	if (include_hunk_headers)
+		out += patch->header_size;
+
+	if (include_file_headers) {
+		git_buf file_header = GIT_BUF_INIT;
+
+		if (git_diff_delta__format_file_header(
+				&file_header, patch->delta, NULL, NULL, 0) < 0)
+			giterr_clear();
+		else
+			out += git_buf_len(&file_header);
+
+		git_buf_free(&file_header);
+	}
+
+	return out;
+}
+
 git_diff_list *git_diff_patch__diff(git_diff_patch *patch)
 {
 	return patch->diff;
@@ -827,6 +937,8 @@ static int diff_patch_hunk_cb(
 	hunk->header[header_len] = '\0';
 	hunk->header_len = header_len;
 
+	patch->header_size += header_len;
+
 	hunk->line_start = git_array_size(patch->lines);
 	hunk->line_count = 0;
 
@@ -847,6 +959,7 @@ static int diff_patch_line_cb(
 	git_diff_patch *patch = payload;
 	diff_patch_hunk *hunk;
 	diff_patch_line *line;
+	const char *content_end = content + content_len;
 
 	GIT_UNUSED(delta);
 	GIT_UNUSED(range);
@@ -861,34 +974,43 @@ static int diff_patch_line_cb(
 	line->len = content_len;
 	line->origin = line_origin;
 
-	patch->content_size += content_len;
-
 	/* do some bookkeeping so we can provide old/new line numbers */
 
-	for (line->lines = 0; content_len > 0; --content_len) {
+	line->lines = 0;
+	while (content < content_end)
 		if (*content++ == '\n')
 			++line->lines;
-	}
+
+	patch->content_size += content_len;
 
 	switch (line_origin) {
 	case GIT_DIFF_LINE_ADDITION:
+		patch->content_size += 1;
 	case GIT_DIFF_LINE_DEL_EOFNL:
 		line->oldno = -1;
 		line->newno = patch->newno;
 		patch->newno += line->lines;
 		break;
 	case GIT_DIFF_LINE_DELETION:
+		patch->content_size += 1;
 	case GIT_DIFF_LINE_ADD_EOFNL:
 		line->oldno = patch->oldno;
 		line->newno = -1;
 		patch->oldno += line->lines;
 		break;
-	default:
+	case GIT_DIFF_LINE_CONTEXT:
+		patch->content_size += 1;
+		patch->context_size += 1;
+	case GIT_DIFF_LINE_CONTEXT_EOFNL:
+		patch->context_size += content_len;
 		line->oldno = patch->oldno;
 		line->newno = patch->newno;
 		patch->oldno += line->lines;
 		patch->newno += line->lines;
 		break;
+	default:
+		assert(false);
+		break;
 	}
 
 	hunk->line_count++;
diff --git a/src/diff_print.c b/src/diff_print.c
index 244aa6e1d..ee4b5fc17 100644
--- a/src/diff_print.c
+++ b/src/diff_print.c
@@ -7,7 +7,7 @@
 #include "common.h"
 #include "diff.h"
 #include "diff_patch.h"
-#include "buffer.h"
+#include "fileops.h"
 
 typedef struct {
 	git_diff_list *diff;
@@ -21,14 +21,15 @@ static int diff_print_info_init(
 	diff_print_info *pi,
 	git_buf *out, git_diff_list *diff, git_diff_data_cb cb, void *payload)
 {
-	assert(diff && diff->repo);
-
 	pi->diff     = diff;
 	pi->print_cb = cb;
 	pi->payload  = payload;
 	pi->buf      = out;
 
-	if (git_repository__cvar(&pi->oid_strlen, diff->repo, GIT_CVAR_ABBREV) < 0)
+	if (!diff || !diff->repo)
+		pi->oid_strlen = GIT_ABBREV_DEFAULT;
+	else if (git_repository__cvar(
+		&pi->oid_strlen, diff->repo, GIT_CVAR_ABBREV) < 0)
 		return -1;
 
 	pi->oid_strlen += 1; /* for NUL byte */
@@ -41,11 +42,11 @@ static int diff_print_info_init(
 	return 0;
 }
 
-static char pick_suffix(int mode)
+static char diff_pick_suffix(int mode)
 {
 	if (S_ISDIR(mode))
 		return '/';
-	else if (mode & 0100) //-V536
+	else if (GIT_PERMS_IS_EXEC(mode)) /* -V536 */
 		/* in git, modes are very regular, so we must have 0100755 mode */
 		return '*';
 	else
@@ -76,45 +77,49 @@ static int callback_error(void)
 	return GIT_EUSER;
 }
 
-static int print_compact(
+static int diff_print_one_compact(
 	const git_diff_delta *delta, float progress, void *data)
 {
 	diff_print_info *pi = data;
+	git_buf *out = pi->buf;
 	char old_suffix, new_suffix, code = git_diff_status_char(delta->status);
+	int (*strcomp)(const char *, const char *) =
+		pi->diff ? pi->diff->strcomp : git__strcmp;
 
 	GIT_UNUSED(progress);
 
 	if (code == ' ')
 		return 0;
 
-	old_suffix = pick_suffix(delta->old_file.mode);
-	new_suffix = pick_suffix(delta->new_file.mode);
+	old_suffix = diff_pick_suffix(delta->old_file.mode);
+	new_suffix = diff_pick_suffix(delta->new_file.mode);
 
-	git_buf_clear(pi->buf);
+	git_buf_clear(out);
 
 	if (delta->old_file.path != delta->new_file.path &&
-		pi->diff->strcomp(delta->old_file.path,delta->new_file.path) != 0)
-		git_buf_printf(pi->buf, "%c\t%s%c -> %s%c\n", code,
+		strcomp(delta->old_file.path,delta->new_file.path) != 0)
+		git_buf_printf(out, "%c\t%s%c %s%c\n", code,
 			delta->old_file.path, old_suffix, delta->new_file.path, new_suffix);
 	else if (delta->old_file.mode != delta->new_file.mode &&
 		delta->old_file.mode != 0 && delta->new_file.mode != 0)
-		git_buf_printf(pi->buf, "%c\t%s%c (%o -> %o)\n", code,
-			delta->old_file.path, new_suffix, delta->old_file.mode, delta->new_file.mode);
+		git_buf_printf(out, "%c\t%s%c %s%c\n", code,
+			delta->old_file.path, old_suffix, delta->new_file.path, new_suffix);
 	else if (old_suffix != ' ')
-		git_buf_printf(pi->buf, "%c\t%s%c\n", code, delta->old_file.path, old_suffix);
+		git_buf_printf(out, "%c\t%s%c\n", code, delta->old_file.path, old_suffix);
 	else
-		git_buf_printf(pi->buf, "%c\t%s\n", code, delta->old_file.path);
+		git_buf_printf(out, "%c\t%s\n", code, delta->old_file.path);
 
-	if (git_buf_oom(pi->buf))
+	if (git_buf_oom(out))
 		return -1;
 
 	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_FILE_HDR,
-			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
+			git_buf_cstr(out), git_buf_len(out), pi->payload))
 		return callback_error();
 
 	return 0;
 }
 
+/* print a git_diff_list to a print callback in compact format */
 int git_diff_print_compact(
 	git_diff_list *diff,
 	git_diff_data_cb print_cb,
@@ -125,17 +130,18 @@ int git_diff_print_compact(
 	diff_print_info pi;
 
 	if (!(error = diff_print_info_init(&pi, &buf, diff, print_cb, payload)))
-		error = git_diff_foreach(diff, print_compact, NULL, NULL, &pi);
+		error = git_diff_foreach(diff, diff_print_one_compact, NULL, NULL, &pi);
 
 	git_buf_free(&buf);
 
 	return error;
 }
 
-static int print_raw(
+static int diff_print_one_raw(
 	const git_diff_delta *delta, float progress, void *data)
 {
 	diff_print_info *pi = data;
+	git_buf *out = pi->buf;
 	char code = git_diff_status_char(delta->status);
 	char start_oid[GIT_OID_HEXSZ+1], end_oid[GIT_OID_HEXSZ+1];
 
@@ -144,36 +150,37 @@ static int print_raw(
 	if (code == ' ')
 		return 0;
 
-	git_buf_clear(pi->buf);
+	git_buf_clear(out);
 
 	git_oid_tostr(start_oid, pi->oid_strlen, &delta->old_file.oid);
 	git_oid_tostr(end_oid, pi->oid_strlen, &delta->new_file.oid);
 
 	git_buf_printf(
-		pi->buf, ":%06o %06o %s... %s... %c",
+		out, ":%06o %06o %s... %s... %c",
 		delta->old_file.mode, delta->new_file.mode, start_oid, end_oid, code);
 
 	if (delta->similarity > 0)
-		git_buf_printf(pi->buf, "%03u", delta->similarity);
+		git_buf_printf(out, "%03u", delta->similarity);
 
-	if (delta->status == GIT_DELTA_RENAMED || delta->status == GIT_DELTA_COPIED)
+	if (delta->old_file.path != delta->new_file.path)
 		git_buf_printf(
-			pi->buf, "\t%s %s\n", delta->old_file.path, delta->new_file.path);
+			out, "\t%s %s\n", delta->old_file.path, delta->new_file.path);
 	else
 		git_buf_printf(
-			pi->buf, "\t%s\n", delta->old_file.path ?
+			out, "\t%s\n", delta->old_file.path ?
 			delta->old_file.path : delta->new_file.path);
 
-	if (git_buf_oom(pi->buf))
+	if (git_buf_oom(out))
 		return -1;
 
 	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_FILE_HDR,
-			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
+			git_buf_cstr(out), git_buf_len(out), pi->payload))
 		return callback_error();
 
 	return 0;
 }
 
+/* print a git_diff_list to a print callback in raw output format */
 int git_diff_print_raw(
 	git_diff_list *diff,
 	git_diff_data_cb print_cb,
@@ -184,87 +191,115 @@ int git_diff_print_raw(
 	diff_print_info pi;
 
 	if (!(error = diff_print_info_init(&pi, &buf, diff, print_cb, payload)))
-		error = git_diff_foreach(diff, print_raw, NULL, NULL, &pi);
+		error = git_diff_foreach(diff, diff_print_one_raw, NULL, NULL, &pi);
 
 	git_buf_free(&buf);
 
 	return error;
 }
 
-static int print_oid_range(diff_print_info *pi, const git_diff_delta *delta)
+static int diff_print_oid_range(
+	git_buf *out, const git_diff_delta *delta, int oid_strlen)
 {
 	char start_oid[GIT_OID_HEXSZ+1], end_oid[GIT_OID_HEXSZ+1];
 
-	git_oid_tostr(start_oid, pi->oid_strlen, &delta->old_file.oid);
-	git_oid_tostr(end_oid, pi->oid_strlen, &delta->new_file.oid);
+	git_oid_tostr(start_oid, oid_strlen, &delta->old_file.oid);
+	git_oid_tostr(end_oid, oid_strlen, &delta->new_file.oid);
 
 	/* TODO: Match git diff more closely */
 	if (delta->old_file.mode == delta->new_file.mode) {
-		git_buf_printf(pi->buf, "index %s..%s %o\n",
+		git_buf_printf(out, "index %s..%s %o\n",
 			start_oid, end_oid, delta->old_file.mode);
 	} else {
 		if (delta->old_file.mode == 0) {
-			git_buf_printf(pi->buf, "new file mode %o\n", delta->new_file.mode);
+			git_buf_printf(out, "new file mode %o\n", delta->new_file.mode);
 		} else if (delta->new_file.mode == 0) {
-			git_buf_printf(pi->buf, "deleted file mode %o\n", delta->old_file.mode);
+			git_buf_printf(out, "deleted file mode %o\n", delta->old_file.mode);
 		} else {
-			git_buf_printf(pi->buf, "old mode %o\n", delta->old_file.mode);
-			git_buf_printf(pi->buf, "new mode %o\n", delta->new_file.mode);
+			git_buf_printf(out, "old mode %o\n", delta->old_file.mode);
+			git_buf_printf(out, "new mode %o\n", delta->new_file.mode);
 		}
-		git_buf_printf(pi->buf, "index %s..%s\n", start_oid, end_oid);
+		git_buf_printf(out, "index %s..%s\n", start_oid, end_oid);
 	}
 
-	if (git_buf_oom(pi->buf))
+	if (git_buf_oom(out))
 		return -1;
 
 	return 0;
 }
 
-static int print_patch_file(
-	const git_diff_delta *delta, float progress, void *data)
+static int diff_delta_format_with_paths(
+	git_buf *out,
+	const git_diff_delta *delta,
+	const char *oldpfx,
+	const char *newpfx,
+	const char *template)
 {
-	diff_print_info *pi = data;
-	const char *oldpfx = pi->diff->opts.old_prefix;
 	const char *oldpath = delta->old_file.path;
-	const char *newpfx = pi->diff->opts.new_prefix;
 	const char *newpath = delta->new_file.path;
 
-	GIT_UNUSED(progress);
+	if (git_oid_iszero(&delta->old_file.oid)) {
+		oldpfx = "";
+		oldpath = "/dev/null";
+	}
+	if (git_oid_iszero(&delta->new_file.oid)) {
+		newpfx = "";
+		newpath = "/dev/null";
+	}
 
-	if (S_ISDIR(delta->new_file.mode) ||
-		delta->status == GIT_DELTA_UNMODIFIED ||
-		delta->status == GIT_DELTA_IGNORED ||
-		(delta->status == GIT_DELTA_UNTRACKED &&
-		 (pi->diff->opts.flags & GIT_DIFF_INCLUDE_UNTRACKED_CONTENT) == 0))
-		return 0;
+	return git_buf_printf(out, template, oldpfx, oldpath, newpfx, newpath);
+}
 
+int git_diff_delta__format_file_header(
+	git_buf *out,
+	const git_diff_delta *delta,
+	const char *oldpfx,
+	const char *newpfx,
+	int oid_strlen)
+{
 	if (!oldpfx)
 		oldpfx = DIFF_OLD_PREFIX_DEFAULT;
-
 	if (!newpfx)
 		newpfx = DIFF_NEW_PREFIX_DEFAULT;
+	if (!oid_strlen)
+		oid_strlen = GIT_ABBREV_DEFAULT + 1;
 
-	git_buf_clear(pi->buf);
-	git_buf_printf(pi->buf, "diff --git %s%s %s%s\n", oldpfx, delta->old_file.path, newpfx, delta->new_file.path);
+	git_buf_clear(out);
 
-	if (print_oid_range(pi, delta) < 0)
+	git_buf_printf(out, "diff --git %s%s %s%s\n",
+		oldpfx, delta->old_file.path, newpfx, delta->new_file.path);
+
+	if (diff_print_oid_range(out, delta, oid_strlen) < 0)
 		return -1;
 
-	if (git_oid_iszero(&delta->old_file.oid)) {
-		oldpfx = "";
-		oldpath = "/dev/null";
-	}
-	if (git_oid_iszero(&delta->new_file.oid)) {
-		newpfx = "";
-		newpath = "/dev/null";
-	}
+	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0)
+		diff_delta_format_with_paths(
+			out, delta, oldpfx, newpfx, "--- %s%s\n+++ %s%s\n");
 
-	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0) {
-		git_buf_printf(pi->buf, "--- %s%s\n", oldpfx, oldpath);
-		git_buf_printf(pi->buf, "+++ %s%s\n", newpfx, newpath);
-	}
+	return git_buf_oom(out) ? -1 : 0;
+}
 
-	if (git_buf_oom(pi->buf))
+static int diff_print_patch_file(
+	const git_diff_delta *delta, float progress, void *data)
+{
+	diff_print_info *pi = data;
+	const char *oldpfx =
+		pi->diff ? pi->diff->opts.old_prefix : DIFF_OLD_PREFIX_DEFAULT;
+	const char *newpfx =
+		pi->diff ? pi->diff->opts.new_prefix : DIFF_NEW_PREFIX_DEFAULT;
+	uint32_t opts_flags = pi->diff ? pi->diff->opts.flags : GIT_DIFF_NORMAL;
+
+	GIT_UNUSED(progress);
+
+	if (S_ISDIR(delta->new_file.mode) ||
+		delta->status == GIT_DELTA_UNMODIFIED ||
+		delta->status == GIT_DELTA_IGNORED ||
+		(delta->status == GIT_DELTA_UNTRACKED &&
+		 (opts_flags & GIT_DIFF_INCLUDE_UNTRACKED_CONTENT) == 0))
+		return 0;
+
+	if (git_diff_delta__format_file_header(
+			pi->buf, delta, oldpfx, newpfx, pi->oid_strlen) < 0)
 		return -1;
 
 	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_FILE_HDR,
@@ -275,10 +310,10 @@ static int print_patch_file(
 		return 0;
 
 	git_buf_clear(pi->buf);
-	git_buf_printf(
-		pi->buf, "Binary files %s%s and %s%s differ\n",
-		oldpfx, oldpath, newpfx, newpath);
-	if (git_buf_oom(pi->buf))
+
+	if (diff_delta_format_with_paths(
+			pi->buf, delta, oldpfx, newpfx,
+			"Binary files %s%s and %s%s differ\n") < 0)
 		return -1;
 
 	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_BINARY,
@@ -288,7 +323,7 @@ static int print_patch_file(
 	return 0;
 }
 
-static int print_patch_hunk(
+static int diff_print_patch_hunk(
 	const git_diff_delta *d,
 	const git_diff_range *r,
 	const char *header,
@@ -311,7 +346,7 @@ static int print_patch_hunk(
 	return 0;
 }
 
-static int print_patch_line(
+static int diff_print_patch_line(
 	const git_diff_delta *delta,
 	const git_diff_range *range,
 	char line_origin, /* GIT_DIFF_LINE value from above */
@@ -343,6 +378,7 @@ static int print_patch_line(
 	return 0;
 }
 
+/* print a git_diff_list to an output callback in patch format */
 int git_diff_print_patch(
 	git_diff_list *diff,
 	git_diff_data_cb print_cb,
@@ -354,27 +390,15 @@ int git_diff_print_patch(
 
 	if (!(error = diff_print_info_init(&pi, &buf, diff, print_cb, payload)))
 		error = git_diff_foreach(
-			diff, print_patch_file, print_patch_hunk, print_patch_line, &pi);
+			diff, diff_print_patch_file, diff_print_patch_hunk,
+			diff_print_patch_line, &pi);
 
 	git_buf_free(&buf);
 
 	return error;
 }
 
-
-static int print_to_buffer_cb(
-	const git_diff_delta *delta,
-	const git_diff_range *range,
-	char line_origin,
-	const char *content,
-	size_t content_len,
-	void *payload)
-{
-	git_buf *output = payload;
-	GIT_UNUSED(delta); GIT_UNUSED(range); GIT_UNUSED(line_origin);
-	return git_buf_put(output, content, content_len);
-}
-
+/* print a git_diff_patch to an output callback */
 int git_diff_patch_print(
 	git_diff_patch *patch,
 	git_diff_data_cb print_cb,
@@ -389,13 +413,28 @@ int git_diff_patch_print(
 	if (!(error = diff_print_info_init(
 			&pi, &temp, git_diff_patch__diff(patch), print_cb, payload)))
 		error = git_diff_patch__invoke_callbacks(
-			patch, print_patch_file, print_patch_hunk, print_patch_line, &pi);
+			patch, diff_print_patch_file, diff_print_patch_hunk,
+			diff_print_patch_line, &pi);
 
 	git_buf_free(&temp);
 
 	return error;
 }
 
+static int diff_print_to_buffer_cb(
+	const git_diff_delta *delta,
+	const git_diff_range *range,
+	char line_origin,
+	const char *content,
+	size_t content_len,
+	void *payload)
+{
+	git_buf *output = payload;
+	GIT_UNUSED(delta); GIT_UNUSED(range); GIT_UNUSED(line_origin);
+	return git_buf_put(output, content, content_len);
+}
+
+/* print a git_diff_patch to a string buffer */
 int git_diff_patch_to_str(
 	char **string,
 	git_diff_patch *patch)
@@ -403,7 +442,7 @@ int git_diff_patch_to_str(
 	int error;
 	git_buf output = GIT_BUF_INIT;
 
-	error = git_diff_patch_print(patch, print_to_buffer_cb, &output);
+	error = git_diff_patch_print(patch, diff_print_to_buffer_cb, &output);
 
 	/* GIT_EUSER means git_buf_put in print_to_buffer_cb returned -1,
 	 * meaning a memory allocation failure, so just map to -1...
diff --git a/src/diff_tform.c b/src/diff_tform.c
index 94fa035f2..cbe8bafbd 100644
--- a/src/diff_tform.c
+++ b/src/diff_tform.c
@@ -408,57 +408,99 @@ GIT_INLINE(git_diff_file *) similarity_get_file(git_diff_list *diff, size_t idx)
 	return (idx & 1) ? &delta->new_file : &delta->old_file;
 }
 
-static int similarity_calc(
-	git_diff_list *diff,
+typedef struct {
+	size_t idx;
+	git_iterator_type_t src;
+	git_repository *repo;
+	git_diff_file *file;
+	git_buf data;
+	git_odb_object *odb_obj;
+	git_blob *blob;
+} similarity_info;
+
+static int similarity_init(
+	similarity_info *info, git_diff_list *diff, size_t file_idx)
+{
+	info->idx  = file_idx;
+	info->src  = (file_idx & 1) ? diff->new_src : diff->old_src;
+	info->repo = diff->repo;
+	info->file = similarity_get_file(diff, file_idx);
+	info->odb_obj = NULL;
+	info->blob = NULL;
+	git_buf_init(&info->data, 0);
+
+	if (info->file->size > 0)
+		return 0;
+
+	return git_diff_file__resolve_zero_size(
+		info->file, &info->odb_obj, info->repo);
+}
+
+static int similarity_sig(
+	similarity_info *info,
 	const git_diff_find_options *opts,
-	size_t file_idx,
 	void **cache)
 {
 	int error = 0;
-	git_diff_file *file = similarity_get_file(diff, file_idx);
-	git_iterator_type_t src = (file_idx & 1) ? diff->new_src : diff->old_src;
-
-	if (src == GIT_ITERATOR_TYPE_WORKDIR) { /* compute hashsig from file */
-		git_buf path = GIT_BUF_INIT;
-
-		/* TODO: apply wd-to-odb filters to file data if necessary */
+	git_diff_file *file = info->file;
 
+	if (info->src == GIT_ITERATOR_TYPE_WORKDIR) {
 		if ((error = git_buf_joinpath(
-				 &path, git_repository_workdir(diff->repo), file->path)) < 0)
+			&info->data, git_repository_workdir(info->repo), file->path)) < 0)
 			return error;
 
 		/* if path is not a regular file, just skip this item */
-		if (git_path_isfile(path.ptr))
-			error = opts->metric->file_signature(
-				&cache[file_idx], file, path.ptr, opts->metric->payload);
+		if (!git_path_isfile(info->data.ptr))
+			return 0;
 
-		git_buf_free(&path);
-	} else { /* compute hashsig from blob buffer */
-		git_blob *blob = NULL;
-		git_off_t blobsize;
+		/* TODO: apply wd-to-odb filters to file data if necessary */
 
-		/* TODO: add max size threshold a la diff? */
+		error = opts->metric->file_signature(
+			&cache[info->idx], info->file,
+			info->data.ptr, opts->metric->payload);
+	} else {
+		/* if we didn't initially know the size, we might have an odb_obj
+		 * around from earlier, so convert that, otherwise load the blob now
+		 */
+		if (info->odb_obj != NULL)
+			error = git_object__from_odb_object(
+				(git_object **)&info->blob, info->repo,
+				info->odb_obj, GIT_OBJ_BLOB);
+		else
+			error = git_blob_lookup(&info->blob, info->repo, &file->oid);
 
-		if (git_blob_lookup(&blob, diff->repo, &file->oid) < 0) {
+		if (error < 0) {
 			/* if lookup fails, just skip this item in similarity calc */
 			giterr_clear();
-			return 0;
-		}
+		} else {
+			size_t sz;
 
-		blobsize = git_blob_rawsize(blob);
-		if (!git__is_sizet(blobsize)) /* ? what to do ? */
-			blobsize = (size_t)-1;
+			/* index size may not be actual blob size if filtered */
+			if (file->size != git_blob_rawsize(info->blob))
+				file->size = git_blob_rawsize(info->blob);
 
-		error = opts->metric->buffer_signature(
-			&cache[file_idx], file, git_blob_rawcontent(blob),
-			(size_t)blobsize, opts->metric->payload);
+			sz = (size_t)(git__is_sizet(file->size) ? file->size : -1);
 
-		git_blob_free(blob);
+			error = opts->metric->buffer_signature(
+				&cache[info->idx], info->file,
+				git_blob_rawcontent(info->blob), sz, opts->metric->payload);
+		}
 	}
 
 	return error;
 }
 
+static void similarity_unload(similarity_info *info)
+{
+	if (info->odb_obj)
+		git_odb_object_free(info->odb_obj);
+
+	if (info->blob)
+		git_blob_free(info->blob);
+	else
+		git_buf_free(&info->data);
+}
+
 #define FLAG_SET(opts,flag_name) (((opts)->flags & flag_name) != 0)
 
 /* - score < 0 means files cannot be compared
@@ -476,6 +518,8 @@ static int similarity_measure(
 	git_diff_file *a_file = similarity_get_file(diff, a_idx);
 	git_diff_file *b_file = similarity_get_file(diff, b_idx);
 	bool exact_match = FLAG_SET(opts, GIT_DIFF_FIND_EXACT_MATCH_ONLY);
+	int error = 0;
+	similarity_info a_info, b_info;
 
 	*score = -1;
 
@@ -483,7 +527,7 @@ static int similarity_measure(
 	if (GIT_MODE_TYPE(a_file->mode) != GIT_MODE_TYPE(b_file->mode))
 		return 0;
 
-	/* if exact match is requested, force calculation of missing OIDs */
+	/* if exact match is requested, force calculation of missing OIDs now */
 	if (exact_match) {
 		if (git_oid_iszero(&a_file->oid) &&
 			diff->old_src == GIT_ITERATOR_TYPE_WORKDIR &&
@@ -510,19 +554,44 @@ static int similarity_measure(
 		return 0;
 	}
 
+	memset(&a_info, 0, sizeof(a_info));
+	memset(&b_info, 0, sizeof(b_info));
+
+	/* set up similarity data (will try to update missing file sizes) */
+	if (!cache[a_idx] && (error = similarity_init(&a_info, diff, a_idx)) < 0)
+		return error;
+	if (!cache[b_idx] && (error = similarity_init(&b_info, diff, b_idx)) < 0)
+		goto cleanup;
+
+	/* check if file sizes are nowhere near each other */
+	if (a_file->size > 127 &&
+		b_file->size > 127 &&
+		(a_file->size > (b_file->size << 3) ||
+		 b_file->size > (a_file->size << 3)))
+		goto cleanup;
+
 	/* update signature cache if needed */
-	if (!cache[a_idx] && similarity_calc(diff, opts, a_idx, cache) < 0)
-		return -1;
-	if (!cache[b_idx] && similarity_calc(diff, opts, b_idx, cache) < 0)
-		return -1;
+	if (!cache[a_idx]) {
+		if ((error = similarity_sig(&a_info, opts, cache)) < 0)
+			goto cleanup;
+	}
+	if (!cache[b_idx]) {
+		if ((error = similarity_sig(&b_info, opts, cache)) < 0)
+			goto cleanup;
+	}
 
-	/* some metrics may not wish to process this file (too big / too small) */
-	if (!cache[a_idx] || !cache[b_idx])
-		return 0;
+	/* calculate similarity provided that the metric choose to process
+	 * both the a and b files (some may not if file is too big, etc).
+	 */
+	if (cache[a_idx] && cache[b_idx])
+		error = opts->metric->similarity(
+			score, cache[a_idx], cache[b_idx], opts->metric->payload);
+
+cleanup:
+	similarity_unload(&a_info);
+	similarity_unload(&b_info);
 
-	/* compare signatures */
-	return opts->metric->similarity(
-		score, cache[a_idx], cache[b_idx], opts->metric->payload);
+	return error;
 }
 
 static int calc_self_similarity(
@@ -590,11 +659,13 @@ static bool is_rename_target(
 		return false;
 
 	case GIT_DELTA_UNTRACKED:
-	case GIT_DELTA_IGNORED:
 		if (!FLAG_SET(opts, GIT_DIFF_FIND_FOR_UNTRACKED))
 			return false;
 		break;
 
+	case GIT_DELTA_IGNORED:
+		return false;
+
 	default: /* all other status values should be checked */
 		break;
 	}
@@ -673,6 +744,15 @@ GIT_INLINE(bool) delta_is_new_only(git_diff_delta *delta)
 			delta->status == GIT_DELTA_IGNORED);
 }
 
+GIT_INLINE(void) delta_make_rename(
+	git_diff_delta *to, const git_diff_delta *from, uint32_t similarity)
+{
+	to->status     = GIT_DELTA_RENAMED;
+	to->similarity = similarity;
+	memcpy(&to->old_file, &from->old_file, sizeof(to->old_file));
+	to->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
+}
+
 typedef struct {
 	uint32_t idx;
 	uint32_t similarity;
@@ -682,85 +762,156 @@ int git_diff_find_similar(
 	git_diff_list *diff,
 	git_diff_find_options *given_opts)
 {
-	size_t i, j, cache_size;
+	size_t s, t;
 	int error = 0, similarity;
-	git_diff_delta *from, *to;
+	git_diff_delta *src, *tgt;
 	git_diff_find_options opts;
-	size_t num_rewrites = 0, num_updates = 0;
-	void **cache; /* cache of similarity metric file signatures */
-	diff_find_match *match_sources, *match_targets; /* cache of best matches */
+	size_t num_deltas, num_srcs = 0, num_tgts = 0;
+	size_t tried_srcs = 0, tried_tgts = 0;
+	size_t num_rewrites = 0, num_updates = 0, num_bumped = 0;
+	void **sigcache; /* cache of similarity metric file signatures */
+	diff_find_match *tgt2src = NULL;
+	diff_find_match *src2tgt = NULL;
+	diff_find_match *tgt2src_copy = NULL;
+	diff_find_match *best_match;
+	git_diff_file swap;
 
 	if ((error = normalize_find_opts(diff, &opts, given_opts)) < 0)
 		return error;
 
+	num_deltas = diff->deltas.length;
+
 	/* TODO: maybe abort if deltas.length > rename_limit ??? */
-	if (!git__is_uint32(diff->deltas.length))
+	if (!git__is_uint32(num_deltas))
 		return 0;
 
-	cache_size = diff->deltas.length * 2; /* must store b/c length may change */
-	cache = git__calloc(cache_size, sizeof(void *));
-	GITERR_CHECK_ALLOC(cache);
+	sigcache = git__calloc(num_deltas * 2, sizeof(void *));
+	GITERR_CHECK_ALLOC(sigcache);
+
+	/* Label rename sources and targets
+	 *
+	 * This will also set self-similarity scores for MODIFIED files and
+	 * mark them for splitting if break-rewrites is enabled
+	 */
+	git_vector_foreach(&diff->deltas, t, tgt) {
+		if (is_rename_source(diff, &opts, t, sigcache))
+			++num_srcs;
+
+		if (is_rename_target(diff, &opts, t, sigcache))
+			++num_tgts;
+
+		if ((tgt->flags & GIT_DIFF_FLAG__TO_SPLIT) != 0)
+			num_rewrites++;
+	}
+
+	/* if there are no candidate srcs or tgts, we're done */
+	if (!num_srcs || !num_tgts)
+		goto cleanup;
 
-	match_sources = git__calloc(diff->deltas.length, sizeof(diff_find_match));
-	match_targets = git__calloc(diff->deltas.length, sizeof(diff_find_match));
-	GITERR_CHECK_ALLOC(match_sources);
-	GITERR_CHECK_ALLOC(match_targets);
+	src2tgt = git__calloc(num_deltas, sizeof(diff_find_match));
+	GITERR_CHECK_ALLOC(src2tgt);
+	tgt2src = git__calloc(num_deltas, sizeof(diff_find_match));
+	GITERR_CHECK_ALLOC(tgt2src);
 
-	/* next find the most similar delta for each rename / copy candidate */
+	if (FLAG_SET(&opts, GIT_DIFF_FIND_COPIES)) {
+		tgt2src_copy = git__calloc(num_deltas, sizeof(diff_find_match));
+		GITERR_CHECK_ALLOC(tgt2src_copy);
+	}
 
-	git_vector_foreach(&diff->deltas, i, to) {
-		size_t tried_sources = 0;
+	/*
+	 * Find best-fit matches for rename / copy candidates
+	 */
 
-		match_targets[i].idx = (uint32_t)i;
-		match_targets[i].similarity = 0;
+find_best_matches:
+	tried_tgts = num_bumped = 0;
 
+	git_vector_foreach(&diff->deltas, t, tgt) {
 		/* skip things that are not rename targets */
-		if (!is_rename_target(diff, &opts, i, cache))
+		if ((tgt->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0)
 			continue;
 
-		git_vector_foreach(&diff->deltas, j, from) {
-			if (i == j)
-				continue;
+		tried_srcs = 0;
 
+		git_vector_foreach(&diff->deltas, s, src) {
 			/* skip things that are not rename sources */
-			if (!is_rename_source(diff, &opts, j, cache))
+			if ((src->flags & GIT_DIFF_FLAG__IS_RENAME_SOURCE) == 0)
 				continue;
 
-			/* cap on maximum targets we'll examine (per "to" file) */
-			if (++tried_sources > opts.rename_limit)
-				break;
-
 			/* calculate similarity for this pair and find best match */
-			if ((error = similarity_measure(
-					&similarity, diff, &opts, cache, 2 * j, 2 * i + 1)) < 0)
+			if (s == t)
+				similarity = -1; /* don't measure self-similarity here */
+			else if ((error = similarity_measure(
+				&similarity, diff, &opts, sigcache, 2 * s, 2 * t + 1)) < 0)
 				goto cleanup;
 
-			if (similarity < 0) { /* not actually comparable */
-				--tried_sources;
+			if (similarity < 0)
 				continue;
+
+			/* is this a better rename? */
+			if (tgt2src[t].similarity < (uint32_t)similarity &&
+				src2tgt[s].similarity < (uint32_t)similarity)
+			{
+				/* eject old mapping */
+				if (src2tgt[s].similarity > 0) {
+					tgt2src[src2tgt[s].idx].similarity = 0;
+					num_bumped++;
+				}
+				if (tgt2src[t].similarity > 0) {
+					src2tgt[tgt2src[t].idx].similarity = 0;
+					num_bumped++;
+				}
+
+				/* write new mapping */
+				tgt2src[t].idx = (uint32_t)s;
+				tgt2src[t].similarity = (uint32_t)similarity;
+				src2tgt[s].idx = (uint32_t)t;
+				src2tgt[s].similarity = (uint32_t)similarity;
 			}
 
-			if (match_targets[i].similarity < (uint32_t)similarity &&
-				match_sources[j].similarity < (uint32_t)similarity) {
-				match_targets[i].similarity = (uint32_t)similarity;
-				match_sources[j].similarity = (uint32_t)similarity;
-				match_targets[i].idx = (uint32_t)j;
-				match_sources[j].idx = (uint32_t)i;
+			/* keep best absolute match for copies */
+			if (tgt2src_copy != NULL &&
+				tgt2src_copy[t].similarity < (uint32_t)similarity)
+			{
+				tgt2src_copy[t].idx = (uint32_t)s;
+				tgt2src_copy[t].similarity = (uint32_t)similarity;
 			}
+
+			if (++tried_srcs >= num_srcs)
+				break;
+
+			/* cap on maximum targets we'll examine (per "tgt" file) */
+			if (tried_srcs > opts.rename_limit)
+				break;
 		}
+
+		if (++tried_tgts >= num_tgts)
+			break;
 	}
 
-	/* next rewrite the diffs with renames / copies */
+	if (num_bumped > 0) /* try again if we bumped some items */
+		goto find_best_matches;
 
-	git_vector_foreach(&diff->deltas, i, to) {
-		/* check if this delta was the target of a similarity */
-		if ((similarity = (int)match_targets[i].similarity) <= 0)
+	/*
+	 * Rewrite the diffs with renames / copies
+	 */
+
+	tried_tgts = 0;
+
+	git_vector_foreach(&diff->deltas, t, tgt) {
+		/* skip things that are not rename targets */
+		if ((tgt->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0)
 			continue;
 
-		assert(to && (to->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) != 0);
+		/* check if this delta was the target of a similarity */
+		if (tgt2src[t].similarity)
+			best_match = &tgt2src[t];
+		else if (tgt2src_copy && tgt2src_copy[t].similarity)
+			best_match = &tgt2src_copy[t];
+		else
+			continue;
 
-		from = GIT_VECTOR_GET(&diff->deltas, match_targets[i].idx);
-		assert(from && (from->flags & GIT_DIFF_FLAG__IS_RENAME_SOURCE) != 0);
+		s = best_match->idx;
+		src = GIT_VECTOR_GET(&diff->deltas, s);
 
 		/* possible scenarios:
 		 * 1. from DELETE to ADD/UNTRACK/IGNORE = RENAME
@@ -770,135 +921,137 @@ int git_diff_find_similar(
 		 * 5. from OTHER to ADD/UNTRACK/IGNORE = OTHER + COPY
 		 */
 
-		if (from->status == GIT_DELTA_DELETED) {
+		if (src->status == GIT_DELTA_DELETED) {
 
-			if (delta_is_new_only(to)) {
+			if (delta_is_new_only(tgt)) {
 
-				if (similarity < (int)opts.rename_threshold)
+				if (best_match->similarity < opts.rename_threshold)
 					continue;
 
-				from->status = GIT_DELTA_RENAMED;
-				from->similarity = (uint32_t)similarity;
-				memcpy(&from->new_file, &to->new_file, sizeof(from->new_file));
-
-				to->flags |= GIT_DIFF_FLAG__TO_DELETE;
+				delta_make_rename(tgt, src, best_match->similarity);
 
+				src->flags |= GIT_DIFF_FLAG__TO_DELETE;
 				num_rewrites++;
 			} else {
-				assert(delta_is_split(to));
+				assert(delta_is_split(tgt));
 
-				if (similarity < (int)opts.rename_from_rewrite_threshold)
+				if (best_match->similarity < opts.rename_from_rewrite_threshold)
 					continue;
 
-				from->status = GIT_DELTA_RENAMED;
-				from->similarity = (uint32_t)similarity;
-				memcpy(&from->new_file, &to->new_file, sizeof(from->new_file));
+				memcpy(&swap, &tgt->old_file, sizeof(swap));
 
-				to->status = GIT_DELTA_DELETED;
-				memset(&to->new_file, 0, sizeof(to->new_file));
-				to->new_file.path = to->old_file.path;
-				to->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
-				if ((to->flags & GIT_DIFF_FLAG__TO_SPLIT) != 0) {
-					to->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
-					num_rewrites--;
-				}
+				delta_make_rename(tgt, src, best_match->similarity);
+				num_rewrites--;
+
+				src->status = GIT_DELTA_DELETED;
+				memcpy(&src->old_file, &swap, sizeof(src->old_file));
+				memset(&src->new_file, 0, sizeof(src->new_file));
+				src->new_file.path = src->old_file.path;
+				src->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
 
 				num_updates++;
+
+				if (src2tgt[t].similarity > 0 && src2tgt[t].idx > t) {
+					/* what used to be at src t is now at src s */
+					tgt2src[src2tgt[t].idx].idx = (uint32_t)s;
+				}
 			}
 		}
 
-		else if (delta_is_split(from)) {
-			git_diff_file swap;
+		else if (delta_is_split(src)) {
 
-			if (delta_is_new_only(to)) {
+			if (delta_is_new_only(tgt)) {
 
-				if (similarity < (int)opts.rename_threshold)
+				if (best_match->similarity < opts.rename_threshold)
 					continue;
 
-				memcpy(&swap, &from->new_file, sizeof(swap));
-
-				from->status = GIT_DELTA_RENAMED;
-				from->similarity = (uint32_t)similarity;
-				memcpy(&from->new_file, &to->new_file, sizeof(from->new_file));
-				if ((from->flags & GIT_DIFF_FLAG__TO_SPLIT) != 0) {
-					from->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
-					num_rewrites--;
-				}
+				delta_make_rename(tgt, src, best_match->similarity);
 
-				to->status = (diff->new_src == GIT_ITERATOR_TYPE_WORKDIR) ?
+				src->status = (diff->new_src == GIT_ITERATOR_TYPE_WORKDIR) ?
 					GIT_DELTA_UNTRACKED : GIT_DELTA_ADDED;
-				memcpy(&to->new_file, &swap, sizeof(to->new_file));
-				to->old_file.path = to->new_file.path;
+				memset(&src->old_file, 0, sizeof(src->old_file));
+				src->old_file.path = src->new_file.path;
+				src->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+
+				src->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
+				num_rewrites--;
 
 				num_updates++;
 			} else {
-				assert(delta_is_split(from));
+				assert(delta_is_split(src));
 
-				if (similarity < (int)opts.rename_from_rewrite_threshold)
+				if (best_match->similarity < opts.rename_from_rewrite_threshold)
 					continue;
 
-				memcpy(&swap, &to->new_file, sizeof(swap));
+				memcpy(&swap, &tgt->old_file, sizeof(swap));
 
-				to->status = GIT_DELTA_RENAMED;
-				to->similarity = (uint32_t)similarity;
-				memcpy(&to->new_file, &from->new_file, sizeof(to->new_file));
-				if ((to->flags & GIT_DIFF_FLAG__TO_SPLIT) != 0) {
-					to->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
-					num_rewrites--;
-				}
+				delta_make_rename(tgt, src, best_match->similarity);
+				num_rewrites--;
+				num_updates++;
 
-				memcpy(&from->new_file, &swap, sizeof(from->new_file));
-				if ((from->flags & GIT_DIFF_FLAG__TO_SPLIT) == 0) {
-					from->flags |= GIT_DIFF_FLAG__TO_SPLIT;
-					num_rewrites++;
-				}
+				memcpy(&src->old_file, &swap, sizeof(src->old_file));
 
-				/* in the off chance that we've just swapped the new
-				 * element into the correct place, clear the SPLIT flag
+				/* if we've just swapped the new element into the correct
+				 * place, clear the SPLIT flag
 				 */
-				if (match_targets[match_targets[i].idx].idx == i &&
-					match_targets[match_targets[i].idx].similarity >
+				if (tgt2src[s].idx == t &&
+					tgt2src[s].similarity >
 					opts.rename_from_rewrite_threshold) {
-
-					from->status = GIT_DELTA_RENAMED;
-					from->similarity =
-						(uint32_t)match_targets[match_targets[i].idx].similarity;
-					match_targets[match_targets[i].idx].similarity = 0;
-					from->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
+					src->status     = GIT_DELTA_RENAMED;
+					src->similarity = tgt2src[s].similarity;
+					tgt2src[s].similarity = 0;
+					src->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
 					num_rewrites--;
 				}
+				/* otherwise, if we just overwrote a source, update mapping */
+				else if (src2tgt[t].similarity > 0 && src2tgt[t].idx > t) {
+					/* what used to be at src t is now at src s */
+					tgt2src[src2tgt[t].idx].idx = (uint32_t)s;
+				}
 
 				num_updates++;
 			}
 		}
 
-		else if (delta_is_new_only(to)) {
-			if (!FLAG_SET(&opts, GIT_DIFF_FIND_COPIES) ||
-				similarity < (int)opts.copy_threshold)
+		else if (delta_is_new_only(tgt)) {
+			if (!FLAG_SET(&opts, GIT_DIFF_FIND_COPIES))
+				continue;
+
+			if (tgt2src_copy[t].similarity < opts.copy_threshold)
 				continue;
 
-			to->status = GIT_DELTA_COPIED;
-			to->similarity = (uint32_t)similarity;
-			memcpy(&to->old_file, &from->old_file, sizeof(to->old_file));
+			/* always use best possible source for copy */
+			best_match = &tgt2src_copy[t];
+			src = GIT_VECTOR_GET(&diff->deltas, best_match->idx);
+
+			tgt->status     = GIT_DELTA_COPIED;
+			tgt->similarity = best_match->similarity;
+			memcpy(&tgt->old_file, &src->old_file, sizeof(tgt->old_file));
 
 			num_updates++;
 		}
 	}
 
+	/*
+	 * Actually split and delete entries as needed
+	 */
+
 	if (num_rewrites > 0 || num_updates > 0)
 		error = apply_splits_and_deletes(
 			diff, diff->deltas.length - num_rewrites,
-			FLAG_SET(&opts, GIT_DIFF_BREAK_REWRITES));
+			FLAG_SET(&opts, GIT_DIFF_BREAK_REWRITES) &&
+			!FLAG_SET(&opts, GIT_DIFF_BREAK_REWRITES_FOR_RENAMES_ONLY));
 
 cleanup:
-	git__free(match_sources);
-	git__free(match_targets);
+	git__free(tgt2src);
+	git__free(src2tgt);
+	git__free(tgt2src_copy);
 
-	for (i = 0; i < cache_size; ++i) {
-		if (cache[i] != NULL)
-			opts.metric->free_signature(cache[i], opts.metric->payload);
+	for (t = 0; t < num_deltas * 2; ++t) {
+		if (sigcache[t] != NULL)
+			opts.metric->free_signature(sigcache[t], opts.metric->payload);
 	}
-	git__free(cache);
+	git__free(sigcache);
 
 	if (!given_opts || !given_opts->metric)
 		git__free(opts.metric);
diff --git a/src/filebuf.c b/src/filebuf.c
index 246ae34e7..714a32395 100644
--- a/src/filebuf.c
+++ b/src/filebuf.c
@@ -53,7 +53,7 @@ static int lock_file(git_filebuf *file, int flags)
 			giterr_clear(); /* actual OS error code just confuses */
 			giterr_set(GITERR_OS,
 				"Failed to lock file '%s' for writing", file->path_lock);
-			return -1;
+			return GIT_ELOCKED;
 		}
 	}
 
@@ -66,7 +66,7 @@ static int lock_file(git_filebuf *file, int flags)
 	}
 
 	if (file->fd < 0)
-		return -1;
+		return file->fd;
 
 	file->fd_is_open = true;
 
@@ -197,7 +197,7 @@ static int write_deflate(git_filebuf *file, void *source, size_t len)
 
 int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 {
-	int compression;
+	int compression, error = -1;
 	size_t path_len;
 
 	/* opening an already open buffer is a programming error;
@@ -282,7 +282,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 		memcpy(file->path_lock + path_len, GIT_FILELOCK_EXTENSION, GIT_FILELOCK_EXTLENGTH);
 
 		/* open the file for locking */
-		if (lock_file(file, flags) < 0)
+		if ((error = lock_file(file, flags)) < 0)
 			goto cleanup;
 	}
 
@@ -290,7 +290,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 
 cleanup:
 	git_filebuf_cleanup(file);
-	return -1;
+	return error;
 }
 
 int git_filebuf_hash(git_oid *oid, git_filebuf *file)
diff --git a/src/fileops.c b/src/fileops.c
index 02f48e120..126d45f26 100644
--- a/src/fileops.c
+++ b/src/fileops.c
@@ -58,17 +58,19 @@ int git_futils_creat_locked(const char *path, const mode_t mode)
 	int fd;
 
 #ifdef GIT_WIN32
-	wchar_t buf[GIT_WIN_PATH];
+	git_win32_path buf;
 
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
-	fd = _wopen(buf, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_EXCL, mode);
+	git_win32_path_from_c(buf, path);
+	fd = _wopen(buf, O_WRONLY | O_CREAT | O_TRUNC |
+		O_EXCL | O_BINARY | O_CLOEXEC, mode);
 #else
-	fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_EXCL, mode);
+	fd = open(path, O_WRONLY | O_CREAT | O_TRUNC |
+		O_EXCL | O_BINARY | O_CLOEXEC, mode);
 #endif
 
 	if (fd < 0) {
 		giterr_set(GITERR_OS, "Failed to create locked file '%s'", path);
-		return -1;
+		return errno == EEXIST ? GIT_ELOCKED : -1;
 	}
 
 	return fd;
@@ -108,7 +110,7 @@ git_off_t git_futils_filesize(git_file fd)
 mode_t git_futils_canonical_mode(mode_t raw_mode)
 {
 	if (S_ISREG(raw_mode))
-		return S_IFREG | GIT_CANONICAL_PERMS(raw_mode);
+		return S_IFREG | GIT_PERMS_CANONICAL(raw_mode);
 	else if (S_ISLNK(raw_mode))
 		return S_IFLNK;
 	else if (S_ISGITLINK(raw_mode))
@@ -145,6 +147,7 @@ int git_futils_readbuffer_fd(git_buf *buf, git_file fd, size_t len)
 int git_futils_readbuffer_updated(
 	git_buf *buf, const char *path, time_t *mtime, size_t *size, int *updated)
 {
+	int error = 0;
 	git_file fd;
 	struct stat st;
 	bool changed = false;
@@ -154,11 +157,15 @@ int git_futils_readbuffer_updated(
 	if (updated != NULL)
 		*updated = 0;
 
-	if ((fd = git_futils_open_ro(path)) < 0)
-		return fd;
+	if (p_stat(path, &st) < 0) {
+		error = errno;
+		giterr_set(GITERR_OS, "Failed to stat '%s'", path);
+		if (error == ENOENT || error == ENOTDIR)
+			return GIT_ENOTFOUND;
+		return -1;
+	}
 
-	if (p_fstat(fd, &st) < 0 || S_ISDIR(st.st_mode) || !git__is_sizet(st.st_size+1)) {
-		p_close(fd);
+	if (S_ISDIR(st.st_mode) || !git__is_sizet(st.st_size+1)) {
 		giterr_set(GITERR_OS, "Invalid regular file stat for '%s'", path);
 		return -1;
 	}
@@ -175,7 +182,6 @@ int git_futils_readbuffer_updated(
 		changed = true;
 
 	if (!changed) {
-		p_close(fd);
 		return 0;
 	}
 
@@ -184,6 +190,9 @@ int git_futils_readbuffer_updated(
 	if (size != NULL)
 		*size = (size_t)st.st_size;
 
+	if ((fd = git_futils_open_ro(path)) < 0)
+		return fd;
+
 	if (git_futils_readbuffer_fd(buf, fd, (size_t)st.st_size) < 0) {
 		p_close(fd);
 		return -1;
@@ -220,6 +229,7 @@ int git_futils_writebuffer(
 	if ((error = p_write(fd, git_buf_cstr(buf), git_buf_len(buf))) < 0) {
 		giterr_set(GITERR_OS, "Could not write to '%s'", path);
 		(void)p_close(fd);
+		return error;
 	}
 
 	if ((error = p_close(fd)) < 0)
@@ -320,7 +330,7 @@ int git_futils_mkdir(
 	min_root_len = git_path_root(make_path.ptr);
 	if (root < min_root_len)
 		root = min_root_len;
-	while (make_path.ptr[root] == '/')
+	while (root >= 0 && make_path.ptr[root] == '/')
 		++root;
 
 	/* clip root to make_path length */
@@ -625,6 +635,18 @@ static git_futils_dirs_guess_cb git_futils__dir_guess[GIT_FUTILS_DIR__MAX] = {
 	git_futils_guess_xdg_dirs,
 };
 
+int git_futils_dirs_global_init(void)
+{
+	git_futils_dir_t i;
+	const git_buf *path;
+	int error = 0;
+
+	for (i = 0; !error && i < GIT_FUTILS_DIR__MAX; i++)
+		error = git_futils_dirs_get(&path, i);
+
+	return error;
+}
+
 static int git_futils_check_selector(git_futils_dir_t which)
 {
 	if (which < GIT_FUTILS_DIR__MAX)
@@ -847,6 +869,7 @@ typedef struct {
 	uint32_t flags;
 	uint32_t mkdir_flags;
 	mode_t dirmode;
+	int error;
 } cp_r_info;
 
 #define GIT_CPDIR__MKDIR_DONE_FOR_TO_ROOT (1u << 10)
@@ -885,20 +908,23 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		return 0;
 
 	if (git_buf_joinpath(
-			&info->to, info->to_root, from->ptr + info->from_prefix) < 0)
-		return -1;
+			&info->to, info->to_root, from->ptr + info->from_prefix) < 0) {
+		error = -1;
+		goto exit;
+	}
 
 	if (p_lstat(info->to.ptr, &to_st) < 0) {
 		if (errno != ENOENT && errno != ENOTDIR) {
 			giterr_set(GITERR_OS,
 				"Could not access %s while copying files", info->to.ptr);
-			return -1;
+			error = -1;
+			goto exit;
 		}
 	} else
 		exists = true;
 
 	if ((error = git_path_lstat(from->ptr, &from_st)) < 0)
-		return error;
+		goto exit;
 
 	if (S_ISDIR(from_st.st_mode)) {
 		mode_t oldmode = info->dirmode;
@@ -912,13 +938,14 @@ static int _cp_r_callback(void *ref, git_buf *from)
 			error = _cp_r_mkdir(info, from);
 
 		/* recurse onto target directory */
-		if (!error && (!exists || S_ISDIR(to_st.st_mode)))
-			error = git_path_direach(from, _cp_r_callback, info);
+		if (!error && (!exists || S_ISDIR(to_st.st_mode)) &&
+			((error = git_path_direach(from, _cp_r_callback, info)) == GIT_EUSER))
+			error = info->error;
 
 		if (oldmode != 0)
 			info->dirmode = oldmode;
 
-		return error;
+		goto exit;
 	}
 
 	if (exists) {
@@ -928,7 +955,8 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		if (p_unlink(info->to.ptr) < 0) {
 			giterr_set(GITERR_OS, "Cannot overwrite existing file '%s'",
 				info->to.ptr);
-			return -1;
+			error = -1;
+			goto exit;
 		}
 	}
 
@@ -941,7 +969,7 @@ static int _cp_r_callback(void *ref, git_buf *from)
 	/* Make container directory on demand if needed */
 	if ((info->flags & GIT_CPDIR_CREATE_EMPTY_DIRS) == 0 &&
 		(error = _cp_r_mkdir(info, from)) < 0)
-		return error;
+		goto exit;
 
 	/* make symlink or regular file */
 	if (S_ISLNK(from_st.st_mode))
@@ -950,11 +978,13 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		mode_t usemode = from_st.st_mode;
 
 		if ((info->flags & GIT_CPDIR_SIMPLE_TO_MODE) != 0)
-			usemode = (usemode & 0111) ? 0777 : 0666;
+			usemode = GIT_PERMS_FOR_WRITE(usemode);
 
 		error = git_futils_cp(from->ptr, info->to.ptr, usemode);
 	}
 
+exit:
+	info->error = error;
 	return error;
 }
 
@@ -975,6 +1005,7 @@ int git_futils_cp_r(
 	info.flags   = flags;
 	info.dirmode = dirmode;
 	info.from_prefix = path.size;
+	info.error = 0;
 	git_buf_init(&info.to, 0);
 
 	/* precalculate mkdir flags */
@@ -996,6 +1027,9 @@ int git_futils_cp_r(
 	git_buf_free(&path);
 	git_buf_free(&info.to);
 
+	if (error == GIT_EUSER)
+		error = info.error;
+
 	return error;
 }
 
diff --git a/src/fileops.h b/src/fileops.h
index f4e059c83..02f79b9e7 100644
--- a/src/fileops.h
+++ b/src/fileops.h
@@ -223,9 +223,13 @@ extern int git_futils_open_ro(const char *path);
  */
 extern git_off_t git_futils_filesize(git_file fd);
 
+#define GIT_PERMS_IS_EXEC(MODE)		(((MODE) & 0111) != 0)
+#define GIT_PERMS_CANONICAL(MODE)	(GIT_PERMS_IS_EXEC(MODE) ? 0755 : 0644)
+#define GIT_PERMS_FOR_WRITE(MODE)   (GIT_PERMS_IS_EXEC(MODE) ? 0777 : 0666)
+
 #define GIT_MODE_PERMS_MASK			0777
-#define GIT_CANONICAL_PERMS(MODE)	(((MODE) & 0100) ? 0755 : 0644)
-#define GIT_MODE_TYPE(MODE)			((MODE) & ~GIT_MODE_PERMS_MASK)
+#define GIT_MODE_TYPE_MASK			0170000
+#define GIT_MODE_TYPE(MODE)			((MODE) & GIT_MODE_TYPE_MASK)
 #define GIT_MODE_ISBLOB(MODE)		(GIT_MODE_TYPE(MODE) == GIT_MODE_TYPE(GIT_FILEMODE_BLOB))
 
 /**
@@ -244,7 +248,7 @@ extern mode_t git_futils_canonical_mode(mode_t raw_mode);
  * @param out buffer to populate with the mapping information.
  * @param fd open descriptor to configure the mapping from.
  * @param begin first byte to map, this should be page aligned.
- * @param end number of bytes to map.
+ * @param len number of bytes to map.
  * @return
  * - 0 on success;
  * - -1 on error.
@@ -278,7 +282,7 @@ extern void git_futils_mmap_free(git_map *map);
 /**
  * Find a "global" file (i.e. one in a user's home directory).
  *
- * @param pathbuf buffer to write the full path into
+ * @param path buffer to write the full path into
  * @param filename name of file to find in the home directory
  * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
  */
@@ -287,7 +291,7 @@ extern int git_futils_find_global_file(git_buf *path, const char *filename);
 /**
  * Find an "XDG" file (i.e. one in user's XDG config path).
  *
- * @param pathbuf buffer to write the full path into
+ * @param path buffer to write the full path into
  * @param filename name of file to find in the home directory
  * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
  */
@@ -296,7 +300,7 @@ extern int git_futils_find_xdg_file(git_buf *path, const char *filename);
 /**
  * Find a "system" file (i.e. one shared for all users of the system).
  *
- * @param pathbuf buffer to write the full path into
+ * @param path buffer to write the full path into
  * @param filename name of file to find in the home directory
  * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
  */
@@ -310,6 +314,13 @@ typedef enum {
 } git_futils_dir_t;
 
 /**
+ * Configures global data for configuration file search paths.
+ *
+ * @return 0 on success, <0 on failure
+ */
+extern int git_futils_dirs_global_init(void);
+
+/**
  * Get the search path for global/system/xdg files
  *
  * @param out pointer to git_buf containing search path
diff --git a/src/global.c b/src/global.c
index 2d40ca2fc..b504e5e0a 100644
--- a/src/global.c
+++ b/src/global.c
@@ -65,26 +65,28 @@ int git_threads_init(void)
 		return -1;
 
 	/* Initialize any other subsystems that have global state */
-	if ((error = git_hash_global_init()) >= 0)
-		_tls_init = 1;
-
-	if (error == 0)
+	if ((error = git_hash_global_init()) >= 0 &&
+		(error = git_futils_dirs_global_init()) >= 0)
 		_tls_init = 1;
 
 	GIT_MEMORY_BARRIER;
 
+	win32_pthread_initialize();
+
 	return error;
 }
 
 void git_threads_shutdown(void)
 {
+	/* Shut down any subsystems that have global state */
+	win32_pthread_shutdown();
+	git_futils_dirs_free();
+	git_hash_global_shutdown();
+
 	TlsFree(_tls_index);
 	_tls_init = 0;
-	git_mutex_free(&git__mwindow_mutex);
 
-	/* Shut down any subsystems that have global state */
-	git_hash_global_shutdown();
-	git_futils_dirs_free();
+	git_mutex_free(&git__mwindow_mutex);
 }
 
 git_global_st *git__global_state(void)
@@ -127,7 +129,8 @@ int git_threads_init(void)
 	pthread_key_create(&_tls_key, &cb__free_status);
 
 	/* Initialize any other subsystems that have global state */
-	if ((error = git_hash_global_init()) >= 0)
+	if ((error = git_hash_global_init()) >= 0 &&
+		(error = git_futils_dirs_global_init()) >= 0)
 		_tls_init = 1;
 
 	GIT_MEMORY_BARRIER;
diff --git a/src/hash/hash_win32.c b/src/hash/hash_win32.c
index 43d54ca6d..095ceb359 100644
--- a/src/hash/hash_win32.c
+++ b/src/hash/hash_win32.c
@@ -20,33 +20,16 @@ static struct git_hash_prov hash_prov = {0};
 /* Initialize CNG, if available */
 GIT_INLINE(int) hash_cng_prov_init(void)
 {
-	OSVERSIONINFOEX version_test = {0};
-	DWORD version_test_mask;
-	DWORDLONG version_condition_mask = 0;
 	char dll_path[MAX_PATH];
 	DWORD dll_path_len, size_len;
 
-	return -1;
-
 	/* Only use CNG on Windows 2008 / Vista SP1  or better (Windows 6.0 SP1) */
-	version_test.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEX);
-	version_test.dwMajorVersion = 6;
-	version_test.dwMinorVersion = 0;
-	version_test.wServicePackMajor = 1;
-	version_test.wServicePackMinor = 0;
-
-	version_test_mask = (VER_MAJORVERSION | VER_MINORVERSION | VER_SERVICEPACKMAJOR | VER_SERVICEPACKMINOR);
-
-	VER_SET_CONDITION(version_condition_mask, VER_MAJORVERSION, VER_GREATER_EQUAL);
-	VER_SET_CONDITION(version_condition_mask, VER_MINORVERSION, VER_GREATER_EQUAL);
-	VER_SET_CONDITION(version_condition_mask, VER_SERVICEPACKMAJOR, VER_GREATER_EQUAL);
-	VER_SET_CONDITION(version_condition_mask, VER_SERVICEPACKMINOR, VER_GREATER_EQUAL);
-
-	if (!VerifyVersionInfo(&version_test, version_test_mask, version_condition_mask))
+	if (!git_has_win32_version(6, 0, 1))
 		return -1;
 
 	/* Load bcrypt.dll explicitly from the system directory */
-	if ((dll_path_len = GetSystemDirectory(dll_path, MAX_PATH)) == 0 || dll_path_len > MAX_PATH ||
+	if ((dll_path_len = GetSystemDirectory(dll_path, MAX_PATH)) == 0 ||
+		dll_path_len > MAX_PATH ||
 		StringCchCat(dll_path, MAX_PATH, "\\") < 0 ||
 		StringCchCat(dll_path, MAX_PATH, GIT_HASH_CNG_DLL_NAME) < 0 ||
 		(hash_prov.prov.cng.dll = LoadLibrary(dll_path)) == NULL)
diff --git a/src/hashsig.c b/src/hashsig.c
index ab8d8b3f0..109f966ba 100644
--- a/src/hashsig.c
+++ b/src/hashsig.c
@@ -13,12 +13,15 @@ typedef uint64_t hashsig_state;
 
 #define HASHSIG_SCALE 100
 
-#define HASHSIG_HASH_WINDOW 32
-#define HASHSIG_HASH_START	0
+#define HASHSIG_MAX_RUN 80
+#define HASHSIG_HASH_START	0x012345678ABCDEF0LL
 #define HASHSIG_HASH_SHIFT  5
-#define HASHSIG_HASH_MASK   0x7FFFFFFF
+
+#define HASHSIG_HASH_MIX(S,CH) \
+	(S) = ((S) << HASHSIG_HASH_SHIFT) - (S) + (hashsig_state)(CH)
 
 #define HASHSIG_HEAP_SIZE ((1 << 7) - 1)
+#define HASHSIG_HEAP_MIN_SIZE 4
 
 typedef int (*hashsig_cmp)(const void *a, const void *b, void *);
 
@@ -28,14 +31,6 @@ typedef struct {
 	hashsig_t values[HASHSIG_HEAP_SIZE];
 } hashsig_heap;
 
-typedef struct {
-	hashsig_state state, shift_n;
-	char window[HASHSIG_HASH_WINDOW];
-	int win_len, win_pos, saw_lf;
-} hashsig_in_progress;
-
-#define HASHSIG_IN_PROGRESS_INIT { HASHSIG_HASH_START, 1, {0}, 0, 0, 1 }
-
 struct git_hashsig {
 	hashsig_heap mins;
 	hashsig_heap maxs;
@@ -43,8 +38,8 @@ struct git_hashsig {
 	int considered;
 };
 
-#define HEAP_LCHILD_OF(I) (((I)*2)+1)
-#define HEAP_RCHILD_OF(I) (((I)*2)+2)
+#define HEAP_LCHILD_OF(I) (((I)<<1)+1)
+#define HEAP_RCHILD_OF(I) (((I)<<1)+2)
 #define HEAP_PARENT_OF(I) (((I)-1)>>1)
 
 static void hashsig_heap_init(hashsig_heap *h, hashsig_cmp cmp)
@@ -115,134 +110,109 @@ static void hashsig_heap_sort(hashsig_heap *h)
 
 static void hashsig_heap_insert(hashsig_heap *h, hashsig_t val)
 {
-	/* if heap is full, pop top if new element should replace it */
-	if (h->size == h->asize && h->cmp(&val, &h->values[0], NULL) > 0) {
-		h->size--;
-		h->values[0] = h->values[h->size];
-		hashsig_heap_down(h, 0);
-	}
-
 	/* if heap is not full, insert new element */
 	if (h->size < h->asize) {
 		h->values[h->size++] = val;
 		hashsig_heap_up(h, h->size - 1);
 	}
-}
-
-GIT_INLINE(bool) hashsig_include_char(
-	char ch, git_hashsig_option_t opt, int *saw_lf)
-{
-	if ((opt & GIT_HASHSIG_IGNORE_WHITESPACE) && git__isspace(ch))
-		return false;
-
-	if (opt & GIT_HASHSIG_SMART_WHITESPACE) {
-		if (ch == '\r' || (*saw_lf && git__isspace(ch)))
-			return false;
 
-		*saw_lf = (ch == '\n');
+	/* if heap is full, pop top if new element should replace it */
+	else if (h->cmp(&val, &h->values[0], NULL) > 0) {
+		h->size--;
+		h->values[0] = h->values[h->size];
+		hashsig_heap_down(h, 0);
 	}
 
-	return true;
 }
 
-static void hashsig_initial_window(
-	git_hashsig *sig,
-	const char **data,
-	size_t size,
-	hashsig_in_progress *prog)
-{
-	hashsig_state state, shift_n;
-	int win_len;
-	const char *scan, *end;
-
-	/* init until we have processed at least HASHSIG_HASH_WINDOW data */
-
-	if (prog->win_len >= HASHSIG_HASH_WINDOW)
-		return;
-
-	state   = prog->state;
-	win_len = prog->win_len;
-	shift_n = prog->shift_n;
-
-	scan = *data;
-	end  = scan + size;
-
-	while (scan < end && win_len < HASHSIG_HASH_WINDOW) {
-		char ch = *scan++;
-
-		if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf))
-			continue;
-
-		state = (state * HASHSIG_HASH_SHIFT + ch) & HASHSIG_HASH_MASK;
-
-		if (!win_len)
-			shift_n = 1;
-		else
-			shift_n = (shift_n * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK;
-
-		prog->window[win_len++] = ch;
-	}
-
-	/* insert initial hash if we just finished */
+typedef struct {
+	int use_ignores;
+	uint8_t ignore_ch[256];
+} hashsig_in_progress;
 
-	if (win_len == HASHSIG_HASH_WINDOW) {
-		hashsig_heap_insert(&sig->mins, (hashsig_t)state);
-		hashsig_heap_insert(&sig->maxs, (hashsig_t)state);
-		sig->considered = 1;
+static void hashsig_in_progress_init(
+	hashsig_in_progress *prog, git_hashsig *sig)
+{
+	int i;
+
+	switch (sig->opt) {
+	case GIT_HASHSIG_IGNORE_WHITESPACE:
+		for (i = 0; i < 256; ++i)
+			prog->ignore_ch[i] = git__isspace_nonlf(i);
+		prog->use_ignores = 1;
+		break;
+	case GIT_HASHSIG_SMART_WHITESPACE:
+		for (i = 0; i < 256; ++i)
+			prog->ignore_ch[i] = git__isspace(i);
+		prog->use_ignores = 1;
+		break;
+	default:
+		memset(prog, 0, sizeof(*prog));
+		break;
 	}
-
-	prog->state   = state;
-	prog->win_len = win_len;
-	prog->shift_n = shift_n;
-
-	*data = scan;
 }
 
+#define HASHSIG_IN_PROGRESS_INIT { 1 }
+
 static int hashsig_add_hashes(
 	git_hashsig *sig,
-	const char *data,
+	const uint8_t *data,
 	size_t size,
 	hashsig_in_progress *prog)
 {
-	const char *scan = data, *end = data + size;
-	hashsig_state state, shift_n, rmv;
-
-	if (prog->win_len < HASHSIG_HASH_WINDOW)
-		hashsig_initial_window(sig, &scan, size, prog);
-
-	state   = prog->state;
-	shift_n = prog->shift_n;
-
-	/* advance window, adding new chars and removing old */
-
-	for (; scan < end; ++scan) {
-		char ch = *scan;
-
-		if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf))
-			continue;
-
-		rmv = shift_n * prog->window[prog->win_pos];
+	const uint8_t *scan = data, *end = data + size;
+	hashsig_state state = HASHSIG_HASH_START;
+	int use_ignores = prog->use_ignores, len;
+	uint8_t ch;
+
+	while (scan < end) {
+		state = HASHSIG_HASH_START;
+
+		for (len = 0; scan < end && len < HASHSIG_MAX_RUN; ) {
+			ch = *scan;
+
+			if (use_ignores)
+				for (; scan < end && git__isspace_nonlf(ch); ch = *scan)
+					++scan;
+			else if (sig->opt != GIT_HASHSIG_NORMAL)
+				for (; scan < end && ch == '\r'; ch = *scan)
+					++scan;
+
+			/* peek at next character to decide what to do next */
+			if (sig->opt == GIT_HASHSIG_SMART_WHITESPACE)
+				use_ignores = (ch == '\n');
+
+			if (scan >= end)
+				break;
+			++scan;
+
+			/* check run terminator */
+			if (ch == '\n' || ch == '\0')
+				break;
+
+			++len;
+			HASHSIG_HASH_MIX(state, ch);
+		}
 
-		state = (state - rmv) & HASHSIG_HASH_MASK;
-		state = (state * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK;
-		state = (state + ch) & HASHSIG_HASH_MASK;
+		if (len > 0) {
+			hashsig_heap_insert(&sig->mins, (hashsig_t)state);
+			hashsig_heap_insert(&sig->maxs, (hashsig_t)state);
 
-		hashsig_heap_insert(&sig->mins, (hashsig_t)state);
-		hashsig_heap_insert(&sig->maxs, (hashsig_t)state);
-		sig->considered++;
+			sig->considered++;
 
-		prog->window[prog->win_pos] = ch;
-		prog->win_pos = (prog->win_pos + 1) % HASHSIG_HASH_WINDOW;
+			while (scan < end && (*scan == '\n' || !*scan))
+				++scan;
+		}
 	}
 
-	prog->state = state;
+	prog->use_ignores = use_ignores;
 
 	return 0;
 }
 
 static int hashsig_finalize_hashes(git_hashsig *sig)
 {
-	if (sig->mins.size < HASHSIG_HEAP_SIZE) {
+	if (sig->mins.size < HASHSIG_HEAP_MIN_SIZE) {
 		giterr_set(GITERR_INVALID,
 			"File too small for similarity signature calculation");
 		return GIT_EBUFS;
@@ -274,11 +244,13 @@ int git_hashsig_create(
 	git_hashsig_option_t opts)
 {
 	int error;
-	hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT;
+	hashsig_in_progress prog;
 	git_hashsig *sig = hashsig_alloc(opts);
 	GITERR_CHECK_ALLOC(sig);
 
-	error = hashsig_add_hashes(sig, buf, buflen, &prog);
+	hashsig_in_progress_init(&prog, sig);
+
+	error = hashsig_add_hashes(sig, (const uint8_t *)buf, buflen, &prog);
 
 	if (!error)
 		error = hashsig_finalize_hashes(sig);
@@ -296,10 +268,10 @@ int git_hashsig_create_fromfile(
 	const char *path,
 	git_hashsig_option_t opts)
 {
-	char buf[4096];
+	uint8_t buf[0x1000];
 	ssize_t buflen = 0;
 	int error = 0, fd;
-	hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT;
+	hashsig_in_progress prog;
 	git_hashsig *sig = hashsig_alloc(opts);
 	GITERR_CHECK_ALLOC(sig);
 
@@ -308,6 +280,8 @@ int git_hashsig_create_fromfile(
 		return fd;
 	}
 
+	hashsig_in_progress_init(&prog, sig);
+
 	while (!error) {
 		if ((buflen = p_read(fd, buf, sizeof(buf))) <= 0) {
 			if ((error = (int)buflen) < 0)
@@ -362,6 +336,12 @@ static int hashsig_heap_compare(const hashsig_heap *a, const hashsig_heap *b)
 
 int git_hashsig_compare(const git_hashsig *a, const git_hashsig *b)
 {
-	return (hashsig_heap_compare(&a->mins, &b->mins) +
-			hashsig_heap_compare(&a->maxs, &b->maxs)) / 2;
+	/* if we have fewer than the maximum number of elements, then just use
+	 * one array since the two arrays will be the same
+	 */
+	if (a->mins.size < HASHSIG_HEAP_SIZE)
+		return hashsig_heap_compare(&a->mins, &b->mins);
+	else
+		return (hashsig_heap_compare(&a->mins, &b->mins) +
+				hashsig_heap_compare(&a->maxs, &b->maxs)) / 2;
 }
diff --git a/src/ignore.c b/src/ignore.c
index e150b9585..0c35d0431 100644
--- a/src/ignore.c
+++ b/src/ignore.c
@@ -37,7 +37,7 @@ static int parse_ignore_file(
 			GITERR_CHECK_ALLOC(match);
 		}
 
-		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE;
+		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE | GIT_ATTR_FNMATCH_ALLOWNEG;
 
 		if (!(error = git_attr_fnmatch__parse(
 			match, ignores->pool, context, &scan)))
@@ -159,17 +159,36 @@ int git_ignore__push_dir(git_ignores *ign, const char *dir)
 {
 	if (git_buf_joinpath(&ign->dir, ign->dir.ptr, dir) < 0)
 		return -1;
-	else
-		return push_ignore_file(
-			ign->repo, ign, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE);
+
+	return push_ignore_file(
+		ign->repo, ign, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE);
 }
 
 int git_ignore__pop_dir(git_ignores *ign)
 {
 	if (ign->ign_path.length > 0) {
 		git_attr_file *file = git_vector_last(&ign->ign_path);
-		if (git__suffixcmp(ign->dir.ptr, file->key + 2) == 0)
+		const char *start, *end, *scan;
+		size_t keylen;
+
+		/* - ign->dir looks something like "a/b" (or "a/b/c/d")
+		 * - file->key looks something like "0#a/b/.gitignore
+		 *
+		 * We are popping the last directory off ign->dir.  We also want to
+		 * remove the file from the vector if the directory part of the key
+		 * matches the ign->dir path.  We need to test if the "a/b" part of
+		 * the file key matches the path we are about to pop.
+		 */
+
+		for (start = end = scan = &file->key[2]; *scan; ++scan)
+			if (*scan == '/')
+				end = scan; /* point 'end' to last '/' in key */
+		keylen = (end - start) + 1;
+
+		if (ign->dir.size >= keylen &&
+			!memcmp(ign->dir.ptr + ign->dir.size - keylen, start, keylen))
 			git_vector_pop(&ign->ign_path);
+
 		git_buf_rtruncate_at_char(&ign->dir, '/');
 	}
 	return 0;
@@ -298,12 +317,9 @@ int git_ignore_path_is_ignored(
 		path.full.size = (tail - path.full.ptr);
 		path.is_dir = (tail == end) ? full_is_dir : true;
 
-		/* update ignores for new path fragment */
-		if (path.basename == path.path)
-			error = git_ignore__for_path(repo, path.path, &ignores);
-		else
-			error = git_ignore__push_dir(&ignores, path.basename);
-		if (error < 0)
+		/* initialize ignores the first time through */
+		if (path.basename == path.path &&
+			(error = git_ignore__for_path(repo, path.path, &ignores)) < 0)
 			break;
 
 		/* first process builtins - success means path was found */
@@ -327,6 +343,10 @@ int git_ignore_path_is_ignored(
 		if (tail == end)
 			break;
 
+		/* now add this directory to list of ignores */
+		if ((error = git_ignore__push_dir(&ignores, path.path)) < 0)
+			break;
+
 		/* reinstate divider in path */
 		*tail = '/';
 		while (*tail == '/') tail++;
@@ -340,3 +360,61 @@ cleanup:
 	return error;
 }
 
+
+int git_ignore__check_pathspec_for_exact_ignores(
+	git_repository *repo,
+	git_vector *vspec,
+	bool no_fnmatch)
+{
+	int error = 0;
+	size_t i;
+	git_attr_fnmatch *match;
+	int ignored;
+	git_buf path = GIT_BUF_INIT;
+	const char *wd, *filename;
+	git_index *idx;
+
+	if ((error = git_repository__ensure_not_bare(
+			repo, "validate pathspec")) < 0 ||
+		(error = git_repository_index(&idx, repo)) < 0)
+		return error;
+
+	wd = git_repository_workdir(repo);
+
+	git_vector_foreach(vspec, i, match) {
+		/* skip wildcard matches (if they are being used) */
+		if ((match->flags & GIT_ATTR_FNMATCH_HASWILD) != 0 &&
+			!no_fnmatch)
+			continue;
+
+		filename = match->pattern;
+
+		/* if file is already in the index, it's fine */
+		if (git_index_get_bypath(idx, filename, 0) != NULL)
+			continue;
+
+		if ((error = git_buf_joinpath(&path, wd, filename)) < 0)
+			break;
+
+		/* is there a file on disk that matches this exactly? */
+		if (!git_path_isfile(path.ptr))
+			continue;
+
+		/* is that file ignored? */
+		if ((error = git_ignore_path_is_ignored(&ignored, repo, filename)) < 0)
+			break;
+
+		if (ignored) {
+			giterr_set(GITERR_INVALID, "pathspec contains ignored file '%s'",
+				filename);
+			error = GIT_EINVALIDSPEC;
+			break;
+		}
+	}
+
+	git_index_free(idx);
+	git_buf_free(&path);
+
+	return error;
+}
+
diff --git a/src/ignore.h b/src/ignore.h
index e00e4a8c8..851c824bf 100644
--- a/src/ignore.h
+++ b/src/ignore.h
@@ -24,14 +24,15 @@
  */
 typedef struct {
 	git_repository *repo;
-	git_buf dir;
+	git_buf dir; /* current directory reflected in ign_path */
 	git_attr_file *ign_internal;
 	git_vector ign_path;
 	git_vector ign_global;
 	int ignore_case;
 } git_ignores;
 
-extern int git_ignore__for_path(git_repository *repo, const char *path, git_ignores *ign);
+extern int git_ignore__for_path(
+	git_repository *repo, const char *path, git_ignores *ign);
 
 extern int git_ignore__push_dir(git_ignores *ign, const char *dir);
 
@@ -41,4 +42,13 @@ extern void git_ignore__free(git_ignores *ign);
 
 extern int git_ignore__lookup(git_ignores *ign, const char *path, int *ignored);
 
+/* command line Git sometimes generates an error message if given a
+ * pathspec that contains an exact match to an ignored file (provided
+ * --force isn't also given).  This makes it easy to check it that has
+ * happened.  Returns GIT_EINVALIDSPEC if the pathspec contains ignored
+ * exact matches (that are not already present in the index).
+ */
+extern int git_ignore__check_pathspec_for_exact_ignores(
+	git_repository *repo, git_vector *pathspec, bool no_fnmatch);
+
 #endif
diff --git a/src/index.c b/src/index.c
index 4f0c70135..9b32222a7 100644
--- a/src/index.c
+++ b/src/index.c
@@ -15,6 +15,9 @@
 #include "hash.h"
 #include "iterator.h"
 #include "pathspec.h"
+#include "ignore.h"
+#include "blob.h"
+
 #include "git2/odb.h"
 #include "git2/oid.h"
 #include "git2/blob.h"
@@ -99,8 +102,6 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 static bool is_index_extended(git_index *index);
 static int write_index(git_index *index, git_filebuf *file);
 
-static int index_find(size_t *at_pos, git_index *index, const char *path, int stage);
-
 static void index_entry_free(git_index_entry *entry);
 static void index_entry_reuc_free(git_index_reuc_entry *reuc);
 
@@ -112,7 +113,7 @@ static int index_srch(const void *key, const void *array_member)
 
 	ret = strcmp(srch_key->path, entry->path);
 
-	if (ret == 0)
+	if (ret == 0 && srch_key->stage != GIT_INDEX_STAGE_ANY)
 		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
 
 	return ret;
@@ -126,7 +127,7 @@ static int index_isrch(const void *key, const void *array_member)
 
 	ret = strcasecmp(srch_key->path, entry->path);
 
-	if (ret == 0)
+	if (ret == 0 && srch_key->stage != GIT_INDEX_STAGE_ANY)
 		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
 
 	return ret;
@@ -259,6 +260,22 @@ static int reuc_icmp(const void *a, const void *b)
 	return strcasecmp(info_a->path, info_b->path);
 }
 
+static void index_entry_reuc_free(git_index_reuc_entry *reuc)
+{
+	if (!reuc)
+		return;
+	git__free(reuc->path);
+	git__free(reuc);
+}
+
+static void index_entry_free(git_index_entry *entry)
+{
+	if (!entry)
+		return;
+	git__free(entry->path);
+	git__free(entry);
+}
+
 static unsigned int index_create_mode(unsigned int mode)
 {
 	if (S_ISLNK(mode))
@@ -267,7 +284,7 @@ static unsigned int index_create_mode(unsigned int mode)
 	if (S_ISDIR(mode) || (mode & S_IFMT) == (S_IFLNK | S_IFDIR))
 		return (S_IFLNK | S_IFDIR);
 
-	return S_IFREG | ((mode & 0100) ? 0755 : 0644);
+	return S_IFREG | GIT_PERMS_CANONICAL(mode);
 }
 
 static unsigned int index_merge_mode(
@@ -288,16 +305,16 @@ void git_index__set_ignore_case(git_index *index, bool ignore_case)
 {
 	index->ignore_case = ignore_case;
 
-	index->entries._cmp = ignore_case ? index_icmp : index_cmp;
 	index->entries_cmp_path = ignore_case ? index_icmp_path : index_cmp_path;
 	index->entries_search = ignore_case ? index_isrch : index_srch;
 	index->entries_search_path = ignore_case ? index_isrch_path : index_srch_path;
-	index->entries.sorted = 0;
+
+	git_vector_set_cmp(&index->entries, ignore_case ? index_icmp : index_cmp);
 	git_vector_sort(&index->entries);
 
-	index->reuc._cmp = ignore_case ? reuc_icmp : reuc_cmp;
 	index->reuc_search = ignore_case ? reuc_isrch : reuc_srch;
-	index->reuc.sorted = 0;
+
+	git_vector_set_cmp(&index->reuc, ignore_case ? reuc_icmp : reuc_cmp);
 	git_vector_sort(&index->reuc);
 }
 
@@ -365,11 +382,8 @@ static void index_entries_free(git_vector *entries)
 {
 	size_t i;
 
-	for (i = 0; i < entries->length; ++i) {
-		git_index_entry *e = git_vector_get(entries, i);
-		git__free(e->path);
-		git__free(e);
-	}
+	for (i = 0; i < entries->length; ++i)
+		index_entry_free(git__swap(entries->contents[i], NULL));
 
 	git_vector_clear(entries);
 }
@@ -484,8 +498,12 @@ int git_index_write(git_index *index)
 	git_vector_sort(&index->reuc);
 
 	if ((error = git_filebuf_open(
-			 &file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS)) < 0)
+		     &file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS)) < 0) {
+		if (error == GIT_ELOCKED)
+			giterr_set(GITERR_INDEX, "The index is locked. This might be due to a concurrrent or crashed process");
+
 		return error;
+	}
 
 	if ((error = write_index(index, &file)) < 0) {
 		git_filebuf_cleanup(&file);
@@ -503,6 +521,12 @@ int git_index_write(git_index *index)
 	return 0;
 }
 
+const char * git_index_path(git_index *index)
+{
+	assert(index);
+	return index->index_file_path;
+}
+
 int git_index_write_tree(git_oid *oid, git_index *index)
 {
 	git_repository *repo;
@@ -547,7 +571,7 @@ const git_index_entry *git_index_get_bypath(
 
 	git_vector_sort(&index->entries);
 
-	if (index_find(&pos, index, path, stage) < 0) {
+	if (git_index__find(&pos, index, path, stage) < 0) {
 		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
 		return NULL;
 	}
@@ -585,42 +609,23 @@ int git_index_entry__cmp_icase(const void *a, const void *b)
 	return strcasecmp(entry_a->path, entry_b->path);
 }
 
-static int index_entry_init(git_index_entry **entry_out, git_index *index, const char *rel_path)
+static int index_entry_init(
+	git_index_entry **entry_out, git_index *index, const char *rel_path)
 {
+	int error = 0;
 	git_index_entry *entry = NULL;
 	struct stat st;
 	git_oid oid;
-	const char *workdir;
-	git_buf full_path = GIT_BUF_INIT;
-	int error;
 
 	if (INDEX_OWNER(index) == NULL)
 		return create_index_error(-1,
 			"Could not initialize index entry. "
 			"Index is not backed up by an existing repository.");
 
-	workdir = git_repository_workdir(INDEX_OWNER(index));
-
-	if (!workdir)
-		return create_index_error(GIT_EBAREREPO,
-			"Could not initialize index entry. Repository is bare");
-
-	if ((error = git_buf_joinpath(&full_path, workdir, rel_path)) < 0)
-		return error;
-
-	if ((error = git_path_lstat(full_path.ptr, &st)) < 0) {
-		git_buf_free(&full_path);
-		return error;
-	}
-
-	git_buf_free(&full_path); /* done with full path */
-
-	/* There is no need to validate the rel_path here, since it will be
-	 * immediately validated by the call to git_blob_create_fromfile.
-	 */
-
-	/* write the blob to disk and get the oid */
-	if ((error = git_blob_create_fromworkdir(&oid, INDEX_OWNER(index), rel_path)) < 0)
+	/* write the blob to disk and get the oid and stat info */
+	error = git_blob__create_from_paths(
+		&oid, &st, INDEX_OWNER(index), NULL, rel_path, 0, true);
+	if (error < 0)
 		return error;
 
 	entry = git__calloc(1, sizeof(git_index_entry));
@@ -668,15 +673,6 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 	return 0;
 }
 
-static void index_entry_reuc_free(git_index_reuc_entry *reuc)
-{
-	if (!reuc)
-		return;
-
-	git__free(reuc->path);
-	git__free(reuc);
-}
-
 static git_index_entry *index_entry_dup(const git_index_entry *source_entry)
 {
 	git_index_entry *entry;
@@ -695,14 +691,6 @@ static git_index_entry *index_entry_dup(const git_index_entry *source_entry)
 	return entry;
 }
 
-static void index_entry_free(git_index_entry *entry)
-{
-	if (!entry)
-		return;
-	git__free(entry->path);
-	git__free(entry);
-}
-
 static int index_insert(git_index *index, git_index_entry *entry, int replace)
 {
 	size_t path_length, position;
@@ -721,7 +709,8 @@ static int index_insert(git_index *index, git_index_entry *entry, int replace)
 		entry->flags |= GIT_IDXENTRY_NAMEMASK;
 
 	/* look if an entry with this path already exists */
-	if (!index_find(&position, index, entry->path, GIT_IDXENTRY_STAGE(entry))) {
+	if (!git_index__find(
+			&position, index, entry->path, GIT_IDXENTRY_STAGE(entry))) {
 		existing = (git_index_entry **)&index->entries.contents[position];
 
 		/* update filemode to existing values if stat is not trusted */
@@ -734,8 +723,9 @@ static int index_insert(git_index *index, git_index_entry *entry, int replace)
 	if (!replace || !existing)
 		return git_vector_insert(&index->entries, entry);
 
-	/* exists, replace it */
-	git__free((*existing)->path);
+	/* exists, replace it (preserving name from existing entry) */
+	git__free(entry->path);
+	entry->path = (*existing)->path;
 	git__free(*existing);
 	*existing = entry;
 
@@ -832,7 +822,7 @@ int git_index_remove(git_index *index, const char *path, int stage)
 
 	git_vector_sort(&index->entries);
 
-	if (index_find(&position, index, path, stage) < 0) {
+	if (git_index__find(&position, index, path, stage) < 0) {
 		giterr_set(GITERR_INDEX, "Index does not contain %s at stage %d",
 			path, stage);
 		return GIT_ENOTFOUND;
@@ -888,7 +878,8 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage)
 	return error;
 }
 
-static int index_find(size_t *at_pos, git_index *index, const char *path, int stage)
+int git_index__find(
+	size_t *at_pos, git_index *index, const char *path, int stage)
 {
 	struct entry_srch_key srch_key;
 
@@ -897,7 +888,8 @@ static int index_find(size_t *at_pos, git_index *index, const char *path, int st
 	srch_key.path = path;
 	srch_key.stage = stage;
 
-	return git_vector_bsearch2(at_pos, &index->entries, index->entries_search, &srch_key);
+	return git_vector_bsearch2(
+		at_pos, &index->entries, index->entries_search, &srch_key);
 }
 
 int git_index_find(size_t *at_pos, git_index *index, const char *path)
@@ -996,7 +988,7 @@ static int index_conflict__get_byindex(
 	int stage, len = 0;
 
 	assert(ancestor_out && our_out && their_out && index);
-	
+
 	*ancestor_out = NULL;
 	*our_out = NULL;
 	*their_out = NULL;
@@ -1009,7 +1001,7 @@ static int index_conflict__get_byindex(
 
 		stage = GIT_IDXENTRY_STAGE(conflict_entry);
 		path = conflict_entry->path;
-		
+
 		switch (stage) {
 		case 3:
 			*their_out = conflict_entry;
@@ -1354,14 +1346,11 @@ int git_index_reuc_remove(git_index *index, size_t position)
 void git_index_reuc_clear(git_index *index)
 {
 	size_t i;
-	git_index_reuc_entry *reuc;
 
 	assert(index);
 
-	git_vector_foreach(&index->reuc, i, reuc) {
-		git__free(reuc->path);
-		git__free(reuc);
-	}
+	for (i = 0; i < index->reuc.length; ++i)
+		index_entry_reuc_free(git__swap(index->reuc.contents[i], NULL));
 
 	git_vector_clear(&index->reuc);
 }
@@ -1386,7 +1375,7 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 	while (size) {
 		git_index_reuc_entry *lost;
 
-		len = strlen(buffer) + 1;
+		len = p_strnlen(buffer, size) + 1;
 		if (size <= len)
 			return index_error_invalid("reading reuc entries");
 
@@ -1406,14 +1395,18 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 
 			if (git__strtol32(&tmp, buffer, &endptr, 8) < 0 ||
 				!endptr || endptr == buffer || *endptr ||
-				(unsigned)tmp > UINT_MAX)
+				(unsigned)tmp > UINT_MAX) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry stage");
+			}
 
 			lost->mode[i] = tmp;
 
 			len = (endptr + 1) - buffer;
-			if (size <= len)
+			if (size <= len) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry stage");
+			}
 
 			size -= len;
 			buffer += len;
@@ -1423,8 +1416,10 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 		for (i = 0; i < 3; i++) {
 			if (!lost->mode[i])
 				continue;
-			if (size < 20)
+			if (size < 20) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry oid");
+			}
 
 			git_oid_fromraw(&lost->oid[i], (const unsigned char *) buffer);
 			size -= 20;
@@ -1453,7 +1448,7 @@ static int read_conflict_names(git_index *index, const char *buffer, size_t size
 		return -1;
 
 #define read_conflict_name(ptr) \
-	len = strlen(buffer) + 1; \
+	len = p_strnlen(buffer, size) + 1; \
 	if (size < len) \
 		return index_error_invalid("reading conflict name entries"); \
 	\
@@ -1580,7 +1575,8 @@ static size_t read_extension(git_index *index, const char *buffer, size_t buffer
 
 	total_size = dest.extension_size + sizeof(struct index_extension);
 
-	if (buffer_size < total_size ||
+	if (dest.extension_size > total_size ||
+		buffer_size < total_size ||
 		buffer_size - total_size < INDEX_FOOTER_SIZE)
 		return 0;
 
@@ -1955,8 +1951,9 @@ int git_index_entry_stage(const git_index_entry *entry)
 }
 
 typedef struct read_tree_data {
-	git_index *index;
 	git_vector *old_entries;
+	git_vector *new_entries;
+	git_vector_cmp entries_search;
 } read_tree_data;
 
 static int read_tree_cb(
@@ -1987,7 +1984,7 @@ static int read_tree_cb(
 		skey.stage = 0;
 
 		if (!git_vector_bsearch2(
-				&pos, data->old_entries, data->index->entries_search, &skey) &&
+				&pos, data->old_entries, data->entries_search, &skey) &&
 			(old_entry = git_vector_get(data->old_entries, pos)) != NULL &&
 			entry->mode == old_entry->mode &&
 			git_oid_equal(&entry->oid, &old_entry->oid))
@@ -2005,7 +2002,7 @@ static int read_tree_cb(
 	entry->path = git_buf_detach(&path);
 	git_buf_free(&path);
 
-	if (git_vector_insert(&data->index->entries, entry) < 0) {
+	if (git_vector_insert(data->new_entries, entry) < 0) {
 		index_entry_free(entry);
 		return -1;
 	}
@@ -2019,27 +2016,243 @@ int git_index_read_tree(git_index *index, const git_tree *tree)
 	git_vector entries = GIT_VECTOR_INIT;
 	read_tree_data data;
 
+	git_vector_set_cmp(&entries, index->entries._cmp); /* match sort */
+
+	data.old_entries = &index->entries;
+	data.new_entries = &entries;
+	data.entries_search = index->entries_search;
+
 	git_vector_sort(&index->entries);
 
-	entries._cmp = index->entries._cmp;
-	git_vector_swap(&entries, &index->entries);
+	error = git_tree_walk(tree, GIT_TREEWALK_POST, read_tree_cb, &data);
+
+	git_vector_sort(&entries);
 
 	git_index_clear(index);
 
-	data.index = index;
-	data.old_entries = &entries;
+	git_vector_swap(&entries, &index->entries);
+	git_vector_free(&entries);
 
-	error = git_tree_walk(tree, GIT_TREEWALK_POST, read_tree_cb, &data);
+	return error;
+}
 
-	index_entries_free(&entries);
-	git_vector_free(&entries);
+git_repository *git_index_owner(const git_index *index)
+{
+	return INDEX_OWNER(index);
+}
+
+int git_index_add_all(
+	git_index *index,
+	const git_strarray *paths,
+	unsigned int flags,
+	git_index_matched_path_cb cb,
+	void *payload)
+{
+	int error;
+	git_repository *repo;
+	git_iterator *wditer = NULL;
+	const git_index_entry *wd = NULL;
+	git_index_entry *entry;
+	git_pathspec ps;
+	const char *match;
+	size_t existing;
+	bool no_fnmatch = (flags & GIT_INDEX_ADD_DISABLE_PATHSPEC_MATCH) != 0;
+	int ignorecase;
+	git_oid blobid;
+
+	assert(index);
+
+	if (INDEX_OWNER(index) == NULL)
+		return create_index_error(-1,
+			"Could not add paths to index. "
+			"Index is not backed up by an existing repository.");
+
+	repo = INDEX_OWNER(index);
+	if ((error = git_repository__ensure_not_bare(repo, "index add all")) < 0)
+		return error;
+
+	if (git_repository__cvar(&ignorecase, repo, GIT_CVAR_IGNORECASE) < 0)
+		return -1;
+
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
+		return error;
+
+	/* optionally check that pathspec doesn't mention any ignored files */
+	if ((flags & GIT_INDEX_ADD_CHECK_PATHSPEC) != 0 &&
+		(flags & GIT_INDEX_ADD_FORCE) == 0 &&
+		(error = git_ignore__check_pathspec_for_exact_ignores(
+			repo, &ps.pathspec, no_fnmatch)) < 0)
+		goto cleanup;
+
+	if ((error = git_iterator_for_workdir(
+			&wditer, repo, 0, ps.prefix, ps.prefix)) < 0)
+		goto cleanup;
+
+	while (!(error = git_iterator_advance(&wd, wditer))) {
+
+		/* check if path actually matches */
+		if (!git_pathspec__match(
+				&ps.pathspec, wd->path, no_fnmatch, ignorecase, &match, NULL))
+			continue;
+
+		/* skip ignored items that are not already in the index */
+		if ((flags & GIT_INDEX_ADD_FORCE) == 0 &&
+			git_iterator_current_is_ignored(wditer) &&
+			git_index__find(&existing, index, wd->path, 0) < 0)
+			continue;
+
+		/* issue notification callback if requested */
+		if (cb && (error = cb(wd->path, match, payload)) != 0) {
+			if (error > 0) /* return > 0 means skip this one */
+				continue;
+			if (error < 0) { /* return < 0 means abort */
+				giterr_clear();
+				error = GIT_EUSER;
+				break;
+			}
+		}
+
+		/* TODO: Should we check if the file on disk is already an exact
+		 * match to the file in the index and skip this work if it is?
+		 */
+
+		/* write the blob to disk and get the oid */
+		if ((error = git_blob_create_fromworkdir(&blobid, repo, wd->path)) < 0)
+			break;
+
+		/* make the new entry to insert */
+		if ((entry = index_entry_dup(wd)) == NULL) {
+			error = -1;
+			break;
+		}
+		entry->oid = blobid;
+
+		/* add working directory item to index */
+		if ((error = index_insert(index, entry, 1)) < 0) {
+			index_entry_free(entry);
+			break;
+		}
+
+		git_tree_cache_invalidate_path(index->tree, wd->path);
+
+		/* add implies conflict resolved, move conflict entries to REUC */
+		if ((error = index_conflict_to_reuc(index, wd->path)) < 0) {
+			if (error != GIT_ENOTFOUND)
+				break;
+			giterr_clear();
+		}
+	}
+
+	if (error == GIT_ITEROVER)
+		error = 0;
+
+cleanup:
+	git_iterator_free(wditer);
+	git_pathspec__clear(&ps);
+
+	return error;
+}
+
+enum {
+	INDEX_ACTION_NONE = 0,
+	INDEX_ACTION_UPDATE = 1,
+	INDEX_ACTION_REMOVE = 2,
+};
+
+static int index_apply_to_all(
+	git_index *index,
+	int action,
+	const git_strarray *paths,
+	git_index_matched_path_cb cb,
+	void *payload)
+{
+	int error = 0;
+	size_t i;
+	git_pathspec ps;
+	const char *match;
+	git_buf path = GIT_BUF_INIT;
+
+	assert(index);
+
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
+		return error;
 
 	git_vector_sort(&index->entries);
 
+	for (i = 0; !error && i < index->entries.length; ++i) {
+		git_index_entry *entry = git_vector_get(&index->entries, i);
+
+		/* check if path actually matches */
+		if (!git_pathspec__match(
+				&ps.pathspec, entry->path, false, index->ignore_case,
+				&match, NULL))
+			continue;
+
+		/* issue notification callback if requested */
+		if (cb && (error = cb(entry->path, match, payload)) != 0) {
+			if (error > 0) { /* return > 0 means skip this one */
+				error = 0;
+				continue;
+			}
+			if (error < 0) { /* return < 0 means abort */
+				giterr_clear();
+				error = GIT_EUSER;
+				break;
+			}
+		}
+
+		/* index manipulation may alter entry, so don't depend on it */
+		if ((error = git_buf_sets(&path, entry->path)) < 0)
+			break;
+
+		switch (action) {
+		case INDEX_ACTION_NONE:
+			break;
+		case INDEX_ACTION_UPDATE:
+			error = git_index_add_bypath(index, path.ptr);
+
+			if (error == GIT_ENOTFOUND) {
+				giterr_clear();
+
+				error = git_index_remove_bypath(index, path.ptr);
+
+				if (!error) /* back up foreach if we removed this */
+					i--;
+			}
+			break;
+		case INDEX_ACTION_REMOVE:
+			if (!(error = git_index_remove_bypath(index, path.ptr)))
+				i--; /* back up foreach if we removed this */
+			break;
+		default:
+			giterr_set(GITERR_INVALID, "Unknown index action %d", action);
+			error = -1;
+			break;
+		}
+	}
+
+	git_buf_free(&path);
+	git_pathspec__clear(&ps);
+
 	return error;
 }
 
-git_repository *git_index_owner(const git_index *index)
+int git_index_remove_all(
+	git_index *index,
+	const git_strarray *pathspec,
+	git_index_matched_path_cb cb,
+	void *payload)
 {
-	return INDEX_OWNER(index);
+	return index_apply_to_all(
+		index, INDEX_ACTION_REMOVE, pathspec, cb, payload);
+}
+
+int git_index_update_all(
+	git_index *index,
+	const git_strarray *pathspec,
+	git_index_matched_path_cb cb,
+	void *payload)
+{
+	return index_apply_to_all(
+		index, INDEX_ACTION_UPDATE, pathspec, cb, payload);
 }
diff --git a/src/index.h b/src/index.h
index a59107a7b..40577e105 100644
--- a/src/index.h
+++ b/src/index.h
@@ -47,13 +47,17 @@ struct git_index_conflict_iterator {
 	size_t cur;
 };
 
-extern void git_index_entry__init_from_stat(git_index_entry *entry, struct stat *st);
+extern void git_index_entry__init_from_stat(
+	git_index_entry *entry, struct stat *st);
 
 extern size_t git_index__prefix_position(git_index *index, const char *path);
 
 extern int git_index_entry__cmp(const void *a, const void *b);
 extern int git_index_entry__cmp_icase(const void *a, const void *b);
 
+extern int git_index__find(
+	size_t *at_pos, git_index *index, const char *path, int stage);
+
 extern void git_index__set_ignore_case(git_index *index, bool ignore_case);
 
 #endif
diff --git a/src/indexer.c b/src/indexer.c
index 1b5339f23..09f962934 100644
--- a/src/indexer.c
+++ b/src/indexer.c
@@ -325,7 +325,7 @@ static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t ent
 	/* FIXME: Parse the object instead of hashing it */
 	if (git_odb__hashobj(&oid, obj) < 0) {
 		giterr_set(GITERR_INDEXER, "Failed to hash object");
-		return -1;
+		goto on_error;
 	}
 
 	pentry = git__calloc(1, sizeof(struct git_pack_entry));
@@ -602,7 +602,7 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_transfer_progress *
 	git_vector_sort(&idx->objects);
 
 	git_buf_sets(&filename, idx->pack->pack_name);
-	git_buf_truncate(&filename, filename.size - strlen("pack"));
+	git_buf_shorten(&filename, strlen("pack"));
 	git_buf_puts(&filename, "idx");
 	if (git_buf_oom(&filename))
 		return -1;
diff --git a/src/iterator.c b/src/iterator.c
index 76b0e41d0..bdc98d22b 100644
--- a/src/iterator.c
+++ b/src/iterator.c
@@ -1321,9 +1321,10 @@ static void workdir_iterator__free(git_iterator *self)
 	git_ignore__free(&wi->ignores);
 }
 
-int git_iterator_for_workdir(
+int git_iterator_for_workdir_ext(
 	git_iterator **out,
 	git_repository *repo,
+	const char *repo_workdir,
 	git_iterator_flag_t flags,
 	const char *start,
 	const char *end)
@@ -1331,8 +1332,11 @@ int git_iterator_for_workdir(
 	int error;
 	workdir_iterator *wi;
 
-	if (git_repository__ensure_not_bare(repo, "scan working directory") < 0)
-		return GIT_EBAREREPO;
+	if (!repo_workdir) {
+		if (git_repository__ensure_not_bare(repo, "scan working directory") < 0)
+			return GIT_EBAREREPO;
+		repo_workdir = git_repository_workdir(repo);
+	}
 
 	/* initialize as an fs iterator then do overrides */
 	wi = git__calloc(1, sizeof(workdir_iterator));
@@ -1346,13 +1350,13 @@ int git_iterator_for_workdir(
 	wi->fi.update_entry_cb = workdir_iterator__update_entry;
 
 	if ((error = iterator__update_ignore_case((git_iterator *)wi, flags)) < 0 ||
-		(error = git_ignore__for_path(repo, "", &wi->ignores)) < 0)
+		(error = git_ignore__for_path(repo, ".gitignore", &wi->ignores)) < 0)
 	{
 		git_iterator_free((git_iterator *)wi);
 		return error;
 	}
 
-	return fs_iterator__initialize(out, &wi->fi, git_repository_workdir(repo));
+	return fs_iterator__initialize(out, &wi->fi, repo_workdir);
 }
 
 
diff --git a/src/iterator.h b/src/iterator.h
index 493ff4b2a..ea88fa6a2 100644
--- a/src/iterator.h
+++ b/src/iterator.h
@@ -79,15 +79,26 @@ extern int git_iterator_for_index(
 	const char *start,
 	const char *end);
 
+extern int git_iterator_for_workdir_ext(
+	git_iterator **out,
+	git_repository *repo,
+	const char *repo_workdir,
+	git_iterator_flag_t flags,
+	const char *start,
+	const char *end);
+
 /* workdir iterators will match the ignore_case value from the index of the
  * repository, unless you override with a non-zero flag value
  */
-extern int git_iterator_for_workdir(
+GIT_INLINE(int) git_iterator_for_workdir(
 	git_iterator **out,
 	git_repository *repo,
 	git_iterator_flag_t flags,
 	const char *start,
-	const char *end);
+	const char *end)
+{
+	return git_iterator_for_workdir_ext(out, repo, NULL, flags, start, end);
+}
 
 /* for filesystem iterators, you have to explicitly pass in the ignore_case
  * behavior that you desire
diff --git a/src/merge.c b/src/merge.c
index 82d2e6f37..2e94ce1cd 100644
--- a/src/merge.c
+++ b/src/merge.c
@@ -1902,8 +1902,10 @@ static int write_merge_msg(
 	entries = git__calloc(heads_len, sizeof(struct merge_msg_entry));
 	GITERR_CHECK_ALLOC(entries); 
 
-	if (git_vector_init(&matching, heads_len, NULL) < 0)
+	if (git_vector_init(&matching, heads_len, NULL) < 0) {
+		git__free(entries);
 		return -1;
+	}
 
 	for (i = 0; i < heads_len; i++)
 		entries[i].merge_head = heads[i];
diff --git a/src/netops.c b/src/netops.c
index 69179dd1c..803c2696a 100644
--- a/src/netops.c
+++ b/src/netops.c
@@ -19,10 +19,6 @@
 #	endif
 #endif
 
-#ifdef __FreeBSD__
-#	include <netinet/in.h>
-#endif
-
 #ifdef GIT_SSL
 # include <openssl/ssl.h>
 # include <openssl/err.h>
diff --git a/src/odb.c b/src/odb.c
index 8e62efd00..a0bfec403 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -232,6 +232,7 @@ int git_odb__hashlink(git_oid *out, const char *path)
 		link_data[size] = '\0';
 		if (read_len != (ssize_t)size) {
 			giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path);
+			git__free(link_data);
 			return -1;
 		}
 
@@ -290,10 +291,10 @@ typedef struct {
 	git_otype type;
 } fake_wstream;
 
-static int fake_wstream__fwrite(git_oid *oid, git_odb_stream *_stream)
+static int fake_wstream__fwrite(git_odb_stream *_stream, const git_oid *oid)
 {
 	fake_wstream *stream = (fake_wstream *)_stream;
-	return _stream->backend->write(oid, _stream->backend, stream->buffer, stream->size, stream->type);
+	return _stream->backend->write(_stream->backend, oid, stream->buffer, stream->size, stream->type);
 }
 
 static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
@@ -444,7 +445,7 @@ int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
 		return 0;
 	}
 
-	giterr_set(GITERR_ODB, "No ODB backend loaded at index " PRIuZ, pos);
+	giterr_set(GITERR_ODB, "No ODB backend loaded at index %" PRIuZ, pos);
 	return GIT_ENOTFOUND;
 }
 
@@ -607,7 +608,6 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 	git_odb_object *object;
 	size_t i;
 	bool found = false;
-	bool refreshed = false;
 
 	assert(db && id);
 
@@ -616,7 +616,6 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 		return (int)true;
 	}
 
-attempt_lookup:
 	for (i = 0; i < db->backends.length && !found; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
@@ -625,16 +624,6 @@ attempt_lookup:
 			found = b->exists(b, id);
 	}
 
-	if (!found && !refreshed) {
-		if (git_odb_refresh(db) < 0) {
-			giterr_clear();
-			return (int)false;
-		}
-
-		refreshed = true;
-		goto attempt_lookup;
-	}
-
 	return (int)found;
 }
 
@@ -699,7 +688,6 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 {
 	size_t i, reads = 0;
 	int error;
-	bool refreshed = false;
 	git_rawobj raw;
 	git_odb_object *object;
 
@@ -709,7 +697,6 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 	if (*out != NULL)
 		return 0;
 
-attempt_lookup:
 	error = GIT_ENOTFOUND;
 
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
@@ -722,14 +709,6 @@ attempt_lookup:
 		}
 	}
 
-	if (error == GIT_ENOTFOUND && !refreshed) {
-		if ((error = git_odb_refresh(db)) < 0)
-			return error;
-
-		refreshed = true;
-		goto attempt_lookup;
-	}
-
 	if (error && error != GIT_PASSTHROUGH) {
 		if (!reads)
 			return git_odb__error_notfound("no match for id", id);
@@ -751,7 +730,7 @@ int git_odb_read_prefix(
 	git_oid found_full_oid = {{0}};
 	git_rawobj raw;
 	void *data = NULL;
-	bool found = false, refreshed = false;
+	bool found = false;
 	git_odb_object *object;
 
 	assert(out && db);
@@ -768,7 +747,6 @@ int git_odb_read_prefix(
 			return 0;
 	}
 
-attempt_lookup:
 	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
@@ -785,22 +763,16 @@ attempt_lookup:
 			git__free(data);
 			data = raw.data;
 
-			if (found && git_oid__cmp(&full_oid, &found_full_oid))
+			if (found && git_oid__cmp(&full_oid, &found_full_oid)) {
+				git__free(raw.data);
 				return git_odb__error_ambiguous("multiple matches for prefix");
+			}
 
 			found_full_oid = full_oid;
 			found = true;
 		}
 	}
 
-	if (!found && !refreshed) {
-		if ((error = git_odb_refresh(db)) < 0)
-			return error;
-
-		refreshed = true;
-		goto attempt_lookup;
-	}
-
 	if (!found)
 		return git_odb__error_notfound("no match for prefix", short_id);
 
@@ -848,7 +820,7 @@ int git_odb_write(
 			continue;
 
 		if (b->write != NULL)
-			error = b->write(oid, b, data, len, type);
+			error = b->write(b, oid, data, len, type);
 	}
 
 	if (!error || error == GIT_PASSTHROUGH)
@@ -862,17 +834,27 @@ int git_odb_write(
 		return error;
 
 	stream->write(stream, data, len);
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
+	error = stream->finalize_write(stream, oid);
+	git_odb_stream_free(stream);
 
 	return error;
 }
 
+static void hash_header(git_hash_ctx *ctx, size_t size, git_otype type)
+{
+	char header[64];
+	int hdrlen;
+
+	hdrlen = git_odb__format_object_header(header, sizeof(header), size, type);
+	git_hash_update(ctx, header, hdrlen);
+}
+
 int git_odb_open_wstream(
 	git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
 {
 	size_t i, writes = 0;
 	int error = GIT_ERROR;
+	git_hash_ctx *ctx;
 
 	assert(stream && db);
 
@@ -898,9 +880,71 @@ int git_odb_open_wstream(
 	if (error < 0 && !writes)
 		error = git_odb__error_unsupported_in_backend("write object");
 
+	ctx = git__malloc(sizeof(git_hash_ctx));
+	GITERR_CHECK_ALLOC(ctx);
+
+
+	git_hash_ctx_init(ctx);
+	hash_header(ctx, size, type);
+	(*stream)->hash_ctx = ctx;
+
+	(*stream)->declared_size = size;
+	(*stream)->received_bytes = 0;
+
 	return error;
 }
 
+static int git_odb_stream__invalid_length(
+	const git_odb_stream *stream,
+	const char *action)
+{
+	giterr_set(GITERR_ODB,
+		"Cannot %s - "
+		"Invalid length. %"PRIuZ" was expected. The "
+		"total size of the received chunks amounts to %"PRIuZ".",
+		action, stream->declared_size, stream->received_bytes);		
+
+	return -1;
+}
+
+int git_odb_stream_write(git_odb_stream *stream, const char *buffer, size_t len)
+{
+	git_hash_update(stream->hash_ctx, buffer, len);
+
+	stream->received_bytes += len;
+
+	if (stream->received_bytes > stream->declared_size)
+		return git_odb_stream__invalid_length(stream,
+			"stream_write()");
+
+	return stream->write(stream, buffer, len);
+}
+
+int git_odb_stream_finalize_write(git_oid *out, git_odb_stream *stream)
+{
+	if (stream->received_bytes != stream->declared_size)
+		return git_odb_stream__invalid_length(stream,
+			"stream_finalize_write()");
+
+	git_hash_final(out, stream->hash_ctx);
+
+	if (git_odb_exists(stream->backend->odb, out))
+		return 0;
+
+	return stream->finalize_write(stream, out);
+}
+
+int git_odb_stream_read(git_odb_stream *stream, char *buffer, size_t len)
+{
+	return stream->read(stream, buffer, len);
+}
+
+void git_odb_stream_free(git_odb_stream *stream)
+{
+	git__free(stream->hash_ctx);
+	stream->free(stream);
+}
+
 int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
 {
 	size_t i, reads = 0;
diff --git a/src/odb_loose.c b/src/odb_loose.c
index e78172cf6..4ff57158d 100644
--- a/src/odb_loose.c
+++ b/src/odb_loose.c
@@ -33,7 +33,9 @@ typedef struct loose_backend {
 
 	int object_zlib_level; /** loose object zlib compression level. */
 	int fsync_object_files; /** loose object file fsync flag. */
-	char *objects_dir;
+
+	size_t objects_dirlen;
+	char objects_dir[GIT_FLEX_ARRAY];
 } loose_backend;
 
 /* State structure for exploring directories,
@@ -56,24 +58,30 @@ typedef struct {
  *
  ***********************************************************/
 
-static int object_file_name(git_buf *name, const char *dir, const git_oid *id)
+static int object_file_name(
+	git_buf *name, const loose_backend *be, const git_oid *id)
 {
-	git_buf_sets(name, dir);
-
-	/* expand length for 40 hex sha1 chars + 2 * '/' + '\0' */
-	if (git_buf_grow(name, git_buf_len(name) + GIT_OID_HEXSZ + 3) < 0)
+	/* expand length for object root + 40 hex sha1 chars + 2 * '/' + '\0' */
+	if (git_buf_grow(name, be->objects_dirlen + GIT_OID_HEXSZ + 3) < 0)
 		return -1;
 
+	git_buf_set(name, be->objects_dir, be->objects_dirlen);
 	git_path_to_dir(name);
 
 	/* loose object filename: aa/aaa... (41 bytes) */
-	git_oid_pathfmt(name->ptr + git_buf_len(name), id);
+	git_oid_pathfmt(name->ptr + name->size, id);
 	name->size += GIT_OID_HEXSZ + 1;
 	name->ptr[name->size] = '\0';
 
 	return 0;
 }
 
+static int object_mkdir(const git_buf *name, const loose_backend *be)
+{
+	return git_futils_mkdir(
+		name->ptr + be->objects_dirlen, be->objects_dir, GIT_OBJECT_DIR_MODE,
+		GIT_MKDIR_PATH | GIT_MKDIR_SKIP_LAST | GIT_MKDIR_VERIFY_DIR);
+}
 
 static size_t get_binary_object_header(obj_hdr *hdr, git_buf *obj)
 {
@@ -457,7 +465,7 @@ static int locate_object(
 	loose_backend *backend,
 	const git_oid *oid)
 {
-	int error = object_file_name(object_location, backend->objects_dir, oid);
+	int error = object_file_name(object_location, backend, oid);
 
 	if (!error && !git_path_exists(object_location->ptr))
 		return GIT_ENOTFOUND;
@@ -491,7 +499,7 @@ static int fn_locate_object_short_oid(void *state, git_buf *pathbuf) {
 	}
 
 	if (sstate->found > 1)
-		return git_odb__error_ambiguous("multiple matches in loose objects");
+		return GIT_EAMBIGUOUS;
 
 	return 0;
 }
@@ -537,12 +545,16 @@ static int locate_object_short_oid(
 	/* Explore directory to find a unique object matching short_oid */
 	error = git_path_direach(
 		object_location, fn_locate_object_short_oid, &state);
-	if (error)
+
+	if (error && error != GIT_EUSER)
 		return error;
 
 	if (!state.found)
 		return git_odb__error_notfound("no matching loose object for prefix", short_oid);
 
+	if (state.found > 1)
+		return git_odb__error_ambiguous("multiple matches in loose objects");
+
 	/* Convert obtained hex formatted oid to raw */
 	error = git_oid_fromstr(res_oid, (char *)state.res_oid);
 	if (error)
@@ -633,10 +645,12 @@ static int loose_backend__read_prefix(
 {
 	int error = 0;
 
+	assert(len <= GIT_OID_HEXSZ);
+
 	if (len < GIT_OID_MINPREFIXLEN)
 		error = git_odb__error_ambiguous("prefix length too short");
 
-	else if (len >= GIT_OID_HEXSZ) {
+	else if (len == GIT_OID_HEXSZ) {
 		/* We can fall back to regular read method */
 		error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
 		if (!error)
@@ -761,24 +775,16 @@ static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb
 	return state.cb_error ? state.cb_error : error;
 }
 
-static int loose_backend__stream_fwrite(git_oid *oid, git_odb_stream *_stream)
+static int loose_backend__stream_fwrite(git_odb_stream *_stream, const git_oid *oid)
 {
 	loose_writestream *stream = (loose_writestream *)_stream;
 	loose_backend *backend = (loose_backend *)_stream->backend;
 	git_buf final_path = GIT_BUF_INIT;
 	int error = 0;
 
-	if (git_filebuf_hash(oid, &stream->fbuf) < 0 ||
-		object_file_name(&final_path, backend->objects_dir, oid) < 0 ||
-		git_futils_mkpath2file(final_path.ptr, GIT_OBJECT_DIR_MODE) < 0)
+	if (object_file_name(&final_path, backend, oid) < 0 ||
+		object_mkdir(&final_path, backend) < 0)
 		error = -1;
-	/*
-	 * Don't try to add an existing object to the repository. This
-	 * is what git does and allows us to sidestep the fact that
-	 * we're not allowed to overwrite a read-only file on Windows.
-	 */
-	else if (git_path_exists(final_path.ptr) == true)
-		git_filebuf_cleanup(&stream->fbuf);
 	else
 		error = git_filebuf_commit_at(
 			&stream->fbuf, final_path.ptr, GIT_OBJECT_FILE_MODE);
@@ -802,17 +808,6 @@ static void loose_backend__stream_free(git_odb_stream *_stream)
 	git__free(stream);
 }
 
-static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
-{
-	const char *type_str = git_object_type2string(obj_type);
-	int len = snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
-
-	assert(len > 0);				/* otherwise snprintf() is broken */
-	assert(((size_t)len) < n);		/* otherwise the caller is broken! */
-
-	return len+1;
-}
-
 static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, size_t length, git_otype type)
 {
 	loose_backend *backend;
@@ -826,7 +821,7 @@ static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_
 	backend = (loose_backend *)_backend;
 	*stream_out = NULL;
 
-	hdrlen = format_object_header(hdr, sizeof(hdr), length, type);
+	hdrlen = git_odb__format_object_header(hdr, sizeof(hdr), length, type);
 
 	stream = git__calloc(1, sizeof(loose_writestream));
 	GITERR_CHECK_ALLOC(stream);
@@ -840,7 +835,6 @@ static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_
 
 	if (git_buf_joinpath(&tmp_path, backend->objects_dir, "tmp_object") < 0 ||
 		git_filebuf_open(&stream->fbuf, tmp_path.ptr,
-			GIT_FILEBUF_HASH_CONTENTS |
 			GIT_FILEBUF_TEMPORARY |
 			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT)) < 0 ||
 		stream->stream.write((git_odb_stream *)stream, hdr, hdrlen) < 0)
@@ -855,7 +849,7 @@ static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_
 	return !stream ? -1 : 0;
 }
 
-static int loose_backend__write(git_oid *oid, git_odb_backend *_backend, const void *data, size_t len, git_otype type)
+static int loose_backend__write(git_odb_backend *_backend, const git_oid *oid, const void *data, size_t len, git_otype type)
 {
 	int error = 0, header_len;
 	git_buf final_path = GIT_BUF_INIT;
@@ -866,7 +860,7 @@ static int loose_backend__write(git_oid *oid, git_odb_backend *_backend, const v
 	backend = (loose_backend *)_backend;
 
 	/* prepare the header for the file */
-	header_len = format_object_header(header, sizeof(header), len, type);
+	header_len = git_odb__format_object_header(header, sizeof(header), len, type);
 
 	if (git_buf_joinpath(&final_path, backend->objects_dir, "tmp_object") < 0 ||
 		git_filebuf_open(&fbuf, final_path.ptr,
@@ -880,8 +874,8 @@ static int loose_backend__write(git_oid *oid, git_odb_backend *_backend, const v
 	git_filebuf_write(&fbuf, header, header_len);
 	git_filebuf_write(&fbuf, data, len);
 
-	if (object_file_name(&final_path, backend->objects_dir, oid) < 0 ||
-		git_futils_mkpath2file(final_path.ptr, GIT_OBJECT_DIR_MODE) < 0 ||
+	if (object_file_name(&final_path, backend, oid) < 0 ||
+		object_mkdir(&final_path, backend) < 0 ||
 		git_filebuf_commit_at(&fbuf, final_path.ptr, GIT_OBJECT_FILE_MODE) < 0)
 		error = -1;
 
@@ -898,7 +892,6 @@ static void loose_backend__free(git_odb_backend *_backend)
 	assert(_backend);
 	backend = (loose_backend *)_backend;
 
-	git__free(backend->objects_dir);
 	git__free(backend);
 }
 
@@ -909,13 +902,20 @@ int git_odb_backend_loose(
 	int do_fsync)
 {
 	loose_backend *backend;
+	size_t objects_dirlen;
+
+	assert(backend_out && objects_dir);
+
+	objects_dirlen = strlen(objects_dir);
 
-	backend = git__calloc(1, sizeof(loose_backend));
+	backend = git__calloc(1, sizeof(loose_backend) + objects_dirlen + 2);
 	GITERR_CHECK_ALLOC(backend);
 
 	backend->parent.version = GIT_ODB_BACKEND_VERSION;
-	backend->objects_dir = git__strdup(objects_dir);
-	GITERR_CHECK_ALLOC(backend->objects_dir);
+	backend->objects_dirlen = objects_dirlen;
+	memcpy(backend->objects_dir, objects_dir, objects_dirlen);
+	if (backend->objects_dir[backend->objects_dirlen - 1] != '/')
+		backend->objects_dir[backend->objects_dirlen++] = '/';
 
 	if (compression_level < 0)
 		compression_level = Z_BEST_SPEED;
diff --git a/src/odb_pack.c b/src/odb_pack.c
index eec79259b..cadc93a65 100644
--- a/src/odb_pack.c
+++ b/src/odb_pack.c
@@ -259,23 +259,26 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen
 	return git_odb__error_notfound("failed to find pack entry", oid);
 }
 
-static unsigned pack_entry_find_prefix_inner(
-		struct git_pack_entry *e,
-		struct pack_backend *backend,
-		const git_oid *short_oid,
-		size_t len,
-		struct git_pack_file *last_found)
+static int pack_entry_find_prefix(
+	struct git_pack_entry *e,
+	struct pack_backend *backend,
+	const git_oid *short_oid,
+	size_t len)
 {
 	int error;
 	size_t i;
-	unsigned found = 0;
+	git_oid found_full_oid = {{0}};
+	bool found = false;
+	struct git_pack_file *last_found = backend->last_found;
 
 	if (last_found) {
 		error = git_pack_entry_find(e, last_found, short_oid, len);
 		if (error == GIT_EAMBIGUOUS)
 			return error;
-		if (!error)
-			found = 1;
+		if (!error) {
+			git_oid_cpy(&found_full_oid, &e->sha1);
+			found = true;
+		}
 	}
 
 	for (i = 0; i < backend->packs.length; ++i) {
@@ -289,28 +292,16 @@ static unsigned pack_entry_find_prefix_inner(
 		if (error == GIT_EAMBIGUOUS)
 			return error;
 		if (!error) {
-			if (++found > 1)
-				break;
+			if (found && git_oid_cmp(&e->sha1, &found_full_oid))
+				return git_odb__error_ambiguous("found multiple pack entries");
+			git_oid_cpy(&found_full_oid, &e->sha1);
+			found = true;
 			backend->last_found = p;
 		}
 	}
 
-	return found;
-}
-
-static int pack_entry_find_prefix(
-	struct git_pack_entry *e,
-	struct pack_backend *backend,
-	const git_oid *short_oid,
-	size_t len)
-{
-	struct git_pack_file *last_found = backend->last_found;
-	unsigned int found = pack_entry_find_prefix_inner(e, backend, short_oid, len, last_found);
-
 	if (!found)
 		return git_odb__error_notfound("no matching pack entry for prefix", short_oid);
-	else if (found > 1)
-		return git_odb__error_ambiguous("found multiple pack entries");
 	else
 		return 0;
 }
@@ -345,14 +336,15 @@ static int pack_backend__refresh(git_odb_backend *_backend)
 	git_buf_free(&path);
 
 	if (error < 0)
-		return error;
+		return -1;
 
 	git_vector_sort(&backend->packs);
 	return 0;
 }
 
-
-static int pack_backend__read_header(size_t *len_p, git_otype *type_p, struct git_odb_backend *backend, const git_oid *oid)
+static int pack_backend__read_header_internal(
+	size_t *len_p, git_otype *type_p,
+	struct git_odb_backend *backend, const git_oid *oid)
 {
 	struct git_pack_entry e;
 	int error;
@@ -365,7 +357,26 @@ static int pack_backend__read_header(size_t *len_p, git_otype *type_p, struct gi
 	return git_packfile_resolve_header(len_p, type_p, e.p, e.offset);
 }
 
-static int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+static int pack_backend__read_header(
+	size_t *len_p, git_otype *type_p,
+	struct git_odb_backend *backend, const git_oid *oid)
+{
+	int error;
+
+	error = pack_backend__read_header_internal(len_p, type_p, backend, oid);
+
+	if (error != GIT_ENOTFOUND)
+		return error;
+
+	if ((error = pack_backend__refresh(backend)) < 0)
+		return error;
+
+	return pack_backend__read_header_internal(len_p, type_p, backend, oid);
+}
+
+static int pack_backend__read_internal(
+	void **buffer_p, size_t *len_p, git_otype *type_p,
+	git_odb_backend *backend, const git_oid *oid)
 {
 	struct git_pack_entry e;
 	git_rawobj raw;
@@ -382,7 +393,24 @@ static int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p,
 	return 0;
 }
 
-static int pack_backend__read_prefix(
+static int pack_backend__read(
+	void **buffer_p, size_t *len_p, git_otype *type_p,
+	git_odb_backend *backend, const git_oid *oid)
+{
+	int error;
+
+	error = pack_backend__read_internal(buffer_p, len_p, type_p, backend, oid);
+
+	if (error != GIT_ENOTFOUND)
+		return error;
+
+	if ((error = pack_backend__refresh(backend)) < 0)
+		return error;
+
+	return pack_backend__read_internal(buffer_p, len_p, type_p, backend, oid);
+}
+
+static int pack_backend__read_prefix_internal(
 	git_oid *out_oid,
 	void **buffer_p,
 	size_t *len_p,
@@ -419,9 +447,45 @@ static int pack_backend__read_prefix(
 	return error;
 }
 
+static int pack_backend__read_prefix(
+	git_oid *out_oid,
+	void **buffer_p,
+	size_t *len_p,
+	git_otype *type_p,
+	git_odb_backend *backend,
+	const git_oid *short_oid,
+	size_t len)
+{
+	int error;
+
+	error = pack_backend__read_prefix_internal(
+		out_oid, buffer_p, len_p, type_p, backend, short_oid, len);
+
+	if (error != GIT_ENOTFOUND)
+		return error;
+
+	if ((error = pack_backend__refresh(backend)) < 0)
+		return error;
+
+	return pack_backend__read_prefix_internal(
+		out_oid, buffer_p, len_p, type_p, backend, short_oid, len);
+}
+
 static int pack_backend__exists(git_odb_backend *backend, const git_oid *oid)
 {
 	struct git_pack_entry e;
+	int error;
+
+	error = pack_entry_find(&e, (struct pack_backend *)backend, oid);
+
+	if (error != GIT_ENOTFOUND)
+		return error == 0;
+
+	if ((error = pack_backend__refresh(backend)) < 0) {
+		giterr_clear();
+		return (int)false;
+	}
+
 	return pack_entry_find(&e, (struct pack_backend *)backend, oid) == 0;
 }
 
diff --git a/src/oid.c b/src/oid.c
index 8300e46c1..a70b7e099 100644
--- a/src/oid.c
+++ b/src/oid.c
@@ -369,8 +369,10 @@ int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid)
 	bool is_leaf;
 	node_index idx;
 
-	if (os->full)
+	if (os->full) {
+		giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
 		return -1;
+	}
 
 	if (text_oid == NULL)
 		return os->min_length;
@@ -396,12 +398,19 @@ int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid)
 			node->tail = NULL;
 
 			node = push_leaf(os, idx, git__fromhex(tail[0]), &tail[1]);
-			GITERR_CHECK_ALLOC(node);
+			if (node == NULL) {
+				if (os->full)
+					giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
+				return -1;
+			}
 		}
 
 		if (node->children[c] == 0) {
-			if (push_leaf(os, idx, c, &text_oid[i + 1]) == NULL)
+			if (push_leaf(os, idx, c, &text_oid[i + 1]) == NULL) {
+				if (os->full)
+					giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
 				return -1;
+			}
 			break;
 		}
 
diff --git a/src/oid.h b/src/oid.h
index 077d0a4c8..cfe7ca1b2 100644
--- a/src/oid.h
+++ b/src/oid.h
@@ -9,17 +9,8 @@
 
 #include "git2/oid.h"
 
-/*
- * Compare two oid structures.
- *
- * @param a first oid structure.
- * @param b second oid structure.
- * @return <0, 0, >0 if a < b, a == b, a > b.
- */
-GIT_INLINE(int) git_oid__cmp(const git_oid *a, const git_oid *b)
+GIT_INLINE(int) git_oid__hashcmp(const unsigned char *sha1, const unsigned char *sha2)
 {
-	const unsigned char *sha1 = a->id;
-	const unsigned char *sha2 = b->id;
 	int i;
 
 	for (i = 0; i < GIT_OID_RAWSZ; i++, sha1++, sha2++) {
@@ -30,4 +21,16 @@ GIT_INLINE(int) git_oid__cmp(const git_oid *a, const git_oid *b)
 	return 0;
 }
 
+/*
+ * Compare two oid structures.
+ *
+ * @param a first oid structure.
+ * @param b second oid structure.
+ * @return <0, 0, >0 if a < b, a == b, a > b.
+ */
+GIT_INLINE(int) git_oid__cmp(const git_oid *a, const git_oid *b)
+{
+	return git_oid__hashcmp(a->id, b->id);
+}
+
 #endif
diff --git a/src/pack-objects.c b/src/pack-objects.c
index 500104c55..7f427e3bd 100644
--- a/src/pack-objects.c
+++ b/src/pack-objects.c
@@ -505,8 +505,10 @@ static git_pobject **compute_write_order(git_packbuilder *pb)
 	/*
 	 * Mark objects that are at the tip of tags.
 	 */
-	if (git_tag_foreach(pb->repo, &cb_tag_foreach, pb) < 0)
+	if (git_tag_foreach(pb->repo, &cb_tag_foreach, pb) < 0) {
+		git__free(wo);
 		return NULL;
+	}
 
 	/*
 	 * Give the objects in the original recency order until
diff --git a/src/pack.c b/src/pack.c
index 7ce7099e0..e7fb9f1ae 100644
--- a/src/pack.c
+++ b/src/pack.c
@@ -329,8 +329,10 @@ static int pack_index_open(struct git_pack_file *p)
 	memcpy(idx_name, p->pack_name, base_len);
 	memcpy(idx_name + base_len, ".idx", sizeof(".idx"));
 
-	if ((error = git_mutex_lock(&p->lock)) < 0)
+	if ((error = git_mutex_lock(&p->lock)) < 0) {
+		git__free(idx_name);
 		return error;
+	}
 
 	if (p->index_version == -1)
 		error = pack_index_check(idx_name, p);
@@ -820,7 +822,7 @@ void git_packfile_free(struct git_pack_file *p)
 
 	git_mwindow_free_all(&p->mwf);
 
-	if (p->mwf.fd != -1)
+	if (p->mwf.fd >= 0)
 		p_close(p->mwf.fd);
 
 	pack_index_free(p);
@@ -903,7 +905,8 @@ static int packfile_open(struct git_pack_file *p)
 cleanup:
 	giterr_set(GITERR_OS, "Invalid packfile '%s'", p->pack_name);
 
-	p_close(p->mwf.fd);
+	if (p->mwf.fd >= 0)
+		p_close(p->mwf.fd);
 	p->mwf.fd = -1;
 
 	git_mutex_unlock(&p->lock);
@@ -1107,8 +1110,11 @@ static int pack_entry_find_offset(
 		short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects);
 #endif
 
-	/* Use git.git lookup code */
+#ifdef GIT_USE_LOOKUP
 	pos = sha1_entry_pos(index, stride, 0, lo, hi, p->num_objects, short_oid->id);
+#else
+	pos = sha1_position(index, stride, lo, hi, short_oid->id);
+#endif
 
 	if (pos >= 0) {
 		/* An object matching exactly the oid was found */
diff --git a/src/path.c b/src/path.c
index 6437979d5..56b0b49ca 100644
--- a/src/path.c
+++ b/src/path.c
@@ -8,7 +8,6 @@
 #include "path.h"
 #include "posix.h"
 #ifdef GIT_WIN32
-#include "win32/dir.h"
 #include "win32/posix.h"
 #else
 #include <dirent.h>
@@ -243,8 +242,8 @@ int git_path_root(const char *path)
 
 #ifdef GIT_WIN32
 	/* Are we dealing with a windows network path? */
-	else if ((path[0] == '/' && path[1] == '/') ||
-		(path[0] == '\\' && path[1] == '\\'))
+	else if ((path[0] == '/' && path[1] == '/' && path[2] != '/') ||
+		(path[0] == '\\' && path[1] == '\\' && path[2] != '\\'))
 	{
 		offset += 2;
 
@@ -486,24 +485,26 @@ bool git_path_is_empty_dir(const char *path)
 {
 	git_buf pathbuf = GIT_BUF_INIT;
 	HANDLE hFind = INVALID_HANDLE_VALUE;
-	wchar_t wbuf[GIT_WIN_PATH];
+	git_win32_path wbuf;
 	WIN32_FIND_DATAW ffd;
 	bool retval = true;
 
 	if (!git_path_isdir(path)) return false;
 
 	git_buf_printf(&pathbuf, "%s\\*", path);
-	git__utf8_to_16(wbuf, GIT_WIN_PATH, git_buf_cstr(&pathbuf));
+	git_win32_path_from_c(wbuf, git_buf_cstr(&pathbuf));
 
 	hFind = FindFirstFileW(wbuf, &ffd);
 	if (INVALID_HANDLE_VALUE == hFind) {
 		giterr_set(GITERR_OS, "Couldn't open '%s'", path);
+		git_buf_free(&pathbuf);
 		return false;
 	}
 
 	do {
 		if (!git_path_is_dot_or_dotdotW(ffd.cFileName)) {
 			retval = false;
+			break;
 		}
 	} while (FindNextFileW(hFind, &ffd) != 0);
 
@@ -603,7 +604,7 @@ int git_path_find_dir(git_buf *dir, const char *path, const char *base)
 	}
 
 	/* call dirname if this is not a directory */
-	if (!error && git_path_isdir(dir->ptr) == false)
+	if (!error) /* && git_path_isdir(dir->ptr) == false) */
 		error = git_path_dirname_r(dir, dir->ptr);
 
 	if (!error)
@@ -645,12 +646,33 @@ int git_path_resolve_relative(git_buf *path, size_t ceiling)
 			/* do nothing with singleton dot */;
 
 		else if (len == 2 && from[0] == '.' && from[1] == '.') {
-			while (to > base && to[-1] == '/') to--;
-			while (to > base && to[-1] != '/') to--;
-		}
+			/* error out if trying to up one from a hard base */
+			if (to == base && ceiling != 0) {
+				giterr_set(GITERR_INVALID,
+					"Cannot strip root component off url");
+				return -1;
+			}
+
+			/* no more path segments to strip,
+			 * use '../' as a new base path */
+			if (to == base) {
+				if (*next == '/')
+					len++;
 
-		else {
-			if (*next == '/')
+				if (to != from)
+					memmove(to, from, len);
+
+				to += len;
+				/* this is now the base, can't back up from a
+				 * relative prefix */
+				base = to;
+			} else {
+				/* back up a path segment */
+				while (to > base && to[-1] == '/') to--;
+				while (to > base && to[-1] != '/') to--;
+			}
+		} else {
+			if (*next == '/' && *from != '/')
 				len++;
 
 			if (to != from)
@@ -743,10 +765,10 @@ int git_path_direach(
 
 		git_buf_truncate(path, wd_len); /* restore path */
 
-		if (result < 0) {
+		if (result) {
 			closedir(dir);
 			git__free(de_buf);
-			return -1;
+			return GIT_EUSER;
 		}
 	}
 
diff --git a/src/path.h b/src/path.h
index ead4fa338..b2899e97f 100644
--- a/src/path.h
+++ b/src/path.h
@@ -175,7 +175,6 @@ extern bool git_path_contains(git_buf *dir, const char *item);
  *
  * @param parent Directory path that might contain subdir
  * @param subdir Subdirectory name to look for in parent
- * @param append_if_exists If true, then subdir will be appended to the parent path if it does exist
  * @return true if subdirectory exists, false otherwise.
  */
 extern bool git_path_contains_dir(git_buf *parent, const char *subdir);
@@ -185,7 +184,6 @@ extern bool git_path_contains_dir(git_buf *parent, const char *subdir);
  *
  * @param dir Directory path that might contain file
  * @param file File name to look for in parent
- * @param append_if_exists If true, then file will be appended to the path if it does exist
  * @return true if file exists, false otherwise.
  */
 extern bool git_path_contains_file(git_buf *dir, const char *file);
diff --git a/src/pathspec.c b/src/pathspec.c
index 35c79ce82..d56d03918 100644
--- a/src/pathspec.c
+++ b/src/pathspec.c
@@ -5,9 +5,16 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
+#include "git2/pathspec.h"
+#include "git2/diff.h"
 #include "pathspec.h"
 #include "buf_text.h"
 #include "attr_file.h"
+#include "iterator.h"
+#include "repository.h"
+#include "index.h"
+#include "bitvec.h"
+#include "diff.h"
 
 /* what is the common non-wildcard prefix for all items in the pathspec */
 char *git_pathspec_prefix(const git_strarray *pathspec)
@@ -56,7 +63,7 @@ bool git_pathspec_is_empty(const git_strarray *pathspec)
 }
 
 /* build a vector of fnmatch patterns to evaluate efficiently */
-int git_pathspec_init(
+int git_pathspec__vinit(
 	git_vector *vspec, const git_strarray *strspec, git_pool *strpool)
 {
 	size_t i;
@@ -76,7 +83,7 @@ int git_pathspec_init(
 		if (!match)
 			return -1;
 
-		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE;
+		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE | GIT_ATTR_FNMATCH_ALLOWNEG;
 
 		ret = git_attr_fnmatch__parse(match, strpool, NULL, &pattern);
 		if (ret == GIT_ENOTFOUND) {
@@ -93,7 +100,7 @@ int git_pathspec_init(
 }
 
 /* free data from the pathspec vector */
-void git_pathspec_free(git_vector *vspec)
+void git_pathspec__vfree(git_vector *vspec)
 {
 	git_attr_fnmatch *match;
 	unsigned int i;
@@ -106,63 +113,612 @@ void git_pathspec_free(git_vector *vspec)
 	git_vector_free(vspec);
 }
 
+struct pathspec_match_context {
+	int fnmatch_flags;
+	int (*strcomp)(const char *, const char *);
+	int (*strncomp)(const char *, const char *, size_t);
+};
+
+static void pathspec_match_context_init(
+	struct pathspec_match_context *ctxt,
+	bool disable_fnmatch,
+	bool casefold)
+{
+	if (disable_fnmatch)
+		ctxt->fnmatch_flags = -1;
+	else if (casefold)
+		ctxt->fnmatch_flags = FNM_CASEFOLD;
+	else
+		ctxt->fnmatch_flags = 0;
+
+	if (casefold) {
+		ctxt->strcomp  = git__strcasecmp;
+		ctxt->strncomp = git__strncasecmp;
+	} else {
+		ctxt->strcomp  = git__strcmp;
+		ctxt->strncomp = git__strncmp;
+	}
+}
+
+static int pathspec_match_one(
+	const git_attr_fnmatch *match,
+	struct pathspec_match_context *ctxt,
+	const char *path)
+{
+	int result = (match->flags & GIT_ATTR_FNMATCH_MATCH_ALL) ? 0 : FNM_NOMATCH;
+
+	if (result == FNM_NOMATCH)
+		result = ctxt->strcomp(match->pattern, path) ? FNM_NOMATCH : 0;
+
+	if (ctxt->fnmatch_flags >= 0 && result == FNM_NOMATCH)
+		result = p_fnmatch(match->pattern, path, ctxt->fnmatch_flags);
+
+	/* if we didn't match, look for exact dirname prefix match */
+	if (result == FNM_NOMATCH &&
+		(match->flags & GIT_ATTR_FNMATCH_HASWILD) == 0 &&
+		ctxt->strncomp(path, match->pattern, match->length) == 0 &&
+		path[match->length] == '/')
+		result = 0;
+
+	/* if we didn't match and this is a negative match, check for exact
+	 * match of filename with leading '!'
+	 */
+	if (result == FNM_NOMATCH &&
+		(match->flags & GIT_ATTR_FNMATCH_NEGATIVE) != 0 &&
+		*path == '!' &&
+		ctxt->strncomp(path + 1, match->pattern, match->length) == 0 &&
+		(!path[match->length + 1] || path[match->length + 1] == '/'))
+		return 1;
+
+	if (result == 0)
+		return (match->flags & GIT_ATTR_FNMATCH_NEGATIVE) ? 0 : 1;
+	return -1;
+}
+
+static int git_pathspec__match_at(
+	size_t *matched_at,
+	const git_vector *vspec,
+	struct pathspec_match_context *ctxt,
+	const char *path0,
+	const char *path1)
+{
+	int result = GIT_ENOTFOUND;
+	size_t i = 0;
+	const git_attr_fnmatch *match;
+
+	git_vector_foreach(vspec, i, match) {
+		if (path0 && (result = pathspec_match_one(match, ctxt, path0)) >= 0)
+			break;
+		if (path1 && (result = pathspec_match_one(match, ctxt, path1)) >= 0)
+			break;
+	}
+
+	*matched_at = i;
+	return result;
+}
+
 /* match a path against the vectorized pathspec */
-bool git_pathspec_match_path(
-	git_vector *vspec,
+bool git_pathspec__match(
+	const git_vector *vspec,
 	const char *path,
 	bool disable_fnmatch,
 	bool casefold,
-	const char **matched_pathspec)
+	const char **matched_pathspec,
+	size_t *matched_at)
 {
-	size_t i;
-	git_attr_fnmatch *match;
-	int fnmatch_flags = 0;
-	int (*use_strcmp)(const char *, const char *);
-	int (*use_strncmp)(const char *, const char *, size_t);
+	int result;
+	size_t pos;
+	struct pathspec_match_context ctxt;
 
 	if (matched_pathspec)
 		*matched_pathspec = NULL;
+	if (matched_at)
+		*matched_at = GIT_PATHSPEC_NOMATCH;
 
 	if (!vspec || !vspec->length)
 		return true;
 
-	if (disable_fnmatch)
-		fnmatch_flags = -1;
-	else if (casefold)
-		fnmatch_flags = FNM_CASEFOLD;
+	pathspec_match_context_init(&ctxt, disable_fnmatch, casefold);
 
-	if (casefold) {
-		use_strcmp  = git__strcasecmp;
-		use_strncmp = git__strncasecmp;
-	} else {
-		use_strcmp  = git__strcmp;
-		use_strncmp = git__strncmp;
+	result = git_pathspec__match_at(&pos, vspec, &ctxt, path, NULL);
+	if (result >= 0) {
+		if (matched_pathspec) {
+			const git_attr_fnmatch *match = git_vector_get(vspec, pos);
+			*matched_pathspec = match->pattern;
+		}
+
+		if (matched_at)
+			*matched_at = pos;
 	}
 
-	git_vector_foreach(vspec, i, match) {
-		int result = (match->flags & GIT_ATTR_FNMATCH_MATCH_ALL) ? 0 : FNM_NOMATCH;
+	return (result > 0);
+}
+
+
+int git_pathspec__init(git_pathspec *ps, const git_strarray *paths)
+{
+	int error = 0;
+
+	memset(ps, 0, sizeof(*ps));
+
+	ps->prefix = git_pathspec_prefix(paths);
+
+	if ((error = git_pool_init(&ps->pool, 1, 0)) < 0 ||
+		(error = git_pathspec__vinit(&ps->pathspec, paths, &ps->pool)) < 0)
+		git_pathspec__clear(ps);
+
+	return error;
+}
+
+void git_pathspec__clear(git_pathspec *ps)
+{
+	git__free(ps->prefix);
+	git_pathspec__vfree(&ps->pathspec);
+	git_pool_clear(&ps->pool);
+	memset(ps, 0, sizeof(*ps));
+}
 
-		if (result == FNM_NOMATCH)
-			result = use_strcmp(match->pattern, path) ? FNM_NOMATCH : 0;
+int git_pathspec_new(git_pathspec **out, const git_strarray *pathspec)
+{
+	int error = 0;
+	git_pathspec *ps = git__malloc(sizeof(git_pathspec));
+	GITERR_CHECK_ALLOC(ps);
+
+	if ((error = git_pathspec__init(ps, pathspec)) < 0) {
+		git__free(ps);
+		return error;
+	}
+
+	GIT_REFCOUNT_INC(ps);
+	*out = ps;
+	return 0;
+}
+
+static void pathspec_free(git_pathspec *ps)
+{
+	git_pathspec__clear(ps);
+	git__free(ps);
+}
 
-		if (fnmatch_flags >= 0 && result == FNM_NOMATCH)
-			result = p_fnmatch(match->pattern, path, fnmatch_flags);
+void git_pathspec_free(git_pathspec *ps)
+{
+	if (!ps)
+		return;
+	GIT_REFCOUNT_DEC(ps, pathspec_free);
+}
+
+int git_pathspec_matches_path(
+	const git_pathspec *ps, uint32_t flags, const char *path)
+{
+	bool no_fnmatch = (flags & GIT_PATHSPEC_NO_GLOB) != 0;
+	bool casefold =  (flags & GIT_PATHSPEC_IGNORE_CASE) != 0;
+
+	assert(ps && path);
+
+	return (0 != git_pathspec__match(
+		&ps->pathspec, path, no_fnmatch, casefold, NULL, NULL));
+}
+
+static void pathspec_match_free(git_pathspec_match_list *m)
+{
+	git_pathspec_free(m->pathspec);
+	m->pathspec = NULL;
+
+	git_array_clear(m->matches);
+	git_array_clear(m->failures);
+	git_pool_clear(&m->pool);
+	git__free(m);
+}
+
+static git_pathspec_match_list *pathspec_match_alloc(
+	git_pathspec *ps, int datatype)
+{
+	git_pathspec_match_list *m = git__calloc(1, sizeof(git_pathspec_match_list));
+
+	if (m != NULL && git_pool_init(&m->pool, 1, 0) < 0) {
+		pathspec_match_free(m);
+		m = NULL;
+	}
+
+	/* need to keep reference to pathspec and increment refcount because
+	 * failures array stores pointers to the pattern strings of the
+	 * pathspec that had no matches
+	 */
+	GIT_REFCOUNT_INC(ps);
+	m->pathspec = ps;
+	m->datatype = datatype;
+
+	return m;
+}
+
+GIT_INLINE(size_t) pathspec_mark_pattern(git_bitvec *used, size_t pos)
+{
+	if (!git_bitvec_get(used, pos)) {
+		git_bitvec_set(used, pos, true);
+		return 1;
+	}
+
+	return 0;
+}
+
+static size_t pathspec_mark_remaining(
+	git_bitvec *used,
+	git_vector *patterns,
+	struct pathspec_match_context *ctxt,
+	size_t start,
+	const char *path0,
+	const char *path1)
+{
+	size_t count = 0;
+
+	if (path1 == path0)
+		path1 = NULL;
+
+	for (; start < patterns->length; ++start) {
+		const git_attr_fnmatch *pat = git_vector_get(patterns, start);
+
+		if (git_bitvec_get(used, start))
+			continue;
+
+		if (path0 && pathspec_match_one(pat, ctxt, path0) > 0)
+			count += pathspec_mark_pattern(used, start);
+		else if (path1 && pathspec_match_one(pat, ctxt, path1) > 0)
+			count += pathspec_mark_pattern(used, start);
+	}
+
+	return count;
+}
+
+static int pathspec_build_failure_array(
+	git_pathspec_string_array_t *failures,
+	git_vector *patterns,
+	git_bitvec *used,
+	git_pool *pool)
+{
+	size_t pos;
+	char **failed;
+	const git_attr_fnmatch *pat;
+
+	for (pos = 0; pos < patterns->length; ++pos) {
+		if (git_bitvec_get(used, pos))
+			continue;
+
+		if ((failed = git_array_alloc(*failures)) == NULL)
+			return -1;
+
+		pat = git_vector_get(patterns, pos);
+
+		if ((*failed = git_pool_strdup(pool, pat->pattern)) == NULL)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int pathspec_match_from_iterator(
+	git_pathspec_match_list **out,
+	git_iterator *iter,
+	uint32_t flags,
+	git_pathspec *ps)
+{
+	int error = 0;
+	git_pathspec_match_list *m = NULL;
+	const git_index_entry *entry = NULL;
+	struct pathspec_match_context ctxt;
+	git_vector *patterns = &ps->pathspec;
+	bool find_failures = out && (flags & GIT_PATHSPEC_FIND_FAILURES) != 0;
+	bool failures_only = !out || (flags & GIT_PATHSPEC_FAILURES_ONLY) != 0;
+	size_t pos, used_ct = 0, found_files = 0;
+	git_index *index = NULL;
+	git_bitvec used_patterns;
+	char **file;
+
+	if (git_bitvec_init(&used_patterns, patterns->length) < 0)
+		return -1;
+
+	if (out) {
+		*out = m = pathspec_match_alloc(ps, PATHSPEC_DATATYPE_STRINGS);
+		GITERR_CHECK_ALLOC(m);
+	}
 
-		/* if we didn't match, look for exact dirname prefix match */
-		if (result == FNM_NOMATCH &&
-			(match->flags & GIT_ATTR_FNMATCH_HASWILD) == 0 &&
-			use_strncmp(path, match->pattern, match->length) == 0 &&
-			path[match->length] == '/')
-			result = 0;
+	if ((error = git_iterator_reset(iter, ps->prefix, ps->prefix)) < 0)
+		goto done;
 
-		if (result == 0) {
-			if (matched_pathspec)
-				*matched_pathspec = match->pattern;
+	if (git_iterator_type(iter) == GIT_ITERATOR_TYPE_WORKDIR &&
+		(error = git_repository_index__weakptr(
+			&index, git_iterator_owner(iter))) < 0)
+		goto done;
 
-			return (match->flags & GIT_ATTR_FNMATCH_NEGATIVE) ? false : true;
+	pathspec_match_context_init(
+		&ctxt, (flags & GIT_PATHSPEC_NO_GLOB) != 0,
+		git_iterator_ignore_case(iter));
+
+	while (!(error = git_iterator_advance(&entry, iter))) {
+		/* search for match with entry->path */
+		int result = git_pathspec__match_at(
+			&pos, patterns, &ctxt, entry->path, NULL);
+
+		/* no matches for this path */
+		if (result < 0)
+			continue;
+
+		/* if result was a negative pattern match, then don't list file */
+		if (!result) {
+			used_ct += pathspec_mark_pattern(&used_patterns, pos);
+			continue;
+		}
+
+		/* check if path is ignored and untracked */
+		if (index != NULL &&
+			git_iterator_current_is_ignored(iter) &&
+			git_index__find(NULL, index, entry->path, GIT_INDEX_STAGE_ANY) < 0)
+			continue;
+
+		/* mark the matched pattern as used */
+		used_ct += pathspec_mark_pattern(&used_patterns, pos);
+		++found_files;
+
+		/* if find_failures is on, check if any later patterns also match */
+		if (find_failures && used_ct < patterns->length)
+			used_ct += pathspec_mark_remaining(
+				&used_patterns, patterns, &ctxt, pos + 1, entry->path, NULL);
+
+		/* if only looking at failures, exit early or just continue */
+		if (failures_only || !out) {
+			if (used_ct == patterns->length)
+				break;
+			continue;
+		}
+
+		/* insert matched path into matches array */
+		if ((file = (char **)git_array_alloc(m->matches)) == NULL ||
+			(*file = git_pool_strdup(&m->pool, entry->path)) == NULL) {
+			error = -1;
+			goto done;
+		}
+	}
+
+	if (error < 0 && error != GIT_ITEROVER)
+		goto done;
+	error = 0;
+
+	/* insert patterns that had no matches into failures array */
+	if (find_failures && used_ct < patterns->length &&
+		(error = pathspec_build_failure_array(
+			&m->failures, patterns, &used_patterns, &m->pool)) < 0)
+		goto done;
+
+	/* if every pattern failed to match, then we have failed */
+	if ((flags & GIT_PATHSPEC_NO_MATCH_ERROR) != 0 && !found_files) {
+		giterr_set(GITERR_INVALID, "No matching files were found");
+		error = GIT_ENOTFOUND;
+	}
+
+done:
+	git_bitvec_free(&used_patterns);
+
+	if (error < 0) {
+		pathspec_match_free(m);
+		if (out) *out = NULL;
+	}
+
+	return error;
+}
+
+static git_iterator_flag_t pathspec_match_iter_flags(uint32_t flags)
+{
+	git_iterator_flag_t f = 0;
+
+	if ((flags & GIT_PATHSPEC_IGNORE_CASE) != 0)
+		f |= GIT_ITERATOR_IGNORE_CASE;
+	else if ((flags & GIT_PATHSPEC_USE_CASE) != 0)
+		f |= GIT_ITERATOR_DONT_IGNORE_CASE;
+
+	return f;
+}
+
+int git_pathspec_match_workdir(
+	git_pathspec_match_list **out,
+	git_repository *repo,
+	uint32_t flags,
+	git_pathspec *ps)
+{
+	int error = 0;
+	git_iterator *iter;
+
+	assert(repo);
+
+	if (!(error = git_iterator_for_workdir(
+			&iter, repo, pathspec_match_iter_flags(flags), NULL, NULL))) {
+
+		error = pathspec_match_from_iterator(out, iter, flags, ps);
+
+		git_iterator_free(iter);
+	}
+
+	return error;
+}
+
+int git_pathspec_match_index(
+	git_pathspec_match_list **out,
+	git_index *index,
+	uint32_t flags,
+	git_pathspec *ps)
+{
+	int error = 0;
+	git_iterator *iter;
+
+	assert(index);
+
+	if (!(error = git_iterator_for_index(
+			&iter, index, pathspec_match_iter_flags(flags), NULL, NULL))) {
+
+		error = pathspec_match_from_iterator(out, iter, flags, ps);
+
+		git_iterator_free(iter);
+	}
+
+	return error;
+}
+
+int git_pathspec_match_tree(
+	git_pathspec_match_list **out,
+	git_tree *tree,
+	uint32_t flags,
+	git_pathspec *ps)
+{
+	int error = 0;
+	git_iterator *iter;
+
+	assert(tree);
+
+	if (!(error = git_iterator_for_tree(
+			&iter, tree, pathspec_match_iter_flags(flags), NULL, NULL))) {
+
+		error = pathspec_match_from_iterator(out, iter, flags, ps);
+
+		git_iterator_free(iter);
+	}
+
+	return error;
+}
+
+int git_pathspec_match_diff(
+	git_pathspec_match_list **out,
+	git_diff_list *diff,
+	uint32_t flags,
+	git_pathspec *ps)
+{
+	int error = 0;
+	git_pathspec_match_list *m = NULL;
+	struct pathspec_match_context ctxt;
+	git_vector *patterns = &ps->pathspec;
+	bool find_failures = out && (flags & GIT_PATHSPEC_FIND_FAILURES) != 0;
+	bool failures_only = !out || (flags & GIT_PATHSPEC_FAILURES_ONLY) != 0;
+	size_t i, pos, used_ct = 0, found_deltas = 0;
+	const git_diff_delta *delta, **match;
+	git_bitvec used_patterns;
+
+	assert(diff);
+
+	if (git_bitvec_init(&used_patterns, patterns->length) < 0)
+		return -1;
+
+	if (out) {
+		*out = m = pathspec_match_alloc(ps, PATHSPEC_DATATYPE_DIFF);
+		GITERR_CHECK_ALLOC(m);
+	}
+
+	pathspec_match_context_init(
+		&ctxt, (flags & GIT_PATHSPEC_NO_GLOB) != 0,
+		git_diff_is_sorted_icase(diff));
+
+	git_vector_foreach(&diff->deltas, i, delta) {
+		/* search for match with delta */
+		int result = git_pathspec__match_at(
+			&pos, patterns, &ctxt, delta->old_file.path, delta->new_file.path);
+
+		/* no matches for this path */
+		if (result < 0)
+			continue;
+
+		/* mark the matched pattern as used */
+		used_ct += pathspec_mark_pattern(&used_patterns, pos);
+
+		/* if result was a negative pattern match, then don't list file */
+		if (!result)
+			continue;
+
+		++found_deltas;
+
+		/* if find_failures is on, check if any later patterns also match */
+		if (find_failures && used_ct < patterns->length)
+			used_ct += pathspec_mark_remaining(
+				&used_patterns, patterns, &ctxt, pos + 1,
+				delta->old_file.path, delta->new_file.path);
+
+		/* if only looking at failures, exit early or just continue */
+		if (failures_only || !out) {
+			if (used_ct == patterns->length)
+				break;
+			continue;
+		}
+
+		/* insert matched delta into matches array */
+		if (!(match = (const git_diff_delta **)git_array_alloc(m->matches))) {
+			error = -1;
+			goto done;
+		} else {
+			*match = delta;
 		}
 	}
 
-	return false;
+	/* insert patterns that had no matches into failures array */
+	if (find_failures && used_ct < patterns->length &&
+		(error = pathspec_build_failure_array(
+			&m->failures, patterns, &used_patterns, &m->pool)) < 0)
+		goto done;
+
+	/* if every pattern failed to match, then we have failed */
+	if ((flags & GIT_PATHSPEC_NO_MATCH_ERROR) != 0 && !found_deltas) {
+		giterr_set(GITERR_INVALID, "No matching deltas were found");
+		error = GIT_ENOTFOUND;
+	}
+
+done:
+	git_bitvec_free(&used_patterns);
+
+	if (error < 0) {
+		pathspec_match_free(m);
+		if (out) *out = NULL;
+	}
+
+	return error;
+}
+
+void git_pathspec_match_list_free(git_pathspec_match_list *m)
+{
+	if (m)
+		pathspec_match_free(m);
+}
+
+size_t git_pathspec_match_list_entrycount(
+	const git_pathspec_match_list *m)
+{
+	return m ? git_array_size(m->matches) : 0;
+}
+
+const char *git_pathspec_match_list_entry(
+	const git_pathspec_match_list *m, size_t pos)
+{
+	if (!m || m->datatype != PATHSPEC_DATATYPE_STRINGS ||
+		!git_array_valid_index(m->matches, pos))
+		return NULL;
+
+	return *((const char **)git_array_get(m->matches, pos));
+}
+
+const git_diff_delta *git_pathspec_match_list_diff_entry(
+	const git_pathspec_match_list *m, size_t pos)
+{
+	if (!m || m->datatype != PATHSPEC_DATATYPE_DIFF ||
+		!git_array_valid_index(m->matches, pos))
+		return NULL;
+
+	return *((const git_diff_delta **)git_array_get(m->matches, pos));
+}
+
+size_t git_pathspec_match_list_failed_entrycount(
+	const git_pathspec_match_list *m)
+{
+	return m ? git_array_size(m->failures) : 0;
+}
+
+const char * git_pathspec_match_list_failed_entry(
+	const git_pathspec_match_list *m, size_t pos)
+{
+	char **entry = m ? git_array_get(m->failures, pos) : NULL;
+
+	return entry ? *entry : NULL;
 }
 
diff --git a/src/pathspec.h b/src/pathspec.h
index 43a94baad..40cd21c3f 100644
--- a/src/pathspec.h
+++ b/src/pathspec.h
@@ -8,9 +8,35 @@
 #define INCLUDE_pathspec_h__
 
 #include "common.h"
+#include <git2/pathspec.h>
 #include "buffer.h"
 #include "vector.h"
 #include "pool.h"
+#include "array.h"
+
+/* public compiled pathspec */
+struct git_pathspec {
+	git_refcount rc;
+	char *prefix;
+	git_vector pathspec;
+	git_pool pool;
+};
+
+enum {
+	PATHSPEC_DATATYPE_STRINGS = 0,
+	PATHSPEC_DATATYPE_DIFF = 1,
+};
+
+typedef git_array_t(char *) git_pathspec_string_array_t;
+
+/* public interface to pathspec matching */
+struct git_pathspec_match_list {
+	git_pathspec *pathspec;
+	git_array_t(void *) matches;
+	git_pathspec_string_array_t failures;
+	git_pool pool;
+	int datatype;
+};
 
 /* what is the common non-wildcard prefix for all items in the pathspec */
 extern char *git_pathspec_prefix(const git_strarray *pathspec);
@@ -19,22 +45,31 @@ extern char *git_pathspec_prefix(const git_strarray *pathspec);
 extern bool git_pathspec_is_empty(const git_strarray *pathspec);
 
 /* build a vector of fnmatch patterns to evaluate efficiently */
-extern int git_pathspec_init(
+extern int git_pathspec__vinit(
 	git_vector *vspec, const git_strarray *strspec, git_pool *strpool);
 
 /* free data from the pathspec vector */
-extern void git_pathspec_free(git_vector *vspec);
+extern void git_pathspec__vfree(git_vector *vspec);
+
+#define GIT_PATHSPEC_NOMATCH ((size_t)-1)
 
 /*
  * Match a path against the vectorized pathspec.
  * The matched pathspec is passed back into the `matched_pathspec` parameter,
  * unless it is passed as NULL by the caller.
  */
-extern bool git_pathspec_match_path(
-	git_vector *vspec,
+extern bool git_pathspec__match(
+	const git_vector *vspec,
 	const char *path,
 	bool disable_fnmatch,
 	bool casefold,
-	const char **matched_pathspec);
+	const char **matched_pathspec,
+	size_t *matched_at);
+
+/* easy pathspec setup */
+
+extern int git_pathspec__init(git_pathspec *ps, const git_strarray *paths);
+
+extern void git_pathspec__clear(git_pathspec *ps);
 
 #endif
diff --git a/src/posix.c b/src/posix.c
index 5d526d33c..b75109b83 100644
--- a/src/posix.c
+++ b/src/posix.c
@@ -111,12 +111,12 @@ int p_open(const char *path, int flags, ...)
 		va_end(arg_list);
 	}
 
-	return open(path, flags | O_BINARY, mode);
+	return open(path, flags | O_BINARY | O_CLOEXEC, mode);
 }
 
 int p_creat(const char *path, mode_t mode)
 {
-	return open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, mode);
+	return open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_CLOEXEC, mode);
 }
 
 int p_getcwd(char *buffer_out, size_t size)
diff --git a/src/posix.h b/src/posix.h
index 719c8a04c..de2aa0b73 100644
--- a/src/posix.h
+++ b/src/posix.h
@@ -25,6 +25,9 @@
 #if !defined(O_BINARY)
 #define O_BINARY 0
 #endif
+#if !defined(O_CLOEXEC)
+#define O_CLOEXEC 0
+#endif
 
 typedef int git_file;
 
@@ -68,16 +71,12 @@ typedef int GIT_SOCKET;
 
 #define p_localtime_r localtime_r
 #define p_gmtime_r gmtime_r
-#define p_gettimeofday gettimeofday
 
 #else
 
 typedef SOCKET GIT_SOCKET;
-struct timezone;
 extern struct tm * p_localtime_r (const time_t *timer, struct tm *result);
 extern struct tm * p_gmtime_r (const time_t *timer, struct tm *result);
-extern int p_gettimeofday(struct timeval *tv, struct timezone *tz);
-
 
 #endif
 
@@ -90,6 +89,10 @@ extern int p_gettimeofday(struct timeval *tv, struct timezone *tz);
 #	include "unix/posix.h"
 #endif
 
+#ifndef __MINGW32__
+# define p_strnlen strnlen
+#endif
+
 #ifdef NO_READDIR_R
 #	include <dirent.h>
 GIT_INLINE(int) p_readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result)
diff --git a/src/pqueue.h b/src/pqueue.h
index ed7139285..9061f8279 100644
--- a/src/pqueue.h
+++ b/src/pqueue.h
@@ -48,7 +48,7 @@ typedef struct {
  *			should be preallocated
  * @param cmppri the callback function to compare two nodes of the queue
  *
- * @Return the handle or NULL for insufficent memory
+ * @return the handle or NULL for insufficent memory
  */
 int git_pqueue_init(git_pqueue *q, size_t n, git_pqueue_cmp cmppri);
 
@@ -83,8 +83,7 @@ int git_pqueue_insert(git_pqueue *q, void *d);
 
 /**
  * pop the highest-ranking item from the queue.
- * @param p the queue
- * @param d where to copy the entry to
+ * @param q the queue
  * @return NULL on error, otherwise the entry
  */
 void *git_pqueue_pop(git_pqueue *q);
@@ -93,7 +92,6 @@ void *git_pqueue_pop(git_pqueue *q);
 /**
  * access highest-ranking item without removing it.
  * @param q the queue
- * @param d the entry
  * @return NULL on error, otherwise the entry
  */
 void *git_pqueue_peek(git_pqueue *q);
diff --git a/src/push.c b/src/push.c
index 452d71789..eaaa46248 100644
--- a/src/push.c
+++ b/src/push.c
@@ -233,6 +233,37 @@ on_error:
 	return error;
 }
 
+/**
+ * Insert all tags until we find a non-tag object, which is returned
+ * in `out`.
+ */
+static int enqueue_tag(git_object **out, git_push *push, git_oid *id)
+{
+	git_object *obj = NULL, *target = NULL;
+	int error;
+
+	if ((error = git_object_lookup(&obj, push->repo, id, GIT_OBJ_TAG)) < 0)
+		return error;
+
+	while (git_object_type(obj) == GIT_OBJ_TAG) {
+		if ((error = git_packbuilder_insert(push->pb, git_object_id(obj), NULL)) < 0)
+			break;
+
+		if ((error = git_tag_target(&target, (git_tag *) obj)) < 0)
+			break;
+
+		git_object_free(obj);
+		obj = target;
+	}
+
+	if (error < 0)
+		git_object_free(obj);
+	else
+		*out = obj;
+
+	return error;
+}
+
 static int revwalk(git_vector *commits, git_push *push)
 {
 	git_remote_head *head;
@@ -265,21 +296,11 @@ static int revwalk(git_vector *commits, git_push *push)
 			goto on_error;
 
 		if (type == GIT_OBJ_TAG) {
-			git_tag *tag;
 			git_object *target;
 
-			if (git_packbuilder_insert(push->pb, &spec->loid, NULL) < 0)
+			if ((error = enqueue_tag(&target, push, &spec->loid)) < 0)
 				goto on_error;
 
-			if (git_tag_lookup(&tag, push->repo, &spec->loid) < 0)
-				goto on_error;
-
-			if (git_tag_peel(&target, tag) < 0) {
-				git_tag_free(tag);
-				goto on_error;
-			}
-			git_tag_free(tag);
-
 			if (git_object_type(target) == GIT_OBJ_COMMIT) {
 				if (git_revwalk_push(rw, git_object_id(target)) < 0) {
 					git_object_free(target);
diff --git a/src/refdb_fs.c b/src/refdb_fs.c
index b9e283ac5..894ff7c84 100644
--- a/src/refdb_fs.c
+++ b/src/refdb_fs.c
@@ -15,6 +15,7 @@
 #include "refdb.h"
 #include "refdb_fs.h"
 #include "iterator.h"
+#include "sortedcache.h"
 
 #include <git2/tag.h>
 #include <git2/object.h>
@@ -53,243 +54,149 @@ typedef struct refdb_fs_backend {
 	git_repository *repo;
 	char *path;
 
-	git_refcache refcache;
+	git_sortedcache *refcache;
 	int peeling_mode;
 } refdb_fs_backend;
 
-static int reference_read(
-	git_buf *file_content,
-	time_t *mtime,
-	const char *repo_path,
-	const char *ref_name,
-	int *updated)
+static int packref_cmp(const void *a_, const void *b_)
 {
-	git_buf path = GIT_BUF_INIT;
-	int result;
-
-	assert(file_content && repo_path && ref_name);
-
-	/* Determine the full path of the file */
-	if (git_buf_joinpath(&path, repo_path, ref_name) < 0)
-		return -1;
-
-	result = git_futils_readbuffer_updated(file_content, path.ptr, mtime, NULL, updated);
-	git_buf_free(&path);
-
-	return result;
-}
-
-static int packed_parse_oid(
-	struct packref **ref_out,
-	const char **buffer_out,
-	const char *buffer_end)
-{
-	struct packref *ref = NULL;
-
-	const char *buffer = *buffer_out;
-	const char *refname_begin, *refname_end;
-
-	size_t refname_len;
-	git_oid id;
-
-	refname_begin = (buffer + GIT_OID_HEXSZ + 1);
-	if (refname_begin >= buffer_end || refname_begin[-1] != ' ')
-		goto corrupt;
-
-	/* Is this a valid object id? */
-	if (git_oid_fromstr(&id, buffer) < 0)
-		goto corrupt;
-
-	refname_end = memchr(refname_begin, '\n', buffer_end - refname_begin);
-	if (refname_end == NULL)
-		refname_end = buffer_end;
-
-	if (refname_end[-1] == '\r')
-		refname_end--;
-
-	refname_len = refname_end - refname_begin;
-
-	ref = git__calloc(1, sizeof(struct packref) + refname_len + 1);
-	GITERR_CHECK_ALLOC(ref);
-
-	memcpy(ref->name, refname_begin, refname_len);
-	ref->name[refname_len] = 0;
-
-	git_oid_cpy(&ref->oid, &id);
-
-	*ref_out = ref;
-	*buffer_out = refname_end + 1;
-	return 0;
-
-corrupt:
-	git__free(ref);
-	giterr_set(GITERR_REFERENCE, "The packed references file is corrupted");
-	return -1;
+	const struct packref *a = a_, *b = b_;
+	return strcmp(a->name, b->name);
 }
 
-static int packed_parse_peel(
-	struct packref *tag_ref,
-	const char **buffer_out,
-	const char *buffer_end)
+static int packed_reload(refdb_fs_backend *backend)
 {
-	const char *buffer = *buffer_out + 1;
-
-	assert(buffer[-1] == '^');
-
-	/* Ensure it's not the first entry of the file */
-	if (tag_ref == NULL)
-		goto corrupt;
-
-	if (buffer + GIT_OID_HEXSZ > buffer_end)
-		goto corrupt;
-
-	/* Is this a valid object id? */
-	if (git_oid_fromstr(&tag_ref->peel, buffer) < 0)
-		goto corrupt;
-
-	buffer = buffer + GIT_OID_HEXSZ;
-	if (*buffer == '\r')
-		buffer++;
-
-	if (buffer != buffer_end) {
-		if (*buffer == '\n')
-			buffer++;
-		else
-			goto corrupt;
-	}
-
-	tag_ref->flags |= PACKREF_HAS_PEEL;
-	*buffer_out = buffer;
-	return 0;
-
-corrupt:
-	giterr_set(GITERR_REFERENCE, "The packed references file is corrupted");
-	return -1;
-}
-
-static int packed_load(refdb_fs_backend *backend)
-{
-	int result, updated;
-	git_buf packfile = GIT_BUF_INIT;
-	const char *buffer_start, *buffer_end;
-	git_refcache *ref_cache = &backend->refcache;
-
-	/* First we make sure we have allocated the hash table */
-	if (ref_cache->packfile == NULL) {
-		ref_cache->packfile = git_strmap_alloc();
-		GITERR_CHECK_ALLOC(ref_cache->packfile);
-	}
+	int error;
+	git_buf packedrefs = GIT_BUF_INIT;
+	char *scan, *eof, *eol;
 
-	if (backend->path == NULL)
+	if (!backend->path)
 		return 0;
 
-	result = reference_read(&packfile, &ref_cache->packfile_time,
-		backend->path, GIT_PACKEDREFS_FILE, &updated);
+	error = git_sortedcache_lockandload(backend->refcache, &packedrefs);
 
 	/*
-	 * If we couldn't find the file, we need to clear the table and
-	 * return. On any other error, we return that error. If everything
-	 * went fine and the file wasn't updated, then there's nothing new
-	 * for us here, so just return. Anything else means we need to
-	 * refresh the packed refs.
+	 * If we can't find the packed-refs, clear table and return.
+	 * Any other error just gets passed through.
+	 * If no error, and file wasn't changed, just return.
+	 * Anything else means we need to refresh the packed refs.
 	 */
-	if (result == GIT_ENOTFOUND) {
-		git_strmap_clear(ref_cache->packfile);
-		return 0;
+	if (error <= 0) {
+		if (error == GIT_ENOTFOUND) {
+			git_sortedcache_clear(backend->refcache, true);
+			giterr_clear();
+			error = 0;
+		}
+		return error;
 	}
 
-	if (result < 0)
-		return -1;
+	/* At this point, refresh the packed refs from the loaded buffer. */
 
-	if (!updated)
-		return 0;
+	git_sortedcache_clear(backend->refcache, false);
 
-	/*
-	 * At this point, we want to refresh the packed refs. We already
-	 * have the contents in our buffer.
-	 */
-	git_strmap_clear(ref_cache->packfile);
-
-	buffer_start = (const char *)packfile.ptr;
-	buffer_end = (const char *)(buffer_start) + packfile.size;
+	scan = (char *)packedrefs.ptr;
+	eof  = scan + packedrefs.size;
 
 	backend->peeling_mode = PEELING_NONE;
 
-	if (buffer_start[0] == '#') {
+	if (*scan == '#') {
 		static const char *traits_header = "# pack-refs with: ";
 
-		if (git__prefixcmp(buffer_start, traits_header) == 0) {
-			char *traits = (char *)buffer_start + strlen(traits_header);
-			char *traits_end = strchr(traits, '\n');
+		if (git__prefixcmp(scan, traits_header) == 0) {
+			scan += strlen(traits_header);
+			eol = strchr(scan, '\n');
 
-			if (traits_end == NULL)
+			if (!eol)
 				goto parse_failed;
+			*eol = '\0';
 
-			*traits_end = '\0';
-
-			if (strstr(traits, " fully-peeled ") != NULL) {
+			if (strstr(scan, " fully-peeled ") != NULL) {
 				backend->peeling_mode = PEELING_FULL;
-			} else if (strstr(traits, " peeled ") != NULL) {
+			} else if (strstr(scan, " peeled ") != NULL) {
 				backend->peeling_mode = PEELING_STANDARD;
 			}
 
-			buffer_start = traits_end + 1;
+			scan = eol + 1;
 		}
 	}
 
-	while (buffer_start < buffer_end && buffer_start[0] == '#') {
-		buffer_start = strchr(buffer_start, '\n');
-		if (buffer_start == NULL)
+	while (scan < eof && *scan == '#') {
+		if (!(eol = strchr(scan, '\n')))
 			goto parse_failed;
-
-		buffer_start++;
+		scan = eol + 1;
 	}
 
-	while (buffer_start < buffer_end) {
-		int err;
-		struct packref *ref = NULL;
+	while (scan < eof) {
+		struct packref *ref;
+		git_oid oid;
 
-		if (packed_parse_oid(&ref, &buffer_start, buffer_end) < 0)
+		/* parse "<OID> <refname>\n" */
+
+		if (git_oid_fromstr(&oid, scan) < 0)
 			goto parse_failed;
+		scan += GIT_OID_HEXSZ;
 
-		if (buffer_start[0] == '^') {
-			if (packed_parse_peel(ref, &buffer_start, buffer_end) < 0)
-				goto parse_failed;
-		} else if (backend->peeling_mode == PEELING_FULL ||
-							(backend->peeling_mode == PEELING_STANDARD &&
-							git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) == 0)) {
-			ref->flags |= PACKREF_CANNOT_PEEL;
-		}
+		if (*scan++ != ' ')
+			goto parse_failed;
+		if (!(eol = strchr(scan, '\n')))
+			goto parse_failed;
+		*eol = '\0';
+		if (eol[-1] == '\r')
+			eol[-1] = '\0';
 
-		git_strmap_insert(ref_cache->packfile, ref->name, ref, err);
-		if (err < 0)
+		if (git_sortedcache_upsert((void **)&ref, backend->refcache, scan) < 0)
 			goto parse_failed;
+		scan = eol + 1;
+
+		git_oid_cpy(&ref->oid, &oid);
+
+		/* look for optional "^<OID>\n" */
+
+		if (*scan == '^') {
+			if (git_oid_fromstr(&oid, scan + 1) < 0)
+				goto parse_failed;
+			scan += GIT_OID_HEXSZ + 1;
+
+			if (scan < eof) {
+				if (!(eol = strchr(scan, '\n')))
+					goto parse_failed;
+				scan = eol + 1;
+			}
+
+			git_oid_cpy(&ref->peel, &oid);
+			ref->flags |= PACKREF_HAS_PEEL;
+		}
+		else if (backend->peeling_mode == PEELING_FULL ||
+				(backend->peeling_mode == PEELING_STANDARD &&
+				 git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) == 0))
+			ref->flags |= PACKREF_CANNOT_PEEL;
 	}
 
-	git_buf_free(&packfile);
+	git_sortedcache_wunlock(backend->refcache);
+	git_buf_free(&packedrefs);
+
 	return 0;
 
 parse_failed:
-	git_strmap_free(ref_cache->packfile);
-	ref_cache->packfile = NULL;
-	git_buf_free(&packfile);
+	giterr_set(GITERR_REFERENCE, "Corrupted packed references file");
+
+	git_sortedcache_clear(backend->refcache, false);
+	git_sortedcache_wunlock(backend->refcache);
+	git_buf_free(&packedrefs);
+
 	return -1;
 }
 
-static int loose_parse_oid(git_oid *oid, const char *filename, git_buf *file_content)
+static int loose_parse_oid(
+	git_oid *oid, const char *filename, git_buf *file_content)
 {
-	size_t len;
-	const char *str;
+	const char *str = git_buf_cstr(file_content);
 
-	len = git_buf_len(file_content);
-	if (len < GIT_OID_HEXSZ)
+	if (git_buf_len(file_content) < GIT_OID_HEXSZ)
 		goto corrupted;
 
-	/* str is guranteed to be zero-terminated */
-	str = git_buf_cstr(file_content);
-
 	/* we need to get 40 OID characters from the file */
-	if (git_oid_fromstr(oid, git_buf_cstr(file_content)) < 0)
+	if (git_oid_fromstr(oid, str) < 0)
 		goto corrupted;
 
 	/* If the file is longer than 40 chars, the 41st must be a space */
@@ -302,68 +209,71 @@ corrupted:
 	return -1;
 }
 
-static int loose_lookup_to_packfile(
-	struct packref **ref_out,
-	refdb_fs_backend *backend,
-	const char *name)
+static int loose_readbuffer(git_buf *buf, const char *base, const char *path)
+{
+	int error;
+
+	/* build full path to file */
+	if ((error = git_buf_joinpath(buf, base, path)) < 0 ||
+		(error = git_futils_readbuffer(buf, buf->ptr)) < 0)
+		git_buf_free(buf);
+
+	return error;
+}
+
+static int loose_lookup_to_packfile(refdb_fs_backend *backend, const char *name)
 {
+	int error = 0;
 	git_buf ref_file = GIT_BUF_INIT;
 	struct packref *ref = NULL;
-	size_t name_len;
+	git_oid oid;
 
-	*ref_out = NULL;
+	/* if we fail to load the loose reference, assume someone changed
+	 * the filesystem under us and skip it...
+	 */
+	if (loose_readbuffer(&ref_file, backend->path, name) < 0) {
+		giterr_clear();
+		goto done;
+	}
 
-	if (reference_read(&ref_file, NULL, backend->path, name, NULL) < 0)
-		return -1;
+	/* skip symbolic refs */
+	if (!git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF))
+		goto done;
 
-	git_buf_rtrim(&ref_file);
+	/* parse OID from file */
+	if ((error = loose_parse_oid(&oid, name, &ref_file)) < 0)
+		goto done;
 
-	name_len = strlen(name);
-	ref = git__calloc(1, sizeof(struct packref) + name_len + 1);
-	GITERR_CHECK_ALLOC(ref);
+	git_sortedcache_wlock(backend->refcache);
 
-	memcpy(ref->name, name, name_len);
-	ref->name[name_len] = 0;
+	if (!(error = git_sortedcache_upsert(
+			(void **)&ref, backend->refcache, name))) {
 
-	if (loose_parse_oid(&ref->oid, name, &ref_file) < 0) {
-		git_buf_free(&ref_file);
-		git__free(ref);
-		return -1;
+		git_oid_cpy(&ref->oid, &oid);
+		ref->flags = PACKREF_WAS_LOOSE;
 	}
 
-	ref->flags = PACKREF_WAS_LOOSE;
+	git_sortedcache_wunlock(backend->refcache);
 
-	*ref_out = ref;
+done:
 	git_buf_free(&ref_file);
-	return 0;
+	return error;
 }
 
-
 static int _dirent_loose_load(void *data, git_buf *full_path)
 {
 	refdb_fs_backend *backend = (refdb_fs_backend *)data;
-	void *old_ref = NULL;
-	struct packref *ref;
 	const char *file_path;
-	int err;
 
-	if (git_path_isdir(full_path->ptr) == true)
+	if (git__suffixcmp(full_path->ptr, ".lock") == 0)
+		return 0;
+
+	if (git_path_isdir(full_path->ptr))
 		return git_path_direach(full_path, _dirent_loose_load, backend);
 
 	file_path = full_path->ptr + strlen(backend->path);
 
-	if (loose_lookup_to_packfile(&ref, backend, file_path) < 0)
-		return -1;
-
-	git_strmap_insert2(
-		backend->refcache.packfile, ref->name, ref, old_ref, err);
-	if (err < 0) {
-		git__free(ref);
-		return -1;
-	}
-
-	git__free(old_ref);
-	return 0;
+	return loose_lookup_to_packfile(backend, file_path);
 }
 
 /*
@@ -374,11 +284,8 @@ static int _dirent_loose_load(void *data, git_buf *full_path)
  */
 static int packed_loadloose(refdb_fs_backend *backend)
 {
+	int error;
 	git_buf refs_path = GIT_BUF_INIT;
-	int result;
-
-	/* the packfile must have been previously loaded! */
-	assert(backend->refcache.packfile);
 
 	if (git_buf_joinpath(&refs_path, backend->path, GIT_REFS_DIR) < 0)
 		return -1;
@@ -388,10 +295,11 @@ static int packed_loadloose(refdb_fs_backend *backend)
 	 * This will overwrite any old packed entries with their
 	 * updated loose versions
 	 */
-	result = git_path_direach(&refs_path, _dirent_loose_load, backend);
+	error = git_path_direach(&refs_path, _dirent_loose_load, backend);
+
 	git_buf_free(&refs_path);
 
-	return result;
+	return (error == GIT_EUSER) ? -1 : error;
 }
 
 static int refdb_fs_backend__exists(
@@ -399,23 +307,17 @@ static int refdb_fs_backend__exists(
 	git_refdb_backend *_backend,
 	const char *ref_name)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	git_buf ref_path = GIT_BUF_INIT;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
-
-	if (packed_load(backend) < 0)
-		return -1;
+	assert(backend);
 
-	if (git_buf_joinpath(&ref_path, backend->path, ref_name) < 0)
+	if (packed_reload(backend) < 0 ||
+		git_buf_joinpath(&ref_path, backend->path, ref_name) < 0)
 		return -1;
 
-	if (git_path_isfile(ref_path.ptr) == true ||
-		git_strmap_exists(backend->refcache.packfile, ref_path.ptr))
-		*exists = 1;
-	else
-		*exists = 0;
+	*exists = git_path_isfile(ref_path.ptr) ||
+		(git_sortedcache_lookup(backend->refcache, ref_name) != NULL);
 
 	git_buf_free(&ref_path);
 	return 0;
@@ -447,64 +349,39 @@ static int loose_lookup(
 	refdb_fs_backend *backend,
 	const char *ref_name)
 {
-	const char *target;
-	git_oid oid;
 	git_buf ref_file = GIT_BUF_INIT;
 	int error = 0;
 
-	error = reference_read(&ref_file, NULL, backend->path, ref_name, NULL);
+	if (out)
+		*out = NULL;
 
-	if (error < 0)
-		goto done;
+	if ((error = loose_readbuffer(&ref_file, backend->path, ref_name)) < 0)
+		/* cannot read loose ref file - gah */;
+	else if (git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF) == 0) {
+		const char *target;
 
-	if (git__prefixcmp((const char *)(ref_file.ptr), GIT_SYMREF) == 0) {
 		git_buf_rtrim(&ref_file);
 
-		if ((target = loose_parse_symbolic(&ref_file)) == NULL) {
+		if (!(target = loose_parse_symbolic(&ref_file)))
 			error = -1;
-			goto done;
-		}
-
-		*out = git_reference__alloc_symbolic(ref_name, target);
+		else if (out != NULL)
+			*out = git_reference__alloc_symbolic(ref_name, target);
 	} else {
-		if ((error = loose_parse_oid(&oid, ref_name, &ref_file)) < 0)
-			goto done;
+		git_oid oid;
 
-		*out = git_reference__alloc(ref_name, &oid, NULL);
+		if (!(error = loose_parse_oid(&oid, ref_name, &ref_file)) &&
+			out != NULL)
+			*out = git_reference__alloc(ref_name, &oid, NULL);
 	}
 
-	if (*out == NULL)
-		error = -1;
-
-done:
 	git_buf_free(&ref_file);
 	return error;
 }
 
-static int packed_map_entry(
-	struct packref **entry,
-	khiter_t *pos,
-	refdb_fs_backend *backend,
-	const char *ref_name)
+static int ref_error_notfound(const char *name)
 {
-	git_strmap *packfile_refs;
-
-	if (packed_load(backend) < 0)
-		return -1;
-
-	/* Look up on the packfile */
-	packfile_refs = backend->refcache.packfile;
-
-	*pos = git_strmap_lookup_index(packfile_refs, ref_name);
-
-	if (!git_strmap_valid_index(packfile_refs, *pos)) {
-		giterr_set(GITERR_REFERENCE, "Reference '%s' not found", ref_name);
-		return GIT_ENOTFOUND;
-	}
-
-	*entry = git_strmap_value_at(packfile_refs, *pos);
-
-	return 0;
+	giterr_set(GITERR_REFERENCE, "Reference '%s' not found", name);
+	return GIT_ENOTFOUND;
 }
 
 static int packed_lookup(
@@ -512,18 +389,27 @@ static int packed_lookup(
 	refdb_fs_backend *backend,
 	const char *ref_name)
 {
-	struct packref *entry;
-	khiter_t pos;
 	int error = 0;
+	struct packref *entry;
 
-	if ((error = packed_map_entry(&entry, &pos, backend, ref_name)) < 0)
-		return error;
+	if (packed_reload(backend) < 0)
+		return -1;
 
-	if ((*out = git_reference__alloc(ref_name,
-		&entry->oid, &entry->peel)) == NULL)
+	if (git_sortedcache_rlock(backend->refcache) < 0)
 		return -1;
 
-	return 0;
+	entry = git_sortedcache_lookup(backend->refcache, ref_name);
+	if (!entry) {
+		error = ref_error_notfound(ref_name);
+	} else {
+		*out = git_reference__alloc(ref_name, &entry->oid, &entry->peel);
+		if (!*out)
+			error = -1;
+	}
+
+	git_sortedcache_runlock(backend->refcache);
+
+	return error;
 }
 
 static int refdb_fs_backend__lookup(
@@ -531,73 +417,68 @@ static int refdb_fs_backend__lookup(
 	git_refdb_backend *_backend,
 	const char *ref_name)
 {
-	refdb_fs_backend *backend;
-	int result;
-
-	assert(_backend);
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
+	int error;
 
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	if ((result = loose_lookup(out, backend, ref_name)) == 0)
+	if (!(error = loose_lookup(out, backend, ref_name)))
 		return 0;
 
 	/* only try to lookup this reference on the packfile if it
 	 * wasn't found on the loose refs; not if there was a critical error */
-	if (result == GIT_ENOTFOUND) {
+	if (error == GIT_ENOTFOUND) {
 		giterr_clear();
-		result = packed_lookup(out, backend, ref_name);
+		error = packed_lookup(out, backend, ref_name);
 	}
 
-	return result;
+	return error;
 }
 
 typedef struct {
 	git_reference_iterator parent;
 
 	char *glob;
+
+	git_pool pool;
 	git_vector loose;
-	unsigned int loose_pos;
-	khiter_t packed_pos;
+
+	size_t loose_pos;
+	size_t packed_pos;
 } refdb_fs_iter;
 
 static void refdb_fs_backend__iterator_free(git_reference_iterator *_iter)
 {
 	refdb_fs_iter *iter = (refdb_fs_iter *) _iter;
-	char *loose_path;
-	size_t i;
-
-	git_vector_foreach(&iter->loose, i, loose_path) {
-		git__free(loose_path);
-	}
 
 	git_vector_free(&iter->loose);
-
-	git__free(iter->glob);
+	git_pool_clear(&iter->pool);
 	git__free(iter);
 }
 
 static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter)
 {
-	git_strmap *packfile = backend->refcache.packfile;
+	int error = 0;
 	git_buf path = GIT_BUF_INIT;
-	git_iterator *fsit;
+	git_iterator *fsit = NULL;
 	const git_index_entry *entry = NULL;
 
 	if (!backend->path) /* do nothing if no path for loose refs */
 		return 0;
 
-	if (git_buf_printf(&path, "%s/refs", backend->path) < 0)
-		return -1;
-
-	if (git_iterator_for_filesystem(&fsit, git_buf_cstr(&path), 0, NULL, NULL) < 0)
-		return -1;
+	if ((error = git_buf_printf(&path, "%s/refs", backend->path)) < 0 ||
+		(error = git_iterator_for_filesystem(
+			&fsit, git_buf_cstr(&path), 0, NULL, NULL)) < 0) {
+		git_buf_free(&path);
+		return error;
+	}
 
-	git_vector_init(&iter->loose, 8, NULL);
-	git_buf_sets(&path, GIT_REFS_DIR);
+	error = git_buf_sets(&path, GIT_REFS_DIR);
 
-	while (!git_iterator_advance(&entry, fsit)) {
+	while (!error && !git_iterator_advance(&entry, fsit)) {
 		const char *ref_name;
-		khiter_t pos;
+		struct packref *ref;
+		char *ref_dup;
 
 		git_buf_truncate(&path, strlen(GIT_REFS_DIR));
 		git_buf_puts(&path, entry->path);
@@ -607,27 +488,32 @@ static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter)
 			(iter->glob && p_fnmatch(iter->glob, ref_name, 0) != 0))
 			continue;
 
-		pos = git_strmap_lookup_index(packfile, ref_name);
-		if (git_strmap_valid_index(packfile, pos)) {
-			struct packref *ref = git_strmap_value_at(packfile, pos);
+		git_sortedcache_rlock(backend->refcache);
+		ref = git_sortedcache_lookup(backend->refcache, ref_name);
+		if (ref)
 			ref->flags |= PACKREF_SHADOWED;
-		}
+		git_sortedcache_runlock(backend->refcache);
 
-		git_vector_insert(&iter->loose, git__strdup(ref_name));
+		ref_dup = git_pool_strdup(&iter->pool, ref_name);
+		if (!ref_dup)
+			error = -1;
+		else
+			error = git_vector_insert(&iter->loose, ref_dup);
 	}
 
 	git_iterator_free(fsit);
 	git_buf_free(&path);
 
-	return 0;
+	return error;
 }
 
 static int refdb_fs_backend__iterator_next(
 	git_reference **out, git_reference_iterator *_iter)
 {
+	int error = GIT_ITEROVER;
 	refdb_fs_iter *iter = (refdb_fs_iter *)_iter;
 	refdb_fs_backend *backend = (refdb_fs_backend *)iter->parent.db->backend;
-	git_strmap *packfile = backend->refcache.packfile;
+	struct packref *ref;
 
 	while (iter->loose_pos < iter->loose.length) {
 		const char *path = git_vector_get(&iter->loose, iter->loose_pos++);
@@ -638,99 +524,102 @@ static int refdb_fs_backend__iterator_next(
 		giterr_clear();
 	}
 
-	while (iter->packed_pos < kh_end(packfile)) {
-		struct packref *ref = NULL;
-
-		while (!kh_exist(packfile, iter->packed_pos)) {
-			iter->packed_pos++;
-			if (iter->packed_pos == kh_end(packfile))
-				return GIT_ITEROVER;
-		}
+	git_sortedcache_rlock(backend->refcache);
 
-		ref = kh_val(packfile, iter->packed_pos);
-		iter->packed_pos++;
+	while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) {
+		ref = git_sortedcache_entry(backend->refcache, iter->packed_pos++);
+		if (!ref) /* stop now if another thread deleted refs and we past end */
+			break;
 
 		if (ref->flags & PACKREF_SHADOWED)
 			continue;
-
 		if (iter->glob && p_fnmatch(iter->glob, ref->name, 0) != 0)
 			continue;
 
 		*out = git_reference__alloc(ref->name, &ref->oid, &ref->peel);
-		if (*out == NULL)
-			return -1;
-
-		return 0;
+		error = (*out != NULL) ? 0 : -1;
+		break;
 	}
 
-	return GIT_ITEROVER;
+	git_sortedcache_runlock(backend->refcache);
+	return error;
 }
 
 static int refdb_fs_backend__iterator_next_name(
 	const char **out, git_reference_iterator *_iter)
 {
+	int error = GIT_ITEROVER;
 	refdb_fs_iter *iter = (refdb_fs_iter *)_iter;
 	refdb_fs_backend *backend = (refdb_fs_backend *)iter->parent.db->backend;
-	git_strmap *packfile = backend->refcache.packfile;
+	struct packref *ref;
 
 	while (iter->loose_pos < iter->loose.length) {
 		const char *path = git_vector_get(&iter->loose, iter->loose_pos++);
 
-		if (git_strmap_exists(packfile, path))
-			continue;
+		if (loose_lookup(NULL, backend, path) == 0) {
+			*out = path;
+			return 0;
+		}
 
-		*out = path;
-		return 0;
+		giterr_clear();
 	}
 
-	while (iter->packed_pos < kh_end(packfile)) {
-		while (!kh_exist(packfile, iter->packed_pos)) {
-			iter->packed_pos++;
-			if (iter->packed_pos == kh_end(packfile))
-				return GIT_ITEROVER;
-		}
+	git_sortedcache_rlock(backend->refcache);
 
-		*out = kh_key(packfile, iter->packed_pos);
-		iter->packed_pos++;
+	while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) {
+		ref = git_sortedcache_entry(backend->refcache, iter->packed_pos++);
+		if (!ref) /* stop now if another thread deleted refs and we past end */
+			break;
 
-		if (iter->glob && p_fnmatch(iter->glob, *out, 0) != 0)
+		if (ref->flags & PACKREF_SHADOWED)
+			continue;
+		if (iter->glob && p_fnmatch(iter->glob, ref->name, 0) != 0)
 			continue;
 
-		return 0;
+		*out = ref->name;
+		error = 0;
+		break;
 	}
 
-	return GIT_ITEROVER;
+	git_sortedcache_runlock(backend->refcache);
+	return error;
 }
 
 static int refdb_fs_backend__iterator(
 	git_reference_iterator **out, git_refdb_backend *_backend, const char *glob)
 {
 	refdb_fs_iter *iter;
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	if (packed_load(backend) < 0)
+	if (packed_reload(backend) < 0)
 		return -1;
 
 	iter = git__calloc(1, sizeof(refdb_fs_iter));
 	GITERR_CHECK_ALLOC(iter);
 
-	if (glob != NULL)
-		iter->glob = git__strdup(glob);
+	if (git_pool_init(&iter->pool, 1, 0) < 0 ||
+		git_vector_init(&iter->loose, 8, NULL) < 0)
+		goto fail;
+
+	if (glob != NULL &&
+		(iter->glob = git_pool_strdup(&iter->pool, glob)) == NULL)
+		goto fail;
 
 	iter->parent.next = refdb_fs_backend__iterator_next;
 	iter->parent.next_name = refdb_fs_backend__iterator_next_name;
 	iter->parent.free = refdb_fs_backend__iterator_free;
 
-	if (iter_load_loose_paths(backend, iter) < 0) {
-		refdb_fs_backend__iterator_free((git_reference_iterator *)iter);
-		return -1;
-	}
+	if (iter_load_loose_paths(backend, iter) < 0)
+		goto fail;
 
 	*out = (git_reference_iterator *)iter;
 	return 0;
+
+fail:
+	refdb_fs_backend__iterator_free((git_reference_iterator *)iter);
+	return -1;
 }
 
 static bool ref_is_available(
@@ -756,33 +645,40 @@ static int reference_path_available(
 	const char* old_ref,
 	int force)
 {
-	struct packref *this_ref;
+	size_t i;
 
-	if (packed_load(backend) < 0)
+	if (packed_reload(backend) < 0)
 		return -1;
 
 	if (!force) {
 		int exists;
 
-		if (refdb_fs_backend__exists(&exists, (git_refdb_backend *)backend, new_ref) < 0)
+		if (refdb_fs_backend__exists(
+				&exists, (git_refdb_backend *)backend, new_ref) < 0)
 			return -1;
 
 		if (exists) {
 			giterr_set(GITERR_REFERENCE,
 				"Failed to write reference '%s': a reference with "
-				" that name already exists.", new_ref);
+				"that name already exists.", new_ref);
 			return GIT_EEXISTS;
 		}
 	}
 
-	git_strmap_foreach_value(backend->refcache.packfile, this_ref, {
-		if (!ref_is_available(old_ref, new_ref, this_ref->name)) {
+	git_sortedcache_rlock(backend->refcache);
+
+	for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) {
+		struct packref *ref = git_sortedcache_entry(backend->refcache, i);
+
+		if (ref && !ref_is_available(old_ref, new_ref, ref->name)) {
+			git_sortedcache_runlock(backend->refcache);
 			giterr_set(GITERR_REFERENCE,
-				"The path to reference '%s' collides with an existing one", new_ref);
+				"Path to reference '%s' collides with existing one", new_ref);
 			return -1;
 		}
-	});
-	
+	}
+
+	git_sortedcache_runlock(backend->refcache);
 	return 0;
 }
 
@@ -809,12 +705,9 @@ static int loose_write(refdb_fs_backend *backend, const git_reference *ref)
 
 	if (ref->type == GIT_REF_OID) {
 		char oid[GIT_OID_HEXSZ + 1];
-
-		git_oid_fmt(oid, &ref->target.oid);
-		oid[GIT_OID_HEXSZ] = '\0';
+		git_oid_nfmt(oid, sizeof(oid), &ref->target.oid);
 
 		git_filebuf_printf(&file, "%s\n", oid);
-
 	} else if (ref->type == GIT_REF_SYMBOLIC) {
 		git_filebuf_printf(&file, GIT_SYMREF "%s\n", ref->target.symbolic);
 	} else {
@@ -824,14 +717,6 @@ static int loose_write(refdb_fs_backend *backend, const git_reference *ref)
 	return git_filebuf_commit(&file, GIT_REFS_FILE_MODE);
 }
 
-static int packed_sort(const void *a, const void *b)
-{
-	const struct packref *ref_a = (const struct packref *)a;
-	const struct packref *ref_b = (const struct packref *)b;
-
-	return strcmp(ref_a->name, ref_b->name);
-}
-
 /*
  * Find out what object this reference resolves to.
  *
@@ -884,9 +769,7 @@ static int packed_find_peel(refdb_fs_backend *backend, struct packref *ref)
 static int packed_write_ref(struct packref *ref, git_filebuf *file)
 {
 	char oid[GIT_OID_HEXSZ + 1];
-
-	git_oid_fmt(oid, &ref->oid);
-	oid[GIT_OID_HEXSZ] = 0;
+	git_oid_nfmt(oid, sizeof(oid), &ref->oid);
 
 	/*
 	 * For references that peel to an object in the repo, we must
@@ -900,8 +783,7 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file)
 	 */
 	if (ref->flags & PACKREF_HAS_PEEL) {
 		char peel[GIT_OID_HEXSZ + 1];
-		git_oid_fmt(peel, &ref->peel);
-		peel[GIT_OID_HEXSZ] = 0;
+		git_oid_nfmt(peel, sizeof(peel), &ref->peel);
 
 		if (git_filebuf_printf(file, "%s %s\n^%s\n", oid, ref->name, peel) < 0)
 			return -1;
@@ -924,31 +806,30 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file)
  * is well-written, because we are destructing references
  * here otherwise.
  */
-static int packed_remove_loose(
-	refdb_fs_backend *backend,
-	git_vector *packing_list)
+static int packed_remove_loose(refdb_fs_backend *backend)
 {
 	size_t i;
 	git_buf full_path = GIT_BUF_INIT;
 	int failed = 0;
 
-	for (i = 0; i < packing_list->length; ++i) {
-		struct packref *ref = git_vector_get(packing_list, i);
+	/* backend->refcache is already locked when this is called */
 
-		if ((ref->flags & PACKREF_WAS_LOOSE) == 0)
+	for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) {
+		struct packref *ref = git_sortedcache_entry(backend->refcache, i);
+
+		if (!ref || !(ref->flags & PACKREF_WAS_LOOSE))
 			continue;
 
 		if (git_buf_joinpath(&full_path, backend->path, ref->name) < 0)
 			return -1; /* critical; do not try to recover on oom */
 
-		if (git_path_exists(full_path.ptr) == true && p_unlink(full_path.ptr) < 0) {
+		if (git_path_exists(full_path.ptr) && p_unlink(full_path.ptr) < 0) {
 			if (failed)
 				continue;
 
 			giterr_set(GITERR_REFERENCE,
 				"Failed to remove loose reference '%s' after packing: %s",
 				full_path.ptr, strerror(errno));
-
 			failed = 1;
 		}
 
@@ -969,85 +850,53 @@ static int packed_remove_loose(
  */
 static int packed_write(refdb_fs_backend *backend)
 {
+	git_sortedcache *refcache = backend->refcache;
 	git_filebuf pack_file = GIT_FILEBUF_INIT;
 	size_t i;
-	git_buf pack_file_path = GIT_BUF_INIT;
-	git_vector packing_list;
-	unsigned int total_refs;
-
-	assert(backend && backend->refcache.packfile);
-
-	total_refs =
-		(unsigned int)git_strmap_num_entries(backend->refcache.packfile);
 
-	if (git_vector_init(&packing_list, total_refs, packed_sort) < 0)
+	/* lock the cache to updates while we do this */
+	if (git_sortedcache_wlock(refcache) < 0)
 		return -1;
 
-	/* Load all the packfile into a vector */
-	{
-		struct packref *reference;
-
-		/* cannot fail: vector already has the right size */
-		git_strmap_foreach_value(backend->refcache.packfile, reference, {
-			git_vector_insert(&packing_list, reference);
-		});
-	}
-
-	/* sort the vector so the entries appear sorted on the packfile */
-	git_vector_sort(&packing_list);
-
-	/* Now we can open the file! */
-	if (git_buf_joinpath(&pack_file_path,
-		backend->path, GIT_PACKEDREFS_FILE) < 0)
-		goto cleanup_memory;
-
-	if (git_filebuf_open(&pack_file, pack_file_path.ptr, 0) < 0)
-		goto cleanup_packfile;
+	/* Open the file! */
+	if (git_filebuf_open(&pack_file, git_sortedcache_path(refcache), 0) < 0)
+		goto fail;
 
 	/* Packfiles have a header... apparently
 	 * This is in fact not required, but we might as well print it
 	 * just for kicks */
 	if (git_filebuf_printf(&pack_file, "%s\n", GIT_PACKEDREFS_HEADER) < 0)
-		goto cleanup_packfile;
+		goto fail;
 
-	for (i = 0; i < packing_list.length; ++i) {
-		struct packref *ref = (struct packref *)git_vector_get(&packing_list, i);
+	for (i = 0; i < git_sortedcache_entrycount(refcache); ++i) {
+		struct packref *ref = git_sortedcache_entry(refcache, i);
 
 		if (packed_find_peel(backend, ref) < 0)
-			goto cleanup_packfile;
+			goto fail;
 
 		if (packed_write_ref(ref, &pack_file) < 0)
-			goto cleanup_packfile;
+			goto fail;
 	}
 
 	/* if we've written all the references properly, we can commit
 	 * the packfile to make the changes effective */
 	if (git_filebuf_commit(&pack_file, GIT_PACKEDREFS_FILE_MODE) < 0)
-		goto cleanup_memory;
+		goto fail;
 
 	/* when and only when the packfile has been properly written,
 	 * we can go ahead and remove the loose refs */
-	 if (packed_remove_loose(backend, &packing_list) < 0)
-		 goto cleanup_memory;
-
-	 {
-		struct stat st;
-		if (p_stat(pack_file_path.ptr, &st) == 0)
-			backend->refcache.packfile_time = st.st_mtime;
-	 }
+	if (packed_remove_loose(backend) < 0)
+		goto fail;
 
-	git_vector_free(&packing_list);
-	git_buf_free(&pack_file_path);
+	git_sortedcache_updated(refcache);
+	git_sortedcache_wunlock(refcache);
 
 	/* we're good now */
 	return 0;
 
-cleanup_packfile:
+fail:
 	git_filebuf_cleanup(&pack_file);
-
-cleanup_memory:
-	git_vector_free(&packing_list);
-	git_buf_free(&pack_file_path);
+	git_sortedcache_wunlock(refcache);
 
 	return -1;
 }
@@ -1057,11 +906,10 @@ static int refdb_fs_backend__write(
 	const git_reference *ref,
 	int force)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	int error;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
 	error = reference_path_available(backend, ref->name, NULL, force);
 	if (error < 0)
@@ -1074,17 +922,13 @@ static int refdb_fs_backend__delete(
 	git_refdb_backend *_backend,
 	const char *ref_name)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	git_buf loose_path = GIT_BUF_INIT;
-	struct packref *pack_ref;
-	khiter_t pack_ref_pos;
+	size_t pack_pos;
 	int error = 0;
 	bool loose_deleted = 0;
 
-	assert(_backend);
-	assert(ref_name);
-
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend && ref_name);
 
 	/* If a loose reference exists, remove it from the filesystem */
 	if (git_buf_joinpath(&loose_path, backend->path, ref_name) < 0)
@@ -1100,19 +944,23 @@ static int refdb_fs_backend__delete(
 	if (error != 0)
 		return error;
 
+	if (packed_reload(backend) < 0)
+		return -1;
+
 	/* If a packed reference exists, remove it from the packfile and repack */
-	error = packed_map_entry(&pack_ref, &pack_ref_pos, backend, ref_name);
+	if (git_sortedcache_wlock(backend->refcache) < 0)
+		return -1;
 
-	if (error == GIT_ENOTFOUND)
-		return loose_deleted ? 0 : GIT_ENOTFOUND;
+	if (!(error = git_sortedcache_lookup_index(
+			&pack_pos, backend->refcache, ref_name)))
+		error = git_sortedcache_remove(backend->refcache, pack_pos);
 
-	if (error == 0) {
-		git_strmap_delete_at(backend->refcache.packfile, pack_ref_pos);
-		git__free(pack_ref);
-		error = packed_write(backend);
-	}
+	git_sortedcache_wunlock(backend->refcache);
 
-	return error;
+	if (error == GIT_ENOTFOUND)
+		return loose_deleted ? 0 : ref_error_notfound(ref_name);
+
+	return packed_write(backend);
 }
 
 static int refdb_fs_backend__rename(
@@ -1122,53 +970,44 @@ static int refdb_fs_backend__rename(
 	const char *new_name,
 	int force)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	git_reference *old, *new;
 	int error;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	error = reference_path_available(backend, new_name, old_name, force);
-	if (error < 0)
-		return error;
-
-	error = refdb_fs_backend__lookup(&old, _backend, old_name);
-	if (error < 0)
+	if ((error = reference_path_available(
+			backend, new_name, old_name, force)) < 0 ||
+		(error = refdb_fs_backend__lookup(&old, _backend, old_name)) < 0)
 		return error;
 
-	error = refdb_fs_backend__delete(_backend, old_name);
-	if (error < 0) {
+	if ((error = refdb_fs_backend__delete(_backend, old_name)) < 0) {
 		git_reference_free(old);
 		return error;
 	}
 
-	new = realloc(old, sizeof(git_reference) + strlen(new_name) + 1);
-	memcpy(new->name, new_name, strlen(new_name) + 1);
-
-	error = loose_write(backend, new);
-	if (error < 0) {
-		git_reference_free(new);
-		return error;
+	new = git_reference__set_name(old, new_name);
+	if (!new) {
+		git_reference_free(old);
+		return -1;
 	}
 
-	if (out) {
-		*out = new;
-	} else {
+	if ((error = loose_write(backend, new)) < 0 || out == NULL) {
 		git_reference_free(new);
+		return error;
 	}
 
+	*out = new;
 	return 0;
 }
 
 static int refdb_fs_backend__compress(git_refdb_backend *_backend)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	if (packed_load(backend) < 0 || /* load the existing packfile */
+	if (packed_reload(backend) < 0 || /* load the existing packfile */
 		packed_loadloose(backend) < 0 || /* add all the loose refs */
 		packed_write(backend) < 0) /* write back to disk */
 		return -1;
@@ -1176,29 +1015,13 @@ static int refdb_fs_backend__compress(git_refdb_backend *_backend)
 	return 0;
 }
 
-static void refcache_free(git_refcache *refs)
-{
-	assert(refs);
-
-	if (refs->packfile) {
-		struct packref *reference;
-
-		git_strmap_foreach_value(refs->packfile, reference, {
-			git__free(reference);
-		});
-
-		git_strmap_free(refs->packfile);
-	}
-}
-
 static void refdb_fs_backend__free(git_refdb_backend *_backend)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	refcache_free(&backend->refcache);
+	git_sortedcache_free(backend->refcache);
 	git__free(backend->path);
 	git__free(backend);
 }
@@ -1222,7 +1045,7 @@ static int setup_namespace(git_buf *path, git_repository *repo)
 	if (parts == NULL)
 		return -1;
 
-	/**
+	/*
 	 * From `man gitnamespaces`:
 	 *  namespaces which include a / will expand to a hierarchy
 	 *  of namespaces; for example, GIT_NAMESPACE=foo/bar will store
@@ -1239,7 +1062,7 @@ static int setup_namespace(git_buf *path, git_repository *repo)
 	if (git_futils_mkdir_r(git_buf_cstr(path), repo->path_repository, 0777) < 0)
 		return -1;
 
-	/* Return the root of the namespaced path, i.e. without the trailing '/refs' */
+	/* Return root of the namespaced path, i.e. without the trailing '/refs' */
 	git_buf_rtruncate_at_char(path, '/');
 	return 0;
 }
@@ -1256,13 +1079,19 @@ int git_refdb_backend_fs(
 
 	backend->repo = repository;
 
-	if (setup_namespace(&path, repository) < 0) {
-		git__free(backend);
-		return -1;
-	}
+	if (setup_namespace(&path, repository) < 0)
+		goto fail;
 
 	backend->path = git_buf_detach(&path);
 
+	if (git_buf_joinpath(&path, backend->path, GIT_PACKEDREFS_FILE) < 0 ||
+		git_sortedcache_new(
+			&backend->refcache, offsetof(struct packref, name),
+			NULL, NULL, packref_cmp, git_buf_cstr(&path)) < 0)
+		goto fail;
+
+	git_buf_free(&path);
+
 	backend->parent.exists = &refdb_fs_backend__exists;
 	backend->parent.lookup = &refdb_fs_backend__lookup;
 	backend->parent.iterator = &refdb_fs_backend__iterator;
@@ -1274,4 +1103,10 @@ int git_refdb_backend_fs(
 
 	*backend_out = (git_refdb_backend *)backend;
 	return 0;
+
+fail:
+	git_buf_free(&path);
+	git__free(backend->path);
+	git__free(backend);
+	return -1;
 }
diff --git a/src/reflog.c b/src/reflog.c
index 4cc20d2c7..a6752f618 100644
--- a/src/reflog.c
+++ b/src/reflog.c
@@ -484,7 +484,7 @@ int git_reflog_drop(
 	entry = (git_reflog_entry *)git_reflog_entry_byindex(reflog, idx);
 
 	if (entry == NULL) {
-		giterr_set(GITERR_REFERENCE, "No reflog entry at index "PRIuZ, idx);
+		giterr_set(GITERR_REFERENCE, "No reflog entry at index %"PRIuZ, idx);
 		return GIT_ENOTFOUND;
 	}
 
diff --git a/src/refs.c b/src/refs.c
index c0e460cc3..c045ab9dc 100644
--- a/src/refs.c
+++ b/src/refs.c
@@ -88,6 +88,17 @@ git_reference *git_reference__alloc(
 	return ref;
 }
 
+git_reference *git_reference__set_name(
+	git_reference *ref, const char *name)
+{
+	size_t namelen = strlen(name);
+	git_reference *rewrite =
+		git__realloc(ref, sizeof(git_reference) + namelen + 1);
+	if (rewrite != NULL)
+		memcpy(rewrite->name, name, namelen + 1);
+	return rewrite;
+}
+
 void git_reference_free(git_reference *reference)
 {
 	if (reference == NULL)
@@ -952,6 +963,17 @@ int git_reference_is_remote(git_reference *ref)
 	return git_reference__is_remote(ref->name);
 }
 
+int git_reference__is_tag(const char *ref_name)
+{
+	return git__prefixcmp(ref_name, GIT_REFS_TAGS_DIR) == 0;
+}
+
+int git_reference_is_tag(git_reference *ref)
+{
+	assert(ref);
+	return git_reference__is_tag(ref->name);
+}
+
 static int peel_error(int error, git_reference *ref, const char* msg)
 {
 	giterr_set(
diff --git a/src/refs.h b/src/refs.h
index f487ee3fc..4b91c25e8 100644
--- a/src/refs.h
+++ b/src/refs.h
@@ -61,12 +61,15 @@ struct git_reference {
 	char name[0];
 };
 
+git_reference *git_reference__set_name(git_reference *ref, const char *name);
+
 int git_reference__normalize_name_lax(char *buffer_out, size_t out_size, const char *name);
 int git_reference__normalize_name(git_buf *buf, const char *name, unsigned int flags);
 int git_reference__update_terminal(git_repository *repo, const char *ref_name, const git_oid *oid);
 int git_reference__is_valid_name(const char *refname, unsigned int flags);
 int git_reference__is_branch(const char *ref_name);
 int git_reference__is_remote(const char *ref_name);
+int git_reference__is_tag(const char *ref_name);
 
 /**
  * Lookup a reference by name and try to resolve to an OID.
diff --git a/src/refspec.c b/src/refspec.c
index a907df84c..492c6ed3f 100644
--- a/src/refspec.c
+++ b/src/refspec.c
@@ -225,25 +225,31 @@ int git_refspec_rtransform(char *out, size_t outlen, const git_refspec *spec, co
 	return refspec_transform_internal(out, outlen, spec->dst, spec->src, name);
 }
 
-static int refspec_transform(git_buf *out, const char *from, const char *to, const char *name)
+static int refspec_transform(
+	git_buf *out, const char *from, const char *to, const char *name)
 {
-	if (git_buf_sets(out, to) < 0)
-		return -1;
+	size_t to_len   = to   ? strlen(to)   : 0;
+	size_t from_len = from ? strlen(from) : 0;
+	size_t name_len = name ? strlen(name) : 0;
 
-	/*
-	 * No '*' at the end means that it's mapped to one specific
-	 * branch, so no actual transformation is needed.
-	 */
-	if (git_buf_len(out) > 0 && out->ptr[git_buf_len(out) - 1] != '*')
-		return 0;
+	if (git_buf_set(out, to, to_len) < 0)
+		return -1;
 
-	git_buf_truncate(out, git_buf_len(out) - 1); /* remove trailing '*' */
-	git_buf_puts(out, name + strlen(from) - 1);
+	if (to_len > 0) {
+		/* No '*' at the end of 'to' means that refspec is mapped to one
+		 * specific branch, so no actual transformation is needed.
+		 */
+		if (out->ptr[to_len - 1] != '*')
+			return 0;
+		git_buf_shorten(out, 1); /* remove trailing '*' copied from 'to' */
+	}
 
-	if (git_buf_oom(out))
-		return -1;
+	if (from_len > 0) /* ignore trailing '*' from 'from' */
+		from_len--;
+	if (from_len > name_len)
+		from_len = name_len;
 
-	return 0;
+	return git_buf_put(out, name + from_len, name_len - from_len);
 }
 
 int git_refspec_transform_r(git_buf *out, const git_refspec *spec, const char *name)
diff --git a/src/remote.c b/src/remote.c
index 0e8354a11..bdc8e0e67 100644
--- a/src/remote.c
+++ b/src/remote.c
@@ -81,6 +81,31 @@ static int ensure_remote_name_is_valid(const char *name)
 	return error;
 }
 
+static int get_check_cert(git_repository *repo)
+{
+	git_config *cfg;
+	const char *val;
+	int check_cert;
+
+	assert(repo);
+
+	/* Go through the possible sources for SSL verification settings, from
+	 * most specific to least specific. */
+
+	/* GIT_SSL_NO_VERIFY environment variable */
+	if ((val = getenv("GIT_SSL_NO_VERIFY")) &&
+		!git_config_parse_bool(&check_cert, val))
+		return !check_cert;
+
+	/* http.sslVerify config setting */
+	if (!git_repository_config__weakptr(&cfg, repo) &&
+		!git_config_get_bool(&check_cert, cfg, "http.sslVerify"))
+		return check_cert;
+
+	/* By default, we *DO* want to verify the certificate. */
+	return 1;
+}
+
 static int create_internal(git_remote **out, git_repository *repo, const char *name, const char *url, const char *fetch)
 {
 	git_remote *remote;
@@ -94,7 +119,7 @@ static int create_internal(git_remote **out, git_repository *repo, const char *n
 	GITERR_CHECK_ALLOC(remote);
 
 	remote->repo = repo;
-	remote->check_cert = 1;
+	remote->check_cert = (unsigned)get_check_cert(repo);
 	remote->update_fetchhead = 1;
 
 	if (git_vector_init(&remote->refs, 32, NULL) < 0)
@@ -208,7 +233,8 @@ static int refspec_cb(const git_config_entry *entry, void *payload)
 }
 
 static int get_optional_config(
-	git_config *config, git_buf *buf, git_config_foreach_cb cb, void *payload)
+	bool *found, git_config *config, git_buf *buf,
+	git_config_foreach_cb cb, void *payload)
 {
 	int error = 0;
 	const char *key = git_buf_cstr(buf);
@@ -217,10 +243,13 @@ static int get_optional_config(
 		return -1;
 
 	if (cb != NULL)
-		error = git_config_get_multivar(config, key, NULL, cb, payload);
+		error = git_config_get_multivar_foreach(config, key, NULL, cb, payload);
 	else
 		error = git_config_get_string(payload, config, key);
 
+	if (found)
+		*found = !error;
+
 	if (error == GIT_ENOTFOUND) {
 		giterr_clear();
 		error = 0;
@@ -240,6 +269,7 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	int error = 0;
 	git_config *config;
 	struct refspec_cb_data data;
+	bool optional_setting_found = false, found;
 
 	assert(out && repo && name);
 
@@ -253,7 +283,7 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	GITERR_CHECK_ALLOC(remote);
 
 	memset(remote, 0x0, sizeof(git_remote));
-	remote->check_cert = 1;
+	remote->check_cert = (unsigned)get_check_cert(repo);
 	remote->update_fetchhead = 1;
 	remote->name = git__strdup(name);
 	GITERR_CHECK_ALLOC(remote->name);
@@ -269,27 +299,33 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 		goto cleanup;
 	}
 
-	if ((error = git_config_get_string(&val, config, git_buf_cstr(&buf))) < 0)
+	if ((error = get_optional_config(&found, config, &buf, NULL, (void *)&val)) < 0)
 		goto cleanup;
 
-	if (strlen(val) == 0) {
-		giterr_set(GITERR_INVALID, "Malformed remote '%s' - missing URL", name);
-		error = -1;
-		goto cleanup;
-	}
+	optional_setting_found |= found;
 
 	remote->repo = repo;
-	remote->url = git__strdup(val);
-	GITERR_CHECK_ALLOC(remote->url);
+
+	if (found && strlen(val) > 0) {
+		remote->url = git__strdup(val);
+		GITERR_CHECK_ALLOC(remote->url);
+	}
 
 	val = NULL;
 	git_buf_clear(&buf);
 	git_buf_printf(&buf, "remote.%s.pushurl", name);
 
-	if ((error = get_optional_config(config, &buf, NULL, (void *)&val)) < 0)
+	if ((error = get_optional_config(&found, config, &buf, NULL, (void *)&val)) < 0)
 		goto cleanup;
 
-	if (val) {
+	optional_setting_found |= found;
+
+	if (!optional_setting_found) {
+		error = GIT_ENOTFOUND;
+		goto cleanup;
+	}
+
+	if (found && strlen(val) > 0) {
 		remote->pushurl = git__strdup(val);
 		GITERR_CHECK_ALLOC(remote->pushurl);
 	}
@@ -299,14 +335,14 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	git_buf_clear(&buf);
 	git_buf_printf(&buf, "remote.%s.fetch", name);
 
-	if ((error = get_optional_config(config, &buf, refspec_cb, &data)) < 0)
+	if ((error = get_optional_config(NULL, config, &buf, refspec_cb, &data)) < 0)
 		goto cleanup;
 
 	data.fetch = false;
 	git_buf_clear(&buf);
 	git_buf_printf(&buf, "remote.%s.push", name);
 
-	if ((error = get_optional_config(config, &buf, refspec_cb, &data)) < 0)
+	if ((error = get_optional_config(NULL, config, &buf, refspec_cb, &data)) < 0)
 		goto cleanup;
 
 	if (download_tags_value(remote, config) < 0)
@@ -467,6 +503,12 @@ const char *git_remote_name(const git_remote *remote)
 	return remote->name;
 }
 
+git_repository *git_remote_owner(const git_remote *remote)
+{
+	assert(remote);
+	return remote->repo;
+}
+
 const char *git_remote_url(const git_remote *remote)
 {
 	assert(remote);
@@ -509,6 +551,8 @@ const char* git_remote__urlfordirection(git_remote *remote, int direction)
 {
 	assert(remote);
 
+	assert(direction == GIT_DIRECTION_FETCH || direction == GIT_DIRECTION_PUSH);
+
 	if (direction == GIT_DIRECTION_FETCH) {
 		return remote->url;
 	}
@@ -531,8 +575,11 @@ int git_remote_connect(git_remote *remote, git_direction direction)
 	t = remote->transport;
 
 	url = git_remote__urlfordirection(remote, direction);
-	if (url == NULL )
+	if (url == NULL ) {
+		giterr_set(GITERR_INVALID,
+			"Malformed remote '%s' - missing URL", remote->name);
 		return -1;
+	}
 
 	/* A transport could have been supplied in advance with
 	 * git_remote_set_transport */
@@ -1062,10 +1109,10 @@ int git_remote_list(git_strarray *remotes_list, git_repository *repo)
 	if (git_repository_config__weakptr(&cfg, repo) < 0)
 		return -1;
 
-	if (git_vector_init(&list, 4, NULL) < 0)
+	if (git_vector_init(&list, 4, git__strcmp_cb) < 0)
 		return -1;
 
-	if (regcomp(&preg, "^remote\\.(.*)\\.url$", REG_EXTENDED) < 0) {
+	if (regcomp(&preg, "^remote\\.(.*)\\.(push)?url$", REG_EXTENDED) < 0) {
 		giterr_set(GITERR_OS, "Remote catch regex failed to compile");
 		return -1;
 	}
@@ -1090,6 +1137,8 @@ int git_remote_list(git_strarray *remotes_list, git_repository *repo)
 		return error;
 	}
 
+	git_vector_uniq(&list, git__free);
+
 	remotes_list->strings = (char **)list.contents;
 	remotes_list->count = list.length;
 
diff --git a/src/repository.c b/src/repository.c
index ed9469c59..eae22ce51 100644
--- a/src/repository.c
+++ b/src/repository.c
@@ -266,7 +266,7 @@ static int find_ceiling_dir_offset(
 			buf[--len] = '\0';
 
 		if (!strncmp(path, buf2, len) &&
-			path[len] == '/' &&
+			(path[len] == '/' || !path[len]) &&
 			len > max_len)
 		{
 			max_len = len;
@@ -322,17 +322,18 @@ static int find_repo(
 	git_buf path = GIT_BUF_INIT;
 	struct stat st;
 	dev_t initial_device = 0;
-	bool try_with_dot_git = false;
+	bool try_with_dot_git = ((flags & GIT_REPOSITORY_OPEN_BARE) != 0);
 	int ceiling_offset;
 
 	git_buf_free(repo_path);
 
-	if ((error = git_path_prettify_dir(&path, start_path, NULL)) < 0)
+	if ((error = git_path_prettify(&path, start_path, NULL)) < 0)
 		return error;
 
 	ceiling_offset = find_ceiling_dir_offset(path.ptr, ceiling_dirs);
 
-	if ((error = git_buf_joinpath(&path, path.ptr, DOT_GIT)) < 0)
+	if (!try_with_dot_git &&
+		(error = git_buf_joinpath(&path, path.ptr, DOT_GIT)) < 0)
 		return error;
 
 	while (!error && !git_buf_len(repo_path)) {
@@ -384,7 +385,7 @@ static int find_repo(
 		try_with_dot_git = !try_with_dot_git;
 	}
 
-	if (!error && parent_path != NULL) {
+	if (!error && parent_path && !(flags & GIT_REPOSITORY_OPEN_BARE)) {
 		if (!git_buf_len(repo_path))
 			git_buf_clear(parent_path);
 		else {
@@ -460,7 +461,9 @@ int git_repository_open_ext(
 	repo->path_repository = git_buf_detach(&path);
 	GITERR_CHECK_ALLOC(repo->path_repository);
 
-	if ((error = load_config_data(repo)) < 0 ||
+	if ((flags & GIT_REPOSITORY_OPEN_BARE) != 0)
+		repo->is_bare = 1;
+	else if ((error = load_config_data(repo)) < 0 ||
 		(error = load_workdir(repo, &parent)) < 0)
 	{
 		git_repository_free(repo);
@@ -1492,24 +1495,20 @@ static int repo_contains_no_reference(git_repository *repo)
 int git_repository_is_empty(git_repository *repo)
 {
 	git_reference *head = NULL;
-	int error;
+	int is_empty = 0;
 
 	if (git_reference_lookup(&head, repo, GIT_HEAD_FILE) < 0)
 		return -1;
 
-	if (!(error = git_reference_type(head) == GIT_REF_SYMBOLIC))
-		goto cleanup;
-
-	if (!(error = strcmp(
-		git_reference_symbolic_target(head),
-		GIT_REFS_HEADS_DIR "master") == 0))
-			goto cleanup;
+	if (git_reference_type(head) == GIT_REF_SYMBOLIC)
+		is_empty =
+			(strcmp(git_reference_symbolic_target(head),
+					GIT_REFS_HEADS_DIR "master") == 0) &&
+			repo_contains_no_reference(repo);
 
-	error = repo_contains_no_reference(repo);
-
-cleanup:
 	git_reference_free(head);
-	return error < 0 ? -1 : error;
+
+	return is_empty;
 }
 
 const char *git_repository_path(git_repository *repo)
diff --git a/src/revparse.c b/src/revparse.c
index bcfb0843f..e470a954d 100644
--- a/src/revparse.c
+++ b/src/revparse.c
@@ -93,11 +93,7 @@ static int revparse_lookup_object(
 	int error;
 	git_reference *ref;
 
-	error = maybe_sha(object_out, repo, spec);
-	if (!error)
-		return 0;
-
-	if (error < 0 && error != GIT_ENOTFOUND)
+	if ((error = maybe_sha(object_out, repo, spec)) != GIT_ENOTFOUND)
 		return error;
 
 	error = git_reference_dwim(&ref, repo, spec);
@@ -112,24 +108,17 @@ static int revparse_lookup_object(
 		return error;
 	}
 
-	if (error < 0 && error != GIT_ENOTFOUND)
-		return error;
-
-	error = maybe_abbrev(object_out, repo, spec);
-	if (!error)
-		return 0;
-
-	if (error < 0 && error != GIT_ENOTFOUND)
+	if (error != GIT_ENOTFOUND)
 		return error;
 
-	error = maybe_describe(object_out, repo, spec);
-	if (!error)
-		return 0;
+	if ((strlen(spec) < GIT_OID_HEXSZ) &&
+		((error = maybe_abbrev(object_out, repo, spec)) != GIT_ENOTFOUND))
+			return error;
 
-	if (error < 0 && error != GIT_ENOTFOUND)
+	if ((error = maybe_describe(object_out, repo, spec)) != GIT_ENOTFOUND)
 		return error;
 
-	giterr_set(GITERR_REFERENCE, "Refspec '%s' not found.", spec);
+	giterr_set(GITERR_REFERENCE, "Revspec '%s' not found.", spec);
 	return GIT_ENOTFOUND;
 }
 
@@ -228,7 +217,7 @@ static int retrieve_oid_from_reflog(git_oid *oid, git_reference *ref, size_t ide
 		if (numentries < identifier + 1) {
 			giterr_set(
 				GITERR_REFERENCE,
-				"Reflog for '%s' has only "PRIuZ" entries, asked for "PRIuZ,
+				"Reflog for '%s' has only %"PRIuZ" entries, asked for %"PRIuZ,
 				git_reference_name(ref), numentries, identifier);
 
 			error = GIT_ENOTFOUND;
@@ -685,6 +674,8 @@ int revparse__ext(
 	git_reference *reference = NULL;
 	git_object *base_rev = NULL;
 
+	bool should_return_reference = true;
+
 	assert(object_out && reference_out && repo && spec);
 
 	*object_out = NULL;
@@ -693,6 +684,8 @@ int revparse__ext(
 	while (spec[pos]) {
 		switch (spec[pos]) {
 		case '^':
+			should_return_reference = false;
+
 			if ((error = ensure_base_rev_loaded(&base_rev, &reference, spec, identifier_len, repo, false)) < 0)
 				goto cleanup;
 
@@ -725,6 +718,8 @@ int revparse__ext(
 		{
 			git_object *temp_object = NULL;
 
+			should_return_reference = false;
+
 			if ((error = extract_how_many(&n, spec, &pos)) < 0)
 				goto cleanup;
 
@@ -743,6 +738,8 @@ int revparse__ext(
 		{
 			git_object *temp_object = NULL;
 
+			should_return_reference = false;
+
 			if ((error = extract_path(&buf, spec, &pos)) < 0)
 				goto cleanup;
 
@@ -807,6 +804,11 @@ int revparse__ext(
 	if ((error = ensure_base_rev_loaded(&base_rev, &reference, spec, identifier_len, repo, false)) < 0)
 		goto cleanup;
 
+	if (!should_return_reference) {
+		git_reference_free(reference);
+		reference = NULL;
+	}
+
 	*object_out = base_rev;
 	*reference_out = reference;
 	*identifier_len_out = identifier_len;
@@ -899,13 +901,9 @@ int git_revparse(
 			rstr++;
 		}
 
-		if ((error = git_revparse_single(&revspec->from, repo, lstr)) < 0) {
-			return error;
-		}
-
-		if ((error = git_revparse_single(&revspec->to, repo, rstr)) < 0) {
-			return error;
-		}
+		error = git_revparse_single(&revspec->from, repo, lstr);
+		if (!error)
+			error = git_revparse_single(&revspec->to, repo, rstr);
 
 		git__free((void*)lstr);
 	} else {
diff --git a/src/revwalk.c b/src/revwalk.c
index 528d02b20..9e1e39ca4 100644
--- a/src/revwalk.c
+++ b/src/revwalk.c
@@ -41,28 +41,50 @@ git_commit_list_node *git_revwalk__commit_lookup(
 	return commit;
 }
 
-static void mark_uninteresting(git_commit_list_node *commit)
+static int mark_uninteresting(git_commit_list_node *commit)
 {
 	unsigned short i;
+	git_array_t(git_commit_list_node *) pending = GIT_ARRAY_INIT;
+	git_commit_list_node **tmp;
+
 	assert(commit);
 
-	commit->uninteresting = 1;
+	git_array_alloc(pending);
+	GITERR_CHECK_ARRAY(pending);
 
-	/* This means we've reached a merge base, so there's no need to walk any more */
-	if ((commit->flags & (RESULT | STALE)) == RESULT)
-		return;
+	do {
+		commit->uninteresting = 1;
+
+		/* This means we've reached a merge base, so there's no need to walk any more */
+		if ((commit->flags & (RESULT | STALE)) == RESULT) {
+			tmp = git_array_pop(pending);
+			commit = tmp ? *tmp : NULL;
+			continue;
+		}
+
+		for (i = 0; i < commit->out_degree; ++i)
+			if (!commit->parents[i]->uninteresting) {
+				git_commit_list_node **node = git_array_alloc(pending);
+				GITERR_CHECK_ALLOC(node);
+				*node = commit->parents[i];
+			}
+
+		tmp = git_array_pop(pending);
+		commit = tmp ? *tmp : NULL;
+
+	} while (git_array_size(pending) > 0);
+
+	git_array_clear(pending);
 
-	for (i = 0; i < commit->out_degree; ++i)
-		if (!commit->parents[i]->uninteresting)
-			mark_uninteresting(commit->parents[i]);
+	return 0;
 }
 
 static int process_commit(git_revwalk *walk, git_commit_list_node *commit, int hide)
 {
 	int error;
 
-	if (hide)
-		mark_uninteresting(commit);
+	if (hide && mark_uninteresting(commit) < 0)
+		return -1;
 
 	if (commit->seen)
 		return 0;
@@ -77,10 +99,14 @@ static int process_commit(git_revwalk *walk, git_commit_list_node *commit, int h
 
 static int process_commit_parents(git_revwalk *walk, git_commit_list_node *commit)
 {
-	unsigned short i;
+	unsigned short i, max;
 	int error = 0;
 
-	for (i = 0; i < commit->out_degree && !error; ++i)
+	max = commit->out_degree;
+	if (walk->first_parent && commit->out_degree)
+		max = 1;
+
+	for (i = 0; i < max && !error; ++i)
 		error = process_commit(walk, commit->parents[i], commit->uninteresting);
 
 	return error;
@@ -311,7 +337,7 @@ static int revwalk_next_unsorted(git_commit_list_node **object_out, git_revwalk
 static int revwalk_next_toposort(git_commit_list_node **object_out, git_revwalk *walk)
 {
 	git_commit_list_node *next;
-	unsigned short i;
+	unsigned short i, max;
 
 	for (;;) {
 		next = git_commit_list_pop(&walk->iterator_topo);
@@ -325,7 +351,12 @@ static int revwalk_next_toposort(git_commit_list_node **object_out, git_revwalk
 			continue;
 		}
 
-		for (i = 0; i < next->out_degree; ++i) {
+
+		max = next->out_degree;
+		if (walk->first_parent && next->out_degree)
+			max = 1;
+
+		for (i = 0; i < max; ++i) {
 			git_commit_list_node *parent = next->parents[i];
 
 			if (--parent->in_degree == 0 && parent->topo_delay) {
@@ -483,6 +514,11 @@ void git_revwalk_sorting(git_revwalk *walk, unsigned int sort_mode)
 	}
 }
 
+void git_revwalk_simplify_first_parent(git_revwalk *walk)
+{
+	walk->first_parent = 1;
+}
+
 int git_revwalk_next(git_oid *oid, git_revwalk *walk)
 {
 	int error;
diff --git a/src/revwalk.h b/src/revwalk.h
index 22696dfcd..8c821d098 100644
--- a/src/revwalk.h
+++ b/src/revwalk.h
@@ -31,7 +31,8 @@ struct git_revwalk {
 	int (*get_next)(git_commit_list_node **, git_revwalk *);
 	int (*enqueue)(git_revwalk *, git_commit_list_node *);
 
-	unsigned walking:1;
+	unsigned walking:1,
+		first_parent: 1;
 	unsigned int sorting;
 
 	/* merge base calculation */
diff --git a/src/sha1_lookup.c b/src/sha1_lookup.c
index b7e66cc69..c6b561340 100644
--- a/src/sha1_lookup.c
+++ b/src/sha1_lookup.c
@@ -9,6 +9,7 @@
 
 #include "sha1_lookup.h"
 #include "common.h"
+#include "oid.h"
 
 /*
  * Conventional binary search loop looks like this:
@@ -108,7 +109,54 @@ int sha1_entry_pos(const void *table,
 			 * byte 0 thru (ofs-1) are the same between
 			 * lo and hi; ofs is the first byte that is
 			 * different.
+			 *
+			 * If ofs==20, then no bytes are different,
+			 * meaning we have entries with duplicate
+			 * keys. We know that we are in a solid run
+			 * of this entry (because the entries are
+			 * sorted, and our lo and hi are the same,
+			 * there can be nothing but this single key
+			 * in between). So we can stop the search.
+			 * Either one of these entries is it (and
+			 * we do not care which), or we do not have
+			 * it.
+			 *
+			 * Furthermore, we know that one of our
+			 * endpoints must be the edge of the run of
+			 * duplicates. For example, given this
+			 * sequence:
+			 *
+			 *     idx 0 1 2 3 4 5
+			 *     key A C C C C D
+			 *
+			 * If we are searching for "B", we might
+			 * hit the duplicate run at lo=1, hi=3
+			 * (e.g., by first mi=3, then mi=0). But we
+			 * can never have lo > 1, because B < C.
+			 * That is, if our key is less than the
+			 * run, we know that "lo" is the edge, but
+			 * we can say nothing of "hi". Similarly,
+			 * if our key is greater than the run, we
+			 * know that "hi" is the edge, but we can
+			 * say nothing of "lo".
+			 *
+			 * Therefore if we do not find it, we also
+			 * know where it would go if it did exist:
+			 * just on the far side of the edge that we
+			 * know about.
 			 */
+			if (ofs == 20) {
+				mi = lo;
+				mi_key = base + elem_size * mi + key_offset;
+				cmp = memcmp(mi_key, key, 20);
+				if (!cmp)
+					return mi;
+				if (cmp < 0)
+					return -1 - hi;
+				else
+					return -1 - lo;
+			}
+
 			hiv = hi_key[ofs_0];
 			if (ofs_0 < 19)
 				hiv = (hiv << 8) | hi_key[ofs_0+1];
@@ -176,3 +224,26 @@ int sha1_entry_pos(const void *table,
 	} while (lo < hi);
 	return -((int)lo)-1;
 }
+
+int sha1_position(const void *table,
+			size_t stride,
+			unsigned lo, unsigned hi,
+			const unsigned char *key)
+{
+	const unsigned char *base = table;
+
+	do {
+		unsigned mi = (lo + hi) / 2;
+		int cmp = git_oid__hashcmp(base + mi * stride, key);
+
+		if (!cmp)
+			return mi;
+
+		if (cmp > 0)
+			hi = mi;
+		else
+			lo = mi+1;
+	} while (lo < hi);
+
+	return -((int)lo)-1;
+}
diff --git a/src/sha1_lookup.h b/src/sha1_lookup.h
index 9a3537273..3799620c7 100644
--- a/src/sha1_lookup.h
+++ b/src/sha1_lookup.h
@@ -15,4 +15,9 @@ int sha1_entry_pos(const void *table,
 			unsigned lo, unsigned hi, unsigned nr,
 			const unsigned char *key);
 
+int sha1_position(const void *table,
+			size_t stride,
+			unsigned lo, unsigned hi,
+			const unsigned char *key);
+
 #endif
diff --git a/src/signature.c b/src/signature.c
index 0a34ccfaa..52ca2b375 100644
--- a/src/signature.c
+++ b/src/signature.c
@@ -74,7 +74,7 @@ int git_signature_new(git_signature **sig_out, const char *name, const char *ema
 		git_signature_free(p);
 		return signature_error("Signature cannot have an empty name");
 	}
-		
+
 	p->when.time = time;
 	p->when.offset = offset;
 
@@ -129,6 +129,23 @@ int git_signature_now(git_signature **sig_out, const char *name, const char *ema
 	return 0;
 }
 
+int git_signature_default(git_signature **out, git_repository *repo)
+{
+	int error;
+	git_config *cfg;
+	const char *user_name, *user_email;
+
+	if ((error = git_repository_config(&cfg, repo)) < 0)
+		return error;
+
+	if (!(error = git_config_get_string(&user_name, cfg, "user.name")) &&
+		!(error = git_config_get_string(&user_email, cfg, "user.email")))
+		error = git_signature_now(out, user_name, user_email);
+
+	git_config_free(cfg);
+	return error;
+}
+
 int git_signature__parse(git_signature *sig, const char **buffer_out,
 		const char *buffer_end, const char *header, char ender)
 {
diff --git a/src/sortedcache.c b/src/sortedcache.c
new file mode 100644
index 000000000..466e55dbe
--- /dev/null
+++ b/src/sortedcache.c
@@ -0,0 +1,380 @@
+#include "sortedcache.h"
+
+GIT__USE_STRMAP;
+
+int git_sortedcache_new(
+	git_sortedcache **out,
+	size_t item_path_offset,
+	git_sortedcache_free_item_fn free_item,
+	void *free_item_payload,
+	git_vector_cmp item_cmp,
+	const char *path)
+{
+	git_sortedcache *sc;
+	size_t pathlen;
+
+	pathlen = path ? strlen(path) : 0;
+
+	sc = git__calloc(sizeof(git_sortedcache) + pathlen + 1, 1);
+	GITERR_CHECK_ALLOC(sc);
+
+	if (git_pool_init(&sc->pool, 1, 0) < 0 ||
+		git_vector_init(&sc->items, 4, item_cmp) < 0 ||
+		(sc->map = git_strmap_alloc()) == NULL)
+		goto fail;
+
+	if (git_rwlock_init(&sc->lock)) {
+		giterr_set(GITERR_OS, "Failed to initialize lock");
+		goto fail;
+	}
+
+	sc->item_path_offset  = item_path_offset;
+	sc->free_item         = free_item;
+	sc->free_item_payload = free_item_payload;
+	GIT_REFCOUNT_INC(sc);
+	if (pathlen)
+		memcpy(sc->path, path, pathlen);
+
+	*out = sc;
+	return 0;
+
+fail:
+	if (sc->map)
+		git_strmap_free(sc->map);
+	git_vector_free(&sc->items);
+	git_pool_clear(&sc->pool);
+	git__free(sc);
+	return -1;
+}
+
+void git_sortedcache_incref(git_sortedcache *sc)
+{
+	GIT_REFCOUNT_INC(sc);
+}
+
+const char *git_sortedcache_path(git_sortedcache *sc)
+{
+	return sc->path;
+}
+
+static void sortedcache_clear(git_sortedcache *sc)
+{
+	git_strmap_clear(sc->map);
+
+	if (sc->free_item) {
+		size_t i;
+		void *item;
+
+		git_vector_foreach(&sc->items, i, item) {
+			sc->free_item(sc->free_item_payload, item);
+		}
+	}
+
+	git_vector_clear(&sc->items);
+
+	git_pool_clear(&sc->pool);
+}
+
+static void sortedcache_free(git_sortedcache *sc)
+{
+	/* acquire write lock to make sure everyone else is done */
+	if (git_sortedcache_wlock(sc) < 0)
+		return;
+
+	sortedcache_clear(sc);
+	git_vector_free(&sc->items);
+	git_strmap_free(sc->map);
+
+	git_sortedcache_wunlock(sc);
+
+	git_rwlock_free(&sc->lock);
+	git__free(sc);
+}
+
+void git_sortedcache_free(git_sortedcache *sc)
+{
+	if (!sc)
+		return;
+	GIT_REFCOUNT_DEC(sc, sortedcache_free);
+}
+
+static int sortedcache_copy_item(void *payload, void *tgt_item, void *src_item)
+{
+	git_sortedcache *sc = payload;
+	/* path will already have been copied by upsert */
+	memcpy(tgt_item, src_item, sc->item_path_offset);
+	return 0;
+}
+
+/* copy a sorted cache */
+int git_sortedcache_copy(
+	git_sortedcache **out,
+	git_sortedcache *src,
+	bool lock,
+	int (*copy_item)(void *payload, void *tgt_item, void *src_item),
+	void *payload)
+{
+	int error = 0;
+	git_sortedcache *tgt;
+	size_t i;
+	void *src_item, *tgt_item;
+
+	/* just use memcpy if no special copy fn is passed in */
+	if (!copy_item) {
+		copy_item = sortedcache_copy_item;
+		payload   = src;
+	}
+
+	if ((error = git_sortedcache_new(
+			&tgt, src->item_path_offset,
+			src->free_item, src->free_item_payload,
+			src->items._cmp, src->path)) < 0)
+		return error;
+
+	if (lock && git_sortedcache_rlock(src) < 0) {
+		git_sortedcache_free(tgt);
+		return -1;
+	}
+
+	git_vector_foreach(&src->items, i, src_item) {
+		char *path = ((char *)src_item) + src->item_path_offset;
+
+		if ((error = git_sortedcache_upsert(&tgt_item, tgt, path)) < 0 ||
+			(error = copy_item(payload, tgt_item, src_item)) < 0)
+			break;
+	}
+
+	if (lock)
+		git_sortedcache_runlock(src);
+	if (error)
+		git_sortedcache_free(tgt);
+
+	*out = !error ? tgt : NULL;
+
+	return error;
+}
+
+/* lock sortedcache while making modifications */
+int git_sortedcache_wlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+
+	if (git_rwlock_wrlock(&sc->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire write lock on cache");
+		return -1;
+	}
+	return 0;
+}
+
+/* unlock sorted cache when done with modifications */
+void git_sortedcache_wunlock(git_sortedcache *sc)
+{
+	git_vector_sort(&sc->items);
+	git_rwlock_wrunlock(&sc->lock);
+}
+
+/* lock sortedcache for read */
+int git_sortedcache_rlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+
+	if (git_rwlock_rdlock(&sc->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire read lock on cache");
+		return -1;
+	}
+	return 0;
+}
+
+/* unlock sorted cache when done reading */
+void git_sortedcache_runlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+	git_rwlock_rdunlock(&sc->lock);
+}
+
+/* if the file has changed, lock cache and load file contents into buf;
+ * returns <0 on error, >0 if file has not changed
+ */
+int git_sortedcache_lockandload(git_sortedcache *sc, git_buf *buf)
+{
+	int error, fd;
+
+	if ((error = git_sortedcache_wlock(sc)) < 0)
+		return error;
+
+	if ((error = git_futils_filestamp_check(&sc->stamp, sc->path)) <= 0)
+		goto unlock;
+
+	if (!git__is_sizet(sc->stamp.size)) {
+		giterr_set(GITERR_INVALID, "Unable to load file larger than size_t");
+		error = -1;
+		goto unlock;
+	}
+
+	if ((fd = git_futils_open_ro(sc->path)) < 0) {
+		error = fd;
+		goto unlock;
+	}
+
+	if (buf)
+		error = git_futils_readbuffer_fd(buf, fd, (size_t)sc->stamp.size);
+
+	(void)p_close(fd);
+
+	if (error < 0)
+		goto unlock;
+
+	return 1; /* return 1 -> file needs reload and was successfully loaded */
+
+unlock:
+	git_sortedcache_wunlock(sc);
+	return error;
+}
+
+void git_sortedcache_updated(git_sortedcache *sc)
+{
+	 /* update filestamp to latest value */
+	if (git_futils_filestamp_check(&sc->stamp, sc->path) < 0)
+		giterr_clear();
+}
+
+/* release all items in sorted cache */
+int git_sortedcache_clear(git_sortedcache *sc, bool wlock)
+{
+	if (wlock && git_sortedcache_wlock(sc) < 0)
+		return -1;
+
+	sortedcache_clear(sc);
+
+	if (wlock)
+		git_sortedcache_wunlock(sc);
+
+	return 0;
+}
+
+/* find and/or insert item, returning pointer to item data */
+int git_sortedcache_upsert(void **out, git_sortedcache *sc, const char *key)
+{
+	int error = 0;
+	khiter_t pos;
+	void *item;
+	size_t keylen, itemlen;
+	char *item_key;
+
+	pos = git_strmap_lookup_index(sc->map, key);
+	if (git_strmap_valid_index(sc->map, pos)) {
+		item = git_strmap_value_at(sc->map, pos);
+		goto done;
+	}
+
+	keylen  = strlen(key);
+	itemlen = sc->item_path_offset + keylen + 1;
+	itemlen = (itemlen + 7) & ~7;
+
+	if ((item = git_pool_mallocz(&sc->pool, (uint32_t)itemlen)) == NULL) {
+		/* don't use GITERR_CHECK_ALLOC b/c of lock */
+		error = -1;
+		goto done;
+	}
+
+	/* one strange thing is that even if the vector or hash table insert
+	 * fail, there is no way to free the pool item so we just abandon it
+	 */
+
+	item_key = ((char *)item) + sc->item_path_offset;
+	memcpy(item_key, key, keylen);
+
+	pos = kh_put(str, sc->map, item_key, &error);
+	if (error < 0)
+		goto done;
+
+	if (!error)
+		kh_key(sc->map, pos) = item_key;
+	kh_val(sc->map, pos) = item;
+
+	error = git_vector_insert(&sc->items, item);
+	if (error < 0)
+		git_strmap_delete_at(sc->map, pos);
+
+done:
+	if (out)
+		*out = !error ? item : NULL;
+	return error;
+}
+
+/* lookup item by key */
+void *git_sortedcache_lookup(const git_sortedcache *sc, const char *key)
+{
+	khiter_t pos = git_strmap_lookup_index(sc->map, key);
+	if (git_strmap_valid_index(sc->map, pos))
+		return git_strmap_value_at(sc->map, pos);
+	return NULL;
+}
+
+/* find out how many items are in the cache */
+size_t git_sortedcache_entrycount(const git_sortedcache *sc)
+{
+	return git_vector_length(&sc->items);
+}
+
+/* lookup item by index */
+void *git_sortedcache_entry(git_sortedcache *sc, size_t pos)
+{
+	/* make sure the items are sorted so this gets the correct item */
+	if (!sc->items.sorted)
+		git_vector_sort(&sc->items);
+
+	return git_vector_get(&sc->items, pos);
+}
+
+/* helper struct so bsearch callback can know offset + key value for cmp */
+struct sortedcache_magic_key {
+	size_t offset;
+	const char *key;
+};
+
+static int sortedcache_magic_cmp(const void *key, const void *value)
+{
+	const struct sortedcache_magic_key *magic = key;
+	const char *value_key = ((const char *)value) + magic->offset;
+	return strcmp(magic->key, value_key);
+}
+
+/* lookup index of item by key */
+int git_sortedcache_lookup_index(
+	size_t *out, git_sortedcache *sc, const char *key)
+{
+	struct sortedcache_magic_key magic;
+
+	magic.offset = sc->item_path_offset;
+	magic.key    = key;
+
+	return git_vector_bsearch2(out, &sc->items, sortedcache_magic_cmp, &magic);
+}
+
+/* remove entry from cache */
+int git_sortedcache_remove(git_sortedcache *sc, size_t pos)
+{
+	char *item;
+	khiter_t mappos;
+
+	/* because of pool allocation, this can't actually remove the item,
+	 * but we can remove it from the items vector and the hash table.
+	 */
+
+	if ((item = git_vector_get(&sc->items, pos)) == NULL) {
+		giterr_set(GITERR_INVALID, "Removing item out of range");
+		return GIT_ENOTFOUND;
+	}
+
+	(void)git_vector_remove(&sc->items, pos);
+
+	mappos = git_strmap_lookup_index(sc->map, item + sc->item_path_offset);
+	git_strmap_delete_at(sc->map, mappos);
+
+	if (sc->free_item)
+		sc->free_item(sc->free_item_payload, item);
+
+	return 0;
+}
+
diff --git a/src/sortedcache.h b/src/sortedcache.h
new file mode 100644
index 000000000..4cacad62b
--- /dev/null
+++ b/src/sortedcache.h
@@ -0,0 +1,178 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_sorted_cache_h__
+#define INCLUDE_sorted_cache_h__
+
+#include "util.h"
+#include "fileops.h"
+#include "vector.h"
+#include "thread-utils.h"
+#include "pool.h"
+#include "strmap.h"
+
+#include <stddef.h>
+
+/*
+ * The purpose of this data structure is to cache the parsed contents of a
+ * file (a.k.a. the backing file) where each item in the file can be
+ * identified by a key string and you want to both look them up by name
+ * and traverse them in sorted order.  Each item is assumed to itself end
+ * in a GIT_FLEX_ARRAY.
+ */
+
+typedef void (*git_sortedcache_free_item_fn)(void *payload, void *item);
+
+typedef struct {
+	git_refcount rc;
+	git_rwlock   lock;
+	size_t       item_path_offset;
+	git_sortedcache_free_item_fn free_item;
+	void         *free_item_payload;
+	git_pool     pool;
+	git_vector   items;
+	git_strmap   *map;
+	git_futils_filestamp stamp;
+	char         path[GIT_FLEX_ARRAY];
+} git_sortedcache;
+
+/* Create a new sortedcache
+ *
+ * Even though every sortedcache stores items with a GIT_FLEX_ARRAY at
+ * the end containing their key string, you have to provide the item_cmp
+ * sorting function because the sorting function doesn't get a payload
+ * and therefore can't know the offset to the item key string. :-(
+ *
+ * @param out The allocated git_sortedcache
+ * @param item_path_offset Offset to the GIT_FLEX_ARRAY item key in the
+ *        struct - use offsetof(struct mine, key-field) to get this
+ * @param free_item Optional callback to free each item
+ * @param free_item_payload Optional payload passed to free_item callback
+ * @param item_cmp Compare the keys of two items
+ * @param path The path to the backing store file for this cache; this
+ *        may be NULL.  The cache makes it easy to load this and check
+ *        if it has been modified since the last load and/or write.
+ */
+int git_sortedcache_new(
+	git_sortedcache **out,
+	size_t item_path_offset, /* use offsetof(struct, path-field) macro */
+	git_sortedcache_free_item_fn free_item,
+	void *free_item_payload,
+	git_vector_cmp item_cmp,
+	const char *path);
+
+/* Copy a sorted cache
+ *
+ * - `copy_item` can be NULL to just use memcpy
+ * - if `lock`, grabs read lock on `src` during copy and releases after
+ */
+int git_sortedcache_copy(
+	git_sortedcache **out,
+	git_sortedcache *src,
+	bool lock,
+	int (*copy_item)(void *payload, void *tgt_item, void *src_item),
+	void *payload);
+
+/* Free sorted cache (first calling `free_item` callbacks)
+ *
+ * Don't call on a locked collection - it may acquire a write lock
+ */
+void git_sortedcache_free(git_sortedcache *sc);
+
+/* Increment reference count - balance with call to free */
+void git_sortedcache_incref(git_sortedcache *sc);
+
+/* Get the pathname associated with this cache at creation time */
+const char *git_sortedcache_path(git_sortedcache *sc);
+
+/*
+ * CACHE WRITE FUNCTIONS
+ *
+ * The following functions require you to have a writer lock to make the
+ * modification.  Some of the functions take a `wlock` parameter and
+ * will optionally lock and unlock for you if that is passed as true.
+ *
+ */
+
+/* Lock sortedcache for write */
+int git_sortedcache_wlock(git_sortedcache *sc);
+
+/* Unlock sorted cache when done with write */
+void git_sortedcache_wunlock(git_sortedcache *sc);
+
+/* Lock cache and load backing file into a buffer.
+ *
+ * This grabs a write lock on the cache then looks at the modification
+ * time and size of the file on disk.
+ *
+ * If the file appears to have changed, this loads the file contents into
+ * the buffer and returns a positive value leaving the cache locked - the
+ * caller should parse the file content, update the cache as needed, then
+ * release the lock.  NOTE: In this case, the caller MUST unlock the cache.
+ *
+ * If the file appears to be unchanged, then this automatically releases
+ * the lock on the cache, clears the buffer, and returns 0.
+ *
+ * @return 0 if up-to-date, 1 if out-of-date, <0 on error
+ */
+int git_sortedcache_lockandload(git_sortedcache *sc, git_buf *buf);
+
+/* Refresh file timestamp after write completes
+ * You should already be holding the write lock when you call this.
+ */
+void git_sortedcache_updated(git_sortedcache *sc);
+
+/* Release all items in sorted cache
+ *
+ * If `wlock` is true, grabs write lock and releases when done, otherwise
+ * you should already be holding a write lock when you call this.
+ */
+int git_sortedcache_clear(git_sortedcache *sc, bool wlock);
+
+/* Find and/or insert item, returning pointer to item data.
+ * You should already be holding the write lock when you call this.
+ */
+int git_sortedcache_upsert(
+	void **out, git_sortedcache *sc, const char *key);
+
+/* Removes entry at pos from cache
+ * You should already be holding the write lock when you call this.
+ */
+int git_sortedcache_remove(git_sortedcache *sc, size_t pos);
+
+/*
+ * CACHE READ FUNCTIONS
+ *
+ * The following functions access items in the cache.  To prevent the
+ * results from being invalidated before they can be used, you should be
+ * holding either a read lock or a write lock when using these functions.
+ *
+ */
+
+/* Lock sortedcache for read */
+int git_sortedcache_rlock(git_sortedcache *sc);
+
+/* Unlock sorted cache when done with read */
+void git_sortedcache_runlock(git_sortedcache *sc);
+
+/* Lookup item by key - returns NULL if not found */
+void *git_sortedcache_lookup(const git_sortedcache *sc, const char *key);
+
+/* Get how many items are in the cache
+ *
+ * You can call this function without holding a lock, but be aware
+ * that it may change before you use it.
+ */
+size_t git_sortedcache_entrycount(const git_sortedcache *sc);
+
+/* Lookup item by index - returns NULL if out of range */
+void *git_sortedcache_entry(git_sortedcache *sc, size_t pos);
+
+/* Lookup index of item by key - returns GIT_ENOTFOUND if not found */
+int git_sortedcache_lookup_index(
+	size_t *out, git_sortedcache *sc, const char *key);
+
+#endif
diff --git a/src/stash.c b/src/stash.c
index 1222634d5..48f19144d 100644
--- a/src/stash.c
+++ b/src/stash.c
@@ -117,7 +117,7 @@ static int build_tree_from_index(git_tree **out, git_index *index)
 static int commit_index(
 	git_commit **i_commit,
 	git_index *index,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message,
 	const git_commit *parent)
 {
@@ -267,7 +267,7 @@ cleanup:
 static int commit_untracked(
 	git_commit **u_commit,
 	git_index *index,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message,
 	git_commit *i_commit,
 	uint32_t flags)
@@ -354,7 +354,7 @@ cleanup:
 static int commit_worktree(
 	git_oid *w_commit_oid,
 	git_index *index,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message,
 	git_commit *i_commit,
 	git_commit *b_commit,
@@ -431,7 +431,7 @@ cleanup:
 static int update_reflog(
 	git_oid *w_commit_oid,
 	git_repository *repo,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message)
 {
 	git_reference *stash = NULL;
@@ -510,7 +510,7 @@ static int reset_index_and_workdir(
 int git_stash_save(
 	git_oid *out,
 	git_repository *repo,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message,
 	uint32_t flags)
 {
diff --git a/src/status.c b/src/status.c
index 712e0d515..4a0d65092 100644
--- a/src/status.c
+++ b/src/status.c
@@ -11,6 +11,7 @@
 #include "hash.h"
 #include "vector.h"
 #include "tree.h"
+#include "status.h"
 #include "git2/status.h"
 #include "repository.h"
 #include "ignore.h"
@@ -19,11 +20,11 @@
 #include "git2/diff.h"
 #include "diff.h"
 
-static unsigned int index_delta2status(git_delta_t index_status)
+static unsigned int index_delta2status(const git_diff_delta *head2idx)
 {
-	unsigned int st = GIT_STATUS_CURRENT;
+	git_status_t st = GIT_STATUS_CURRENT;
 
-	switch (index_status) {
+	switch (head2idx->status) {
 	case GIT_DELTA_ADDED:
 	case GIT_DELTA_COPIED:
 		st = GIT_STATUS_INDEX_NEW;
@@ -36,6 +37,9 @@ static unsigned int index_delta2status(git_delta_t index_status)
 		break;
 	case GIT_DELTA_RENAMED:
 		st = GIT_STATUS_INDEX_RENAMED;
+
+		if (!git_oid_equal(&head2idx->old_file.oid, &head2idx->new_file.oid))
+			st |= GIT_STATUS_INDEX_MODIFIED;
 		break;
 	case GIT_DELTA_TYPECHANGE:
 		st = GIT_STATUS_INDEX_TYPECHANGE;
@@ -47,13 +51,13 @@ static unsigned int index_delta2status(git_delta_t index_status)
 	return st;
 }
 
-static unsigned int workdir_delta2status(git_delta_t workdir_status)
+static unsigned int workdir_delta2status(
+	git_diff_list *diff, git_diff_delta *idx2wd)
 {
-	unsigned int st = GIT_STATUS_CURRENT;
+	git_status_t st = GIT_STATUS_CURRENT;
 
-	switch (workdir_status) {
+	switch (idx2wd->status) {
 	case GIT_DELTA_ADDED:
-	case GIT_DELTA_RENAMED:
 	case GIT_DELTA_COPIED:
 	case GIT_DELTA_UNTRACKED:
 		st = GIT_STATUS_WT_NEW;
@@ -67,6 +71,31 @@ static unsigned int workdir_delta2status(git_delta_t workdir_status)
 	case GIT_DELTA_IGNORED:
 		st = GIT_STATUS_IGNORED;
 		break;
+	case GIT_DELTA_RENAMED:
+		st = GIT_STATUS_WT_RENAMED;
+
+		if (!git_oid_equal(&idx2wd->old_file.oid, &idx2wd->new_file.oid)) {
+			/* if OIDs don't match, we might need to calculate them now to
+			 * discern between RENAMED vs RENAMED+MODIFED
+			 */
+			if (git_oid_iszero(&idx2wd->old_file.oid) &&
+				diff->old_src == GIT_ITERATOR_TYPE_WORKDIR &&
+				!git_diff__oid_for_file(
+					diff->repo, idx2wd->old_file.path, idx2wd->old_file.mode,
+					idx2wd->old_file.size, &idx2wd->old_file.oid))
+			idx2wd->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+
+			if (git_oid_iszero(&idx2wd->new_file.oid) &&
+				diff->new_src == GIT_ITERATOR_TYPE_WORKDIR &&
+				!git_diff__oid_for_file(
+					diff->repo, idx2wd->new_file.path, idx2wd->new_file.mode,
+					idx2wd->new_file.size, &idx2wd->new_file.oid))
+				idx2wd->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+
+			if (!git_oid_equal(&idx2wd->old_file.oid, &idx2wd->new_file.oid))
+				st |= GIT_STATUS_WT_MODIFIED;
+		}
+		break;
 	case GIT_DELTA_TYPECHANGE:
 		st = GIT_STATUS_WT_TYPECHANGE;
 		break;
@@ -77,142 +106,307 @@ static unsigned int workdir_delta2status(git_delta_t workdir_status)
 	return st;
 }
 
-typedef struct {
-	git_status_cb cb;
-	void *payload;
-	const git_status_options *opts;
-} status_user_callback;
-
-static int status_invoke_cb(
-	git_diff_delta *h2i, git_diff_delta *i2w, void *payload)
+static bool status_is_included(
+	git_status_list *status,
+	git_diff_delta *head2idx,
+	git_diff_delta *idx2wd)
 {
-	status_user_callback *usercb = payload;
-	const char *path = NULL;
-	unsigned int status = 0;
+	if (!(status->opts.flags & GIT_STATUS_OPT_EXCLUDE_SUBMODULES))
+		return 1;
 
-	if (i2w) {
-		path = i2w->old_file.path;
-		status |= workdir_delta2status(i2w->status);
+	/* if excluding submodules and this is a submodule everywhere */
+	if (head2idx) {
+		if (head2idx->status != GIT_DELTA_ADDED &&
+			head2idx->old_file.mode != GIT_FILEMODE_COMMIT)
+			return 1;
+		if (head2idx->status != GIT_DELTA_DELETED &&
+			head2idx->new_file.mode != GIT_FILEMODE_COMMIT)
+			return 1;
 	}
-	if (h2i) {
-		path = h2i->old_file.path;
-		status |= index_delta2status(h2i->status);
+	if (idx2wd) {
+		if (idx2wd->status != GIT_DELTA_ADDED &&
+			idx2wd->old_file.mode != GIT_FILEMODE_COMMIT)
+			return 1;
+		if (idx2wd->status != GIT_DELTA_DELETED &&
+			idx2wd->new_file.mode != GIT_FILEMODE_COMMIT)
+			return 1;
 	}
 
-	/* if excluding submodules and this is a submodule everywhere */
-	if (usercb->opts &&
-		(usercb->opts->flags & GIT_STATUS_OPT_EXCLUDE_SUBMODULES) != 0)
-	{
-		bool in_tree  = (h2i && h2i->status != GIT_DELTA_ADDED);
-		bool in_index = (h2i && h2i->status != GIT_DELTA_DELETED);
-		bool in_wd    = (i2w && i2w->status != GIT_DELTA_DELETED);
-
-		if ((!in_tree || h2i->old_file.mode == GIT_FILEMODE_COMMIT) &&
-			(!in_index || h2i->new_file.mode == GIT_FILEMODE_COMMIT) &&
-			(!in_wd || i2w->new_file.mode == GIT_FILEMODE_COMMIT))
-			return 0;
+	/* only get here if every valid mode is GIT_FILEMODE_COMMIT */
+	return 0;
+}
+
+static git_status_t status_compute(
+	git_status_list *status,
+	git_diff_delta *head2idx,
+	git_diff_delta *idx2wd)
+{
+	git_status_t st = GIT_STATUS_CURRENT;
+
+	if (head2idx)
+		st |= index_delta2status(head2idx);
+
+	if (idx2wd)
+		st |= workdir_delta2status(status->idx2wd, idx2wd);
+
+	return st;
+}
+
+static int status_collect(
+	git_diff_delta *head2idx,
+	git_diff_delta *idx2wd,
+	void *payload)
+{
+	git_status_list *status = payload;
+	git_status_entry *status_entry;
+
+	if (!status_is_included(status, head2idx, idx2wd))
+		return 0;
+
+	status_entry = git__malloc(sizeof(git_status_entry));
+	GITERR_CHECK_ALLOC(status_entry);
+
+	status_entry->status = status_compute(status, head2idx, idx2wd);
+	status_entry->head_to_index = head2idx;
+	status_entry->index_to_workdir = idx2wd;
+
+	return git_vector_insert(&status->paired, status_entry);
+}
+
+GIT_INLINE(int) status_entry_cmp_base(
+	const void *a,
+	const void *b,
+	int (*strcomp)(const char *a, const char *b))
+{
+	const git_status_entry *entry_a = a;
+	const git_status_entry *entry_b = b;
+	const git_diff_delta *delta_a, *delta_b;
+
+	delta_a = entry_a->index_to_workdir ? entry_a->index_to_workdir :
+		entry_a->head_to_index;
+	delta_b = entry_b->index_to_workdir ? entry_b->index_to_workdir :
+		entry_b->head_to_index;
+
+	if (!delta_a && delta_b)
+		return -1;
+	if (delta_a && !delta_b)
+		return 1;
+	if (!delta_a && !delta_b)
+		return 0;
+
+	return strcomp(delta_a->new_file.path, delta_b->new_file.path);
+}
+
+static int status_entry_icmp(const void *a, const void *b)
+{
+	return status_entry_cmp_base(a, b, git__strcasecmp);
+}
+
+static int status_entry_cmp(const void *a, const void *b)
+{
+	return status_entry_cmp_base(a, b, git__strcmp);
+}
+
+static git_status_list *git_status_list_alloc(git_index *index)
+{
+	git_status_list *status = NULL;
+	int (*entrycmp)(const void *a, const void *b);
+
+	if (!(status = git__calloc(1, sizeof(git_status_list))))
+		return NULL;
+
+	entrycmp = index->ignore_case ? status_entry_icmp : status_entry_cmp;
+
+	if (git_vector_init(&status->paired, 0, entrycmp) < 0) {
+		git__free(status);
+		return NULL;
 	}
 
-	return usercb->cb(path, status, usercb->payload);
+	return status;
 }
 
-int git_status_foreach_ext(
+int git_status_list_new(
+	git_status_list **out,
 	git_repository *repo,
-	const git_status_options *opts,
-	git_status_cb cb,
-	void *payload)
+	const git_status_options *opts)
 {
-	int err = 0;
+	git_index *index = NULL;
+	git_status_list *status = NULL;
 	git_diff_options diffopt = GIT_DIFF_OPTIONS_INIT;
-	git_diff_list *head2idx = NULL, *idx2wd = NULL;
+	git_diff_find_options findopt = GIT_DIFF_FIND_OPTIONS_INIT;
 	git_tree *head = NULL;
 	git_status_show_t show =
 		opts ? opts->show : GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
-	status_user_callback usercb;
+	int error = 0;
+	unsigned int flags = opts ? opts->flags : GIT_STATUS_OPT_DEFAULTS;
+
+	assert(show <= GIT_STATUS_SHOW_WORKDIR_ONLY);
 
-	assert(show <= GIT_STATUS_SHOW_INDEX_THEN_WORKDIR);
+	*out = NULL;
 
 	GITERR_CHECK_VERSION(opts, GIT_STATUS_OPTIONS_VERSION, "git_status_options");
 
-	if (show != GIT_STATUS_SHOW_INDEX_ONLY &&
-		(err = git_repository__ensure_not_bare(repo, "status")) < 0)
-		return err;
+	if ((error = git_repository__ensure_not_bare(repo, "status")) < 0 ||
+		(error = git_repository_index(&index, repo)) < 0)
+		return error;
 
 	/* if there is no HEAD, that's okay - we'll make an empty iterator */
-	if (((err = git_repository_head_tree(&head, repo)) < 0) &&
-		!(err == GIT_ENOTFOUND || err == GIT_EORPHANEDHEAD))
-		return err;
+	if (((error = git_repository_head_tree(&head, repo)) < 0) &&
+		error != GIT_ENOTFOUND && error != GIT_EORPHANEDHEAD) {
+		git_index_free(index); /* release index */
+		return error;
+	}
 
-	memcpy(&diffopt.pathspec, &opts->pathspec, sizeof(diffopt.pathspec));
+	status = git_status_list_alloc(index);
+	GITERR_CHECK_ALLOC(status);
+
+	if (opts) {
+		memcpy(&status->opts, opts, sizeof(git_status_options));
+		memcpy(&diffopt.pathspec, &opts->pathspec, sizeof(diffopt.pathspec));
+	}
 
 	diffopt.flags = GIT_DIFF_INCLUDE_TYPECHANGE;
+	findopt.flags = GIT_DIFF_FIND_FOR_UNTRACKED;
 
-	if ((opts->flags & GIT_STATUS_OPT_INCLUDE_UNTRACKED) != 0)
+	if ((flags & GIT_STATUS_OPT_INCLUDE_UNTRACKED) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_INCLUDE_UNTRACKED;
-	if ((opts->flags & GIT_STATUS_OPT_INCLUDE_IGNORED) != 0)
+	if ((flags & GIT_STATUS_OPT_INCLUDE_IGNORED) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_INCLUDE_IGNORED;
-	if ((opts->flags & GIT_STATUS_OPT_INCLUDE_UNMODIFIED) != 0)
+	if ((flags & GIT_STATUS_OPT_INCLUDE_UNMODIFIED) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_INCLUDE_UNMODIFIED;
-	if ((opts->flags & GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS) != 0)
+	if ((flags & GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_RECURSE_UNTRACKED_DIRS;
-	if ((opts->flags & GIT_STATUS_OPT_DISABLE_PATHSPEC_MATCH) != 0)
+	if ((flags & GIT_STATUS_OPT_DISABLE_PATHSPEC_MATCH) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_DISABLE_PATHSPEC_MATCH;
-	if ((opts->flags & GIT_STATUS_OPT_RECURSE_IGNORED_DIRS) != 0)
+	if ((flags & GIT_STATUS_OPT_RECURSE_IGNORED_DIRS) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_RECURSE_IGNORED_DIRS;
-	if ((opts->flags & GIT_STATUS_OPT_EXCLUDE_SUBMODULES) != 0)
+	if ((flags & GIT_STATUS_OPT_EXCLUDE_SUBMODULES) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_IGNORE_SUBMODULES;
 
+	if ((flags & GIT_STATUS_OPT_RENAMES_FROM_REWRITES) != 0)
+		findopt.flags = findopt.flags |
+			GIT_DIFF_FIND_AND_BREAK_REWRITES |
+			GIT_DIFF_FIND_RENAMES_FROM_REWRITES |
+			GIT_DIFF_BREAK_REWRITES_FOR_RENAMES_ONLY;
+
 	if (show != GIT_STATUS_SHOW_WORKDIR_ONLY) {
-		err = git_diff_tree_to_index(&head2idx, repo, head, NULL, &diffopt);
-		if (err < 0)
-			goto cleanup;
-	}
+		if ((error = git_diff_tree_to_index(
+				&status->head2idx, repo, head, NULL, &diffopt)) < 0)
+			goto done;
 
-	if (show != GIT_STATUS_SHOW_INDEX_ONLY) {
-		err = git_diff_index_to_workdir(&idx2wd, repo, NULL, &diffopt);
-		if (err < 0)
-			goto cleanup;
+		if ((flags & GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX) != 0 &&
+			(error = git_diff_find_similar(status->head2idx, &findopt)) < 0)
+			goto done;
 	}
 
-	usercb.cb = cb;
-	usercb.payload = payload;
-	usercb.opts = opts;
+	if (show != GIT_STATUS_SHOW_INDEX_ONLY) {
+		if ((error = git_diff_index_to_workdir(
+				&status->idx2wd, repo, NULL, &diffopt)) < 0)
+			goto done;
 
-	if (show == GIT_STATUS_SHOW_INDEX_THEN_WORKDIR) {
-		if ((err = git_diff__paired_foreach(
-				 head2idx, NULL, status_invoke_cb, &usercb)) < 0)
-			goto cleanup;
+		if ((flags & GIT_STATUS_OPT_RENAMES_INDEX_TO_WORKDIR) != 0 &&
+			(error = git_diff_find_similar(status->idx2wd, &findopt)) < 0)
+			goto done;
+	}
 
-		git_diff_list_free(head2idx);
-		head2idx = NULL;
+	if ((error = git_diff__paired_foreach(
+			status->head2idx, status->idx2wd, status_collect, status)) < 0)
+		goto done;
+
+	if (flags & GIT_STATUS_OPT_SORT_CASE_SENSITIVELY)
+		git_vector_set_cmp(&status->paired, status_entry_cmp);
+	if (flags & GIT_STATUS_OPT_SORT_CASE_INSENSITIVELY)
+		git_vector_set_cmp(&status->paired, status_entry_icmp);
+
+	if ((flags &
+		 (GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX |
+		  GIT_STATUS_OPT_RENAMES_INDEX_TO_WORKDIR |
+		  GIT_STATUS_OPT_SORT_CASE_SENSITIVELY |
+		  GIT_STATUS_OPT_SORT_CASE_INSENSITIVELY)) != 0)
+		git_vector_sort(&status->paired);
+
+done:
+	if (error < 0) {
+		git_status_list_free(status);
+		status = NULL;
 	}
 
-	err = git_diff__paired_foreach(head2idx, idx2wd, status_invoke_cb, &usercb);
+	*out = status;
 
-cleanup:
 	git_tree_free(head);
-	git_diff_list_free(head2idx);
-	git_diff_list_free(idx2wd);
+	git_index_free(index);
+
+	return error;
+}
+
+size_t git_status_list_entrycount(git_status_list *status)
+{
+	assert(status);
+
+	return status->paired.length;
+}
+
+const git_status_entry *git_status_byindex(git_status_list *status, size_t i)
+{
+	assert(status);
 
-	if (err == GIT_EUSER)
-		giterr_clear();
+	return git_vector_get(&status->paired, i);
+}
+
+void git_status_list_free(git_status_list *status)
+{
+	git_status_entry *status_entry;
+	size_t i;
+
+	if (status == NULL)
+		return;
+
+	git_diff_list_free(status->head2idx);
+	git_diff_list_free(status->idx2wd);
+
+	git_vector_foreach(&status->paired, i, status_entry)
+		git__free(status_entry);
 
-	return err;
+	git_vector_free(&status->paired);
+
+	git__memzero(status, sizeof(*status));
+	git__free(status);
 }
 
-int git_status_foreach(
+int git_status_foreach_ext(
 	git_repository *repo,
-	git_status_cb callback,
+	const git_status_options *opts,
+	git_status_cb cb,
 	void *payload)
 {
-	git_status_options opts = GIT_STATUS_OPTIONS_INIT;
+	git_status_list *status;
+	const git_status_entry *status_entry;
+	size_t i;
+	int error = 0;
 
-	opts.show  = GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
-	opts.flags = GIT_STATUS_OPT_INCLUDE_IGNORED |
-		GIT_STATUS_OPT_INCLUDE_UNTRACKED |
-		GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS;
+	if ((error = git_status_list_new(&status, repo, opts)) < 0)
+		return error;
+
+	git_vector_foreach(&status->paired, i, status_entry) {
+		const char *path = status_entry->head_to_index ?
+			status_entry->head_to_index->old_file.path :
+			status_entry->index_to_workdir->old_file.path;
+
+		if (cb(path, status_entry->status, payload) != 0) {
+			error = GIT_EUSER;
+			giterr_clear();
+			break;
+		}
+	}
 
-	return git_status_foreach_ext(repo, &opts, callback, payload);
+	git_status_list_free(status);
+
+	return error;
+}
+
+int git_status_foreach(git_repository *repo, git_status_cb cb, void *payload)
+{
+	return git_status_foreach_ext(repo, NULL, cb, payload);
 }
 
 struct status_file_info {
@@ -264,7 +458,7 @@ int git_status_file(
 	if (index->ignore_case)
 		sfi.fnm_flags = FNM_CASEFOLD;
 
-	opts.show  = GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
+	opts.show = GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
 	opts.flags = GIT_STATUS_OPT_INCLUDE_IGNORED |
 		GIT_STATUS_OPT_RECURSE_IGNORED_DIRS |
 		GIT_STATUS_OPT_INCLUDE_UNTRACKED |
diff --git a/src/status.h b/src/status.h
new file mode 100644
index 000000000..b58e0ebd6
--- /dev/null
+++ b/src/status.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_status_h__
+#define INCLUDE_status_h__
+
+#include "diff.h"
+#include "git2/status.h"
+#include "git2/diff.h"
+
+struct git_status_list {
+	git_status_options opts;
+
+	git_diff_list *head2idx;
+	git_diff_list *idx2wd;
+
+	git_vector paired;
+};
+
+#endif
diff --git a/src/strmap.c b/src/strmap.c
new file mode 100644
index 000000000..b26a13d1f
--- /dev/null
+++ b/src/strmap.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "strmap.h"
+
+int git_strmap_next(
+	void **data,
+	git_strmap_iter* iter,
+	git_strmap *map)
+{
+	if (!map)
+		return GIT_ERROR;
+
+	while (*iter != git_strmap_end(map)) {
+		if (!(git_strmap_has_data(map, *iter))) {
+			++(*iter);
+			continue;
+		}
+
+		*data = git_strmap_value_at(map, *iter);
+
+		++(*iter);
+
+		return GIT_OK;
+	}
+
+	return GIT_ITEROVER;
+}
diff --git a/src/strmap.h b/src/strmap.h
index 44176a0fc..8276ab468 100644
--- a/src/strmap.h
+++ b/src/strmap.h
@@ -17,6 +17,7 @@
 
 __KHASH_TYPE(str, const char *, void *);
 typedef khash_t(str) git_strmap;
+typedef khiter_t git_strmap_iter;
 
 #define GIT__USE_STRMAP \
 	__KHASH_IMPL(str, static kh_inline, const char *, void *, 1, kh_str_hash_func, kh_str_hash_equal)
@@ -31,7 +32,9 @@ typedef khash_t(str) git_strmap;
 #define git_strmap_valid_index(h, idx) (idx != kh_end(h))
 
 #define git_strmap_exists(h, k) (kh_get(str, h, k) != kh_end(h))
+#define git_strmap_has_data(h, idx) kh_exist(h, idx)
 
+#define git_strmap_key(h, idx)             kh_key(h, idx)
 #define git_strmap_value_at(h, idx)        kh_val(h, idx)
 #define git_strmap_set_value_at(h, idx, v) kh_val(h, idx) = v
 #define git_strmap_delete_at(h, idx)       kh_del(str, h, idx)
@@ -61,4 +64,12 @@ typedef khash_t(str) git_strmap;
 #define git_strmap_foreach		kh_foreach
 #define git_strmap_foreach_value	kh_foreach_value
 
+#define git_strmap_begin		kh_begin
+#define git_strmap_end		kh_end
+
+int git_strmap_next(
+	void **data,
+	git_strmap_iter* iter,
+	git_strmap *map);
+
 #endif
diff --git a/src/submodule.c b/src/submodule.c
index af488b7f3..40bda9a41 100644
--- a/src/submodule.c
+++ b/src/submodule.c
@@ -9,9 +9,7 @@
 #include "git2/config.h"
 #include "git2/sys/config.h"
 #include "git2/types.h"
-#include "git2/repository.h"
 #include "git2/index.h"
-#include "git2/submodule.h"
 #include "buffer.h"
 #include "buf_text.h"
 #include "vector.h"
@@ -32,6 +30,8 @@ static git_cvar_map _sm_update_map[] = {
 	{GIT_CVAR_STRING, "rebase", GIT_SUBMODULE_UPDATE_REBASE},
 	{GIT_CVAR_STRING, "merge", GIT_SUBMODULE_UPDATE_MERGE},
 	{GIT_CVAR_STRING, "none", GIT_SUBMODULE_UPDATE_NONE},
+	{GIT_CVAR_FALSE, NULL, GIT_SUBMODULE_UPDATE_NONE},
+	{GIT_CVAR_TRUE, NULL, GIT_SUBMODULE_UPDATE_CHECKOUT},
 };
 
 static git_cvar_map _sm_ignore_map[] = {
@@ -39,6 +39,8 @@ static git_cvar_map _sm_ignore_map[] = {
 	{GIT_CVAR_STRING, "untracked", GIT_SUBMODULE_IGNORE_UNTRACKED},
 	{GIT_CVAR_STRING, "dirty", GIT_SUBMODULE_IGNORE_DIRTY},
 	{GIT_CVAR_STRING, "all", GIT_SUBMODULE_IGNORE_ALL},
+	{GIT_CVAR_FALSE, NULL, GIT_SUBMODULE_IGNORE_NONE},
+	{GIT_CVAR_TRUE, NULL, GIT_SUBMODULE_IGNORE_ALL},
 };
 
 static kh_inline khint_t str_hash_no_trailing_slash(const char *s)
@@ -73,15 +75,11 @@ static int load_submodule_config(git_repository *repo);
 static git_config_backend *open_gitmodules(git_repository *, bool, const git_oid *);
 static int lookup_head_remote(git_buf *url, git_repository *repo);
 static int submodule_get(git_submodule **, git_repository *, const char *, const char *);
-static void submodule_release(git_submodule *sm, int decr);
-static int submodule_load_from_index(git_repository *, const git_index_entry *);
-static int submodule_load_from_head(git_repository*, const char*, const git_oid*);
 static int submodule_load_from_config(const git_config_entry *, void *);
 static int submodule_load_from_wd_lite(git_submodule *, const char *, void *);
 static int submodule_update_config(git_submodule *, const char *, const char *, bool, bool);
-static void submodule_mode_mismatch(git_repository *, const char *, unsigned int);
-static int submodule_index_status(unsigned int *status, git_submodule *sm);
-static int submodule_wd_status(unsigned int *status, git_submodule *sm);
+static void submodule_get_index_status(unsigned int *, git_submodule *);
+static void submodule_get_wd_status(unsigned int *, git_submodule *, git_repository *, git_submodule_ignore_t);
 
 static int submodule_cmp(const void *a, const void *b)
 {
@@ -151,7 +149,7 @@ int git_submodule_foreach(
 	int error;
 	git_submodule *sm;
 	git_vector seen = GIT_VECTOR_INIT;
-	seen._cmp = submodule_cmp;
+	git_vector_set_cmp(&seen, submodule_cmp);
 
 	assert(repo && callback);
 
@@ -163,7 +161,7 @@ int git_submodule_foreach(
 		 * us from issuing a callback twice for a submodule where the name
 		 * and path are not the same.
 		 */
-		if (sm->refcount > 1) {
+		if (GIT_REFCOUNT_VAL(sm) > 1) {
 			if (git_vector_bsearch(NULL, &seen, sm) != GIT_ENOTFOUND)
 				continue;
 			if ((error = git_vector_insert(&seen, sm)) < 0)
@@ -195,9 +193,7 @@ void git_submodule_config_free(git_repository *repo)
 	if (smcfg == NULL)
 		return;
 
-	git_strmap_foreach_value(smcfg, sm, {
-		submodule_release(sm,1);
-	});
+	git_strmap_foreach_value(smcfg, sm, { git_submodule_free(sm); });
 	git_strmap_free(smcfg);
 }
 
@@ -338,7 +334,7 @@ int git_submodule_add_finalize(git_submodule *sm)
 
 	assert(sm);
 
-	if ((error = git_repository_index__weakptr(&index, sm->owner)) < 0 ||
+	if ((error = git_repository_index__weakptr(&index, sm->repo)) < 0 ||
 		(error = git_index_add_bypath(index, GIT_MODULES_FILE)) < 0)
 		return error;
 
@@ -348,7 +344,7 @@ int git_submodule_add_finalize(git_submodule *sm)
 int git_submodule_add_to_index(git_submodule *sm, int write_index)
 {
 	int error;
-	git_repository *repo, *sm_repo = NULL;
+	git_repository *sm_repo = NULL;
 	git_index *index;
 	git_buf path = GIT_BUF_INIT;
 	git_commit *head;
@@ -357,14 +353,12 @@ int git_submodule_add_to_index(git_submodule *sm, int write_index)
 
 	assert(sm);
 
-	repo = sm->owner;
-
 	/* force reload of wd OID by git_submodule_open */
 	sm->flags = sm->flags & ~GIT_SUBMODULE_STATUS__WD_OID_VALID;
 
-	if ((error = git_repository_index__weakptr(&index, repo)) < 0 ||
+	if ((error = git_repository_index__weakptr(&index, sm->repo)) < 0 ||
 		(error = git_buf_joinpath(
-			&path, git_repository_workdir(repo), sm->path)) < 0 ||
+			&path, git_repository_workdir(sm->repo), sm->path)) < 0 ||
 		(error = git_submodule_open(&sm_repo, sm)) < 0)
 		goto cleanup;
 
@@ -416,15 +410,34 @@ cleanup:
 	return error;
 }
 
+const char *git_submodule_ignore_to_str(git_submodule_ignore_t ignore)
+{
+	int i;
+	for (i = 0; i < (int)ARRAY_SIZE(_sm_ignore_map); ++i)
+		if (_sm_ignore_map[i].map_value == ignore)
+			return _sm_ignore_map[i].str_match;
+	return NULL;
+}
+
+const char *git_submodule_update_to_str(git_submodule_update_t update)
+{
+	int i;
+	for (i = 0; i < (int)ARRAY_SIZE(_sm_update_map); ++i)
+		if (_sm_update_map[i].map_value == update)
+			return _sm_update_map[i].str_match;
+	return NULL;
+}
+
 int git_submodule_save(git_submodule *submodule)
 {
 	int error = 0;
 	git_config_backend *mods;
 	git_buf key = GIT_BUF_INIT;
+	const char *val;
 
 	assert(submodule);
 
-	mods = open_gitmodules(submodule->owner, true, NULL);
+	mods = open_gitmodules(submodule->repo, true, NULL);
 	if (!mods) {
 		giterr_set(GITERR_SUBMODULE,
 			"Adding submodules to a bare repository is not supported (for now)");
@@ -445,22 +458,14 @@ int git_submodule_save(git_submodule *submodule)
 		goto cleanup;
 
 	if (!(error = submodule_config_key_trunc_puts(&key, "update")) &&
-		submodule->update != GIT_SUBMODULE_UPDATE_DEFAULT)
-	{
-		const char *val = (submodule->update == GIT_SUBMODULE_UPDATE_CHECKOUT) ?
-			NULL : _sm_update_map[submodule->update].str_match;
+		(val = git_submodule_update_to_str(submodule->update)) != NULL)
 		error = git_config_file_set_string(mods, key.ptr, val);
-	}
 	if (error < 0)
 		goto cleanup;
 
 	if (!(error = submodule_config_key_trunc_puts(&key, "ignore")) &&
-		submodule->ignore != GIT_SUBMODULE_IGNORE_DEFAULT)
-	{
-		const char *val = (submodule->ignore == GIT_SUBMODULE_IGNORE_NONE) ?
-			NULL : _sm_ignore_map[submodule->ignore].str_match;
+		(val = git_submodule_ignore_to_str(submodule->ignore)) != NULL)
 		error = git_config_file_set_string(mods, key.ptr, val);
-	}
 	if (error < 0)
 		goto cleanup;
 
@@ -487,7 +492,7 @@ cleanup:
 git_repository *git_submodule_owner(git_submodule *submodule)
 {
 	assert(submodule);
-	return submodule->owner;
+	return submodule->repo;
 }
 
 const char *git_submodule_name(git_submodule *submodule)
@@ -544,11 +549,12 @@ const git_oid *git_submodule_wd_id(git_submodule *submodule)
 {
 	assert(submodule);
 
+	/* load unless we think we have a valid oid */
 	if (!(submodule->flags & GIT_SUBMODULE_STATUS__WD_OID_VALID)) {
 		git_repository *subrepo;
 
 		/* calling submodule open grabs the HEAD OID if possible */
-		if (!git_submodule_open(&subrepo, submodule))
+		if (!git_submodule_open_bare(&subrepo, submodule))
 			git_repository_free(subrepo);
 		else
 			giterr_clear();
@@ -563,7 +569,8 @@ const git_oid *git_submodule_wd_id(git_submodule *submodule)
 git_submodule_ignore_t git_submodule_ignore(git_submodule *submodule)
 {
 	assert(submodule);
-	return submodule->ignore;
+	return (submodule->ignore < GIT_SUBMODULE_IGNORE_NONE) ?
+		GIT_SUBMODULE_IGNORE_NONE : submodule->ignore;
 }
 
 git_submodule_ignore_t git_submodule_set_ignore(
@@ -573,7 +580,7 @@ git_submodule_ignore_t git_submodule_set_ignore(
 
 	assert(submodule);
 
-	if (ignore == GIT_SUBMODULE_IGNORE_DEFAULT)
+	if (ignore == GIT_SUBMODULE_IGNORE_RESET)
 		ignore = submodule->ignore_default;
 
 	old = submodule->ignore;
@@ -584,7 +591,8 @@ git_submodule_ignore_t git_submodule_set_ignore(
 git_submodule_update_t git_submodule_update(git_submodule *submodule)
 {
 	assert(submodule);
-	return submodule->update;
+	return (submodule->update < GIT_SUBMODULE_UPDATE_CHECKOUT) ?
+		GIT_SUBMODULE_UPDATE_CHECKOUT : submodule->update;
 }
 
 git_submodule_update_t git_submodule_set_update(
@@ -594,7 +602,7 @@ git_submodule_update_t git_submodule_set_update(
 
 	assert(submodule);
 
-	if (update == GIT_SUBMODULE_UPDATE_DEFAULT)
+	if (update == GIT_SUBMODULE_UPDATE_RESET)
 		update = submodule->update_default;
 
 	old = submodule->update;
@@ -625,6 +633,7 @@ int git_submodule_set_fetch_recurse_submodules(
 int git_submodule_init(git_submodule *submodule, int overwrite)
 {
 	int error;
+	const char *val;
 
 	/* write "submodule.NAME.url" */
 
@@ -641,14 +650,10 @@ int git_submodule_init(git_submodule *submodule, int overwrite)
 
 	/* write "submodule.NAME.update" if not default */
 
-	if (submodule->update == GIT_SUBMODULE_UPDATE_CHECKOUT)
-		error = submodule_update_config(
-			submodule, "update", NULL, (overwrite != 0), false);
-	else if (submodule->update != GIT_SUBMODULE_UPDATE_DEFAULT)
-		error = submodule_update_config(
-			submodule, "update",
-			_sm_update_map[submodule->update].str_match,
-			(overwrite != 0), false);
+	val = (submodule->update == GIT_SUBMODULE_UPDATE_CHECKOUT) ?
+		NULL : git_submodule_update_to_str(submodule->update);
+	error = submodule_update_config(
+		submodule, "update", val, (overwrite != 0), false);
 
 	return error;
 }
@@ -667,51 +672,70 @@ int git_submodule_sync(git_submodule *submodule)
 		submodule, "url", submodule->url, true, true);
 }
 
-int git_submodule_open(
-	git_repository **subrepo,
-	git_submodule *submodule)
+static int git_submodule__open(
+	git_repository **subrepo, git_submodule *sm, bool bare)
 {
 	int error;
 	git_buf path = GIT_BUF_INIT;
-	git_repository *repo;
-	const char *workdir;
+	unsigned int flags = GIT_REPOSITORY_OPEN_NO_SEARCH;
+	const char *wd;
 
-	assert(submodule && subrepo);
+	assert(sm && subrepo);
 
-	repo = submodule->owner;
-	workdir = git_repository_workdir(repo);
+	if (git_repository__ensure_not_bare(
+			sm->repo, "open submodule repository") < 0)
+		return GIT_EBAREREPO;
 
-	if (!workdir) {
-		giterr_set(GITERR_REPOSITORY,
-			"Cannot open submodule repository in a bare repo");
-		return GIT_ENOTFOUND;
-	}
-
-	if ((submodule->flags & GIT_SUBMODULE_STATUS_IN_WD) == 0) {
-		giterr_set(GITERR_REPOSITORY,
-			"Cannot open submodule repository that is not checked out");
-		return GIT_ENOTFOUND;
-	}
+	wd = git_repository_workdir(sm->repo);
 
-	if (git_buf_joinpath(&path, workdir, submodule->path) < 0)
+	if (git_buf_joinpath(&path, wd, sm->path) < 0 ||
+		git_buf_joinpath(&path, path.ptr, DOT_GIT) < 0)
 		return -1;
 
-	error = git_repository_open(subrepo, path.ptr);
+	sm->flags = sm->flags &
+		~(GIT_SUBMODULE_STATUS_IN_WD |
+		  GIT_SUBMODULE_STATUS__WD_OID_VALID |
+		  GIT_SUBMODULE_STATUS__WD_SCANNED);
 
-	git_buf_free(&path);
+	if (bare)
+		flags |= GIT_REPOSITORY_OPEN_BARE;
+
+	error = git_repository_open_ext(subrepo, path.ptr, flags, wd);
 
-	/* if we have opened the submodule successfully, let's grab the HEAD OID */
+	/* if we opened the submodule successfully, grab HEAD OID, etc. */
 	if (!error) {
-		if (!git_reference_name_to_id(
-				&submodule->wd_oid, *subrepo, GIT_HEAD_FILE))
-			submodule->flags |= GIT_SUBMODULE_STATUS__WD_OID_VALID;
+		sm->flags |= GIT_SUBMODULE_STATUS_IN_WD |
+			GIT_SUBMODULE_STATUS__WD_SCANNED;
+
+		if (!git_reference_name_to_id(&sm->wd_oid, *subrepo, GIT_HEAD_FILE))
+			sm->flags |= GIT_SUBMODULE_STATUS__WD_OID_VALID;
 		else
 			giterr_clear();
+	} else if (git_path_exists(path.ptr)) {
+		sm->flags |= GIT_SUBMODULE_STATUS__WD_SCANNED |
+			GIT_SUBMODULE_STATUS_IN_WD;
+	} else {
+		git_buf_rtruncate_at_char(&path, '/'); /* remove "/.git" */
+
+		if (git_path_isdir(path.ptr))
+			sm->flags |= GIT_SUBMODULE_STATUS__WD_SCANNED;
 	}
 
+	git_buf_free(&path);
+
 	return error;
 }
 
+int git_submodule_open_bare(git_repository **subrepo, git_submodule *sm)
+{
+	return git_submodule__open(subrepo, sm, true);
+}
+
+int git_submodule_open(git_repository **subrepo, git_submodule *sm)
+{
+	return git_submodule__open(subrepo, sm, false);
+}
+
 int git_submodule_reload_all(git_repository *repo)
 {
 	assert(repo);
@@ -719,74 +743,100 @@ int git_submodule_reload_all(git_repository *repo)
 	return load_submodule_config(repo);
 }
 
-int git_submodule_reload(git_submodule *submodule)
+static void submodule_update_from_index_entry(
+	git_submodule *sm, const git_index_entry *ie)
 {
-	git_repository *repo;
-	git_index *index;
-	int error;
-	size_t pos;
-	git_tree *head;
-	git_config_backend *mods;
+	bool already_found = (sm->flags & GIT_SUBMODULE_STATUS_IN_INDEX) != 0;
 
-	assert(submodule);
+	if (!S_ISGITLINK(ie->mode)) {
+		if (!already_found)
+			sm->flags |= GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE;
+	} else {
+		if (already_found)
+			sm->flags |= GIT_SUBMODULE_STATUS__INDEX_MULTIPLE_ENTRIES;
+		else
+			git_oid_cpy(&sm->index_oid, &ie->oid);
 
-	/* refresh index data */
+		sm->flags |= GIT_SUBMODULE_STATUS_IN_INDEX |
+			GIT_SUBMODULE_STATUS__INDEX_OID_VALID;
+	}
+}
+
+static int submodule_update_index(git_submodule *sm)
+{
+	git_index *index;
+	const git_index_entry *ie;
 
-	repo = submodule->owner;
-	if (git_repository_index__weakptr(&index, repo) < 0)
+	if (git_repository_index__weakptr(&index, sm->repo) < 0)
 		return -1;
 
-	submodule->flags = submodule->flags &
+	sm->flags = sm->flags &
 		~(GIT_SUBMODULE_STATUS_IN_INDEX |
 		  GIT_SUBMODULE_STATUS__INDEX_OID_VALID);
 
-	if (!git_index_find(&pos, index, submodule->path)) {
-		const git_index_entry *entry = git_index_get_byindex(index, pos);
+	if (!(ie = git_index_get_bypath(index, sm->path, 0)))
+		return 0;
 
-		if (S_ISGITLINK(entry->mode)) {
-			if ((error = submodule_load_from_index(repo, entry)) < 0)
-				return error;
-		} else {
-			submodule_mode_mismatch(
-				repo, entry->path, GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE);
-		}
+	submodule_update_from_index_entry(sm, ie);
+
+	return 0;
+}
+
+static void submodule_update_from_head_data(
+	git_submodule *sm, mode_t mode, const git_oid *id)
+{
+	if (!S_ISGITLINK(mode))
+		sm->flags |= GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE;
+	else {
+		git_oid_cpy(&sm->head_oid, id);
+
+		sm->flags |= GIT_SUBMODULE_STATUS_IN_HEAD |
+			GIT_SUBMODULE_STATUS__HEAD_OID_VALID;
 	}
+}
 
-	/* refresh HEAD tree data */
+static int submodule_update_head(git_submodule *submodule)
+{
+	git_tree *head = NULL;
+	git_tree_entry *te = NULL;
 
-	if (!(error = git_repository_head_tree(&head, repo))) {
-		git_tree_entry *te;
+	submodule->flags = submodule->flags &
+		~(GIT_SUBMODULE_STATUS_IN_HEAD |
+		  GIT_SUBMODULE_STATUS__HEAD_OID_VALID);
 
-		submodule->flags = submodule->flags &
-			~(GIT_SUBMODULE_STATUS_IN_HEAD |
-			  GIT_SUBMODULE_STATUS__HEAD_OID_VALID);
+	/* if we can't look up file in current head, then done */
+	if (git_repository_head_tree(&head, submodule->repo) < 0 ||
+		git_tree_entry_bypath(&te, head, submodule->path) < 0)
+		giterr_clear();
+	else
+		submodule_update_from_head_data(submodule, te->attr, &te->oid);
 
-		if (!(error = git_tree_entry_bypath(&te, head, submodule->path))) {
+	git_tree_entry_free(te);
+	git_tree_free(head);
+	return 0;
+}
 
-			if (S_ISGITLINK(te->attr)) {
-				error = submodule_load_from_head(repo, submodule->path, &te->oid);
-			} else {
-				submodule_mode_mismatch(
-					repo, submodule->path,
-					GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE);
-			}
+int git_submodule_reload(git_submodule *submodule)
+{
+	int error = 0;
+	git_config_backend *mods;
 
-			git_tree_entry_free(te);
-		}
-		else if (error == GIT_ENOTFOUND) {
-			giterr_clear();
-			error = 0;
-		}
+	assert(submodule);
 
-		git_tree_free(head);
-	}
+	/* refresh index data */
 
-	if (error < 0)
-		return error;
+	if (submodule_update_index(submodule) < 0)
+		return -1;
+
+	/* refresh HEAD tree data */
+
+	if (submodule_update_head(submodule) < 0)
+		return -1;
 
 	/* refresh config data */
 
-	if ((mods = open_gitmodules(repo, false, NULL)) != NULL) {
+	mods = open_gitmodules(submodule->repo, false, NULL);
+	if (mods != NULL) {
 		git_buf path = GIT_BUF_INIT;
 
 		git_buf_sets(&path, "submodule\\.");
@@ -797,7 +847,7 @@ int git_submodule_reload(git_submodule *submodule)
 			error = -1;
 		else
 			error = git_config_file_foreach_match(
-				mods, path.ptr, submodule_load_from_config, repo);
+				mods, path.ptr, submodule_load_from_config, submodule->repo);
 
 		git_buf_free(&path);
 		git_config_file_free(mods);
@@ -816,38 +866,90 @@ int git_submodule_reload(git_submodule *submodule)
 	return error;
 }
 
-int git_submodule_status(
-	unsigned int *status,
-	git_submodule *submodule)
+static void submodule_copy_oid_maybe(
+	git_oid *tgt, const git_oid *src, bool valid)
 {
-	int error = 0;
-	unsigned int status_val;
+	if (tgt) {
+		if (valid)
+			memcpy(tgt, src, sizeof(*tgt));
+		else
+			memset(tgt, 0, sizeof(*tgt));
+	}
+}
 
-	assert(status && submodule);
+int git_submodule__status(
+	unsigned int *out_status,
+	git_oid *out_head_id,
+	git_oid *out_index_id,
+	git_oid *out_wd_id,
+	git_submodule *sm,
+	git_submodule_ignore_t ign)
+{
+	unsigned int status;
+	git_repository *smrepo = NULL;
 
-	status_val = GIT_SUBMODULE_STATUS__CLEAR_INTERNAL(submodule->flags);
+	if (ign < GIT_SUBMODULE_IGNORE_NONE)
+		ign = sm->ignore;
 
-	if (submodule->ignore != GIT_SUBMODULE_IGNORE_ALL) {
-		if (!(error = submodule_index_status(&status_val, submodule)))
-			error = submodule_wd_status(&status_val, submodule);
+	/* only return location info if ignore == all */
+	if (ign == GIT_SUBMODULE_IGNORE_ALL) {
+		*out_status = (sm->flags & GIT_SUBMODULE_STATUS__IN_FLAGS);
+		return 0;
 	}
 
-	*status = status_val;
+	/* refresh the index OID */
+	if (submodule_update_index(sm) < 0)
+		return -1;
 
-	return error;
+	/* refresh the HEAD OID */
+	if (submodule_update_head(sm) < 0)
+		return -1;
+
+	/* for ignore == dirty, don't scan the working directory */
+	if (ign == GIT_SUBMODULE_IGNORE_DIRTY) {
+		/* git_submodule_open_bare will load WD OID data */
+		if (git_submodule_open_bare(&smrepo, sm) < 0)
+			giterr_clear();
+		else
+			git_repository_free(smrepo);
+		smrepo = NULL;
+	} else if (git_submodule_open(&smrepo, sm) < 0) {
+		giterr_clear();
+		smrepo = NULL;
+	}
+
+	status = GIT_SUBMODULE_STATUS__CLEAR_INTERNAL(sm->flags);
+
+	submodule_get_index_status(&status, sm);
+	submodule_get_wd_status(&status, sm, smrepo, ign);
+
+	git_repository_free(smrepo);
+
+	*out_status = status;
+
+	submodule_copy_oid_maybe(out_head_id, &sm->head_oid,
+		(sm->flags & GIT_SUBMODULE_STATUS__HEAD_OID_VALID) != 0);
+	submodule_copy_oid_maybe(out_index_id, &sm->index_oid,
+		(sm->flags & GIT_SUBMODULE_STATUS__INDEX_OID_VALID) != 0);
+	submodule_copy_oid_maybe(out_wd_id, &sm->wd_oid,
+		(sm->flags & GIT_SUBMODULE_STATUS__WD_OID_VALID) != 0);
+
+	return 0;
 }
 
-int git_submodule_location(
-	unsigned int *location_status,
-	git_submodule *submodule)
+int git_submodule_status(unsigned int *status, git_submodule *sm)
 {
-	assert(location_status && submodule);
+	assert(status && sm);
 
-	*location_status = submodule->flags &
-		(GIT_SUBMODULE_STATUS_IN_HEAD | GIT_SUBMODULE_STATUS_IN_INDEX |
-		 GIT_SUBMODULE_STATUS_IN_CONFIG | GIT_SUBMODULE_STATUS_IN_WD);
+	return git_submodule__status(status, NULL, NULL, NULL, sm, 0);
+}
 
-	return 0;
+int git_submodule_location(unsigned int *location, git_submodule *sm)
+{
+	assert(location && sm);
+
+	return git_submodule__status(
+		location, NULL, NULL, NULL, sm, GIT_SUBMODULE_IGNORE_ALL);
 }
 
 
@@ -857,54 +959,50 @@ int git_submodule_location(
 
 static git_submodule *submodule_alloc(git_repository *repo, const char *name)
 {
+	size_t namelen;
 	git_submodule *sm;
 
-	if (!name || !strlen(name)) {
+	if (!name || !(namelen = strlen(name))) {
 		giterr_set(GITERR_SUBMODULE, "Invalid submodule name");
 		return NULL;
 	}
 
 	sm = git__calloc(1, sizeof(git_submodule));
 	if (sm == NULL)
-		goto fail;
+		return NULL;
 
-	sm->path = sm->name = git__strdup(name);
-	if (!sm->name)
-		goto fail;
+	sm->name = sm->path = git__strdup(name);
+	if (!sm->name) {
+		git__free(sm);
+		return NULL;
+	}
 
-	sm->owner = repo;
-	sm->refcount = 1;
+	GIT_REFCOUNT_INC(sm);
+	sm->ignore = sm->ignore_default = GIT_SUBMODULE_IGNORE_NONE;
+	sm->update = sm->update_default = GIT_SUBMODULE_UPDATE_CHECKOUT;
+	sm->repo   = repo;
 
 	return sm;
-
-fail:
-	submodule_release(sm, 0);
-	return NULL;
 }
 
-static void submodule_release(git_submodule *sm, int decr)
+static void submodule_release(git_submodule *sm)
 {
 	if (!sm)
 		return;
 
-	sm->refcount -= decr;
-
-	if (sm->refcount == 0) {
-		if (sm->name != sm->path) {
-			git__free(sm->path);
-			sm->path = NULL;
-		}
-
-		git__free(sm->name);
-		sm->name = NULL;
-
-		git__free(sm->url);
-		sm->url = NULL;
-
-		sm->owner = NULL;
+	if (sm->path != sm->name)
+		git__free(sm->path);
+	git__free(sm->name);
+	git__free(sm->url);
+	git__memzero(sm, sizeof(*sm));
+	git__free(sm);
+}
 
-		git__free(sm);
-	}
+void git_submodule_free(git_submodule *sm)
+{
+	if (!sm)
+		return;
+	GIT_REFCOUNT_DEC(sm, submodule_release);
 }
 
 static int submodule_get(
@@ -927,6 +1025,7 @@ static int submodule_get(
 
 	if (!git_strmap_valid_index(smcfg, pos)) {
 		sm = submodule_alloc(repo, name);
+		GITERR_CHECK_ALLOC(sm);
 
 		/* insert value at name - if another thread beats us to it, then use
 		 * their record and release our own.
@@ -934,10 +1033,10 @@ static int submodule_get(
 		pos = kh_put(str, smcfg, sm->name, &error);
 
 		if (error < 0) {
-			submodule_release(sm, 1);
+			git_submodule_free(sm);
 			sm = NULL;
 		} else if (error == 0) {
-			submodule_release(sm, 1);
+			git_submodule_free(sm);
 			sm = git_strmap_value_at(smcfg, pos);
 		} else {
 			git_strmap_set_value_at(smcfg, pos, sm);
@@ -951,50 +1050,41 @@ static int submodule_get(
 	return (sm != NULL) ? 0 : -1;
 }
 
-static int submodule_load_from_index(
-	git_repository *repo, const git_index_entry *entry)
+static int submodule_config_error(const char *property, const char *value)
 {
-	git_submodule *sm;
+	giterr_set(GITERR_INVALID,
+		"Invalid value for submodule '%s' property: '%s'", property, value);
+	return -1;
+}
 
-	if (submodule_get(&sm, repo, entry->path, NULL) < 0)
-		return -1;
+int git_submodule_parse_ignore(git_submodule_ignore_t *out, const char *value)
+{
+	int val;
 
-	if (sm->flags & GIT_SUBMODULE_STATUS_IN_INDEX) {
-		sm->flags |= GIT_SUBMODULE_STATUS__INDEX_MULTIPLE_ENTRIES;
-		return 0;
+	if (git_config_lookup_map_value(
+			&val, _sm_ignore_map, ARRAY_SIZE(_sm_ignore_map), value) < 0) {
+		*out = GIT_SUBMODULE_IGNORE_NONE;
+		return submodule_config_error("ignore", value);
 	}
 
-	sm->flags |= GIT_SUBMODULE_STATUS_IN_INDEX;
-
-	git_oid_cpy(&sm->index_oid, &entry->oid);
-	sm->flags |= GIT_SUBMODULE_STATUS__INDEX_OID_VALID;
-
+	*out = (git_submodule_ignore_t)val;
 	return 0;
 }
 
-static int submodule_load_from_head(
-	git_repository *repo, const char *path, const git_oid *oid)
+int git_submodule_parse_update(git_submodule_update_t *out, const char *value)
 {
-	git_submodule *sm;
-
-	if (submodule_get(&sm, repo, path, NULL) < 0)
-		return -1;
-
-	sm->flags |= GIT_SUBMODULE_STATUS_IN_HEAD;
+	int val;
 
-	git_oid_cpy(&sm->head_oid, oid);
-	sm->flags |= GIT_SUBMODULE_STATUS__HEAD_OID_VALID;
+	if (git_config_lookup_map_value(
+			&val, _sm_update_map, ARRAY_SIZE(_sm_update_map), value) < 0) {
+		*out = GIT_SUBMODULE_UPDATE_CHECKOUT;
+		return submodule_config_error("update", value);
+	}
 
+	*out = (git_submodule_update_t)val;
 	return 0;
 }
 
-static int submodule_config_error(const char *property, const char *value)
-{
-	giterr_set(GITERR_INVALID,
-		"Invalid value for submodule '%s' property: '%s'", property, value);
-	return -1;
-}
-
 static int submodule_load_from_config(
 	const git_config_entry *entry, void *data)
 {
@@ -1012,8 +1102,10 @@ static int submodule_load_from_config(
 
 	namestart = key + strlen("submodule.");
 	property  = strrchr(namestart, '.');
-	if (property == NULL)
+
+	if (!property || (property == namestart))
 		return 0;
+
 	property++;
 	is_path = (strcasecmp(property, "path") == 0);
 
@@ -1047,11 +1139,11 @@ static int submodule_load_from_config(
 		git_strmap_insert2(smcfg, alternate, sm, old_sm, error);
 
 		if (error >= 0)
-			sm->refcount++; /* inserted under a new key */
+			GIT_REFCOUNT_INC(sm); /* inserted under a new key */
 
 		/* if we replaced an old module under this key, release the old one */
 		if (old_sm && ((git_submodule *)old_sm) != sm) {
-			submodule_release(old_sm, 1);
+			git_submodule_free(old_sm);
 			/* TODO: log warning about multiple submodules with same path */
 		}
 	}
@@ -1076,22 +1168,18 @@ static int submodule_load_from_config(
 			return -1;
 	}
 	else if (strcasecmp(property, "update") == 0) {
-		int val;
-		if (git_config_lookup_map_value(
-			&val, _sm_update_map, ARRAY_SIZE(_sm_update_map), value) < 0)
-			return submodule_config_error("update", value);
-		sm->update_default = sm->update = (git_submodule_update_t)val;
+		if (git_submodule_parse_update(&sm->update, value) < 0)
+			return -1;
+		sm->update_default = sm->update;
 	}
 	else if (strcasecmp(property, "fetchRecurseSubmodules") == 0) {
 		if (git__parse_bool(&sm->fetch_recurse, value) < 0)
 			return submodule_config_error("fetchRecurseSubmodules", value);
 	}
 	else if (strcasecmp(property, "ignore") == 0) {
-		int val;
-		if (git_config_lookup_map_value(
-			&val, _sm_ignore_map, ARRAY_SIZE(_sm_ignore_map), value) < 0)
-			return submodule_config_error("ignore", value);
-		sm->ignore_default = sm->ignore = (git_submodule_ignore_t)val;
+		if (git_submodule_parse_ignore(&sm->ignore, value) < 0)
+			return -1;
+		sm->ignore_default = sm->ignore;
 	}
 	/* ignore other unknown submodule properties */
 
@@ -1101,13 +1189,15 @@ static int submodule_load_from_config(
 static int submodule_load_from_wd_lite(
 	git_submodule *sm, const char *name, void *payload)
 {
-	git_repository *repo = git_submodule_owner(sm);
 	git_buf path = GIT_BUF_INIT;
 
 	GIT_UNUSED(name);
 	GIT_UNUSED(payload);
 
-	if (git_buf_joinpath(&path, git_repository_workdir(repo), sm->path) < 0)
+	if (git_repository_is_bare(sm->repo))
+		return 0;
+
+	if (git_buf_joinpath(&path, git_repository_workdir(sm->repo), sm->path) < 0)
 		return -1;
 
 	if (git_path_isdir(path.ptr))
@@ -1121,18 +1211,6 @@ static int submodule_load_from_wd_lite(
 	return 0;
 }
 
-static void submodule_mode_mismatch(
-	git_repository *repo, const char *path, unsigned int flag)
-{
-	khiter_t pos = git_strmap_lookup_index(repo->submodules, path);
-
-	if (git_strmap_valid_index(repo->submodules, pos)) {
-		git_submodule *sm = git_strmap_value_at(repo->submodules, pos);
-
-		sm->flags |= flag;
-	}
-}
-
 static int load_submodule_config_from_index(
 	git_repository *repo, git_oid *gitmodules_oid)
 {
@@ -1146,18 +1224,21 @@ static int load_submodule_config_from_index(
 		return error;
 
 	while (!(error = git_iterator_advance(&entry, i))) {
-
-		if (S_ISGITLINK(entry->mode)) {
-			error = submodule_load_from_index(repo, entry);
-			if (error < 0)
-				break;
-		} else {
-			submodule_mode_mismatch(
-				repo, entry->path, GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE);
-
-			if (strcmp(entry->path, GIT_MODULES_FILE) == 0)
-				git_oid_cpy(gitmodules_oid, &entry->oid);
-		}
+		khiter_t pos = git_strmap_lookup_index(repo->submodules, entry->path);
+		git_submodule *sm;
+
+		if (git_strmap_valid_index(repo->submodules, pos)) {
+			sm = git_strmap_value_at(repo->submodules, pos);
+
+			if (S_ISGITLINK(entry->mode))
+				submodule_update_from_index_entry(sm, entry);
+			else
+				sm->flags |= GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE;
+		} else if (S_ISGITLINK(entry->mode)) {
+			if (!submodule_get(&sm, repo, entry->path, NULL))
+				submodule_update_from_index_entry(sm, entry);
+		} else if (strcmp(entry->path, GIT_MODULES_FILE) == 0)
+			git_oid_cpy(gitmodules_oid, &entry->oid);
 	}
 
 	if (error == GIT_ITEROVER)
@@ -1176,8 +1257,11 @@ static int load_submodule_config_from_head(
 	git_iterator *i;
 	const git_index_entry *entry;
 
-	if ((error = git_repository_head_tree(&head, repo)) < 0)
-		return error;
+	/* if we can't look up current head, then there's no submodule in it */
+	if (git_repository_head_tree(&head, repo) < 0) {
+		giterr_clear();
+		return 0;
+	}
 
 	if ((error = git_iterator_for_tree(&i, head, 0, NULL, NULL)) < 0) {
 		git_tree_free(head);
@@ -1185,18 +1269,24 @@ static int load_submodule_config_from_head(
 	}
 
 	while (!(error = git_iterator_advance(&entry, i))) {
-
-		if (S_ISGITLINK(entry->mode)) {
-			error = submodule_load_from_head(repo, entry->path, &entry->oid);
-			if (error < 0)
-				break;
-		} else {
-			submodule_mode_mismatch(
-				repo, entry->path, GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE);
-
-			if (strcmp(entry->path, GIT_MODULES_FILE) == 0 &&
-				git_oid_iszero(gitmodules_oid))
-				git_oid_cpy(gitmodules_oid, &entry->oid);
+		khiter_t pos = git_strmap_lookup_index(repo->submodules, entry->path);
+		git_submodule *sm;
+
+		if (git_strmap_valid_index(repo->submodules, pos)) {
+			sm = git_strmap_value_at(repo->submodules, pos);
+
+			if (S_ISGITLINK(entry->mode))
+				submodule_update_from_head_data(
+					sm, entry->mode, &entry->oid);
+			else
+				sm->flags |= GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE;
+		} else if (S_ISGITLINK(entry->mode)) {
+			if (!submodule_get(&sm, repo, entry->path, NULL))
+				submodule_update_from_head_data(
+					sm, entry->mode, &entry->oid);
+		} else if (strcmp(entry->path, GIT_MODULES_FILE) == 0 &&
+				   git_oid_iszero(gitmodules_oid)) {
+			git_oid_cpy(gitmodules_oid, &entry->oid);
 		}
 	}
 
@@ -1381,7 +1471,7 @@ static int submodule_update_config(
 
 	assert(submodule);
 
-	error = git_repository_config__weakptr(&config, submodule->owner);
+	error = git_repository_config__weakptr(&config, submodule->repo);
 	if (error < 0)
 		return error;
 
@@ -1409,11 +1499,13 @@ cleanup:
 	return error;
 }
 
-static int submodule_index_status(unsigned int *status, git_submodule *sm)
+static void submodule_get_index_status(unsigned int *status, git_submodule *sm)
 {
 	const git_oid *head_oid  = git_submodule_head_id(sm);
 	const git_oid *index_oid = git_submodule_index_id(sm);
 
+	*status = *status & ~GIT_SUBMODULE_STATUS__INDEX_FLAGS;
+
 	if (!head_oid) {
 		if (index_oid)
 			*status |= GIT_SUBMODULE_STATUS_INDEX_ADDED;
@@ -1422,27 +1514,22 @@ static int submodule_index_status(unsigned int *status, git_submodule *sm)
 		*status |= GIT_SUBMODULE_STATUS_INDEX_DELETED;
 	else if (!git_oid_equal(head_oid, index_oid))
 		*status |= GIT_SUBMODULE_STATUS_INDEX_MODIFIED;
-
-	return 0;
 }
 
-static int submodule_wd_status(unsigned int *status, git_submodule *sm)
+static void submodule_get_wd_status(
+	unsigned int *status,
+	git_submodule *sm,
+	git_repository *sm_repo,
+	git_submodule_ignore_t ign)
 {
-	int error = 0;
-	const git_oid *wd_oid, *index_oid;
-	git_repository *sm_repo = NULL;
-
-	/* open repo now if we need it (so wd_id() call won't reopen) */
-	if ((sm->ignore == GIT_SUBMODULE_IGNORE_NONE ||
-		 sm->ignore == GIT_SUBMODULE_IGNORE_UNTRACKED) &&
-		(sm->flags & GIT_SUBMODULE_STATUS_IN_WD) != 0)
-	{
-		if ((error = git_submodule_open(&sm_repo, sm)) < 0)
-			return error;
-	}
+	const git_oid *index_oid = git_submodule_index_id(sm);
+	const git_oid *wd_oid =
+		(sm->flags & GIT_SUBMODULE_STATUS__WD_OID_VALID) ? &sm->wd_oid : NULL;
+	git_tree *sm_head = NULL;
+	git_diff_options opt = GIT_DIFF_OPTIONS_INIT;
+	git_diff_list *diff;
 
-	index_oid = git_submodule_index_id(sm);
-	wd_oid    = git_submodule_wd_id(sm);
+	*status = *status & ~GIT_SUBMODULE_STATUS__WD_FLAGS;
 
 	if (!index_oid) {
 		if (wd_oid)
@@ -1458,59 +1545,49 @@ static int submodule_wd_status(unsigned int *status, git_submodule *sm)
 	else if (!git_oid_equal(index_oid, wd_oid))
 		*status |= GIT_SUBMODULE_STATUS_WD_MODIFIED;
 
-	if (sm_repo != NULL) {
-		git_tree *sm_head;
-		git_diff_options opt = GIT_DIFF_OPTIONS_INIT;
-		git_diff_list *diff;
-
-		/* the diffs below could be optimized with an early termination
-		 * option to the git_diff functions, but for now this is sufficient
-		 * (and certainly no worse that what core git does).
-		 */
-
-		/* perform head-to-index diff on submodule */
-
-		if ((error = git_repository_head_tree(&sm_head, sm_repo)) < 0)
-			return error;
+	/* if we have no repo, then we're done */
+	if (!sm_repo)
+		return;
 
-		if (sm->ignore == GIT_SUBMODULE_IGNORE_NONE)
-			opt.flags |= GIT_DIFF_INCLUDE_UNTRACKED;
+	/* the diffs below could be optimized with an early termination
+	 * option to the git_diff functions, but for now this is sufficient
+	 * (and certainly no worse that what core git does).
+	 */
 
-		error = git_diff_tree_to_index(&diff, sm_repo, sm_head, NULL, &opt);
+	if (ign == GIT_SUBMODULE_IGNORE_NONE)
+		opt.flags |= GIT_DIFF_INCLUDE_UNTRACKED;
 
-		if (!error) {
+	/* if we don't have an orphaned head, check diff with index */
+	if (git_repository_head_tree(&sm_head, sm_repo) < 0)
+		giterr_clear();
+	else {
+		/* perform head to index diff on submodule */
+		if (git_diff_tree_to_index(&diff, sm_repo, sm_head, NULL, &opt) < 0)
+			giterr_clear();
+		else {
 			if (git_diff_num_deltas(diff) > 0)
 				*status |= GIT_SUBMODULE_STATUS_WD_INDEX_MODIFIED;
-
 			git_diff_list_free(diff);
 			diff = NULL;
 		}
 
 		git_tree_free(sm_head);
+	}
 
-		if (error < 0)
-			return error;
-
-		/* perform index-to-workdir diff on submodule */
-
-		error = git_diff_index_to_workdir(&diff, sm_repo, NULL, &opt);
-
-		if (!error) {
-			size_t untracked =
-				git_diff_num_deltas_of_type(diff, GIT_DELTA_UNTRACKED);
-
-			if (untracked > 0)
-				*status |= GIT_SUBMODULE_STATUS_WD_UNTRACKED;
+	/* perform index-to-workdir diff on submodule */
+	if (git_diff_index_to_workdir(&diff, sm_repo, NULL, &opt) < 0)
+		giterr_clear();
+	else {
+		size_t untracked =
+			git_diff_num_deltas_of_type(diff, GIT_DELTA_UNTRACKED);
 
-			if (git_diff_num_deltas(diff) != untracked)
-				*status |= GIT_SUBMODULE_STATUS_WD_WD_MODIFIED;
+		if (untracked > 0)
+			*status |= GIT_SUBMODULE_STATUS_WD_UNTRACKED;
 
-			git_diff_list_free(diff);
-			diff = NULL;
-		}
+		if (git_diff_num_deltas(diff) != untracked)
+			*status |= GIT_SUBMODULE_STATUS_WD_WD_MODIFIED;
 
-		git_repository_free(sm_repo);
+		git_diff_list_free(diff);
+		diff = NULL;
 	}
-
-	return error;
 }
diff --git a/src/submodule.h b/src/submodule.h
index ba8e2518e..b05937503 100644
--- a/src/submodule.h
+++ b/src/submodule.h
@@ -7,6 +7,10 @@
 #ifndef INCLUDE_submodule_h__
 #define INCLUDE_submodule_h__
 
+#include "git2/submodule.h"
+#include "git2/repository.h"
+#include "fileops.h"
+
 /* Notes:
  *
  * Submodule information can be in four places: the index, the config files
@@ -44,44 +48,51 @@
  * an entry for every submodule found in the HEAD and index, and for every
  * submodule described in .gitmodules.  The fields are as follows:
  *
- * - `owner` is the git_repository containing this submodule
+ * - `rc` tracks the refcount of how many hash table entries in the
+ *   git_submodule_cache there are for this submodule.  It only comes into
+ *   play if the name and path of the submodule differ.
+ *
  * - `name` is the name of the submodule from .gitmodules.
  * - `path` is the path to the submodule from the repo root.  It is almost
  *    always the same as `name`.
  * - `url` is the url for the submodule.
- * - `tree_oid` is the SHA1 for the submodule path in the repo HEAD.
- * - `index_oid` is the SHA1 for the submodule recorded in the index.
- * - `workdir_oid` is the SHA1 for the HEAD of the checked out submodule.
  * - `update` is a git_submodule_update_t value - see gitmodules(5) update.
+ * - `update_default` is the update value from the config
  * - `ignore` is a git_submodule_ignore_t value - see gitmodules(5) ignore.
+ * - `ignore_default` is the ignore value from the config
  * - `fetch_recurse` is 0 or 1 - see gitmodules(5) fetchRecurseSubmodules.
- * - `refcount` tracks how many hashmap entries there are for this submodule.
- *   It only comes into play if the name and path of the submodule differ.
- * - `flags` is for internal use, tracking where this submodule has been
- *   found (head, index, config, workdir) and other misc info about it.
+ *
+ * - `repo` is the parent repository that contains this submodule.
+ * - `flags` after for internal use, tracking where this submodule has been
+ *   found (head, index, config, workdir) and known status info, etc.
+ * - `head_oid` is the SHA1 for the submodule path in the repo HEAD.
+ * - `index_oid` is the SHA1 for the submodule recorded in the index.
+ * - `wd_oid` is the SHA1 for the HEAD of the checked out submodule.
  *
  * If the submodule has been added to .gitmodules but not yet git added,
- * then the `index_oid` will be valid and zero.  If the submodule has been
- * deleted, but the delete has not been committed yet, then the `index_oid`
- * will be set, but the `url` will be NULL.
+ * then the `index_oid` will be zero but still marked valid.  If the
+ * submodule has been deleted, but the delete has not been committed yet,
+ * then the `index_oid` will be set, but the `url` will be NULL.
  */
 struct git_submodule {
-	git_repository *owner;
+	git_refcount rc;
+
+	/* information from config */
 	char *name;
-	char *path; /* important: may point to same string data as "name" */
+	char *path; /* important: may just point to "name" string */
 	char *url;
-	uint32_t flags;
-	git_oid head_oid;
-	git_oid index_oid;
-	git_oid wd_oid;
-	/* information from config */
 	git_submodule_update_t update;
 	git_submodule_update_t update_default;
 	git_submodule_ignore_t ignore;
 	git_submodule_ignore_t ignore_default;
 	int fetch_recurse;
+
 	/* internal information */
-	int refcount;
+	git_repository *repo;
+	uint32_t flags;
+	git_oid head_oid;
+	git_oid index_oid;
+	git_oid wd_oid;
 };
 
 /* Additional flags on top of public GIT_SUBMODULE_STATUS values */
@@ -99,4 +110,29 @@ enum {
 #define GIT_SUBMODULE_STATUS__CLEAR_INTERNAL(S) \
 	((S) & ~(0xFFFFFFFFu << 20))
 
+/* Internal status fn returns status and optionally the various OIDs */
+extern int git_submodule__status(
+	unsigned int *out_status,
+	git_oid *out_head_id,
+	git_oid *out_index_id,
+	git_oid *out_wd_id,
+	git_submodule *sm,
+	git_submodule_ignore_t ign);
+
+/* Open submodule repository as bare repo for quick HEAD check, etc. */
+extern int git_submodule_open_bare(
+	git_repository **repo,
+	git_submodule *submodule);
+
+/* Release reference to submodule object - not currently for external use */
+extern void git_submodule_free(git_submodule *sm);
+
+extern int git_submodule_parse_ignore(
+	git_submodule_ignore_t *out, const char *value);
+extern int git_submodule_parse_update(
+	git_submodule_update_t *out, const char *value);
+
+extern const char *git_submodule_ignore_to_str(git_submodule_ignore_t);
+extern const char *git_submodule_update_to_str(git_submodule_update_t);
+
 #endif
diff --git a/src/tag.c b/src/tag.c
index 71f4c1eb1..31a3c8b80 100644
--- a/src/tag.c
+++ b/src/tag.c
@@ -366,10 +366,10 @@ int git_tag_create_frombuffer(git_oid *oid, git_repository *repo, const char *bu
 	if (git_odb_open_wstream(&stream, odb, strlen(buffer), GIT_OBJ_TAG) < 0)
 		return -1;
 
-	stream->write(stream, buffer, strlen(buffer));
+	git_odb_stream_write(stream, buffer, strlen(buffer));
 
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
+	error = git_odb_stream_finalize_write(oid, stream);
+	git_odb_stream_free(stream);
 
 	if (error < 0) {
 		git_buf_free(&ref_name);
diff --git a/src/thread-utils.h b/src/thread-utils.h
index 83148188d..914c1357d 100644
--- a/src/thread-utils.h
+++ b/src/thread-utils.h
@@ -38,15 +38,11 @@ typedef git_atomic git_atomic_ssize;
 
 #endif
 
-GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
-{
-	a->val = val;
-}
-
 #ifdef GIT_THREADS
 
 #define git_thread pthread_t
-#define git_thread_create(thread, attr, start_routine, arg) pthread_create(thread, attr, start_routine, arg)
+#define git_thread_create(thread, attr, start_routine, arg) \
+	pthread_create(thread, attr, start_routine, arg)
 #define git_thread_kill(thread) pthread_cancel(thread)
 #define git_thread_exit(status)	pthread_exit(status)
 #define git_thread_join(id, status) pthread_join(id, status)
@@ -66,6 +62,41 @@ GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
 #define git_cond_signal(c)	pthread_cond_signal(c)
 #define git_cond_broadcast(c)	pthread_cond_broadcast(c)
 
+/* Pthread (-ish) rwlock
+ *
+ * This differs from normal pthreads rwlocks in two ways:
+ * 1. Separate APIs for releasing read locks and write locks (as
+ *    opposed to the pure POSIX API which only has one unlock fn)
+ * 2. You should not use recursive read locks (i.e. grabbing a read
+ *    lock in a thread that already holds a read lock) because the
+ *    Windows implementation doesn't support it
+ */
+#define git_rwlock pthread_rwlock_t
+#define git_rwlock_init(a)		pthread_rwlock_init(a, NULL)
+#define git_rwlock_rdlock(a)	pthread_rwlock_rdlock(a)
+#define git_rwlock_rdunlock(a)	pthread_rwlock_rdunlock(a)
+#define git_rwlock_wrlock(a)	pthread_rwlock_wrlock(a)
+#define git_rwlock_wrunlock(a)	pthread_rwlock_wrunlock(a)
+#define git_rwlock_free(a)		pthread_rwlock_destroy(a)
+#define GIT_RWLOCK_STATIC_INIT	PTHREAD_RWLOCK_INITIALIZER
+
+#ifndef GIT_WIN32
+#define pthread_rwlock_rdunlock pthread_rwlock_unlock
+#define pthread_rwlock_wrunlock pthread_rwlock_unlock
+#endif
+
+
+GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
+{
+#if defined(GIT_WIN32)
+	InterlockedExchange(&a->val, (LONG)val);
+#elif defined(__GNUC__)
+	__sync_lock_test_and_set(&a->val, val);
+#else
+#	error "Unsupported architecture for atomic operations"
+#endif
+}
+
 GIT_INLINE(int) git_atomic_inc(git_atomic *a)
 {
 #if defined(GIT_WIN32)
@@ -100,11 +131,11 @@ GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 }
 
 GIT_INLINE(void *) git___compare_and_swap(
-	volatile void **ptr, void *oldval, void *newval)
+	void * volatile *ptr, void *oldval, void *newval)
 {
 	volatile void *foundval;
 #if defined(GIT_WIN32)
-	foundval = InterlockedCompareExchangePointer(ptr, newval, oldval);
+	foundval = InterlockedCompareExchangePointer((volatile PVOID *)ptr, newval, oldval);
 #elif defined(__GNUC__)
 	foundval = __sync_val_compare_and_swap(ptr, oldval, newval);
 #else
@@ -113,6 +144,16 @@ GIT_INLINE(void *) git___compare_and_swap(
 	return (foundval == oldval) ? oldval : newval;
 }
 
+GIT_INLINE(volatile void *) git___swap(
+	void * volatile *ptr, void *newval)
+{
+#if defined(GIT_WIN32)
+	return InterlockedExchangePointer(ptr, newval);
+#else
+	return __sync_lock_test_and_set(ptr, newval);
+#endif
+}
+
 #ifdef GIT_ARCH_64
 
 GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
@@ -131,7 +172,7 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 #else
 
 #define git_thread unsigned int
-#define git_thread_create(thread, attr, start_routine, arg) (void)0
+#define git_thread_create(thread, attr, start_routine, arg) 0
 #define git_thread_kill(thread) (void)0
 #define git_thread_exit(status) (void)0
 #define git_thread_join(id, status) (void)0
@@ -151,6 +192,22 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 #define git_cond_signal(c) (void)0
 #define git_cond_broadcast(c) (void)0
 
+/* Pthreads rwlock */
+#define git_rwlock unsigned int
+#define git_rwlock_init(a)		0
+#define git_rwlock_rdlock(a)	0
+#define git_rwlock_rdunlock(a)	(void)0
+#define git_rwlock_wrlock(a)	0
+#define git_rwlock_wrunlock(a)	(void)0
+#define git_rwlock_free(a)		(void)0
+#define GIT_RWLOCK_STATIC_INIT	0
+
+
+GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
+{
+	a->val = val;
+}
+
 GIT_INLINE(int) git_atomic_inc(git_atomic *a)
 {
 	return ++a->val;
@@ -168,7 +225,7 @@ GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 }
 
 GIT_INLINE(void *) git___compare_and_swap(
-	volatile void **ptr, void *oldval, void *newval)
+	void * volatile *ptr, void *oldval, void *newval)
 {
 	if (*ptr == oldval)
 		*ptr = newval;
@@ -177,6 +234,14 @@ GIT_INLINE(void *) git___compare_and_swap(
 	return oldval;
 }
 
+GIT_INLINE(volatile void *) git___swap(
+	void * volatile *ptr, void *newval)
+{
+	volatile void *old = *ptr;
+	*ptr = newval;
+	return old;
+}
+
 #ifdef GIT_ARCH_64
 
 GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
@@ -189,13 +254,18 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 
 #endif
 
+GIT_INLINE(int) git_atomic_get(git_atomic *a)
+{
+	return (int)a->val;
+}
+
 /* Atomically replace oldval with newval
  * @return oldval if it was replaced or newval if it was not
  */
 #define git__compare_and_swap(P,O,N) \
-	git___compare_and_swap((volatile void **)P, O, N)
+	git___compare_and_swap((void * volatile *)P, O, N)
 
-#define git__swap(ptr, val) git__compare_and_swap(&ptr, ptr, val)
+#define git__swap(ptr, val) (void *)git___swap((void * volatile *)&ptr, val)
 
 extern int git_online_cpus(void);
 
diff --git a/src/transport.c b/src/transport.c
index 37c244c97..354789db1 100644
--- a/src/transport.c
+++ b/src/transport.c
@@ -73,7 +73,7 @@ static int transport_find_fn(const char *url, git_transport_cb *callback, void *
 	/* It could be a SSH remote path. Check to see if there's a :
 	 * SSH is an unsupported transport mechanism in this version of libgit2 */
 	if (!definition && strrchr(url, ':'))
-		definition = &dummy_transport_definition; 
+		definition = &dummy_transport_definition;
 #else
 	/* For other systems, perform the SSH check first, to avoid going to the
 	 * filesystem if it is not necessary */
@@ -97,7 +97,7 @@ static int transport_find_fn(const char *url, git_transport_cb *callback, void *
 
 	*callback = definition->fn;
 	*param = definition->param;
-	
+
 	return 0;
 }
 
diff --git a/src/transports/cred.c b/src/transports/cred.c
index 4916c6e18..35aaf4f91 100644
--- a/src/transports/cred.c
+++ b/src/transports/cred.c
@@ -9,19 +9,45 @@
 #include "smart.h"
 #include "git2/cred_helpers.h"
 
+int git_cred_has_username(git_cred *cred)
+{
+	int ret = 0;
+
+	switch (cred->credtype) {
+	case GIT_CREDTYPE_USERPASS_PLAINTEXT: {
+		git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
+		ret = !!c->username;
+		break;
+	}
+	case GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE: {
+		git_cred_ssh_keyfile_passphrase *c = (git_cred_ssh_keyfile_passphrase *)cred;
+		ret = !!c->username;
+		break;
+	}
+	case GIT_CREDTYPE_SSH_PUBLICKEY: {
+		git_cred_ssh_publickey *c = (git_cred_ssh_publickey *)cred;
+		ret = !!c->username;
+		break;
+	}
+	}
+
+	return ret;
+}
+
 static void plaintext_free(struct git_cred *cred)
 {
 	git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
-	size_t pass_len = strlen(c->password);
 
 	git__free(c->username);
 
 	/* Zero the memory which previously held the password */
-	memset(c->password, 0x0, pass_len);
-	git__free(c->password);
-
-	memset(c, 0, sizeof(*c));
+	if (c->password) {
+		size_t pass_len = strlen(c->password);
+		git__memzero(c->password, pass_len);
+		git__free(c->password);
+	}
 
+	git__memzero(c, sizeof(*c));
 	git__free(c);
 }
 
@@ -32,8 +58,7 @@ int git_cred_userpass_plaintext_new(
 {
 	git_cred_userpass_plaintext *c;
 
-	if (!cred)
-		return -1;
+	assert(cred);
 
 	c = git__malloc(sizeof(git_cred_userpass_plaintext));
 	GITERR_CHECK_ALLOC(c);
@@ -59,31 +84,40 @@ int git_cred_userpass_plaintext_new(
 	return 0;
 }
 
-#ifdef GIT_SSH
 static void ssh_keyfile_passphrase_free(struct git_cred *cred)
 {
-	git_cred_ssh_keyfile_passphrase *c = (git_cred_ssh_keyfile_passphrase *)cred;
-	size_t pass_len = strlen(c->passphrase);
+	git_cred_ssh_keyfile_passphrase *c =
+		(git_cred_ssh_keyfile_passphrase *)cred;
 
-    if (c->publickey) {
-        git__free(c->publickey);
-    }
-    
+	git__free(c->username);
+	git__free(c->publickey);
 	git__free(c->privatekey);
 
-    if (c->passphrase) {
-        /* Zero the memory which previously held the passphrase */
-        memset(c->passphrase, 0x0, pass_len);
-        git__free(c->passphrase);
-    }
+	if (c->passphrase) {
+		/* Zero the memory which previously held the passphrase */
+		size_t pass_len = strlen(c->passphrase);
+		git__memzero(c->passphrase, pass_len);
+		git__free(c->passphrase);
+	}
 
-	memset(c, 0, sizeof(*c));
+	git__memzero(c, sizeof(*c));
+	git__free(c);
+}
 
+static void ssh_publickey_free(struct git_cred *cred)
+{
+	git_cred_ssh_publickey *c = (git_cred_ssh_publickey *)cred;
+
+	git__free(c->username);
+	git__free(c->publickey);
+
+	git__memzero(c, sizeof(*c));
 	git__free(c);
 }
 
 int git_cred_ssh_keyfile_passphrase_new(
 	git_cred **cred,
+	const char *username,
 	const char *publickey,
 	const char *privatekey,
 	const char *passphrase)
@@ -97,15 +131,20 @@ int git_cred_ssh_keyfile_passphrase_new(
 
 	c->parent.credtype = GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE;
 	c->parent.free = ssh_keyfile_passphrase_free;
-    
-    c->privatekey = git__strdup(privatekey);
+
+	if (username) {
+		c->username = git__strdup(username);
+		GITERR_CHECK_ALLOC(c->username);
+	}
+
+	c->privatekey = git__strdup(privatekey);
 	GITERR_CHECK_ALLOC(c->privatekey);
-    
-    if (publickey) {
+
+	if (publickey) {
 		c->publickey = git__strdup(publickey);
 		GITERR_CHECK_ALLOC(c->publickey);
 	}
-	
+
 	if (passphrase) {
 		c->passphrase = git__strdup(passphrase);
 		GITERR_CHECK_ALLOC(c->passphrase);
@@ -115,48 +154,40 @@ int git_cred_ssh_keyfile_passphrase_new(
 	return 0;
 }
 
-static void ssh_publickey_free(struct git_cred *cred)
-{
-	git_cred_ssh_publickey *c = (git_cred_ssh_publickey *)cred;
-
-    git__free(c->publickey);
-
-    c->sign_callback = NULL;
-    c->sign_data = NULL;
-    
-	memset(c, 0, sizeof(*c));
-
-	git__free(c);
-}
-
 int git_cred_ssh_publickey_new(
 	git_cred **cred,
+	const char *username,
 	const char *publickey,
-    size_t publickey_len,
-	LIBSSH2_USERAUTH_PUBLICKEY_SIGN_FUNC((*sign_callback)),
-    void *sign_data)
+	size_t publickey_len,
+	git_cred_sign_callback sign_callback,
+	void *sign_data)
 {
 	git_cred_ssh_publickey *c;
 
-	if (!cred)
-		return -1;
+	assert(cred);
 
-	c = git__malloc(sizeof(git_cred_ssh_publickey));
+	c = git__calloc(1, sizeof(git_cred_ssh_publickey));
 	GITERR_CHECK_ALLOC(c);
 
 	c->parent.credtype = GIT_CREDTYPE_SSH_PUBLICKEY;
 	c->parent.free = ssh_publickey_free;
-    
-    c->publickey = git__malloc(publickey_len);
-	GITERR_CHECK_ALLOC(c->publickey);
-	
-    memcpy(c->publickey, publickey, publickey_len);
-    
-    c->publickey_len = publickey_len;
-    c->sign_callback = sign_callback;
-    c->sign_data = sign_data;
+
+	if (username) {
+		c->username = git__strdup(username);
+		GITERR_CHECK_ALLOC(c->username);
+	}
+
+	if (publickey_len > 0) {
+		c->publickey = git__malloc(publickey_len);
+		GITERR_CHECK_ALLOC(c->publickey);
+
+		memcpy(c->publickey, publickey, publickey_len);
+	}
+
+	c->publickey_len = publickey_len;
+	c->sign_callback = sign_callback;
+	c->sign_data = sign_data;
 
 	*cred = &c->parent;
 	return 0;
 }
-#endif
diff --git a/src/transports/local.c b/src/transports/local.c
index 4bf1c876a..9ebea979c 100644
--- a/src/transports/local.c
+++ b/src/transports/local.c
@@ -119,15 +119,24 @@ on_error:
 
 static int store_refs(transport_local *t)
 {
-	unsigned int i;
+	size_t i;
+	git_remote_head *head;
 	git_strarray ref_names = {0};
 
 	assert(t);
 
-	if (git_reference_list(&ref_names, t->repo) < 0 ||
-		git_vector_init(&t->refs, ref_names.count, NULL) < 0)
+	if (git_reference_list(&ref_names, t->repo) < 0)
 		goto on_error;
 
+	/* Clear all heads we might have fetched in a previous connect */
+	git_vector_foreach(&t->refs, i, head) {
+		git__free(head->name);
+		git__free(head);
+	}
+
+	/* Clear the vector so we can reuse it */
+	git_vector_clear(&t->refs);
+
 	/* Sort the references first */
 	git__tsort((void **)ref_names.strings, ref_names.count, &git__strcmp_cb);
 
@@ -278,9 +287,9 @@ static int local_push_copy_object(
 		odb_obj_size, odb_obj_type)) < 0)
 		goto on_error;
 
-	if (odb_stream->write(odb_stream, (char *)git_odb_object_data(odb_obj),
+	if (git_odb_stream_write(odb_stream, (char *)git_odb_object_data(odb_obj),
 		odb_obj_size) < 0 ||
-		odb_stream->finalize_write(&remote_odb_obj_oid, odb_stream) < 0) {
+		git_odb_stream_finalize_write(&remote_odb_obj_oid, odb_stream) < 0) {
 		error = -1;
 	} else if (git_oid__cmp(&obj->id, &remote_odb_obj_oid) != 0) {
 		giterr_set(GITERR_ODB, "Error when writing object to remote odb "
@@ -289,7 +298,7 @@ static int local_push_copy_object(
 		error = -1;
 	}
 
-	odb_stream->free(odb_stream);
+	git_odb_stream_free(odb_stream);
 
 on_error:
 	git_odb_object_free(odb_obj);
@@ -352,7 +361,8 @@ static int local_push(
 	   non-bare repo push support would require checking configs to see if
 	   we should override the default 'don't let this happen' behavior */
 	if (!remote_repo->is_bare) {
-		error = -1;
+		error = GIT_EBAREREPO;
+		giterr_set(GITERR_INVALID, "Local push doesn't (yet) support pushing to non-bare repos.");
 		goto on_error;
 	}
 
@@ -593,9 +603,6 @@ static void local_free(git_transport *transport)
 	size_t i;
 	git_remote_head *head;
 
-	/* Close the transport, if it's still open. */
-	local_close(transport);
-
 	git_vector_foreach(&t->refs, i, head) {
 		git__free(head->name);
 		git__free(head);
@@ -603,6 +610,9 @@ static void local_free(git_transport *transport)
 
 	git_vector_free(&t->refs);
 
+	/* Close the transport, if it's still open. */
+	local_close(transport);
+
 	/* Free the transport */
 	git__free(t);
 }
@@ -632,6 +642,7 @@ int git_transport_local(git_transport **out, git_remote *owner, void *param)
 	t->parent.read_flags = local_read_flags;
 	t->parent.cancel = local_cancel;
 
+	git_vector_init(&t->refs, 0, NULL);
 	t->owner = owner;
 
 	*out = (git_transport *) t;
diff --git a/src/transports/smart_protocol.c b/src/transports/smart_protocol.c
index 636616717..0cd5e831d 100644
--- a/src/transports/smart_protocol.c
+++ b/src/transports/smart_protocol.c
@@ -372,7 +372,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 				return error;
 
 			if (pkt->type == GIT_PKT_NAK ||
-			    (pkt->type == GIT_PKT_ACK && pkt->status != GIT_ACK_CONTINUE)) {
+				(pkt->type == GIT_PKT_ACK && pkt->status != GIT_ACK_CONTINUE)) {
 				git__free(pkt);
 				break;
 			}
diff --git a/src/transports/ssh.c b/src/transports/ssh.c
index a312c8d08..bf62bd185 100644
--- a/src/transports/ssh.c
+++ b/src/transports/ssh.c
@@ -5,19 +5,18 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
-#ifdef GIT_SSH
-
 #include "git2.h"
 #include "buffer.h"
 #include "netops.h"
 #include "smart.h"
 
+#ifdef GIT_SSH
+
 #include <libssh2.h>
 
 #define OWNING_SUBTRANSPORT(s) ((ssh_subtransport *)(s)->parent.subtransport)
 
 static const char prefix_ssh[] = "ssh://";
-static const char default_user[] = "git";
 static const char cmd_uploadpack[] = "git-upload-pack";
 static const char cmd_receivepack[] = "git-receive-pack";
 
@@ -46,27 +45,29 @@ typedef struct {
 static int gen_proto(git_buf *request, const char *cmd, const char *url)
 {
 	char *repo;
-	
+
 	if (!git__prefixcmp(url, prefix_ssh)) {
 		url = url + strlen(prefix_ssh);
 		repo = strchr(url, '/');
 	} else {
 		repo = strchr(url, ':');
+		if (repo) repo++;
 	}
-	
+
 	if (!repo) {
+		giterr_set(GITERR_NET, "Malformed git protocol URL");
 		return -1;
 	}
-	
+
 	int len = strlen(cmd) + 1 /* Space */ + 1 /* Quote */ + strlen(repo) + 1 /* Quote */ + 1;
-	
+
 	git_buf_grow(request, len);
 	git_buf_printf(request, "%s '%s'", cmd, repo);
 	git_buf_putc(request, '\0');
-	
+
 	if (git_buf_oom(request))
 		return -1;
-	
+
 	return 0;
 }
 
@@ -74,21 +75,19 @@ static int send_command(ssh_stream *s)
 {
 	int error;
 	git_buf request = GIT_BUF_INIT;
-	
+
 	error = gen_proto(&request, s->cmd, s->url);
 	if (error < 0)
 		goto cleanup;
-	
-	error = libssh2_channel_exec(
-		s->channel,
-		request.ptr
-	);
 
-	if (0 != error)
+	error = libssh2_channel_exec(s->channel, request.ptr);
+	if (error < 0) {
+		giterr_set(GITERR_NET, "SSH could not execute request");
 		goto cleanup;
-	
+	}
+
 	s->sent_command = 1;
-	
+
 cleanup:
 	git_buf_free(&request);
 	return error;
@@ -100,19 +99,21 @@ static int ssh_stream_read(
 	size_t buf_size,
 	size_t *bytes_read)
 {
+	int rc;
 	ssh_stream *s = (ssh_stream *)stream;
-	
+
 	*bytes_read = 0;
-	
+
 	if (!s->sent_command && send_command(s) < 0)
 		return -1;
-	
-	int rc = libssh2_channel_read(s->channel, buffer, buf_size);
-	if (rc < 0)
+
+	if ((rc = libssh2_channel_read(s->channel, buffer, buf_size)) < 0) {
+		giterr_set(GITERR_NET, "SSH could not read data");
 		return -1;
-	
+	}
+
 	*bytes_read = rc;
-	
+
 	return 0;
 }
 
@@ -122,16 +123,16 @@ static int ssh_stream_write(
 	size_t len)
 {
 	ssh_stream *s = (ssh_stream *)stream;
-	
+
 	if (!s->sent_command && send_command(s) < 0)
 		return -1;
-	
-	int rc = libssh2_channel_write(s->channel, buffer, len);
-	if (rc < 0) {
+
+	if (libssh2_channel_write(s->channel, buffer, len) < 0) {
+		giterr_set(GITERR_NET, "SSH could not write data");
 		return -1;
 	}
-	
-	return rc;
+
+	return 0;
 }
 
 static void ssh_stream_free(git_smart_subtransport_stream *stream)
@@ -139,26 +140,27 @@ static void ssh_stream_free(git_smart_subtransport_stream *stream)
 	ssh_stream *s = (ssh_stream *)stream;
 	ssh_subtransport *t = OWNING_SUBTRANSPORT(s);
 	int ret;
-	
+
 	GIT_UNUSED(ret);
-	
+
 	t->current_stream = NULL;
-	
+
 	if (s->channel) {
 		libssh2_channel_close(s->channel);
-        libssh2_channel_free(s->channel);
-        s->channel = NULL;
+		libssh2_channel_free(s->channel);
+		s->channel = NULL;
 	}
-	
+
 	if (s->session) {
-		libssh2_session_free(s->session), s->session = NULL;
+		libssh2_session_free(s->session);
+		s->session = NULL;
 	}
-	
+
 	if (s->socket.socket) {
-		ret = gitno_close(&s->socket);
-		assert(!ret);
+		(void)gitno_close(&s->socket);
+		/* can't do anything here with error return value */
 	}
-	
+
 	git__free(s->url);
 	git__free(s);
 }
@@ -170,26 +172,25 @@ static int ssh_stream_alloc(
 	git_smart_subtransport_stream **stream)
 {
 	ssh_stream *s;
-	
-	if (!stream)
-		return -1;
-	
+
+	assert(stream);
+
 	s = git__calloc(sizeof(ssh_stream), 1);
 	GITERR_CHECK_ALLOC(s);
-	
+
 	s->parent.subtransport = &t->parent;
 	s->parent.read = ssh_stream_read;
 	s->parent.write = ssh_stream_write;
 	s->parent.free = ssh_stream_free;
-	
+
 	s->cmd = cmd;
+
 	s->url = git__strdup(url);
-	
 	if (!s->url) {
 		git__free(s);
 		return -1;
 	}
-	
+
 	*stream = &s->parent;
 	return 0;
 }
@@ -201,133 +202,124 @@ static int git_ssh_extract_url_parts(
 {
 	char *colon, *at;
 	const char *start;
-    
-    colon = strchr(url, ':');
-	
+
+	colon = strchr(url, ':');
+
 	if (colon == NULL) {
 		giterr_set(GITERR_NET, "Malformed URL: missing :");
 		return -1;
 	}
-	
+
 	at = strchr(url, '@');
 	if (at) {
 		start = at+1;
 		*username = git__substrdup(url, at - url);
+		GITERR_CHECK_ALLOC(*username);
 	} else {
 		start = url;
-		*username = git__strdup(default_user);
+		*username = NULL;
 	}
-	
+
 	*host = git__substrdup(start, colon - start);
-	
+	GITERR_CHECK_ALLOC(*host);
+
 	return 0;
 }
 
 static int _git_ssh_authenticate_session(
 	LIBSSH2_SESSION* session,
 	const char *user,
-	git_cred* cred
-)
+	git_cred* cred)
 {
 	int rc;
+
 	do {
 		switch (cred->credtype) {
-			case GIT_CREDTYPE_USERPASS_PLAINTEXT: {
-				git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
-				rc = libssh2_userauth_password(
-					session, 
-					c->username,
-					c->password
-				);
-				break;
-			}
-			case GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE: {
-				git_cred_ssh_keyfile_passphrase *c = (git_cred_ssh_keyfile_passphrase *)cred;
-				rc = libssh2_userauth_publickey_fromfile(
-					session, 
-					user,
-					c->publickey,
-					c->privatekey,
-					c->passphrase
-				);
-				break;
-			}
-			case GIT_CREDTYPE_SSH_PUBLICKEY: {
-				git_cred_ssh_publickey *c = (git_cred_ssh_publickey *)cred;
-				rc = libssh2_userauth_publickey(
-					session,
-					user,
-					(const unsigned char *)c->publickey,
-					c->publickey_len,
-					c->sign_callback,
-					&c->sign_data
-				);
-				break;
-			}
-			default:
-				rc = LIBSSH2_ERROR_AUTHENTICATION_FAILED;
+		case GIT_CREDTYPE_USERPASS_PLAINTEXT: {
+			git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
+			user = c->username ? c->username : user;
+			rc = libssh2_userauth_password(session, user, c->password);
+			break;
+		}
+		case GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE: {
+			git_cred_ssh_keyfile_passphrase *c = (git_cred_ssh_keyfile_passphrase *)cred;
+			user = c->username ? c->username : user;
+			rc = libssh2_userauth_publickey_fromfile(
+				session, c->username, c->publickey, c->privatekey, c->passphrase);
+			break;
 		}
-    } while (LIBSSH2_ERROR_EAGAIN == rc || LIBSSH2_ERROR_TIMEOUT == rc);
-	
-    return rc;
+		case GIT_CREDTYPE_SSH_PUBLICKEY: {
+			git_cred_ssh_publickey *c = (git_cred_ssh_publickey *)cred;
+
+			user = c->username ? c->username : user;
+			rc = libssh2_userauth_publickey(
+				session, c->username, (const unsigned char *)c->publickey,
+				c->publickey_len, c->sign_callback, &c->sign_data);
+			break;
+		}
+		default:
+			rc = LIBSSH2_ERROR_AUTHENTICATION_FAILED;
+		}
+	} while (LIBSSH2_ERROR_EAGAIN == rc || LIBSSH2_ERROR_TIMEOUT == rc);
+
+	if (rc != 0) {
+		giterr_set(GITERR_NET, "Failed to authenticate SSH session");
+		return -1;
+	}
+
+	return 0;
 }
 
-static int _git_ssh_session_create
-(
+static int _git_ssh_session_create(
 	LIBSSH2_SESSION** session,
-	gitno_socket socket
-)
+	gitno_socket socket)
 {
-	if (!session) {
+	int rc = 0;
+	LIBSSH2_SESSION* s;
+
+	assert(session);
+
+	s = libssh2_session_init();
+	if (!s) {
+		giterr_set(GITERR_NET, "Failed to initialize SSH session");
 		return -1;
 	}
-	
-	LIBSSH2_SESSION* s = libssh2_session_init();
-    if (!s)
-        return -1;
-	
-    int rc = 0;
-    do {
-        rc = libssh2_session_startup(s, socket.socket);
-    } while (LIBSSH2_ERROR_EAGAIN == rc || LIBSSH2_ERROR_TIMEOUT == rc);
-	
+
+	do {
+		rc = libssh2_session_startup(s, socket.socket);
+	} while (LIBSSH2_ERROR_EAGAIN == rc || LIBSSH2_ERROR_TIMEOUT == rc);
+
 	if (0 != rc) {
-        goto on_error;
-    }
-	
+		libssh2_session_free(s);
+		giterr_set(GITERR_NET, "Failed to start SSH session");
+		return -1;
+	}
+
 	libssh2_session_set_blocking(s, 1);
-	
+
 	*session = s;
-	
+
 	return 0;
-	
-on_error:
-	if (s) {
-		libssh2_session_free(s), s = NULL;
-	}
-	
-	return -1;
 }
 
 static int _git_ssh_setup_conn(
 	ssh_subtransport *t,
 	const char *url,
 	const char *cmd,
-	git_smart_subtransport_stream **stream
-)
+	git_smart_subtransport_stream **stream)
 {
 	char *host, *port=NULL, *user=NULL, *pass=NULL;
 	const char *default_port="22";
 	ssh_stream *s;
 	LIBSSH2_SESSION* session=NULL;
 	LIBSSH2_CHANNEL* channel=NULL;
-	
+
 	*stream = NULL;
 	if (ssh_stream_alloc(t, url, cmd, stream) < 0)
 		return -1;
-	
+
 	s = (ssh_stream *)*stream;
-	
+
 	if (!git__prefixcmp(url, prefix_ssh)) {
 		url = url + strlen(prefix_ssh);
 		if (gitno_extract_url_parts(&host, &port, &user, &pass, url, default_port) < 0)
@@ -338,42 +330,53 @@ static int _git_ssh_setup_conn(
 		port = git__strdup(default_port);
 		GITERR_CHECK_ALLOC(port);
 	}
-	
+
 	if (gitno_connect(&s->socket, host, port, 0) < 0)
 		goto on_error;
-	
+
 	if (user && pass) {
 		if (git_cred_userpass_plaintext_new(&t->cred, user, pass) < 0)
 			goto on_error;
-	} else {
-		if (t->owner->cred_acquire_cb(&t->cred,
-				t->owner->url,
-				user,
-				GIT_CREDTYPE_USERPASS_PLAINTEXT | GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE,
+	} else if (t->owner->cred_acquire_cb) {
+		if (t->owner->cred_acquire_cb(
+				&t->cred, t->owner->url, user,
+				GIT_CREDTYPE_USERPASS_PLAINTEXT |
+				GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE,
 				t->owner->cred_acquire_payload) < 0)
-			return -1;
+			goto on_error;
+
+		if (!t->cred) {
+			giterr_set(GITERR_NET, "Callback failed to initialize SSH credentials");
+			goto on_error;
+		}
+	} else {
+		giterr_set(GITERR_NET, "Cannot set up SSH connection without credentials");
+		goto on_error;
 	}
 	assert(t->cred);
-	
-	if (!user) {
-		user = git__strdup(default_user);
+
+	if (!user && !git_cred_has_username(t->cred)) {
+		giterr_set_str(GITERR_NET, "Cannot authenticate without a username");
+		goto on_error;
 	}
-	
+
 	if (_git_ssh_session_create(&session, s->socket) < 0)
 		goto on_error;
-	
-    if (_git_ssh_authenticate_session(session, user, t->cred) < 0)
+
+	if (_git_ssh_authenticate_session(session, user, t->cred) < 0)
 		goto on_error;
-	
+
 	channel = libssh2_channel_open_session(session);
-	if (!channel)
-        goto on_error;
-	
+	if (!channel) {
+		giterr_set(GITERR_NET, "Failed to open SSH channel");
+		goto on_error;
+	}
+
 	libssh2_channel_set_blocking(channel, 1);
-	
+
 	s->session = session;
 	s->channel = channel;
-	
+
 	t->current_stream = s;
 	git__free(host);
 	git__free(port);
@@ -381,18 +384,22 @@ static int _git_ssh_setup_conn(
 	git__free(pass);
 
 	return 0;
-	
+
 on_error:
+	s->session = NULL;
+	s->channel = NULL;
+	t->current_stream = NULL;
+
 	if (*stream)
 		ssh_stream_free(*stream);
-	
+
 	git__free(host);
 	git__free(port);
 	git__free(user);
 	git__free(pass);
 
 	if (session)
-		libssh2_session_free(session), session = NULL;
+		libssh2_session_free(session);
 
 	return -1;
 }
@@ -404,7 +411,7 @@ static int ssh_uploadpack_ls(
 {
 	if (_git_ssh_setup_conn(t, url, cmd_uploadpack, stream) < 0)
 		return -1;
-	
+
 	return 0;
 }
 
@@ -414,12 +421,12 @@ static int ssh_uploadpack(
 	git_smart_subtransport_stream **stream)
 {
 	GIT_UNUSED(url);
-	
+
 	if (t->current_stream) {
 		*stream = &t->current_stream->parent;
 		return 0;
 	}
-	
+
 	giterr_set(GITERR_NET, "Must call UPLOADPACK_LS before UPLOADPACK");
 	return -1;
 }
@@ -431,7 +438,7 @@ static int ssh_receivepack_ls(
 {
 	if (_git_ssh_setup_conn(t, url, cmd_receivepack, stream) < 0)
 		return -1;
-	
+
 	return 0;
 }
 
@@ -441,12 +448,12 @@ static int ssh_receivepack(
 	git_smart_subtransport_stream **stream)
 {
 	GIT_UNUSED(url);
-	
+
 	if (t->current_stream) {
 		*stream = &t->current_stream->parent;
 		return 0;
 	}
-	
+
 	giterr_set(GITERR_NET, "Must call RECEIVEPACK_LS before RECEIVEPACK");
 	return -1;
 }
@@ -458,21 +465,21 @@ static int _ssh_action(
 	git_smart_service_t action)
 {
 	ssh_subtransport *t = (ssh_subtransport *) subtransport;
-	
+
 	switch (action) {
 		case GIT_SERVICE_UPLOADPACK_LS:
 			return ssh_uploadpack_ls(t, url, stream);
-			
+
 		case GIT_SERVICE_UPLOADPACK:
 			return ssh_uploadpack(t, url, stream);
-			
+
 		case GIT_SERVICE_RECEIVEPACK_LS:
 			return ssh_receivepack_ls(t, url, stream);
-			
+
 		case GIT_SERVICE_RECEIVEPACK:
 			return ssh_receivepack(t, url, stream);
 	}
-	
+
 	*stream = NULL;
 	return -1;
 }
@@ -480,40 +487,49 @@ static int _ssh_action(
 static int _ssh_close(git_smart_subtransport *subtransport)
 {
 	ssh_subtransport *t = (ssh_subtransport *) subtransport;
-	
+
 	assert(!t->current_stream);
-	
+
 	GIT_UNUSED(t);
-	
+
 	return 0;
 }
 
 static void _ssh_free(git_smart_subtransport *subtransport)
 {
 	ssh_subtransport *t = (ssh_subtransport *) subtransport;
-	
+
 	assert(!t->current_stream);
-	
+
 	git__free(t);
 }
+#endif
 
-int git_smart_subtransport_ssh(git_smart_subtransport **out, git_transport *owner)
+int git_smart_subtransport_ssh(
+	git_smart_subtransport **out, git_transport *owner)
 {
+#ifdef GIT_SSH
 	ssh_subtransport *t;
-	
-	if (!out)
-		return -1;
-	
+
+	assert(out);
+
 	t = git__calloc(sizeof(ssh_subtransport), 1);
 	GITERR_CHECK_ALLOC(t);
-	
+
 	t->owner = (transport_smart *)owner;
 	t->parent.action = _ssh_action;
 	t->parent.close = _ssh_close;
 	t->parent.free = _ssh_free;
-	
+
 	*out = (git_smart_subtransport *) t;
 	return 0;
-}
+#else
+	GIT_UNUSED(owner);
 
+	assert(out);
+	*out = NULL;
+
+	giterr_set(GITERR_INVALID, "Cannot create SSH transport. Library was built without SSH support");
+	return -1;
 #endif
+}
diff --git a/src/transports/winhttp.c b/src/transports/winhttp.c
index 95e422dc0..29d4ba619 100644
--- a/src/transports/winhttp.c
+++ b/src/transports/winhttp.c
@@ -511,7 +511,7 @@ replay:
 
 			/* Check for Windows 7. This workaround is only necessary on
 			 * Windows Vista and earlier. Windows 7 is version 6.1. */
-			if (!git_has_win32_version(6, 1)) {
+			if (!git_has_win32_version(6, 1, 0)) {
 				wchar_t *location;
 				DWORD location_length;
 				int redirect_cmp;
@@ -893,7 +893,7 @@ static int winhttp_connect(
 	const char *url)
 {
 	wchar_t *ua = L"git/1.0 (libgit2 " WIDEN(LIBGIT2_VERSION) L")";
-	wchar_t host[GIT_WIN_PATH];
+	git_win32_path host;
 	int32_t port;
 	const char *default_port = "80";
 	int ret;
@@ -920,7 +920,7 @@ static int winhttp_connect(
 		return -1;
 
 	/* Prepare host */
-	git__utf8_to_16(host, GIT_WIN_PATH, t->host);
+	git_win32_path_from_c(host, t->host);
 
 	/* Establish session */
 	t->session = WinHttpOpen(
@@ -934,7 +934,7 @@ static int winhttp_connect(
 		giterr_set(GITERR_OS, "Failed to init WinHTTP");
 		return -1;
 	}
-	
+
 	/* Establish connection */
 	t->connection = WinHttpConnect(
 			t->session,
@@ -989,7 +989,7 @@ static int winhttp_receivepack(
 {
 	/* WinHTTP only supports Transfer-Encoding: chunked
 	 * on Windows Vista (NT 6.0) and higher. */
-	s->chunked = git_has_win32_version(6, 0);
+	s->chunked = git_has_win32_version(6, 0, 0);
 
 	if (s->chunked)
 		s->parent.write = winhttp_stream_write_chunked;
diff --git a/src/tree.c b/src/tree.c
index 65d01b4d5..0bdf9a93e 100644
--- a/src/tree.c
+++ b/src/tree.c
@@ -10,7 +10,7 @@
 #include "tree.h"
 #include "git2/repository.h"
 #include "git2/object.h"
-#include "path.h"
+#include "fileops.h"
 #include "tree-cache.h"
 #include "index.h"
 
@@ -29,19 +29,19 @@ static bool valid_filemode(const int filemode)
 GIT_INLINE(git_filemode_t) normalize_filemode(git_filemode_t filemode)
 {
 	/* Tree bits set, but it's not a commit */
-	if (filemode & GIT_FILEMODE_TREE && !(filemode & 0100000))
+	if (GIT_MODE_TYPE(filemode) == GIT_FILEMODE_TREE)
 		return GIT_FILEMODE_TREE;
 
-	/* If any of the x bits is set */
-	if (filemode & 0111)
+	/* If any of the x bits are set */
+	if (GIT_PERMS_IS_EXEC(filemode))
 		return GIT_FILEMODE_BLOB_EXECUTABLE;
 
 	/* 16XXXX means commit */
-	if ((filemode & GIT_FILEMODE_COMMIT) == GIT_FILEMODE_COMMIT)
+	if (GIT_MODE_TYPE(filemode) == GIT_FILEMODE_COMMIT)
 		return GIT_FILEMODE_COMMIT;
 
 	/* 12XXXX means commit */
-	if ((filemode & GIT_FILEMODE_LINK) == GIT_FILEMODE_LINK)
+	if (GIT_MODE_TYPE(filemode) == GIT_FILEMODE_LINK)
 		return GIT_FILEMODE_LINK;
 
 	/* Otherwise, return a blob */
@@ -881,8 +881,10 @@ static int tree_walk(
 	git_vector_foreach(&tree->entries, i, entry) {
 		if (preorder) {
 			error = callback(path->ptr, entry, payload);
-			if (error > 0)
+			if (error > 0) {
+				error = 0;
 				continue;
+			}
 			if (error < 0) {
 				giterr_clear();
 				return GIT_EUSER;
@@ -905,11 +907,12 @@ static int tree_walk(
 				return -1;
 
 			error = tree_walk(subtree, callback, path, payload, preorder);
+			git_tree_free(subtree);
+
 			if (error != 0)
 				break;
 
 			git_buf_truncate(path, path_len);
-			git_tree_free(subtree);
 		}
 
 		if (!preorder && callback(path->ptr, entry, payload) < 0) {
diff --git a/src/util.c b/src/util.c
index 1d084daa8..d0c326ae5 100644
--- a/src/util.c
+++ b/src/util.c
@@ -33,6 +33,9 @@ int git_libgit2_capabilities()
 #if defined(GIT_SSL) || defined(GIT_WINHTTP)
 		| GIT_CAP_HTTPS
 #endif
+#if defined(GIT_SSH)
+		| GIT_CAP_SSH
+#endif
 	;
 }
 
@@ -279,6 +282,28 @@ int git__strcasecmp(const char *a, const char *b)
 	return (tolower(*a) - tolower(*b));
 }
 
+int git__strcasesort_cmp(const char *a, const char *b)
+{
+	int cmp = 0;
+
+	while (*a && *b) {
+		if (*a != *b) {
+			if (tolower(*a) != tolower(*b))
+				break;
+			/* use case in sort order even if not in equivalence */
+			if (!cmp)
+				cmp = (int)(*(const uint8_t *)a) - (int)(*(const uint8_t *)b);
+		}
+
+		++a, ++b;
+	}
+
+	if (*a || *b)
+		return tolower(*a) - tolower(*b);
+
+	return cmp;
+}
+
 int git__strncmp(const char *a, const char *b, size_t sz)
 {
 	while (sz && *a && *b && *a == *b)
@@ -686,7 +711,7 @@ void git__qsort_r(
 	void *els, size_t nel, size_t elsize, git__sort_r_cmp cmp, void *payload)
 {
 #if defined(__MINGW32__) || defined(__OpenBSD__) || defined(AMIGA) || \
-	defined(__gnu_hurd__) || \
+	defined(__gnu_hurd__) || defined(__ANDROID_API__) || \
 	(__GLIBC__ == 2 && __GLIBC_MINOR__ < 8)
 	git__insertsort_r(els, nel, elsize, NULL, cmp, payload);
 #elif defined(GIT_WIN32)
@@ -722,12 +747,3 @@ void git__insertsort_r(
 	if (freeswap)
 		git__free(swapel);
 }
-
-void git__memzero(volatile void *data, size_t size)
-{
-	volatile uint8_t *scan = data;
-	uint8_t *end = scan + size;
-
-	while (scan < end)
-		*scan++ = 0x0;
-}
diff --git a/src/util.h b/src/util.h
index 0de466677..bd93b46b5 100644
--- a/src/util.h
+++ b/src/util.h
@@ -55,6 +55,9 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n)
 
 	ptr = (char*)git__malloc(length + 1);
 
+	if (!ptr)
+		return NULL;
+
 	if (length)
 		memcpy(ptr, str, length);
 
@@ -79,7 +82,10 @@ GIT_INLINE(void *) git__realloc(void *ptr, size_t size)
 	return new_ptr;
 }
 
-#define git__free(ptr) free(ptr)
+GIT_INLINE(void) git__free(void *ptr)
+{
+	free(ptr);
+}
 
 #define STRCMP_CASESELECT(IGNORE_CASE, STR1, STR2) \
 	((IGNORE_CASE) ? strcasecmp((STR1), (STR2)) : strcmp((STR1), (STR2)))
@@ -194,6 +200,8 @@ extern int git__strcasecmp(const char *a, const char *b);
 extern int git__strncmp(const char *a, const char *b, size_t sz);
 extern int git__strncasecmp(const char *a, const char *b, size_t sz);
 
+extern int git__strcasesort_cmp(const char *a, const char *b);
+
 #include "thread-utils.h"
 
 typedef struct {
@@ -219,6 +227,9 @@ typedef void (*git_refcount_freeptr)(void *r);
 
 #define GIT_REFCOUNT_OWNER(r) (((git_refcount *)(r))->owner)
 
+#define GIT_REFCOUNT_VAL(r) git_atomic_get(&((git_refcount *)(r))->refcount)
+
+
 static signed char from_hex[] = {
 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 00 */
 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 10 */
@@ -289,6 +300,11 @@ GIT_INLINE(bool) git__isspace(int c)
 	return (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == '\v' || c == 0x85 /* Unicode CR+LF */);
 }
 
+GIT_INLINE(bool) git__isspace_nonlf(int c)
+{
+	return (c == ' ' || c == '\t' || c == '\f' || c == '\r' || c == '\v' || c == 0x85 /* Unicode CR+LF */);
+}
+
 GIT_INLINE(bool) git__iswildcard(int c)
 {
 	return (c == '*' || c == '?' || c == '[');
@@ -325,6 +341,16 @@ extern size_t git__unescape(char *str);
  * Safely zero-out memory, making sure that the compiler
  * doesn't optimize away the operation.
  */
-extern void git__memzero(volatile void *data, size_t size);
+GIT_INLINE(void) git__memzero(void *data, size_t size)
+{
+#ifdef _MSC_VER
+	SecureZeroMemory((PVOID)data, size);
+#else
+	volatile uint8_t *scan = (volatile uint8_t *)data;
+
+	while (size--)
+		*scan++ = 0x0;
+#endif
+}
 
 #endif /* INCLUDE_util_h__ */
diff --git a/src/vector.c b/src/vector.c
index 5ba2fab18..362e7b0c0 100644
--- a/src/vector.c
+++ b/src/vector.c
@@ -220,7 +220,7 @@ void git_vector_pop(git_vector *v)
 		v->length--;
 }
 
-void git_vector_uniq(git_vector *v)
+void git_vector_uniq(git_vector *v, void  (*git_free_cb)(void *))
 {
 	git_vector_cmp cmp;
 	size_t i, j;
@@ -232,9 +232,12 @@ void git_vector_uniq(git_vector *v)
 	cmp = v->_cmp ? v->_cmp : strict_comparison;
 
 	for (i = 0, j = 1 ; j < v->length; ++j)
-		if (!cmp(v->contents[i], v->contents[j]))
+		if (!cmp(v->contents[i], v->contents[j])) {
+			if (git_free_cb)
+				git_free_cb(v->contents[i]);
+
 			v->contents[i] = v->contents[j];
-		else
+		} else
 			v->contents[++i] = v->contents[j];
 
 	v->length -= j - i - 1;
diff --git a/src/vector.h b/src/vector.h
index e2f729b83..279f5c6ee 100644
--- a/src/vector.h
+++ b/src/vector.h
@@ -55,6 +55,11 @@ GIT_INLINE(void *) git_vector_get(const git_vector *v, size_t position)
 
 #define GIT_VECTOR_GET(V,I) ((I) < (V)->length ? (V)->contents[(I)] : NULL)
 
+GIT_INLINE(size_t) git_vector_length(const git_vector *v)
+{
+	return v->length;
+}
+
 GIT_INLINE(void *) git_vector_last(const git_vector *v)
 {
 	return (v->length > 0) ? git_vector_get(v, v->length - 1) : NULL;
@@ -71,11 +76,20 @@ int git_vector_insert_sorted(git_vector *v, void *element,
 	int (*on_dup)(void **old, void *new));
 int git_vector_remove(git_vector *v, size_t idx);
 void git_vector_pop(git_vector *v);
-void git_vector_uniq(git_vector *v);
+void git_vector_uniq(git_vector *v, void  (*git_free_cb)(void *));
 void git_vector_remove_matching(
 	git_vector *v, int (*match)(const git_vector *v, size_t idx));
 
 int git_vector_resize_to(git_vector *v, size_t new_length);
 int git_vector_set(void **old, git_vector *v, size_t position, void *value);
 
+/** Set the comparison function used for sorting the vector */
+GIT_INLINE(void) git_vector_set_cmp(git_vector *v, git_vector_cmp cmp)
+{
+	if (cmp != v->_cmp) {
+		v->_cmp = cmp;
+		v->sorted = 0;
+	}
+}
+
 #endif
diff --git a/src/win32/dir.c b/src/win32/dir.c
index 8c51d8378..f7859b73f 100644
--- a/src/win32/dir.c
+++ b/src/win32/dir.c
@@ -5,8 +5,7 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 #define GIT__WIN32_NO_WRAP_DIR
-#include "dir.h"
-#include "utf-conv.h"
+#include "posix.h"
 
 static int init_filter(char *filter, size_t n, const char *dir)
 {
@@ -25,36 +24,32 @@ static int init_filter(char *filter, size_t n, const char *dir)
 
 git__DIR *git__opendir(const char *dir)
 {
-	char filter[GIT_WIN_PATH];
-	wchar_t filter_w[GIT_WIN_PATH];
+	git_win32_path_as_utf8 filter;
+	git_win32_path filter_w;
 	git__DIR *new = NULL;
+	size_t dirlen;
 
 	if (!dir || !init_filter(filter, sizeof(filter), dir))
 		return NULL;
 
-	new = git__calloc(1, sizeof(*new));
+	dirlen = strlen(dir);
+
+	new = git__calloc(sizeof(*new) + dirlen + 1, 1);
 	if (!new)
 		return NULL;
+	memcpy(new->dir, dir, dirlen);
 
-	new->dir = git__strdup(dir);
-	if (!new->dir)
-		goto fail;
-
-	git__utf8_to_16(filter_w, GIT_WIN_PATH, filter);
+	git_win32_path_from_c(filter_w, filter);
 	new->h = FindFirstFileW(filter_w, &new->f);
 
 	if (new->h == INVALID_HANDLE_VALUE) {
 		giterr_set(GITERR_OS, "Could not open directory '%s'", dir);
-		goto fail;
+		git__free(new);
+		return NULL;
 	}
 
 	new->first = 1;
 	return new;
-
-fail:
-	git__free(new->dir);
-	git__free(new);
-	return NULL;
 }
 
 int git__readdir_ext(
@@ -80,7 +75,7 @@ int git__readdir_ext(
 	if (wcslen(d->f.cFileName) >= sizeof(entry->d_name))
 		return -1;
 
-	git__utf16_to_8(entry->d_name, d->f.cFileName);
+	git_win32_path_to_c(entry->d_name, d->f.cFileName);
 	entry->d_ino = 0;
 
 	*result = entry;
@@ -101,8 +96,8 @@ struct git__dirent *git__readdir(git__DIR *d)
 
 void git__rewinddir(git__DIR *d)
 {
-	char filter[GIT_WIN_PATH];
-	wchar_t filter_w[GIT_WIN_PATH];
+	git_win32_path_as_utf8 filter;
+	git_win32_path filter_w;
 
 	if (!d)
 		return;
@@ -116,7 +111,7 @@ void git__rewinddir(git__DIR *d)
 	if (!init_filter(filter, sizeof(filter), d->dir))
 		return;
 
-	git__utf8_to_16(filter_w, GIT_WIN_PATH, filter);
+	git_win32_path_from_c(filter_w, filter);
 	d->h = FindFirstFileW(filter_w, &d->f);
 
 	if (d->h == INVALID_HANDLE_VALUE)
@@ -134,8 +129,7 @@ int git__closedir(git__DIR *d)
 		FindClose(d->h);
 		d->h = INVALID_HANDLE_VALUE;
 	}
-	git__free(d->dir);
-	d->dir = NULL;
+
 	git__free(d);
 	return 0;
 }
diff --git a/src/win32/dir.h b/src/win32/dir.h
index 7696d468e..24d48f6ba 100644
--- a/src/win32/dir.h
+++ b/src/win32/dir.h
@@ -11,15 +11,15 @@
 
 struct git__dirent {
 	int d_ino;
-	char d_name[261];
+	git_win32_path_as_utf8 d_name;
 };
 
 typedef struct {
 	HANDLE h;
 	WIN32_FIND_DATAW f;
 	struct git__dirent entry;
-	char *dir;
 	int first;
+	char dir[GIT_FLEX_ARRAY];
 } git__DIR;
 
 extern git__DIR *git__opendir(const char *);
diff --git a/src/win32/error.c b/src/win32/error.c
index 4a9a0631f..bc598ae32 100644
--- a/src/win32/error.c
+++ b/src/win32/error.c
@@ -12,7 +12,9 @@
 # include <winhttp.h>
 #endif
 
+#ifndef WC_ERR_INVALID_CHARS
 #define WC_ERR_INVALID_CHARS	0x80
+#endif
 
 char *git_win32_get_error_message(DWORD error_code)
 {
@@ -45,7 +47,7 @@ char *git_win32_get_error_message(DWORD error_code)
 		(LPWSTR)&lpMsgBuf, 0, NULL)) {
 
 		/* Invalid code point check supported on Vista+ only */
-		if (git_has_win32_version(6, 0))
+		if (git_has_win32_version(6, 0, 0))
 			dwFlags = WC_ERR_INVALID_CHARS;
 		else
 			dwFlags = 0;
diff --git a/src/win32/findfile.c b/src/win32/findfile.c
index 5dd3de13d..a1c11fcfb 100644
--- a/src/win32/findfile.c
+++ b/src/win32/findfile.c
@@ -23,11 +23,11 @@ int git_win32__expand_path(struct git_win32__path *s_root, const wchar_t *templ)
 	return s_root->len ? 0 : -1;
 }
 
-static int win32_path_utf16_to_8(git_buf *path_utf8, const wchar_t *path_utf16)
+static int win32_path_to_8(git_buf *path_utf8, const wchar_t *path)
 {
 	char temp_utf8[GIT_PATH_MAX];
 
-	git__utf16_to_8(temp_utf8, path_utf16);
+	git__utf16_to_8(temp_utf8, GIT_PATH_MAX, path);
 	git_path_mkposix(temp_utf8);
 
 	return git_buf_sets(path_utf8, temp_utf8);
@@ -53,7 +53,7 @@ int git_win32__find_file(
 	if (*filename == '/' || *filename == '\\')
 		filename++;
 
-	git__utf8_to_16(file_utf16 + root->len - 1, alloc_len, filename);
+	git__utf8_to_16(file_utf16 + root->len - 1, alloc_len - root->len, filename);
 
 	/* check access */
 	if (_waccess(file_utf16, F_OK) < 0) {
@@ -61,7 +61,7 @@ int git_win32__find_file(
 		return GIT_ENOTFOUND;
 	}
 
-	win32_path_utf16_to_8(path, file_utf16);
+	win32_path_to_8(path, file_utf16);
 	git__free(file_utf16);
 
 	return 0;
@@ -113,7 +113,7 @@ static int win32_find_git_in_path(git_buf *buf, const wchar_t *gitexe)
 			/* replace "bin\\" or "cmd\\" with "etc\\" */
 			wcscpy(&root.path[root.len - 4], L"etc\\");
 
-			win32_path_utf16_to_8(buf, root.path);
+			win32_path_to_8(buf, root.path);
 			return 0;
 		}
 	}
@@ -146,7 +146,7 @@ static int win32_find_git_in_registry(
 			wcscat(path16.path, L"etc\\");
 			path16.len += 4;
 
-			win32_path_utf16_to_8(buf, path16.path);
+			win32_path_to_8(buf, path16.path);
 		}
 
 		RegCloseKey(hKey);
@@ -156,7 +156,7 @@ static int win32_find_git_in_registry(
 }
 
 static int win32_find_existing_dirs(
-	git_buf *out, const wchar_t *tmpl[], char *temp[])
+	git_buf *out, const wchar_t *tmpl[])
 {
 	struct git_win32__path path16;
 	git_buf buf = GIT_BUF_INIT;
@@ -168,7 +168,7 @@ static int win32_find_existing_dirs(
 			path16.path[0] != L'%' &&
 			!_waccess(path16.path, F_OK))
 		{
-			win32_path_utf16_to_8(&buf, path16.path);
+			win32_path_to_8(&buf, path16.path);
 
 			if (buf.size)
 				git_buf_join(out, GIT_PATH_LIST_SEPARATOR, out->ptr, buf.ptr);
@@ -209,7 +209,6 @@ int git_win32__find_system_dirs(git_buf *out)
 
 int git_win32__find_global_dirs(git_buf *out)
 {
-	char *temp[3];
 	static const wchar_t *global_tmpls[4] = {
 		L"%HOME%\\",
 		L"%HOMEDRIVE%%HOMEPATH%\\",
@@ -217,12 +216,11 @@ int git_win32__find_global_dirs(git_buf *out)
 		NULL,
 	};
 
-	return win32_find_existing_dirs(out, global_tmpls, temp);
+	return win32_find_existing_dirs(out, global_tmpls);
 }
 
 int git_win32__find_xdg_dirs(git_buf *out)
 {
-	char *temp[6];
 	static const wchar_t *global_tmpls[7] = {
 		L"%XDG_CONFIG_HOME%\\git",
 		L"%APPDATA%\\git",
@@ -233,5 +231,5 @@ int git_win32__find_xdg_dirs(git_buf *out)
 		NULL,
 	};
 
-	return win32_find_existing_dirs(out, global_tmpls, temp);
+	return win32_find_existing_dirs(out, global_tmpls);
 }
diff --git a/src/win32/mingw-compat.h b/src/win32/mingw-compat.h
index 7b97b48db..fe0abfb54 100644
--- a/src/win32/mingw-compat.h
+++ b/src/win32/mingw-compat.h
@@ -19,6 +19,11 @@
 # define S_IFLNK _S_IFLNK
 # define S_ISLNK(m) (((m) & _S_IFMT) == _S_IFLNK)
 
+GIT_INLINE(size_t) p_strnlen(const char *s, size_t maxlen) {
+	const char *end = memchr(s, 0, maxlen);
+	return end ? (size_t)(end - s) : maxlen;
+}
+
 #endif
 
 #endif /* INCLUDE_mingw_compat__ */
diff --git a/src/win32/posix.h b/src/win32/posix.h
index c49c2175c..24cba23e0 100644
--- a/src/win32/posix.h
+++ b/src/win32/posix.h
@@ -8,7 +8,16 @@
 #define INCLUDE_posix__w32_h__
 
 #include "common.h"
+#include "../posix.h"
 #include "utf-conv.h"
+#include "dir.h"
+
+/* define some standard errnos that the runtime may be missing.  for example,
+ * mingw lacks EAFNOSUPPORT. */
+
+#ifndef EAFNOSUPPORT
+# define EAFNOSUPPORT (INT_MAX-1)
+#endif
 
 GIT_INLINE(int) p_link(const char *old, const char *new)
 {
@@ -20,9 +29,9 @@ GIT_INLINE(int) p_link(const char *old, const char *new)
 
 GIT_INLINE(int) p_mkdir(const char *path, mode_t mode)
 {
-	wchar_t buf[GIT_WIN_PATH];
+	git_win32_path buf;
 	GIT_UNUSED(mode);
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path_from_c(buf, path);
 	return _wmkdir(buf);
 }
 
diff --git a/src/win32/posix_w32.c b/src/win32/posix_w32.c
index f04974428..2f490529c 100644
--- a/src/win32/posix_w32.c
+++ b/src/win32/posix_w32.c
@@ -16,8 +16,8 @@
 
 int p_unlink(const char *path)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	_wchmod(buf, 0666);
 	return _wunlink(buf);
 }
@@ -59,10 +59,11 @@ static int do_lstat(
 	const char *file_name, struct stat *buf, int posix_enotdir)
 {
 	WIN32_FILE_ATTRIBUTE_DATA fdata;
-	wchar_t fbuf[GIT_WIN_PATH], lastch;
+	git_win32_path fbuf;
+	wchar_t lastch;
 	int flen;
 
-	flen = git__utf8_to_16(fbuf, GIT_WIN_PATH, file_name);
+	flen = git_win32_path_from_c(fbuf, file_name);
 
 	/* truncate trailing slashes */
 	for (; flen > 0; --flen) {
@@ -90,6 +91,9 @@ static int do_lstat(
 		if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
 			fMode |= S_IFLNK;
 
+		if ((fMode & (S_IFDIR | S_IFLNK)) == (S_IFDIR | S_IFLNK)) // junction
+			fMode ^= S_IFLNK;
+
 		buf->st_ino = 0;
 		buf->st_gid = 0;
 		buf->st_uid = 0;
@@ -105,10 +109,10 @@ static int do_lstat(
 		 * the length of the path pointed to, which we expect everywhere else
 		 */
 		if (S_ISLNK(fMode)) {
-			char target[GIT_WIN_PATH];
+			git_win32_path_as_utf8 target;
 			int readlink_result;
 
-			readlink_result = p_readlink(file_name, target, GIT_WIN_PATH);
+			readlink_result = p_readlink(file_name, target, sizeof(target));
 
 			if (readlink_result == -1)
 				return -1;
@@ -156,13 +160,22 @@ int p_lstat_posixly(const char *filename, struct stat *buf)
 	return do_lstat(filename, buf, 1);
 }
 
+
+/*
+ * Parts of the The p_readlink function are heavily inspired by the php 
+ * readlink function in link_win32.c
+ *
+ * Copyright (c) 1999 - 2012 The PHP Group. All rights reserved.
+ *
+ * For details of the PHP license see http://www.php.net/license/3_01.txt
+ */
 int p_readlink(const char *link, char *target, size_t target_len)
 {
 	typedef DWORD (WINAPI *fpath_func)(HANDLE, LPWSTR, DWORD, DWORD);
 	static fpath_func pGetFinalPath = NULL;
 	HANDLE hFile;
 	DWORD dwRet;
-	wchar_t link_w[GIT_WIN_PATH];
+	git_win32_path link_w;
 	wchar_t* target_w;
 	int error = 0;
 
@@ -185,7 +198,7 @@ int p_readlink(const char *link, char *target, size_t target_len)
 		}
 	}
 
-	git__utf8_to_16(link_w, GIT_WIN_PATH, link);
+	git_win32_path_from_c(link_w, link);
 
 	hFile = CreateFileW(link_w,			// file to open
 			GENERIC_READ,			// open for reading
@@ -251,10 +264,10 @@ int p_symlink(const char *old, const char *new)
 
 int p_open(const char *path, int flags, ...)
 {
-	wchar_t buf[GIT_WIN_PATH];
+	git_win32_path buf;
 	mode_t mode = 0;
 
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path_from_c(buf, path);
 
 	if (flags & O_CREAT) {
 		va_list arg_list;
@@ -269,8 +282,8 @@ int p_open(const char *path, int flags, ...)
 
 int p_creat(const char *path, mode_t mode)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return _wopen(buf, _O_WRONLY | _O_CREAT | _O_TRUNC | _O_BINARY, mode);
 }
 
@@ -296,7 +309,7 @@ int p_getcwd(char *buffer_out, size_t size)
 
 int p_stat(const char* path, struct stat* buf)
 {
-	char target[GIT_WIN_PATH];
+	git_win32_path_as_utf8 target;
 	int error = 0;
 
 	error = do_lstat(path, buf, 0);
@@ -304,7 +317,7 @@ int p_stat(const char* path, struct stat* buf)
 	/* We need not do this in a loop to unwind chains of symlinks since
 	 * p_readlink calls GetFinalPathNameByHandle which does it for us. */
 	if (error >= 0 && S_ISLNK(buf->st_mode) &&
-		(error = p_readlink(path, target, GIT_WIN_PATH)) >= 0)
+		(error = p_readlink(path, target, sizeof(target))) >= 0)
 		error = do_lstat(target, buf, 0);
 
 	return error;
@@ -312,23 +325,23 @@ int p_stat(const char* path, struct stat* buf)
 
 int p_chdir(const char* path)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return _wchdir(buf);
 }
 
 int p_chmod(const char* path, mode_t mode)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return _wchmod(buf, mode);
 }
 
 int p_rmdir(const char* path)
 {
 	int error;
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 
 	error = _wrmdir(buf);
 
@@ -344,24 +357,24 @@ int p_rmdir(const char* path)
 
 int p_hide_directory__w32(const char *path)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return (SetFileAttributesW(buf, FILE_ATTRIBUTE_HIDDEN) != 0) ? 0 : -1;
 }
 
 char *p_realpath(const char *orig_path, char *buffer)
 {
 	int ret;
-	wchar_t orig_path_w[GIT_WIN_PATH];
-	wchar_t buffer_w[GIT_WIN_PATH];
+	git_win32_path orig_path_w;
+	git_win32_path buffer_w;
 
-	git__utf8_to_16(orig_path_w, GIT_WIN_PATH, orig_path);
+	git_win32_path_from_c(orig_path_w, orig_path);
 
 	/* Implicitly use GetCurrentDirectory which can be a threading issue */
-	ret = GetFullPathNameW(orig_path_w, GIT_WIN_PATH, buffer_w, NULL);
+	ret = GetFullPathNameW(orig_path_w, GIT_WIN_PATH_UTF16, buffer_w, NULL);
 
 	/* According to MSDN, a return value equals to zero means a failure. */
-	if (ret == 0 || ret > GIT_WIN_PATH)
+	if (ret == 0 || ret > GIT_WIN_PATH_UTF16)
 		buffer = NULL;
 
 	else if (GetFileAttributesW(buffer_w) == INVALID_FILE_ATTRIBUTES) {
@@ -445,18 +458,18 @@ int p_setenv(const char* name, const char* value, int overwrite)
 
 int p_access(const char* path, mode_t mode)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return _waccess(buf, mode);
 }
 
 int p_rename(const char *from, const char *to)
 {
-	wchar_t wfrom[GIT_WIN_PATH];
-	wchar_t wto[GIT_WIN_PATH];
+	git_win32_path wfrom;
+	git_win32_path wto;
 
-	git__utf8_to_16(wfrom, GIT_WIN_PATH, from);
-	git__utf8_to_16(wto, GIT_WIN_PATH, to);
+	git_win32_path_from_c(wfrom, from);
+	git_win32_path_from_c(wto, to);
 	return MoveFileExW(wfrom, wto, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) ? 0 : -1;
 }
 
@@ -505,94 +518,40 @@ p_gmtime_r (const time_t *timer, struct tm *result)
 	return result;
 }
 
-#if defined(_MSC_VER) || defined(_MSC_EXTENSIONS)
-#define DELTA_EPOCH_IN_MICROSECS  11644473600000000Ui64
-#else
-#define DELTA_EPOCH_IN_MICROSECS  11644473600000000ULL
-#endif
- 
-#ifndef _TIMEZONE_DEFINED
-#define _TIMEZONE_DEFINED
-struct timezone 
-{
-	int  tz_minuteswest; /* minutes W of Greenwich */
-	int  tz_dsttime;     /* type of dst correction */
-};
-#endif
-
-int p_gettimeofday(struct timeval *tv, struct timezone *tz)
+int p_inet_pton(int af, const char *src, void *dst)
 {
-	FILETIME ft;
-	unsigned __int64 tmpres = 0;
-	static int tzflag;
-
-	if (NULL != tv)
-	{
-		GetSystemTimeAsFileTime(&ft);
-
-		tmpres |= ft.dwHighDateTime;
-		tmpres <<= 32;
-		tmpres |= ft.dwLowDateTime;
-
-		/*converting file time to unix epoch*/
-		tmpres /= 10;  /*convert into microseconds*/
-		tmpres -= DELTA_EPOCH_IN_MICROSECS;
-		tv->tv_sec = (long)(tmpres / 1000000UL);
-		tv->tv_usec = (long)(tmpres % 1000000UL);
-	}
+	struct sockaddr_storage sin;
+	void *addr;
+	int sin_len = sizeof(struct sockaddr_storage), addr_len;
+	int error = 0;
 
-	if (NULL != tz)
-	{
-		 if (!tzflag)
-		{
-			_tzset();
-			tzflag++;
-		}
-		tz->tz_minuteswest = _timezone / 60;
-		tz->tz_dsttime = _daylight;
+	if (af == AF_INET) {
+		addr = &((struct sockaddr_in *)&sin)->sin_addr;
+		addr_len = sizeof(struct in_addr);
+	} else if (af == AF_INET6) {
+		addr = &((struct sockaddr_in6 *)&sin)->sin6_addr;
+		addr_len = sizeof(struct in6_addr);
+	} else {
+		errno = EAFNOSUPPORT;
+		return -1;
 	}
 
-	return 0;
-}
-
-int p_inet_pton(int af, const char* src, void* dst)
-{
-	union {
-		struct sockaddr_in6 sin6;
-		struct sockaddr_in sin;
-	} sa;
-	int srcsize;
-
-	switch(af)
-	{
-		case AF_INET:
-			sa.sin.sin_family = AF_INET;
-			srcsize = (int)sizeof(sa.sin);
-		break;
-		case AF_INET6:
-			sa.sin6.sin6_family = AF_INET6;
-			srcsize = (int)sizeof(sa.sin6);
-		break;
-		default:
-			errno = WSAEPFNOSUPPORT;
-			return -1;
+	if ((error = WSAStringToAddressA((LPSTR)src, af, NULL, (LPSOCKADDR)&sin, &sin_len)) == 0) {
+		memcpy(dst, addr, addr_len);
+		return 1;
 	}
 
-	if (WSAStringToAddress((LPSTR)src, af, NULL, (struct sockaddr *) &sa, &srcsize) != 0)
-	{
-		errno = WSAGetLastError();
+	switch(WSAGetLastError()) {
+	case WSAEINVAL:
+		return 0;
+	case WSAEFAULT:
+		errno = ENOSPC;
+		return -1;
+	case WSA_NOT_ENOUGH_MEMORY:
+		errno = ENOMEM;
 		return -1;
 	}
 
-	switch(af)
-	{
-		case AF_INET:
-			memcpy(dst, &sa.sin.sin_addr, sizeof(sa.sin.sin_addr));
-		break;
-		case AF_INET6:
-			memcpy(dst, &sa.sin6.sin6_addr, sizeof(sa.sin6.sin6_addr));
-		break;
-	}
-
-	return 1;
+	errno = EINVAL;
+	return -1;
 }
diff --git a/src/win32/precompiled.h b/src/win32/precompiled.h
index 5de7e6f34..cbfe98812 100644
--- a/src/win32/precompiled.h
+++ b/src/win32/precompiled.h
@@ -1,4 +1,5 @@
 #include "git2.h"
+#include "common.h"
 
 #include <assert.h>
 #include <errno.h>
@@ -6,6 +7,8 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <fcntl.h>
+#include <time.h>
 
 #include <sys/types.h>
 #include <sys/stat.h>
diff --git a/src/win32/pthread.c b/src/win32/pthread.c
index 2f263b3e0..d50ace695 100644
--- a/src/win32/pthread.c
+++ b/src/win32/pthread.c
@@ -127,9 +127,10 @@ int pthread_cond_signal(pthread_cond_t *cond)
 	return 0;
 }
 
-/* pthread_cond_broadcast is not implemented because doing so with just Win32 events
- * is quite complicated, and no caller in libgit2 uses it yet. */
-
+/* pthread_cond_broadcast is not implemented because doing so with just
+ * Win32 events is quite complicated, and no caller in libgit2 uses it
+ * yet.
+ */
 int pthread_num_processors_np(void)
 {
 	DWORD_PTR p, s;
@@ -142,3 +143,111 @@ int pthread_num_processors_np(void)
 	return n ? n : 1;
 }
 
+
+static HINSTANCE win32_kernel32_dll;
+
+typedef void (WINAPI *win32_srwlock_fn)(GIT_SRWLOCK *);
+
+static win32_srwlock_fn win32_srwlock_initialize;
+static win32_srwlock_fn win32_srwlock_acquire_shared;
+static win32_srwlock_fn win32_srwlock_release_shared;
+static win32_srwlock_fn win32_srwlock_acquire_exclusive;
+static win32_srwlock_fn win32_srwlock_release_exclusive;
+
+int pthread_rwlock_init(
+	pthread_rwlock_t *GIT_RESTRICT lock,
+	const pthread_rwlockattr_t *GIT_RESTRICT attr)
+{
+	(void)attr;
+
+	if (win32_srwlock_initialize)
+		win32_srwlock_initialize(&lock->native.srwl);
+	else
+		InitializeCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_rdlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_acquire_shared)
+		win32_srwlock_acquire_shared(&lock->native.srwl);
+	else
+		EnterCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_rdunlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_release_shared)
+		win32_srwlock_release_shared(&lock->native.srwl);
+	else
+		LeaveCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_wrlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_acquire_exclusive)
+		win32_srwlock_acquire_exclusive(&lock->native.srwl);
+	else
+		EnterCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_wrunlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_release_exclusive)
+		win32_srwlock_release_exclusive(&lock->native.srwl);
+	else
+		LeaveCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_destroy(pthread_rwlock_t *lock)
+{
+	if (!win32_srwlock_initialize)
+		DeleteCriticalSection(&lock->native.csec);
+	git__memzero(lock, sizeof(*lock));
+	return 0;
+}
+
+
+int win32_pthread_initialize(void)
+{
+	if (win32_kernel32_dll)
+		return 0;
+
+	win32_kernel32_dll = LoadLibrary("Kernel32.dll");
+	if (!win32_kernel32_dll) {
+		giterr_set(GITERR_OS, "Could not load Kernel32.dll!");
+		return -1;
+	}
+
+	win32_srwlock_initialize = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "InitializeSRWLock");
+	win32_srwlock_acquire_shared = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "AcquireSRWLockShared");
+	win32_srwlock_release_shared = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockShared");
+	win32_srwlock_acquire_exclusive = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "AcquireSRWLockExclusive");
+	win32_srwlock_release_exclusive = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockExclusive");
+
+	return 0;
+}
+
+int win32_pthread_shutdown(void)
+{
+	if (win32_kernel32_dll) {
+		FreeLibrary(win32_kernel32_dll);
+		win32_kernel32_dll = NULL;
+	}
+
+	return 0;
+}
diff --git a/src/win32/pthread.h b/src/win32/pthread.h
index 8277ecf6e..2ba2ca552 100644
--- a/src/win32/pthread.h
+++ b/src/win32/pthread.h
@@ -19,22 +19,34 @@
 typedef int pthread_mutexattr_t;
 typedef int pthread_condattr_t;
 typedef int pthread_attr_t;
+typedef int pthread_rwlockattr_t;
+
 typedef CRITICAL_SECTION pthread_mutex_t;
 typedef HANDLE pthread_t;
 typedef HANDLE pthread_cond_t;
 
-#define PTHREAD_MUTEX_INITIALIZER {(void*)-1};
+typedef struct { void *Ptr; } GIT_SRWLOCK;
+
+typedef struct {
+	union {
+		GIT_SRWLOCK srwl;
+		CRITICAL_SECTION csec;
+	} native;
+} pthread_rwlock_t;
+
+#define PTHREAD_MUTEX_INITIALIZER  {(void*)-1}
 
 int pthread_create(
-	pthread_t *GIT_RESTRICT,
-	const pthread_attr_t *GIT_RESTRICT,
+	pthread_t *GIT_RESTRICT thread,
+	const pthread_attr_t *GIT_RESTRICT attr,
 	void *(*start_routine)(void*),
-	void *__restrict);
+	void *GIT_RESTRICT arg);
 
 int pthread_join(pthread_t, void **);
 
 int pthread_mutex_init(
-	pthread_mutex_t *GIT_RESTRICT, const pthread_mutexattr_t *GIT_RESTRICT);
+	pthread_mutex_t *GIT_RESTRICT mutex,
+	const pthread_mutexattr_t *GIT_RESTRICT mutexattr);
 int pthread_mutex_destroy(pthread_mutex_t *);
 int pthread_mutex_lock(pthread_mutex_t *);
 int pthread_mutex_unlock(pthread_mutex_t *);
@@ -47,4 +59,16 @@ int pthread_cond_signal(pthread_cond_t *);
 
 int pthread_num_processors_np(void);
 
+int pthread_rwlock_init(
+	pthread_rwlock_t *GIT_RESTRICT lock,
+	const pthread_rwlockattr_t *GIT_RESTRICT attr);
+int pthread_rwlock_rdlock(pthread_rwlock_t *);
+int pthread_rwlock_rdunlock(pthread_rwlock_t *);
+int pthread_rwlock_wrlock(pthread_rwlock_t *);
+int pthread_rwlock_wrunlock(pthread_rwlock_t *);
+int pthread_rwlock_destroy(pthread_rwlock_t *);
+
+extern int win32_pthread_initialize(void);
+extern int win32_pthread_shutdown(void);
+
 #endif
diff --git a/src/win32/utf-conv.c b/src/win32/utf-conv.c
index c06f3a8c2..d4dbfbab9 100644
--- a/src/win32/utf-conv.c
+++ b/src/win32/utf-conv.c
@@ -70,12 +70,12 @@ void git__utf8_to_16(wchar_t *dest, size_t length, const char *src)
 }
 #endif
 
-int git__utf8_to_16(wchar_t *dest, size_t length, const char *src)
+int git__utf8_to_16(wchar_t * dest, size_t dest_size, const char *src)
 {
-	return MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)length);
+	return MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)dest_size);
 }
 
-int git__utf16_to_8(char *out, const wchar_t *input)
+int git__utf16_to_8(char *dest, size_t dest_size, const wchar_t *src)
 {
-	return WideCharToMultiByte(CP_UTF8, 0, input, -1, out, GIT_WIN_PATH, NULL, NULL);
+	return WideCharToMultiByte(CP_UTF8, 0, src, -1, dest, (int)dest_size, NULL, NULL);
 }
diff --git a/src/win32/utf-conv.h b/src/win32/utf-conv.h
index 6cc9205f7..3af77580e 100644
--- a/src/win32/utf-conv.h
+++ b/src/win32/utf-conv.h
@@ -4,16 +4,35 @@
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
  */
+#ifndef INCLUDE_git_utfconv_h__
+#define INCLUDE_git_utfconv_h__
 
 #include <wchar.h>
+#include "common.h"
 
-#ifndef INCLUDE_git_utfconv_h__
-#define INCLUDE_git_utfconv_h__
+/* Maximum characters in a Windows path plus one for NUL byte */
+#define GIT_WIN_PATH_UTF16 (260 + 1)
 
-#define GIT_WIN_PATH (260 + 1)
+/* Maximum bytes necessary to convert a full-length UTF16 path to UTF8 */
+#define GIT_WIN_PATH_UTF8  (260 * 4 + 1)
 
-int git__utf8_to_16(wchar_t *dest, size_t length, const char *src);
-int git__utf16_to_8(char *dest, const wchar_t *src);
+typedef wchar_t git_win32_path[GIT_WIN_PATH_UTF16];
 
-#endif
+typedef char git_win32_path_as_utf8[GIT_WIN_PATH_UTF8];
 
+/* dest_size is the size of dest in wchar_t's */
+int git__utf8_to_16(wchar_t * dest, size_t dest_size, const char *src);
+/* dest_size is the size of dest in char's */
+int git__utf16_to_8(char *dest, size_t dest_size, const wchar_t *src);
+
+GIT_INLINE(int) git_win32_path_from_c(git_win32_path dest, const char *src)
+{
+	return git__utf8_to_16(dest, GIT_WIN_PATH_UTF16, src);
+}
+
+GIT_INLINE(int) git_win32_path_to_c(git_win32_path_as_utf8 dest, const wchar_t *src)
+{
+	return git__utf16_to_8(dest, GIT_WIN_PATH_UTF8, src);
+}
+
+#endif
diff --git a/src/win32/version.h b/src/win32/version.h
index 483962b57..79667697f 100644
--- a/src/win32/version.h
+++ b/src/win32/version.h
@@ -9,12 +9,29 @@
 
 #include <windows.h>
 
-GIT_INLINE(int) git_has_win32_version(int major, int minor)
+GIT_INLINE(int) git_has_win32_version(int major, int minor, int service_pack)
 {
-	WORD wVersion = LOWORD(GetVersion());
+	OSVERSIONINFOEX version_test = {0};
+	DWORD version_test_mask;
+	DWORDLONG version_condition_mask = 0;
+	
+	version_test.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEX);
+	version_test.dwMajorVersion = major;
+	version_test.dwMinorVersion = minor;
+	version_test.wServicePackMajor = (WORD)service_pack;
+	version_test.wServicePackMinor = 0;
 
-	return LOBYTE(wVersion) > major ||
-		(LOBYTE(wVersion) == major && HIBYTE(wVersion) >= minor);
+	version_test_mask = (VER_MAJORVERSION | VER_MINORVERSION | VER_SERVICEPACKMAJOR | VER_SERVICEPACKMINOR);
+
+	VER_SET_CONDITION(version_condition_mask, VER_MAJORVERSION, VER_GREATER_EQUAL);
+	VER_SET_CONDITION(version_condition_mask, VER_MINORVERSION, VER_GREATER_EQUAL);
+	VER_SET_CONDITION(version_condition_mask, VER_SERVICEPACKMAJOR, VER_GREATER_EQUAL);
+	VER_SET_CONDITION(version_condition_mask, VER_SERVICEPACKMINOR, VER_GREATER_EQUAL);
+
+	if (!VerifyVersionInfo(&version_test, version_test_mask, version_condition_mask))
+		return 0;
+
+	return 1;
 }
 
 #endif