Merge branch 'development'

152 files changed, 12815 insertions, 5375 deletions

diff --git a/src/array.h b/src/array.h
index 2d77c71a0..1d4e1c224 100644
--- a/src/array.h
+++ b/src/array.h
@@ -30,37 +30,45 @@
 #define git_array_init(a) \
 	do { (a).size = (a).asize = 0; (a).ptr = NULL; } while (0)
 
+#define git_array_init_to_size(a, desired) \
+	do { (a).size = 0; (a).asize = desired; (a).ptr = git__calloc(desired, sizeof(*(a).ptr)); } while (0)
+
 #define git_array_clear(a) \
 	do { git__free((a).ptr); git_array_init(a); } while (0)
 
 #define GITERR_CHECK_ARRAY(a) GITERR_CHECK_ALLOC((a).ptr)
 
 
-typedef git_array_t(void) git_array_generic_t;
+typedef git_array_t(char) git_array_generic_t;
 
 /* use a generic array for growth so this can return the new item */
-GIT_INLINE(void *) git_array_grow(git_array_generic_t *a, size_t item_size)
+GIT_INLINE(void *) git_array_grow(void *_a, size_t item_size)
 {
+	git_array_generic_t *a = _a;
 	uint32_t new_size = (a->size < 8) ? 8 : a->asize * 3 / 2;
-	void *new_array = git__realloc(a->ptr, new_size * item_size);
+	char *new_array = git__realloc(a->ptr, new_size * item_size);
 	if (!new_array) {
 		git_array_clear(*a);
 		return NULL;
 	} else {
 		a->ptr = new_array; a->asize = new_size; a->size++;
-		return (((char *)a->ptr) + (a->size - 1) * item_size);
+		return a->ptr + (a->size - 1) * item_size;
 	}
 }
 
 #define git_array_alloc(a) \
-	((a).size >= (a).asize) ? \
-	git_array_grow((git_array_generic_t *)&(a), sizeof(*(a).ptr)) :	\
-	(a).ptr ? &(a).ptr[(a).size++] : NULL
+	(((a).size >= (a).asize) ? \
+	git_array_grow(&(a), sizeof(*(a).ptr)) : \
+	((a).ptr ? &(a).ptr[(a).size++] : NULL))
 
 #define git_array_last(a) ((a).size ? &(a).ptr[(a).size - 1] : NULL)
 
+#define git_array_pop(a) ((a).size ? &(a).ptr[--(a).size] : NULL)
+
 #define git_array_get(a, i) (((i) < (a).size) ? &(a).ptr[(i)] : NULL)
 
 #define git_array_size(a) (a).size
 
+#define git_array_valid_index(a, i) ((i) < (a).size)
+
 #endif
diff --git a/src/attr.c b/src/attr.c
index 6cdff29f9..98a328a55 100644
--- a/src/attr.c
+++ b/src/attr.c
@@ -1,3 +1,4 @@
+#include "common.h"
 #include "repository.h"
 #include "fileops.h"
 #include "config.h"
@@ -26,7 +27,6 @@ git_attr_t git_attr_value(const char *attr)
 	return GIT_ATTR_VALUE_T;
 }
 
-
 static int collect_attr_files(
 	git_repository *repo,
 	uint32_t flags,
@@ -103,8 +103,6 @@ int git_attr_get_many(
 	attr_get_many_info *info = NULL;
 	size_t num_found = 0;
 
-	memset((void *)values, 0, sizeof(const char *) * num_attr);
-
 	if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0)
 		return -1;
 
@@ -141,6 +139,11 @@ int git_attr_get_many(
 		}
 	}
 
+	for (k = 0; k < num_attr; k++) {
+		if (!info[k].found)
+			values[k] = NULL;
+	}
+
 cleanup:
 	git_vector_free(&files);
 	git_attr_path__free(&path);
diff --git a/src/attr_file.c b/src/attr_file.c
index d880398e8..4eb732436 100644
--- a/src/attr_file.c
+++ b/src/attr_file.c
@@ -39,7 +39,7 @@ int git_attr_file__new(
 		attrs->key = git_pool_malloc(attrs->pool, (uint32_t)len + 3);
 		GITERR_CHECK_ALLOC(attrs->key);
 
-		attrs->key[0] = '0' + from;
+		attrs->key[0] = '0' + (char)from;
 		attrs->key[1] = '#';
 		memcpy(&attrs->key[2], path, len);
 		attrs->key[len + 2] = '\0';
@@ -79,9 +79,13 @@ int git_attr_file__parse_buffer(
 
 	while (!error && *scan) {
 		/* allocate rule if needed */
-		if (!rule && !(rule = git__calloc(1, sizeof(git_attr_rule)))) {
-			error = -1;
-			break;
+		if (!rule) {
+			if (!(rule = git__calloc(1, sizeof(git_attr_rule)))) {
+				error = -1;
+				break;
+			}
+			rule->match.flags = GIT_ATTR_FNMATCH_ALLOWNEG |
+				GIT_ATTR_FNMATCH_ALLOWMACRO;
 		}
 
 		/* parse the next "pattern attr attr attr" line */
@@ -351,8 +355,8 @@ int git_attr_fnmatch__parse(
 	if (parse_optimized_patterns(spec, pool, *base))
 		return 0;
 
-	spec->flags = (spec->flags & GIT_ATTR_FNMATCH_ALLOWSPACE);
-	allow_space = (spec->flags != 0);
+	spec->flags = (spec->flags & GIT_ATTR_FNMATCH__INCOMING);
+	allow_space = ((spec->flags & GIT_ATTR_FNMATCH_ALLOWSPACE) != 0);
 
 	pattern = *base;
 
@@ -362,7 +366,7 @@ int git_attr_fnmatch__parse(
 		return GIT_ENOTFOUND;
 	}
 
-	if (*pattern == '[') {
+	if (*pattern == '[' && (spec->flags & GIT_ATTR_FNMATCH_ALLOWMACRO) != 0) {
 		if (strncmp(pattern, "[attr]", 6) == 0) {
 			spec->flags = spec->flags | GIT_ATTR_FNMATCH_MACRO;
 			pattern += 6;
@@ -370,7 +374,7 @@ int git_attr_fnmatch__parse(
 		/* else a character range like [a-e]* which is accepted */
 	}
 
-	if (*pattern == '!') {
+	if (*pattern == '!' && (spec->flags & GIT_ATTR_FNMATCH_ALLOWNEG) != 0) {
 		spec->flags = spec->flags | GIT_ATTR_FNMATCH_NEGATIVE;
 		pattern++;
 	}
diff --git a/src/attr_file.h b/src/attr_file.h
index 15bba1c6a..3bc7c6cb8 100644
--- a/src/attr_file.h
+++ b/src/attr_file.h
@@ -28,6 +28,12 @@
 #define GIT_ATTR_FNMATCH_ALLOWSPACE	(1U << 6)
 #define GIT_ATTR_FNMATCH_ICASE		(1U << 7)
 #define GIT_ATTR_FNMATCH_MATCH_ALL	(1U << 8)
+#define GIT_ATTR_FNMATCH_ALLOWNEG   (1U << 9)
+#define GIT_ATTR_FNMATCH_ALLOWMACRO (1U << 10)
+
+#define GIT_ATTR_FNMATCH__INCOMING \
+	(GIT_ATTR_FNMATCH_ALLOWSPACE | \
+	 GIT_ATTR_FNMATCH_ALLOWNEG | GIT_ATTR_FNMATCH_ALLOWMACRO)
 
 extern const char *git_attr__true;
 extern const char *git_attr__false;
diff --git a/src/bitvec.h b/src/bitvec.h
new file mode 100644
index 000000000..fd6f0ccf8
--- /dev/null
+++ b/src/bitvec.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_bitvec_h__
+#define INCLUDE_bitvec_h__
+
+#include "util.h"
+
+/*
+ * This is a silly little fixed length bit vector type that will store
+ * vectors of 64 bits or less directly in the structure and allocate
+ * memory for vectors longer than 64 bits.  You can use the two versions
+ * transparently through the API and avoid heap allocation completely when
+ * using a short bit vector as a result.
+ */
+typedef struct {
+	size_t length;
+	union {
+		uint64_t *words;
+		uint64_t bits;
+	} u;
+} git_bitvec;
+
+GIT_INLINE(int) git_bitvec_init(git_bitvec *bv, size_t capacity)
+{
+	memset(bv, 0x0, sizeof(*bv));
+
+	if (capacity >= 64) {
+		bv->length = (capacity / 64) + 1;
+		bv->u.words = git__calloc(bv->length, sizeof(uint64_t));
+		if (!bv->u.words)
+			return -1;
+	}
+
+	return 0;
+}
+
+#define GIT_BITVEC_MASK(BIT) ((uint64_t)1 << (BIT % 64))
+#define GIT_BITVEC_WORD(BV, BIT) (BV->length ? &BV->u.words[BIT / 64] : &BV->u.bits)
+
+GIT_INLINE(void) git_bitvec_set(git_bitvec *bv, size_t bit, bool on)
+{
+	uint64_t *word = GIT_BITVEC_WORD(bv, bit);
+	uint64_t mask = GIT_BITVEC_MASK(bit);
+
+	if (on)
+		*word |= mask;
+	else
+		*word &= ~mask;
+}
+
+GIT_INLINE(bool) git_bitvec_get(git_bitvec *bv, size_t bit)
+{
+	uint64_t *word = GIT_BITVEC_WORD(bv, bit);
+	return (*word & GIT_BITVEC_MASK(bit)) != 0;
+}
+
+GIT_INLINE(void) git_bitvec_clear(git_bitvec *bv)
+{
+	if (!bv->length)
+		bv->u.bits = 0;
+	else
+		memset(bv->u.words, 0x0, bv->length * sizeof(uint64_t));
+}
+
+GIT_INLINE(void) git_bitvec_free(git_bitvec *bv)
+{
+	if (bv->length)
+		git__free(bv->u.words);
+}
+
+#endif
diff --git a/src/blame.c b/src/blame.c
new file mode 100644
index 000000000..219a6bfe4
--- /dev/null
+++ b/src/blame.c
@@ -0,0 +1,476 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "blame.h"
+#include "git2/commit.h"
+#include "git2/revparse.h"
+#include "git2/revwalk.h"
+#include "git2/tree.h"
+#include "git2/diff.h"
+#include "git2/blob.h"
+#include "git2/signature.h"
+#include "util.h"
+#include "repository.h"
+#include "blame_git.h"
+
+
+static int hunk_byfinalline_search_cmp(const void *key, const void *entry)
+{
+	uint16_t lineno = (uint16_t)*(size_t*)key;
+	git_blame_hunk *hunk = (git_blame_hunk*)entry;
+
+	if (lineno < hunk->final_start_line_number)
+		return -1;
+	if (lineno >= hunk->final_start_line_number + hunk->lines_in_hunk)
+		return 1;
+	return 0;
+}
+
+static int paths_cmp(const void *a, const void *b) { return git__strcmp((char*)a, (char*)b); }
+static int hunk_cmp(const void *_a, const void *_b)
+{
+	git_blame_hunk *a = (git_blame_hunk*)_a,
+						*b = (git_blame_hunk*)_b;
+
+	return a->final_start_line_number - b->final_start_line_number;
+}
+
+static bool hunk_ends_at_or_before_line(git_blame_hunk *hunk, size_t line)
+{
+	return line >= (size_t)(hunk->final_start_line_number + hunk->lines_in_hunk - 1);
+}
+
+static bool hunk_starts_at_or_after_line(git_blame_hunk *hunk, size_t line)
+{
+	return line <= hunk->final_start_line_number;
+}
+
+static git_blame_hunk* new_hunk(
+		uint16_t start,
+		uint16_t lines,
+		uint16_t orig_start,
+		const char *path)
+{
+	git_blame_hunk *hunk = git__calloc(1, sizeof(git_blame_hunk));
+	if (!hunk) return NULL;
+
+	hunk->lines_in_hunk = lines;
+	hunk->final_start_line_number = start;
+	hunk->orig_start_line_number = orig_start;
+	hunk->orig_path = path ? git__strdup(path) : NULL;
+
+	return hunk;
+}
+
+static git_blame_hunk* dup_hunk(git_blame_hunk *hunk)
+{
+	git_blame_hunk *newhunk = new_hunk(
+			hunk->final_start_line_number,
+			hunk->lines_in_hunk,
+			hunk->orig_start_line_number,
+			hunk->orig_path);
+	git_oid_cpy(&newhunk->orig_commit_id, &hunk->orig_commit_id);
+	git_oid_cpy(&newhunk->final_commit_id, &hunk->final_commit_id);
+	newhunk->boundary = hunk->boundary;
+	newhunk->final_signature = git_signature_dup(hunk->final_signature);
+	newhunk->orig_signature = git_signature_dup(hunk->orig_signature);
+	return newhunk;
+}
+
+static void free_hunk(git_blame_hunk *hunk)
+{
+	git__free((void*)hunk->orig_path);
+	git_signature_free(hunk->final_signature);
+	git_signature_free(hunk->orig_signature);
+	git__free(hunk);
+}
+
+/* Starting with the hunk that includes start_line, shift all following hunks'
+ * final_start_line by shift_by lines */
+static void shift_hunks_by(git_vector *v, size_t start_line, int shift_by)
+{
+	size_t i;
+
+	if (!git_vector_bsearch2( &i, v, hunk_byfinalline_search_cmp, &start_line)) {
+		for (; i < v->length; i++) {
+			git_blame_hunk *hunk = (git_blame_hunk*)v->contents[i];
+			hunk->final_start_line_number += shift_by;
+		}
+	}
+}
+
+git_blame* git_blame__alloc(
+	git_repository *repo,
+	git_blame_options opts,
+	const char *path)
+{
+	git_blame *gbr = (git_blame*)git__calloc(1, sizeof(git_blame));
+	if (!gbr) {
+		giterr_set_oom();
+		return NULL;
+	}
+	git_vector_init(&gbr->hunks, 8, hunk_cmp);
+	git_vector_init(&gbr->paths, 8, paths_cmp);
+	gbr->repository = repo;
+	gbr->options = opts;
+	gbr->path = git__strdup(path);
+	git_vector_insert(&gbr->paths, git__strdup(path));
+	return gbr;
+}
+
+void git_blame_free(git_blame *blame)
+{
+	size_t i;
+	git_blame_hunk *hunk;
+	char *path;
+
+	if (!blame) return;
+
+	git_vector_foreach(&blame->hunks, i, hunk)
+		free_hunk(hunk);
+	git_vector_free(&blame->hunks);
+
+	git_vector_foreach(&blame->paths, i, path)
+		git__free(path);
+	git_vector_free(&blame->paths);
+
+	git_array_clear(blame->line_index);
+
+	git__free((void*)blame->path);
+	git_blob_free(blame->final_blob);
+	git__free(blame);
+}
+
+uint32_t git_blame_get_hunk_count(git_blame *blame)
+{
+	assert(blame);
+	return (uint32_t)blame->hunks.length;
+}
+
+const git_blame_hunk *git_blame_get_hunk_byindex(git_blame *blame, uint32_t index)
+{
+	assert(blame);
+	return (git_blame_hunk*)git_vector_get(&blame->hunks, index);
+}
+
+const git_blame_hunk *git_blame_get_hunk_byline(git_blame *blame, uint32_t lineno)
+{
+	size_t i;
+	assert(blame);
+
+	if (!git_vector_bsearch2( &i, &blame->hunks, hunk_byfinalline_search_cmp, &lineno)) {
+		return git_blame_get_hunk_byindex(blame, (uint32_t)i);
+	}
+
+	return NULL;
+}
+
+static void normalize_options(
+		git_blame_options *out,
+		const git_blame_options *in,
+		git_repository *repo)
+{
+	git_blame_options dummy = GIT_BLAME_OPTIONS_INIT;
+	if (!in) in = &dummy;
+
+	memcpy(out, in, sizeof(git_blame_options));
+
+	/* No newest_commit => HEAD */
+	if (git_oid_iszero(&out->newest_commit)) {
+		git_reference_name_to_id(&out->newest_commit, repo, "HEAD");
+	}
+
+	/* min_line 0 really means 1 */
+	if (!out->min_line) out->min_line = 1;
+	/* max_line 0 really means N, but we don't know N yet */
+
+	/* Fix up option implications */
+	if (out->flags & GIT_BLAME_TRACK_COPIES_ANY_COMMIT_COPIES)
+		out->flags |= GIT_BLAME_TRACK_COPIES_SAME_COMMIT_COPIES;
+	if (out->flags & GIT_BLAME_TRACK_COPIES_SAME_COMMIT_COPIES)
+		out->flags |= GIT_BLAME_TRACK_COPIES_SAME_COMMIT_MOVES;
+	if (out->flags & GIT_BLAME_TRACK_COPIES_SAME_COMMIT_MOVES)
+		out->flags |= GIT_BLAME_TRACK_COPIES_SAME_FILE;
+}
+
+static git_blame_hunk *split_hunk_in_vector(
+		git_vector *vec,
+		git_blame_hunk *hunk,
+		size_t rel_line,
+		bool return_new)
+{
+	size_t new_line_count;
+	git_blame_hunk *nh;
+
+	/* Don't split if already at a boundary */
+	if (rel_line <= 0 ||
+	    rel_line >= hunk->lines_in_hunk)
+	{
+		return hunk;
+	}
+
+	new_line_count = hunk->lines_in_hunk - rel_line;
+	nh = new_hunk((uint16_t)(hunk->final_start_line_number+rel_line), (uint16_t)new_line_count,
+			(uint16_t)(hunk->orig_start_line_number+rel_line), hunk->orig_path);
+	git_oid_cpy(&nh->final_commit_id, &hunk->final_commit_id);
+	git_oid_cpy(&nh->orig_commit_id, &hunk->orig_commit_id);
+
+	/* Adjust hunk that was split */
+	hunk->lines_in_hunk -= (uint16_t)new_line_count;
+	git_vector_insert_sorted(vec, nh, NULL);
+	{
+		git_blame_hunk *ret = return_new ? nh : hunk;
+		return ret;
+	}
+}
+
+/*
+ * Construct a list of char indices for where lines begin
+ * Adapted from core git:
+ * https://github.com/gitster/git/blob/be5c9fb9049ed470e7005f159bb923a5f4de1309/builtin/blame.c#L1760-L1789
+ */
+static int index_blob_lines(git_blame *blame)
+{
+    const char *buf = blame->final_buf;
+    git_off_t len = blame->final_buf_size;
+    int num = 0, incomplete = 0, bol = 1;
+    size_t *i;
+ 
+    if (len && buf[len-1] != '\n')
+        incomplete++; /* incomplete line at the end */
+    while (len--) {
+        if (bol) {
+            i = git_array_alloc(blame->line_index);
+            GITERR_CHECK_ALLOC(i);
+            *i = buf - blame->final_buf;
+            bol = 0;
+        }
+        if (*buf++ == '\n') {
+            num++;
+            bol = 1;
+        }
+    }
+    i = git_array_alloc(blame->line_index);
+    GITERR_CHECK_ALLOC(i);
+    *i = buf - blame->final_buf;
+    blame->num_lines = num + incomplete;
+    return blame->num_lines;
+}
+ 
+static git_blame_hunk* hunk_from_entry(git_blame__entry *e)
+{
+	git_blame_hunk *h = new_hunk(
+			e->lno+1, e->num_lines, e->s_lno+1, e->suspect->path);
+	git_oid_cpy(&h->final_commit_id, git_commit_id(e->suspect->commit));
+	h->final_signature = git_signature_dup(git_commit_author(e->suspect->commit));
+	h->boundary = e->is_boundary ? 1 : 0;
+	return h;
+}
+
+static int load_blob(git_blame *blame)
+{
+	int error;
+
+	if (blame->final_blob) return 0;
+
+	error = git_commit_lookup(&blame->final, blame->repository, &blame->options.newest_commit);
+	if (error < 0)
+		goto cleanup;
+	error = git_object_lookup_bypath((git_object**)&blame->final_blob,
+			(git_object*)blame->final, blame->path, GIT_OBJ_BLOB);
+	if (error < 0)
+		goto cleanup;
+
+cleanup:
+	return error;
+}
+
+static int blame_internal(git_blame *blame)
+{
+	int error;
+	git_blame__entry *ent = NULL;
+	git_blame__origin *o;
+
+	if ((error = load_blob(blame)) < 0 ||
+	    (error = git_blame__get_origin(&o, blame, blame->final, blame->path)) < 0)
+		goto cleanup;
+	blame->final_buf = git_blob_rawcontent(blame->final_blob);
+	blame->final_buf_size = git_blob_rawsize(blame->final_blob);
+
+	ent = git__calloc(1, sizeof(git_blame__entry));
+	ent->num_lines = index_blob_lines(blame);
+	ent->lno = blame->options.min_line - 1;
+	ent->num_lines = ent->num_lines - blame->options.min_line + 1;
+	if (blame->options.max_line > 0)
+		ent->num_lines = blame->options.max_line - blame->options.min_line + 1;
+	ent->s_lno = ent->lno;
+	ent->suspect = o;
+
+	blame->ent = ent;
+	blame->path = blame->path;
+
+	git_blame__like_git(blame, blame->options.flags);
+
+cleanup:
+	for (ent = blame->ent; ent; ) {
+		git_blame__entry *e = ent->next;
+
+		git_vector_insert(&blame->hunks, hunk_from_entry(ent));
+
+		git_blame__free_entry(ent);
+		ent = e;
+	}
+
+	return error;
+}
+
+/*******************************************************************************
+ * File blaming
+ ******************************************************************************/
+
+int git_blame_file(
+		git_blame **out,
+		git_repository *repo,
+		const char *path,
+		git_blame_options *options)
+{
+	int error = -1;
+	git_blame_options normOptions = GIT_BLAME_OPTIONS_INIT;
+	git_blame *blame = NULL;
+
+	assert(out && repo && path);
+	normalize_options(&normOptions, options, repo);
+
+	blame = git_blame__alloc(repo, normOptions, path);
+	GITERR_CHECK_ALLOC(blame);
+
+	if ((error = load_blob(blame)) < 0)
+		goto on_error;
+
+	if ((error = blame_internal(blame)) < 0)
+		goto on_error;
+
+	*out = blame;
+	return 0;
+
+on_error:
+	git_blame_free(blame);
+	return error;
+}
+
+/*******************************************************************************
+ * Buffer blaming
+ *******************************************************************************/
+
+static bool hunk_is_bufferblame(git_blame_hunk *hunk)
+{
+	return git_oid_iszero(&hunk->final_commit_id);
+}
+
+static int buffer_hunk_cb(
+	const git_diff_delta *delta,
+	const git_diff_hunk *hunk,
+	void *payload)
+{
+	git_blame *blame = (git_blame*)payload;
+	uint32_t wedge_line;
+
+	GIT_UNUSED(delta);
+
+	wedge_line = (hunk->old_lines == 0) ? hunk->new_start : hunk->old_start;
+	blame->current_diff_line = wedge_line;
+
+	blame->current_hunk = (git_blame_hunk*)git_blame_get_hunk_byline(blame, wedge_line);
+	if (!blame->current_hunk) {
+		/* Line added at the end of the file */
+		blame->current_hunk = new_hunk(wedge_line, 0, wedge_line, blame->path);
+		git_vector_insert(&blame->hunks, blame->current_hunk);
+	} else if (!hunk_starts_at_or_after_line(blame->current_hunk, wedge_line)){
+		/* If this hunk doesn't start between existing hunks, split a hunk up so it does */
+		blame->current_hunk = split_hunk_in_vector(&blame->hunks, blame->current_hunk,
+				wedge_line - blame->current_hunk->orig_start_line_number, true);
+	}
+
+	return 0;
+}
+
+static int ptrs_equal_cmp(const void *a, const void *b) { return a<b ? -1 : a>b ? 1 : 0; }
+static int buffer_line_cb(
+	const git_diff_delta *delta,
+	const git_diff_hunk *hunk,
+	const git_diff_line *line,
+	void *payload)
+{
+	git_blame *blame = (git_blame*)payload;
+
+	GIT_UNUSED(delta);
+	GIT_UNUSED(hunk);
+	GIT_UNUSED(line);
+
+	if (line->origin == GIT_DIFF_LINE_ADDITION) {
+		if (hunk_is_bufferblame(blame->current_hunk) &&
+		    hunk_ends_at_or_before_line(blame->current_hunk, blame->current_diff_line)) {
+			/* Append to the current buffer-blame hunk */
+			blame->current_hunk->lines_in_hunk++;
+			shift_hunks_by(&blame->hunks, blame->current_diff_line+1, 1);
+		} else {
+			/* Create a new buffer-blame hunk with this line */
+			shift_hunks_by(&blame->hunks, blame->current_diff_line, 1);
+			blame->current_hunk = new_hunk((uint16_t)blame->current_diff_line, 1, 0, blame->path);
+			git_vector_insert_sorted(&blame->hunks, blame->current_hunk, NULL);
+		}
+		blame->current_diff_line++;
+	}
+
+	if (line->origin == GIT_DIFF_LINE_DELETION) {
+		/* Trim the line from the current hunk; remove it if it's now empty */
+		size_t shift_base = blame->current_diff_line + blame->current_hunk->lines_in_hunk+1;
+
+		if (--(blame->current_hunk->lines_in_hunk) == 0) {
+			size_t i;
+			shift_base--;
+			if (!git_vector_search2(&i, &blame->hunks, ptrs_equal_cmp, blame->current_hunk)) {
+				git_vector_remove(&blame->hunks, i);
+				free_hunk(blame->current_hunk);
+				blame->current_hunk = (git_blame_hunk*)git_blame_get_hunk_byindex(blame, (uint32_t)i);
+			}
+		}
+		shift_hunks_by(&blame->hunks, shift_base, -1);
+	}
+	return 0;
+}
+
+int git_blame_buffer(
+		git_blame **out,
+		git_blame *reference,
+		const char *buffer,
+		uint32_t buffer_len)
+{
+	git_blame *blame;
+	git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT;
+	size_t i;
+	git_blame_hunk *hunk;
+
+	diffopts.context_lines = 0;
+
+	assert(out && reference && buffer && buffer_len);
+
+	blame = git_blame__alloc(reference->repository, reference->options, reference->path);
+
+	/* Duplicate all of the hunk structures in the reference blame */
+	git_vector_foreach(&reference->hunks, i, hunk) {
+		git_vector_insert(&blame->hunks, dup_hunk(hunk));
+	}
+
+	/* Diff to the reference blob */
+	git_diff_blob_to_buffer(reference->final_blob, blame->path,
+			buffer, buffer_len, blame->path,
+			&diffopts, NULL, buffer_hunk_cb, buffer_line_cb, blame);
+
+	*out = blame;
+	return 0;
+}
diff --git a/src/blame.h b/src/blame.h
new file mode 100644
index 000000000..637e43985
--- /dev/null
+++ b/src/blame.h
@@ -0,0 +1,93 @@
+#ifndef INCLUDE_blame_h__
+#define INCLUDE_blame_h__
+
+#include "git2/blame.h"
+#include "common.h"
+#include "vector.h"
+#include "diff.h"
+#include "array.h"
+#include "git2/oid.h"
+
+/*
+ * One blob in a commit that is being suspected
+ */
+typedef struct git_blame__origin {
+	int refcnt;
+	struct git_blame__origin *previous;
+	git_commit *commit;
+	git_blob *blob;
+	char path[GIT_FLEX_ARRAY];
+} git_blame__origin;
+
+/*
+ * Each group of lines is described by a git_blame__entry; it can be split
+ * as we pass blame to the parents.  They form a linked list in the
+ * scoreboard structure, sorted by the target line number.
+ */
+typedef struct git_blame__entry {
+	struct git_blame__entry *prev;
+	struct git_blame__entry *next;
+
+	/* the first line of this group in the final image;
+	 * internally all line numbers are 0 based.
+	 */
+	int lno;
+
+	/* how many lines this group has */
+	int num_lines;
+
+	/* the commit that introduced this group into the final image */
+	git_blame__origin *suspect;
+
+	/* true if the suspect is truly guilty; false while we have not
+	 * checked if the group came from one of its parents.
+	 */
+	bool guilty;
+
+	/* true if the entry has been scanned for copies in the current parent
+	 */
+	bool scanned;
+
+	/* the line number of the first line of this group in the
+	 * suspect's file; internally all line numbers are 0 based.
+	 */
+	int s_lno;
+
+	/* how significant this entry is -- cached to avoid
+	 * scanning the lines over and over.
+	 */
+	unsigned score;
+
+	/* Whether this entry has been tracked to a boundary commit.
+	 */
+	bool is_boundary;
+} git_blame__entry;
+
+struct git_blame {
+	const char *path;
+	git_repository *repository;
+	git_blame_options options;
+
+	git_vector hunks;
+	git_vector paths;
+
+	git_blob *final_blob;
+	git_array_t(size_t) line_index;
+
+	size_t current_diff_line;
+	git_blame_hunk *current_hunk;
+
+	/* Scoreboard fields */
+	git_commit *final;
+	git_blame__entry *ent;
+	int num_lines;
+	const char *final_buf;
+	git_off_t final_buf_size;
+};
+
+git_blame *git_blame__alloc(
+	git_repository *repo,
+	git_blame_options opts,
+	const char *path);
+
+#endif
diff --git a/src/blame_git.c b/src/blame_git.c
new file mode 100644
index 000000000..800f1f039
--- /dev/null
+++ b/src/blame_git.c
@@ -0,0 +1,622 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "blame_git.h"
+#include "commit.h"
+#include "blob.h"
+#include "xdiff/xinclude.h"
+
+/*
+ * Origin is refcounted and usually we keep the blob contents to be
+ * reused.
+ */
+static git_blame__origin *origin_incref(git_blame__origin *o)
+{
+	if (o)
+		o->refcnt++;
+	return o;
+}
+
+static void origin_decref(git_blame__origin *o)
+{
+	if (o && --o->refcnt <= 0) {
+		if (o->previous)
+			origin_decref(o->previous);
+		git_blob_free(o->blob);
+		git_commit_free(o->commit);
+		git__free(o);
+	}
+}
+
+/* Given a commit and a path in it, create a new origin structure. */
+static int make_origin(git_blame__origin **out, git_commit *commit, const char *path)
+{
+	int error = 0;
+	git_blame__origin *o;
+
+	o = git__calloc(1, sizeof(*o) + strlen(path) + 1);
+	GITERR_CHECK_ALLOC(o);
+	o->commit = commit;
+	o->refcnt = 1;
+	strcpy(o->path, path);
+
+	if (!(error = git_object_lookup_bypath((git_object**)&o->blob, (git_object*)commit,
+			path, GIT_OBJ_BLOB))) {
+		*out = o;
+	} else {
+		origin_decref(o);
+	}
+	return error;
+}
+
+/* Locate an existing origin or create a new one. */
+int git_blame__get_origin(
+		git_blame__origin **out,
+		git_blame *blame,
+		git_commit *commit,
+		const char *path)
+{
+	git_blame__entry *e;
+
+	for (e = blame->ent; e; e = e->next) {
+		if (e->suspect->commit == commit && !strcmp(e->suspect->path, path)) {
+			*out = origin_incref(e->suspect);
+		}
+	}
+	return make_origin(out, commit, path);
+}
+
+typedef struct blame_chunk_cb_data {
+	git_blame *blame;
+	git_blame__origin *target;
+	git_blame__origin *parent;
+	long tlno;
+	long plno;
+}blame_chunk_cb_data;
+
+static bool same_suspect(git_blame__origin *a, git_blame__origin *b)
+{
+	if (a == b)
+		return true;
+	if (git_oid_cmp(git_commit_id(a->commit), git_commit_id(b->commit)))
+		return false;
+	return 0 == strcmp(a->path, b->path);
+}
+
+/* find the line number of the last line the target is suspected for */
+static int find_last_in_target(git_blame *blame, git_blame__origin *target)
+{
+	git_blame__entry *e;
+	int last_in_target = -1;
+
+	for (e=blame->ent; e; e=e->next) {
+		if (e->guilty || !same_suspect(e->suspect, target))
+			continue;
+		if (last_in_target < e->s_lno + e->num_lines)
+			last_in_target = e->s_lno + e->num_lines;
+	}
+	return last_in_target;
+}
+
+/*
+ * It is known that lines between tlno to same came from parent, and e
+ * has an overlap with that range.  it also is known that parent's
+ * line plno corresponds to e's line tlno.
+ *
+ *                <---- e ----->
+ *                   <------>         (entirely within)
+ *                   <------------>   (extends past)
+ *             <------------>         (starts before)
+ *             <------------------>   (entirely encloses)
+ *
+ * Split e into potentially three parts; before this chunk, the chunk
+ * to be blamed for the parent, and after that portion.
+ */
+static void split_overlap(git_blame__entry *split, git_blame__entry *e,
+		int tlno, int plno, int same, git_blame__origin *parent)
+{
+	int chunk_end_lno;
+
+	if (e->s_lno < tlno) {
+		/* there is a pre-chunk part not blamed on the parent */
+		split[0].suspect = origin_incref(e->suspect);
+		split[0].lno = e->lno;
+		split[0].s_lno = e->s_lno;
+		split[0].num_lines = tlno - e->s_lno;
+		split[1].lno = e->lno + tlno - e->s_lno;
+		split[1].s_lno = plno;
+	} else {
+		split[1].lno = e->lno;
+		split[1].s_lno = plno + (e->s_lno - tlno);
+	}
+
+	if (same < e->s_lno + e->num_lines) {
+		/* there is a post-chunk part not blamed on parent */
+		split[2].suspect = origin_incref(e->suspect);
+		split[2].lno = e->lno + (same - e->s_lno);
+		split[2].s_lno = e->s_lno + (same - e->s_lno);
+		split[2].num_lines = e->s_lno + e->num_lines - same;
+		chunk_end_lno = split[2].lno;
+	} else {
+		chunk_end_lno = e->lno + e->num_lines;
+	}
+	split[1].num_lines = chunk_end_lno - split[1].lno;
+
+	/*
+	 * if it turns out there is nothing to blame the parent for, forget about
+	 * the splitting. !split[1].suspect signals this.
+	 */
+	if (split[1].num_lines < 1)
+		return;
+	split[1].suspect = origin_incref(parent);
+}
+
+/*
+ * Link in a new blame entry to the scoreboard. Entries that cover the same
+ * line range have been removed from the scoreboard previously.
+ */
+static void add_blame_entry(git_blame *blame, git_blame__entry *e)
+{
+	git_blame__entry *ent, *prev = NULL;
+
+	origin_incref(e->suspect);
+
+	for (ent = blame->ent; ent && ent->lno < e->lno; ent = ent->next)
+		prev = ent;
+
+	/* prev, if not NULL, is the last one that is below e */
+	e->prev = prev;
+	if (prev) {
+		e->next = prev->next;
+		prev->next = e;
+	} else {
+		e->next = blame->ent;
+		blame->ent = e;
+	}
+	if (e->next)
+		e->next->prev = e;
+}
+
+/*
+ * src typically is on-stack; we want to copy the information in it to
+ * a malloced blame_entry that is already on the linked list of the scoreboard.
+ * The origin of dst loses a refcnt while the origin of src gains one.
+ */
+static void dup_entry(git_blame__entry *dst, git_blame__entry *src)
+{
+	git_blame__entry *p, *n;
+
+	p = dst->prev;
+	n = dst->next;
+	origin_incref(src->suspect);
+	origin_decref(dst->suspect);
+	memcpy(dst, src, sizeof(*src));
+	dst->prev = p;
+	dst->next = n;
+	dst->score = 0;
+}
+
+/*
+ * split_overlap() divided an existing blame e into up to three parts in split.
+ * Adjust the linked list of blames in the scoreboard to reflect the split.
+ */
+static void split_blame(git_blame *blame, git_blame__entry *split, git_blame__entry *e)
+{
+	git_blame__entry *new_entry;
+
+	if (split[0].suspect && split[2].suspect) {
+		/* The first part (reuse storage for the existing entry e */
+		dup_entry(e, &split[0]);
+
+		/* The last part -- me */
+		new_entry = git__malloc(sizeof(*new_entry));
+		memcpy(new_entry, &(split[2]), sizeof(git_blame__entry));
+		add_blame_entry(blame, new_entry);
+
+		/* ... and the middle part -- parent */
+		new_entry = git__malloc(sizeof(*new_entry));
+		memcpy(new_entry, &(split[1]), sizeof(git_blame__entry));
+		add_blame_entry(blame, new_entry);
+	} else if (!split[0].suspect && !split[2].suspect) {
+		/*
+		 * The parent covers the entire area; reuse storage for e and replace it
+		 * with the parent
+		 */
+		dup_entry(e, &split[1]);
+	} else if (split[0].suspect) {
+		/* me and then parent */
+		dup_entry(e, &split[0]);
+		new_entry = git__malloc(sizeof(*new_entry));
+		memcpy(new_entry, &(split[1]), sizeof(git_blame__entry));
+		add_blame_entry(blame, new_entry);
+	} else {
+		/* parent and then me */
+		dup_entry(e, &split[1]);
+		new_entry = git__malloc(sizeof(*new_entry));
+		memcpy(new_entry, &(split[2]), sizeof(git_blame__entry));
+		add_blame_entry(blame, new_entry);
+	}
+}
+
+/* 
+ * After splitting the blame, the origins used by the on-stack blame_entry
+ * should lose one refcnt each.
+ */
+static void decref_split(git_blame__entry *split)
+{
+	int i;
+	for (i=0; i<3; i++)
+		origin_decref(split[i].suspect);
+}
+
+/*
+ * Helper for blame_chunk(). blame_entry e is known to overlap with the patch
+ * hunk; split it and pass blame to the parent.
+ */
+static void blame_overlap(
+		git_blame *blame,
+		git_blame__entry *e,
+		int tlno,
+		int plno,
+		int same,
+		git_blame__origin *parent)
+{
+	git_blame__entry split[3] = {{0}};
+
+	split_overlap(split, e, tlno, plno, same, parent);
+	if (split[1].suspect)
+		split_blame(blame, split, e);
+	decref_split(split);
+}
+
+/*
+ * Process one hunk from the patch between the current suspect for blame_entry
+ * e and its parent. Find and split the overlap, and pass blame to the
+ * overlapping part to the parent.
+ */
+static void blame_chunk(
+		git_blame *blame,
+		int tlno,
+		int plno,
+		int same,
+		git_blame__origin *target,
+		git_blame__origin *parent)
+{
+	git_blame__entry *e;
+
+	for (e = blame->ent; e; e = e->next) {
+		if (e->guilty || !same_suspect(e->suspect, target))
+			continue;
+		if (same <= e->s_lno)
+			continue;
+		if (tlno < e->s_lno + e->num_lines) {
+			blame_overlap(blame, e, tlno, plno, same, parent);
+		}
+	}
+}
+
+static int my_emit(
+		xdfenv_t *xe,
+		xdchange_t *xscr,
+		xdemitcb_t *ecb,
+		xdemitconf_t const *xecfg)
+{
+	xdchange_t *xch = xscr;
+	GIT_UNUSED(xe);
+	GIT_UNUSED(xecfg);
+	while (xch) {
+		blame_chunk_cb_data *d = ecb->priv;
+		blame_chunk(d->blame, d->tlno, d->plno, xch->i2, d->target, d->parent);
+		d->plno = xch->i1 + xch->chg1;
+		d->tlno = xch->i2 + xch->chg2;
+		xch = xch->next;
+	}
+	return 0;
+}
+
+static void trim_common_tail(mmfile_t *a, mmfile_t *b, long ctx)
+{
+	const int blk = 1024;
+	long trimmed = 0, recovered = 0;
+	char *ap = a->ptr + a->size;
+	char *bp = b->ptr + b->size;
+	long smaller = (long)((a->size < b->size) ? a->size : b->size);
+
+	if (ctx)
+		return;
+
+	while (blk + trimmed <= smaller && !memcmp(ap - blk, bp - blk, blk)) {
+		trimmed += blk;
+		ap -= blk;
+		bp -= blk;
+	}
+
+	while (recovered < trimmed)
+		if (ap[recovered++] == '\n')
+			break;
+	a->size -= trimmed - recovered;
+	b->size -= trimmed - recovered;
+}
+
+static int diff_hunks(mmfile_t file_a, mmfile_t file_b, void *cb_data)
+{
+	xpparam_t xpp = {0};
+	xdemitconf_t xecfg = {0};
+	xdemitcb_t ecb = {0};
+
+	xecfg.emit_func = (void(*)(void))my_emit;
+	ecb.priv = cb_data;
+
+	trim_common_tail(&file_a, &file_b, 0);
+	return xdl_diff(&file_a, &file_b, &xpp, &xecfg, &ecb);
+}
+
+static void fill_origin_blob(git_blame__origin *o, mmfile_t *file)
+{
+	memset(file, 0, sizeof(*file));
+	if (o->blob) {
+		file->ptr = (char*)git_blob_rawcontent(o->blob);
+		file->size = (size_t)git_blob_rawsize(o->blob);
+	}
+}
+
+static int pass_blame_to_parent(
+		git_blame *blame,
+		git_blame__origin *target,
+		git_blame__origin *parent)
+{
+	int last_in_target;
+	mmfile_t file_p, file_o;
+	blame_chunk_cb_data d = { blame, target, parent, 0, 0 };
+
+	last_in_target = find_last_in_target(blame, target);
+	if (last_in_target < 0)
+		return 1; /* nothing remains for this target */
+
+	fill_origin_blob(parent, &file_p);
+	fill_origin_blob(target, &file_o);
+
+	diff_hunks(file_p, file_o, &d);
+	/* The reset (i.e. anything after tlno) are the same as the parent */
+	blame_chunk(blame, d.tlno, d.plno, last_in_target, target, parent);
+
+	return 0;
+}
+
+static int paths_on_dup(void **old, void *new)
+{
+	GIT_UNUSED(old);
+	git__free(new);
+	return -1;
+}
+
+static git_blame__origin* find_origin(
+		git_blame *blame,
+		git_commit *parent,
+		git_blame__origin *origin)
+{
+	git_blame__origin *porigin = NULL;
+	git_diff *difflist = NULL;
+	git_diff_options diffopts = GIT_DIFF_OPTIONS_INIT;
+	git_tree *otree=NULL, *ptree=NULL;
+
+	/* Get the trees from this commit and its parent */
+	if (0 != git_commit_tree(&otree, origin->commit) ||
+	    0 != git_commit_tree(&ptree, parent))
+		goto cleanup;
+
+	/* Configure the diff */
+	diffopts.context_lines = 0;
+	diffopts.flags = GIT_DIFF_SKIP_BINARY_CHECK;
+
+	/* Check to see if files we're interested have changed */
+	diffopts.pathspec.count = blame->paths.length;
+	diffopts.pathspec.strings = (char**)blame->paths.contents;
+	if (0 != git_diff_tree_to_tree(&difflist, blame->repository, ptree, otree, &diffopts))
+			goto cleanup;
+
+	if (!git_diff_num_deltas(difflist)) {
+		/* No changes; copy data */
+		git_blame__get_origin(&porigin, blame, parent, origin->path);
+	} else {
+		git_diff_find_options findopts = GIT_DIFF_FIND_OPTIONS_INIT;
+		int i;
+
+		/* Generate a full diff between the two trees */
+		git_diff_free(difflist);
+		diffopts.pathspec.count = 0;
+		if (0 != git_diff_tree_to_tree(&difflist, blame->repository, ptree, otree, &diffopts))
+			goto cleanup;
+
+		/* Let diff find renames */
+		findopts.flags = GIT_DIFF_FIND_RENAMES;
+		if (0 != git_diff_find_similar(difflist, &findopts))
+			goto cleanup;
+
+		/* Find one that matches */
+		for (i=0; i<(int)git_diff_num_deltas(difflist); i++) {
+			const git_diff_delta *delta = git_diff_get_delta(difflist, i);
+
+			if (!git_vector_bsearch(NULL, &blame->paths, delta->new_file.path))
+			{
+				git_vector_insert_sorted(&blame->paths, (void*)git__strdup(delta->old_file.path),
+						paths_on_dup);
+				make_origin(&porigin, parent, delta->old_file.path);
+			}
+		}
+	}
+
+cleanup:
+	git_diff_free(difflist);
+	git_tree_free(otree);
+	git_tree_free(ptree);
+	return porigin;
+}
+
+/*
+ * The blobs of origin and porigin exactly match, so everything origin is
+ * suspected for can be blamed on the parent.
+ */
+static void pass_whole_blame(git_blame *blame,
+		git_blame__origin *origin, git_blame__origin *porigin)
+{
+	git_blame__entry *e;
+
+	if (!porigin->blob)
+		git_object_lookup((git_object**)&porigin->blob, blame->repository,
+				git_blob_id(origin->blob), GIT_OBJ_BLOB);
+	for (e=blame->ent; e; e=e->next) {
+		if (!same_suspect(e->suspect, origin))
+			continue;
+		origin_incref(porigin);
+		origin_decref(e->suspect);
+		e->suspect = porigin;
+	}
+}
+
+static void pass_blame(git_blame *blame, git_blame__origin *origin, uint32_t opt)
+{
+	git_commit *commit = origin->commit;
+	int i, num_parents;
+	git_blame__origin *sg_buf[16];
+	git_blame__origin *porigin, **sg_origin = sg_buf;
+
+	GIT_UNUSED(opt);
+
+	num_parents = git_commit_parentcount(commit);
+	if (!git_oid_cmp(git_commit_id(commit), &blame->options.oldest_commit))
+		/* Stop at oldest specified commit */
+		num_parents = 0;
+	if (!num_parents) {
+		git_oid_cpy(&blame->options.oldest_commit, git_commit_id(commit));
+		goto finish;
+	}
+	else if (num_parents < (int)ARRAY_SIZE(sg_buf))
+		memset(sg_buf, 0, sizeof(sg_buf));
+	else
+		sg_origin = git__calloc(num_parents, sizeof(*sg_origin));
+
+	for (i=0; i<num_parents; i++) {
+		git_commit *p;
+		int j, same;
+
+		if (sg_origin[i])
+			continue;
+
+		git_commit_parent(&p, origin->commit, i);
+		porigin = find_origin(blame, p, origin);
+
+		if (!porigin)
+			continue;
+		if (porigin->blob && origin->blob &&
+		    !git_oid_cmp(git_blob_id(porigin->blob), git_blob_id(origin->blob))) {
+			pass_whole_blame(blame, origin, porigin);
+			origin_decref(porigin);
+			goto finish;
+		}
+		for (j = same = 0; j<i; j++)
+			if (sg_origin[j] &&
+				 !git_oid_cmp(git_blob_id(sg_origin[j]->blob), git_blob_id(porigin->blob))) {
+				same = 1;
+				break;
+			}
+		if (!same)
+			sg_origin[i] = porigin;
+		else
+			origin_decref(porigin);
+	}
+
+	/* Standard blame */
+	for (i=0; i<num_parents; i++) {
+		git_blame__origin *porigin = sg_origin[i];
+		if (!porigin)
+			continue;
+		if (!origin->previous) {
+			origin_incref(porigin);
+			origin->previous = porigin;
+		}
+		if (pass_blame_to_parent(blame, origin, porigin))
+			goto finish;
+	}
+
+	/* TODO: optionally find moves in parents' files */
+
+	/* TODO: optionally find copies in parents' files */
+
+finish:
+	for (i=0; i<num_parents; i++)
+		if (sg_origin[i])
+			origin_decref(sg_origin[i]);
+	if (sg_origin != sg_buf)
+		git__free(sg_origin);
+	return;
+}
+
+/*
+ * If two blame entries that are next to each other came from
+ * contiguous lines in the same origin (i.e. <commit, path> pair),
+ * merge them together.
+ */
+static void coalesce(git_blame *blame)
+{
+	git_blame__entry *ent, *next;
+
+	for (ent=blame->ent; ent && (next = ent->next); ent = next) {
+		if (same_suspect(ent->suspect, next->suspect) &&
+		    ent->guilty == next->guilty &&
+		    ent->s_lno + ent->num_lines == next->s_lno)
+		{
+			ent->num_lines += next->num_lines;
+			ent->next = next->next;
+			if (ent->next)
+				ent->next->prev = ent;
+			origin_decref(next->suspect);
+			git__free(next);
+			ent->score = 0;
+			next = ent; /* again */
+		}
+	}
+}
+
+void git_blame__like_git(git_blame *blame, uint32_t opt)
+{
+	while (true) {
+		git_blame__entry *ent;
+		git_blame__origin *suspect = NULL;
+
+		/* Find a suspect to break down */
+		for (ent = blame->ent; !suspect && ent; ent = ent->next)
+			if (!ent->guilty)
+				suspect = ent->suspect;
+		if (!suspect)
+			return; /* all done */
+
+		/* We'll use this suspect later in the loop, so hold on to it for now. */
+		origin_incref(suspect);
+		pass_blame(blame, suspect, opt);
+
+		/* Take responsibility for the remaining entries */
+		for (ent = blame->ent; ent; ent = ent->next) {
+			if (same_suspect(ent->suspect, suspect)) {
+				ent->guilty = true;
+				ent->is_boundary = !git_oid_cmp(
+						git_commit_id(suspect->commit),
+						&blame->options.oldest_commit);
+			}
+		}
+		origin_decref(suspect);
+	}
+
+	coalesce(blame);
+}
+
+void git_blame__free_entry(git_blame__entry *ent)
+{
+	if (!ent) return;
+	origin_decref(ent->suspect);
+	git__free(ent);
+}
diff --git a/src/blame_git.h b/src/blame_git.h
new file mode 100644
index 000000000..3ec2710b8
--- /dev/null
+++ b/src/blame_git.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_blame_git__
+#define INCLUDE_blame_git__
+
+#include "blame.h"
+
+int git_blame__get_origin(
+		git_blame__origin **out,
+		git_blame *sb,
+		git_commit *commit,
+		const char *path);
+void git_blame__free_entry(git_blame__entry *ent);
+void git_blame__like_git(git_blame *sb, uint32_t flags);
+
+#endif
diff --git a/src/blob.c b/src/blob.c
index 2e4d5f479..2c6d52800 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -60,10 +60,10 @@ int git_blob_create_frombuffer(git_oid *oid, git_repository *repo, const void *b
 		(error = git_odb_open_wstream(&stream, odb, len, GIT_OBJ_BLOB)) < 0)
 		return error;
 
-	if ((error = stream->write(stream, buffer, len)) == 0)
-		error = stream->finalize_write(oid, stream);
+	if ((error = git_odb_stream_write(stream, buffer, len)) == 0)
+		error = git_odb_stream_finalize_write(oid, stream);
 
-	stream->free(stream);
+	git_odb_stream_free(stream);
 	return error;
 }
 
@@ -80,12 +80,12 @@ static int write_file_stream(
 		return error;
 
 	if ((fd = git_futils_open_ro(path)) < 0) {
-		stream->free(stream);
+		git_odb_stream_free(stream);
 		return -1;
 	}
 
 	while (!error && (read_len = p_read(fd, buffer, sizeof(buffer))) > 0) {
-		error = stream->write(stream, buffer, read_len);
+		error = git_odb_stream_write(stream, buffer, read_len);
 		written += read_len;
 	}
 
@@ -97,36 +97,32 @@ static int write_file_stream(
 	}
 
 	if (!error)
-		error = stream->finalize_write(oid, stream);
+		error = git_odb_stream_finalize_write(oid, stream);
 
-	stream->free(stream);
+	git_odb_stream_free(stream);
 	return error;
 }
 
 static int write_file_filtered(
 	git_oid *oid,
+	git_off_t *size,
 	git_odb *odb,
 	const char *full_path,
-	git_vector *filters)
+	git_filter_list *fl)
 {
 	int error;
-	git_buf source = GIT_BUF_INIT;
-	git_buf dest = GIT_BUF_INIT;
+	git_buf tgt = GIT_BUF_INIT;
 
-	if ((error = git_futils_readbuffer(&source, full_path)) < 0)
-		return error;
-
-	error = git_filters_apply(&dest, &source, filters);
-
-	/* Free the source as soon as possible. This can be big in memory,
-	 * and we don't want to ODB write to choke */
-	git_buf_free(&source);
+	error = git_filter_list_apply_to_file(&tgt, fl, NULL, full_path);
 
 	/* Write the file to disk if it was properly filtered */
-	if (!error)
-		error = git_odb_write(oid, odb, dest.ptr, dest.size, GIT_OBJ_BLOB);
+	if (!error) {
+		*size = tgt.size;
+
+		error = git_odb_write(oid, odb, tgt.ptr, tgt.size, GIT_OBJ_BLOB);
+	}
 
-	git_buf_free(&dest);
+	git_buf_free(&tgt);
 	return error;
 }
 
@@ -152,45 +148,67 @@ static int write_symlink(
 	return error;
 }
 
-static int blob_create_internal(git_oid *oid, git_repository *repo, const char *content_path, const char *hint_path, bool try_load_filters)
+int git_blob__create_from_paths(
+	git_oid *oid,
+	struct stat *out_st,
+	git_repository *repo,
+	const char *content_path,
+	const char *hint_path,
+	mode_t hint_mode,
+	bool try_load_filters)
 {
 	int error;
 	struct stat st;
 	git_odb *odb = NULL;
 	git_off_t size;
+	mode_t mode;
+	git_buf path = GIT_BUF_INIT;
 
 	assert(hint_path || !try_load_filters);
 
-	if ((error = git_path_lstat(content_path, &st)) < 0 || (error = git_repository_odb__weakptr(&odb, repo)) < 0)
-		return error;
+	if (!content_path) {
+		if (git_repository__ensure_not_bare(repo, "create blob from file") < 0)
+			return GIT_EBAREREPO;
+
+		if (git_buf_joinpath(
+				&path, git_repository_workdir(repo), hint_path) < 0)
+			return -1;
+
+		content_path = path.ptr;
+	}
+
+	if ((error = git_path_lstat(content_path, &st)) < 0 ||
+		(error = git_repository_odb(&odb, repo)) < 0)
+		goto done;
+
+	if (out_st)
+		memcpy(out_st, &st, sizeof(st));
 
 	size = st.st_size;
+	mode = hint_mode ? hint_mode : st.st_mode;
 
-	if (S_ISLNK(st.st_mode)) {
+	if (S_ISLNK(mode)) {
 		error = write_symlink(oid, odb, content_path, (size_t)size);
 	} else {
-		git_vector write_filters = GIT_VECTOR_INIT;
-		int filter_count = 0;
+		git_filter_list *fl = NULL;
 
-		if (try_load_filters) {
+		if (try_load_filters)
 			/* Load the filters for writing this file to the ODB */
-			filter_count = git_filters_load(
-				&write_filters, repo, hint_path, GIT_FILTER_TO_ODB);
-		}
+			error = git_filter_list_load(
+				&fl, repo, NULL, hint_path, GIT_FILTER_TO_ODB);
 
-		if (filter_count < 0) {
-			/* Negative value means there was a critical error */
-			error = filter_count;
-		} else if (filter_count == 0) {
+		if (error < 0)
+			/* well, that didn't work */;
+		else if (fl == NULL)
 			/* No filters need to be applied to the document: we can stream
 			 * directly from disk */
 			error = write_file_stream(oid, odb, content_path, size);
-		} else {
+		else {
 			/* We need to apply one or more filters */
-			error = write_file_filtered(oid, odb, content_path, &write_filters);
-		}
+			error = write_file_filtered(oid, &size, odb, content_path, fl);
 
-		git_filters_free(&write_filters);
+			git_filter_list_free(fl);
+		}
 
 		/*
 		 * TODO: eventually support streaming filtered files, for files
@@ -207,34 +225,21 @@ static int blob_create_internal(git_oid *oid, git_repository *repo, const char *
 		 */
 	}
 
+done:
+	git_odb_free(odb);
+	git_buf_free(&path);
+
 	return error;
 }
 
-int git_blob_create_fromworkdir(git_oid *oid, git_repository *repo, const char *path)
+int git_blob_create_fromworkdir(
+	git_oid *oid, git_repository *repo, const char *path)
 {
-	git_buf full_path = GIT_BUF_INIT;
-	const char *workdir;
-	int error;
-
-	if ((error = git_repository__ensure_not_bare(repo, "create blob from file")) < 0)
-		return error;
-
-	workdir = git_repository_workdir(repo);
-
-	if (git_buf_joinpath(&full_path, workdir, path) < 0) {
-		git_buf_free(&full_path);
-		return -1;
-	}
-
-	error = blob_create_internal(
-		oid, repo, git_buf_cstr(&full_path),
-		git_buf_cstr(&full_path) + strlen(workdir), true);
-
-	git_buf_free(&full_path);
-	return error;
+	return git_blob__create_from_paths(oid, NULL, repo, NULL, path, 0, true);
 }
 
-int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *path)
+int git_blob_create_fromdisk(
+	git_oid *oid, git_repository *repo, const char *path)
 {
 	int error;
 	git_buf full_path = GIT_BUF_INIT;
@@ -251,8 +256,8 @@ int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *pat
 	if (workdir && !git__prefixcmp(hintpath, workdir))
 		hintpath += strlen(workdir);
 
-	error = blob_create_internal(
-		oid, repo, git_buf_cstr(&full_path), hintpath, true);
+	error = git_blob__create_from_paths(
+		oid, NULL, repo, git_buf_cstr(&full_path), hintpath, 0, true);
 
 	git_buf_free(&full_path);
 	return error;
@@ -272,17 +277,14 @@ int git_blob_create_fromchunks(
 	git_filebuf file = GIT_FILEBUF_INIT;
 	git_buf path = GIT_BUF_INIT;
 
-	if (git_buf_join_n(
-		&path, '/', 3, 
-		git_repository_path(repo),
-		GIT_OBJECTS_DIR, 
-		"streamed") < 0)
-			goto cleanup;
+	if (git_buf_joinpath(
+			&path, git_repository_path(repo), GIT_OBJECTS_DIR "streamed") < 0)
+		goto cleanup;
 
 	content = git__malloc(BUFFER_SIZE);
 	GITERR_CHECK_ALLOC(content);
 
-	if (git_filebuf_open(&file, git_buf_cstr(&path), GIT_FILEBUF_TEMPORARY) < 0)
+	if (git_filebuf_open(&file, git_buf_cstr(&path), GIT_FILEBUF_TEMPORARY, 0666) < 0)
 		goto cleanup;
 
 	while (1) {
@@ -303,7 +305,8 @@ int git_blob_create_fromchunks(
 	if (git_filebuf_flush(&file) < 0)
 		goto cleanup;
 
-	error = blob_create_internal(oid, repo, file.path_lock, hintpath, hintpath != NULL);
+	error = git_blob__create_from_paths(
+		oid, NULL, repo, file.path_lock, hintpath, 0, hintpath != NULL);
 
 cleanup:
 	git_buf_free(&path);
@@ -318,8 +321,34 @@ int git_blob_is_binary(git_blob *blob)
 
 	assert(blob);
 
-	content.ptr = blob->odb_object->buffer;
-	content.size = min(blob->odb_object->cached.size, 4000);
+	content.ptr   = blob->odb_object->buffer;
+	content.size  = min(blob->odb_object->cached.size, 4000);
+	content.asize = 0;
 
 	return git_buf_text_is_binary(&content);
 }
+
+int git_blob_filtered_content(
+	git_buf *out,
+	git_blob *blob,
+	const char *path,
+	int check_for_binary_data)
+{
+	int error = 0;
+	git_filter_list *fl = NULL;
+
+	assert(blob && path && out);
+
+	if (check_for_binary_data && git_blob_is_binary(blob))
+		return 0;
+
+	if (!(error = git_filter_list_load(
+			&fl, git_blob_owner(blob), blob, path, GIT_FILTER_TO_WORKTREE))) {
+
+		error = git_filter_list_apply_to_blob(out, fl, blob);
+
+		git_filter_list_free(fl);
+	}
+
+	return error;
+}
diff --git a/src/blob.h b/src/blob.h
index 22e37cc3a..4cd9f1e0c 100644
--- a/src/blob.h
+++ b/src/blob.h
@@ -21,4 +21,13 @@ void git_blob__free(void *blob);
 int git_blob__parse(void *blob, git_odb_object *obj);
 int git_blob__getbuf(git_buf *buffer, git_blob *blob);
 
+extern int git_blob__create_from_paths(
+	git_oid *out_oid,
+	struct stat *out_st,
+	git_repository *repo,
+	const char *full_path,
+	const char *hint_path,
+	mode_t hint_mode,
+	bool apply_filters);
+
 #endif
diff --git a/src/branch.c b/src/branch.c
index 7064fa7fc..95b3fd980 100644
--- a/src/branch.c
+++ b/src/branch.c
@@ -124,48 +124,66 @@ on_error:
 	return error;
 }
 
-int git_branch_foreach(
-	git_repository *repo,
-	unsigned int list_flags,
-	git_branch_foreach_cb callback,
-	void *payload)
-{
+typedef struct {
 	git_reference_iterator *iter;
-	git_reference *ref;
-	int error = 0;
+	unsigned int flags;
+} branch_iter;
 
-	if (git_reference_iterator_new(&iter, repo) < 0)
-		return -1;
+int git_branch_next(git_reference **out, git_branch_t *out_type, git_branch_iterator *_iter)
+{
+	branch_iter *iter = (branch_iter *) _iter;
+	git_reference *ref;
+	int error;
 
-	while ((error = git_reference_next(&ref, iter)) == 0) {
-		if (list_flags & GIT_BRANCH_LOCAL &&
-		    git__prefixcmp(ref->name, GIT_REFS_HEADS_DIR) == 0) {
-			if (callback(ref->name + strlen(GIT_REFS_HEADS_DIR),
-					GIT_BRANCH_LOCAL, payload)) {
-				error = GIT_EUSER;
-			}
+	while ((error = git_reference_next(&ref, iter->iter)) == 0) {
+		if ((iter->flags & GIT_BRANCH_LOCAL) &&
+		    !git__prefixcmp(ref->name, GIT_REFS_HEADS_DIR)) {
+			*out = ref;
+			*out_type = GIT_BRANCH_LOCAL;
+
+			return 0;
+		} else  if ((iter->flags & GIT_BRANCH_REMOTE) &&
+			    !git__prefixcmp(ref->name, GIT_REFS_REMOTES_DIR)) {
+			*out = ref;
+			*out_type = GIT_BRANCH_REMOTE;
+
+			return 0;
+		} else {
+			git_reference_free(ref);
 		}
+	}
 
-		if (list_flags & GIT_BRANCH_REMOTE &&
-		    git__prefixcmp(ref->name, GIT_REFS_REMOTES_DIR) == 0) {
-			if (callback(ref->name + strlen(GIT_REFS_REMOTES_DIR),
-					GIT_BRANCH_REMOTE, payload)) {
-				error = GIT_EUSER;
-			}
-		}
+	return error;
+}
+
+int git_branch_iterator_new(
+	git_branch_iterator **out,
+	git_repository *repo,
+	git_branch_t list_flags)
+{
+	branch_iter *iter;
+
+	iter = git__calloc(1, sizeof(branch_iter));
+	GITERR_CHECK_ALLOC(iter);
 
-		git_reference_free(ref);
+	iter->flags = list_flags;
 
-		/* check if the callback has cancelled iteration */
-		if (error == GIT_EUSER)
-			break;
+	if (git_reference_iterator_new(&iter->iter, repo) < 0) {
+		git__free(iter);
+		return -1;
 	}
 
-	if (error == GIT_ITEROVER)
-		error = 0;
+	*out = (git_branch_iterator *) iter;
 
-	git_reference_iterator_free(iter);
-	return error;
+	return 0;
+}
+
+void git_branch_iterator_free(git_branch_iterator *_iter)
+{
+	branch_iter *iter = (branch_iter *) _iter;
+
+	git_reference_iterator_free(iter->iter);
+	git__free(iter);
 }
 
 int git_branch_move(
@@ -585,7 +603,7 @@ int git_branch_is_head(
 
 	error = git_repository_head(&head, git_reference_owner(branch));
 
-	if (error == GIT_EORPHANEDHEAD || error == GIT_ENOTFOUND)
+	if (error == GIT_EUNBORNBRANCH || error == GIT_ENOTFOUND)
 		return false;
 
 	if (error < 0)
diff --git a/src/buf_text.c b/src/buf_text.c
index 443454b5f..631feb3f8 100644
--- a/src/buf_text.c
+++ b/src/buf_text.c
@@ -70,10 +70,10 @@ int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src)
 	assert(tgt != src);
 
 	if (!next)
-		return GIT_ENOTFOUND;
+		return git_buf_set(tgt, src->ptr, src->size);
 
 	/* reduce reallocs while in the loop */
-	if (git_buf_grow(tgt, src->size) < 0)
+	if (git_buf_grow(tgt, src->size + 1) < 0)
 		return -1;
 	out = tgt->ptr;
 	tgt->size = 0;
@@ -81,20 +81,25 @@ int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src)
 	/* Find the next \r and copy whole chunk up to there to tgt */
 	for (; next; scan = next + 1, next = memchr(scan, '\r', scan_end - scan)) {
 		if (next > scan) {
-			size_t copylen = next - scan;
+			size_t copylen = (size_t)(next - scan);
 			memcpy(out, scan, copylen);
 			out += copylen;
 		}
 
 		/* Do not drop \r unless it is followed by \n */
-		if (next[1] != '\n')
+		if (next + 1 == scan_end || next[1] != '\n')
 			*out++ = '\r';
 	}
 
 	/* Copy remaining input into dest */
-	memcpy(out, scan, scan_end - scan + 1); /* +1 for NUL byte */
-	out += (scan_end - scan);
-	tgt->size = out - tgt->ptr;
+	if (scan < scan_end) {
+		size_t remaining = (size_t)(scan_end - scan);
+		memcpy(out, scan, remaining);
+		out += remaining;
+	}
+
+	tgt->size = (size_t)(out - tgt->ptr);
+	tgt->ptr[tgt->size] = '\0';
 
 	return 0;
 }
@@ -109,7 +114,7 @@ int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src)
 	assert(tgt != src);
 
 	if (!next)
-		return GIT_ENOTFOUND;
+		return git_buf_set(tgt, src->ptr, src->size);
 
 	/* attempt to reduce reallocs while in the loop */
 	if (git_buf_grow(tgt, src->size + (src->size >> 4) + 1) < 0)
@@ -170,8 +175,14 @@ int git_buf_text_common_prefix(git_buf *buf, const git_strarray *strings)
 bool git_buf_text_is_binary(const git_buf *buf)
 {
 	const char *scan = buf->ptr, *end = buf->ptr + buf->size;
+	git_bom_t bom;
 	int printable = 0, nonprintable = 0;
 
+	scan += git_buf_text_detect_bom(&bom, buf, 0);
+
+	if (bom > GIT_BOM_UTF8)
+		return 1;
+
 	while (scan < end) {
 		unsigned char c = *scan++;
 
@@ -262,7 +273,7 @@ bool git_buf_text_gather_stats(
 	while (scan < end) {
 		unsigned char c = *scan++;
 
-		if ((c > 0x1F && c < 0x7F) || c > 0x9f)
+		if (c > 0x1F && c != 0x7F)
 			stats->printable++;
 		else switch (c) {
 			case '\0':
diff --git a/src/buf_text.h b/src/buf_text.h
index 58e4e26a7..3ac9d1443 100644
--- a/src/buf_text.h
+++ b/src/buf_text.h
@@ -56,16 +56,16 @@ GIT_INLINE(int) git_buf_text_puts_escape_regex(git_buf *buf, const char *string)
 extern void git_buf_text_unescape(git_buf *buf);
 
 /**
- * Replace all \r\n with \n (or do nothing if no \r\n are found)
+ * Replace all \r\n with \n. Does not modify \r without trailing \n.
  *
- * @return 0 on success, GIT_ENOTFOUND if no \r\n, -1 on memory error
+ * @return 0 on success, -1 on memory error
  */
 extern int git_buf_text_crlf_to_lf(git_buf *tgt, const git_buf *src);
 
 /**
- * Replace all \n with \r\n (or do nothing if no \n are found)
+ * Replace all \n with \r\n. Does not modify existing \r\n.
  *
- * @return 0 on success, GIT_ENOTFOUND if no \n, -1 on memory error
+ * @return 0 on success, -1 on memory error
  */
 extern int git_buf_text_lf_to_crlf(git_buf *tgt, const git_buf *src);
 
diff --git a/src/buffer.c b/src/buffer.c
index 6e3ffe560..20682322e 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -6,6 +6,7 @@
  */
 #include "buffer.h"
 #include "posix.h"
+#include "git2/buffer.h"
 #include <stdarg.h>
 #include <ctype.h>
 
@@ -31,7 +32,8 @@ void git_buf_init(git_buf *buf, size_t initial_size)
 		git_buf_grow(buf, initial_size);
 }
 
-int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom)
+int git_buf_try_grow(
+	git_buf *buf, size_t target_size, bool mark_oom, bool preserve_external)
 {
 	char *new_ptr;
 	size_t new_size;
@@ -39,6 +41,9 @@ int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom)
 	if (buf->ptr == git_buf__oom)
 		return -1;
 
+	if (!target_size)
+		target_size = buf->size;
+
 	if (target_size <= buf->asize)
 		return 0;
 
@@ -66,6 +71,9 @@ int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom)
 		return -1;
 	}
 
+	if (preserve_external && !buf->asize && buf->ptr != NULL && buf->size > 0)
+		memcpy(new_ptr, buf->ptr, min(buf->size, new_size));
+
 	buf->asize = new_size;
 	buf->ptr   = new_ptr;
 
@@ -77,11 +85,16 @@ int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom)
 	return 0;
 }
 
+int git_buf_grow(git_buf *buffer, size_t target_size)
+{
+	return git_buf_try_grow(buffer, target_size, true, true);
+}
+
 void git_buf_free(git_buf *buf)
 {
 	if (!buf) return;
 
-	if (buf->ptr != git_buf__initbuf && buf->ptr != git_buf__oom)
+	if (buf->asize > 0 && buf->ptr != NULL && buf->ptr != git_buf__oom)
 		git__free(buf->ptr);
 
 	git_buf_init(buf, 0);
@@ -90,11 +103,15 @@ void git_buf_free(git_buf *buf)
 void git_buf_clear(git_buf *buf)
 {
 	buf->size = 0;
+
+	if (!buf->ptr)
+		buf->ptr = git_buf__initbuf;
+
 	if (buf->asize > 0)
 		buf->ptr[0] = '\0';
 }
 
-int git_buf_set(git_buf *buf, const char *data, size_t len)
+int git_buf_set(git_buf *buf, const void *data, size_t len)
 {
 	if (len == 0 || data == NULL) {
 		git_buf_clear(buf);
@@ -137,7 +154,7 @@ int git_buf_puts(git_buf *buf, const char *string)
 	return git_buf_put(buf, string, strlen(string));
 }
 
-static const char b64str[64] =
+static const char b64str[] =
 	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 int git_buf_put_base64(git_buf *buf, const char *data, size_t len)
@@ -194,6 +211,8 @@ int git_buf_vprintf(git_buf *buf, const char *format, va_list ap)
 			format, args
 		);
 
+		va_end(args);
+
 		if (len < 0) {
 			git__free(buf->ptr);
 			buf->ptr = git_buf__oom;
@@ -259,6 +278,15 @@ void git_buf_truncate(git_buf *buf, size_t len)
 	}
 }
 
+void git_buf_shorten(git_buf *buf, size_t amount)
+{
+	if (amount > buf->size)
+		amount = buf->size;
+
+	buf->size = buf->size - amount;
+	buf->ptr[buf->size] = '\0';
+}
+
 void git_buf_rtruncate_at_char(git_buf *buf, char separator)
 {
 	ssize_t idx = git_buf_rfind_next(buf, separator);
diff --git a/src/buffer.h b/src/buffer.h
index 5402f3827..4ca9d4d94 100644
--- a/src/buffer.h
+++ b/src/buffer.h
@@ -9,18 +9,26 @@
 
 #include "common.h"
 #include "git2/strarray.h"
+#include "git2/buffer.h"
 #include <stdarg.h>
 
-typedef struct {
-	char *ptr;
-	size_t asize, size;
-} git_buf;
+/* typedef struct {
+ *  	char   *ptr;
+ *  	size_t asize, size;
+ * } git_buf;
+ */
 
 extern char git_buf__initbuf[];
 extern char git_buf__oom[];
 
+/* Use to initialize buffer structure when git_buf is on stack */
 #define GIT_BUF_INIT { git_buf__initbuf, 0, 0 }
 
+GIT_INLINE(bool) git_buf_is_allocated(const git_buf *buf)
+{
+	return (buf->ptr != NULL && buf->asize > 0);
+}
+
 /**
  * Initialize a git_buf structure.
  *
@@ -32,27 +40,16 @@ extern void git_buf_init(git_buf *buf, size_t initial_size);
 /**
  * Attempt to grow the buffer to hold at least `target_size` bytes.
  *
- * If the allocation fails, this will return an error.  If mark_oom is true,
+ * If the allocation fails, this will return an error.  If `mark_oom` is true,
  * this will mark the buffer as invalid for future operations; if false,
  * existing buffer content will be preserved, but calling code must handle
- * that buffer was not expanded.
+ * that buffer was not expanded.  If `preserve_external` is true, then any
+ * existing data pointed to be `ptr` even if `asize` is zero will be copied
+ * into the newly allocated buffer.
  */
-extern int git_buf_try_grow(git_buf *buf, size_t target_size, bool mark_oom);
-
-/**
- * Grow the buffer to hold at least `target_size` bytes.
- *
- * If the allocation fails, this will return an error and the buffer will be
- * marked as invalid for future operations, invaliding contents.
- *
- * @return 0 on success or -1 on failure
- */
-GIT_INLINE(int) git_buf_grow(git_buf *buf, size_t target_size)
-{
-	return git_buf_try_grow(buf, target_size, true);
-}
+extern int git_buf_try_grow(
+	git_buf *buf, size_t target_size, bool mark_oom, bool preserve_external);
 
-extern void git_buf_free(git_buf *buf);
 extern void git_buf_swap(git_buf *buf_a, git_buf *buf_b);
 extern char *git_buf_detach(git_buf *buf);
 extern void git_buf_attach(git_buf *buf, char *ptr, size_t asize);
@@ -81,7 +78,6 @@ GIT_INLINE(bool) git_buf_oom(const git_buf *buf)
  * return code of these functions and call them in a series then just call
  * git_buf_oom at the end.
  */
-int git_buf_set(git_buf *buf, const char *data, size_t len);
 int git_buf_sets(git_buf *buf, const char *string);
 int git_buf_putc(git_buf *buf, char c);
 int git_buf_put(git_buf *buf, const char *data, size_t len);
@@ -91,6 +87,7 @@ int git_buf_vprintf(git_buf *buf, const char *format, va_list ap);
 void git_buf_clear(git_buf *buf);
 void git_buf_consume(git_buf *buf, const char *end);
 void git_buf_truncate(git_buf *buf, size_t len);
+void git_buf_shorten(git_buf *buf, size_t amount);
 void git_buf_rtruncate_at_char(git_buf *path, char separator);
 
 int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...);
diff --git a/src/cc-compat.h b/src/cc-compat.h
index a5e4ce17e..37f1ea81e 100644
--- a/src/cc-compat.h
+++ b/src/cc-compat.h
@@ -54,8 +54,12 @@
 
 #if defined (_MSC_VER)
 	typedef unsigned char bool;
-#	define true 1
-#	define false 0
+#	ifndef true
+#		define true 1
+#	endif
+#	ifndef false
+#		define false 0
+#	endif
 #else
 #	include <stdbool.h>
 #endif
diff --git a/src/checkout.c b/src/checkout.c
index 8f9ec64e4..6d7e3cfd4 100644
--- a/src/checkout.c
+++ b/src/checkout.c
@@ -26,6 +26,8 @@
 #include "diff.h"
 #include "pathspec.h"
 #include "buf_text.h"
+#include "merge_file.h"
+#include "path.h"
 
 /* See docs/checkout-internals.md for more information */
 
@@ -35,21 +37,23 @@ enum {
 	CHECKOUT_ACTION__UPDATE_BLOB = 2,
 	CHECKOUT_ACTION__UPDATE_SUBMODULE = 4,
 	CHECKOUT_ACTION__CONFLICT = 8,
-	CHECKOUT_ACTION__MAX = 8,
-	CHECKOUT_ACTION__DEFER_REMOVE = 16,
+	CHECKOUT_ACTION__UPDATE_CONFLICT = 16,
+	CHECKOUT_ACTION__MAX = 16,
+	CHECKOUT_ACTION__DEFER_REMOVE = 32,
 	CHECKOUT_ACTION__REMOVE_AND_UPDATE =
 		(CHECKOUT_ACTION__UPDATE_BLOB | CHECKOUT_ACTION__REMOVE),
 };
 
 typedef struct {
 	git_repository *repo;
-	git_diff_list *diff;
+	git_diff *diff;
 	git_checkout_opts opts;
 	bool opts_free_baseline;
 	char *pfx;
 	git_index *index;
 	git_pool pool;
 	git_vector removes;
+	git_vector conflicts;
 	git_buf path;
 	size_t workdir_len;
 	unsigned int strategy;
@@ -59,6 +63,16 @@ typedef struct {
 	size_t completed_steps;
 } checkout_data;
 
+typedef struct {
+	const git_index_entry *ancestor;
+	const git_index_entry *ours;
+	const git_index_entry *theirs;
+
+	int name_collision:1,
+		directoryfile:1,
+		one_to_two:1;
+} checkout_conflictdata;
+
 static int checkout_notify(
 	checkout_data *data,
 	git_checkout_notify_t why,
@@ -246,10 +260,10 @@ static int checkout_action_wd_only(
 	bool remove = false;
 	git_checkout_notify_t notify = GIT_CHECKOUT_NOTIFY_NONE;
 
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			pathspec, wd->path,
 			(data->strategy & GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH) != 0,
-			git_iterator_ignore_case(workdir), NULL))
+			git_iterator_ignore_case(workdir), NULL, NULL))
 		return 0;
 
 	/* check if item is tracked in the index but not in the checkout diff */
@@ -592,6 +606,359 @@ static int checkout_remaining_wd_items(
 	return error;
 }
 
+GIT_INLINE(int) checkout_idxentry_cmp(
+	const git_index_entry *a,
+	const git_index_entry *b)
+{
+	if (!a && !b)
+		return 0;
+	else if (!a && b)
+		return -1;
+	else if(a && !b)
+		return 1;
+	else
+		return strcmp(a->path, b->path);
+}
+
+static int checkout_conflictdata_cmp(const void *a, const void *b)
+{
+	const checkout_conflictdata *ca = a;
+	const checkout_conflictdata *cb = b;
+	int diff;
+
+	if ((diff = checkout_idxentry_cmp(ca->ancestor, cb->ancestor)) == 0 &&
+		(diff = checkout_idxentry_cmp(ca->ours, cb->theirs)) == 0)
+		diff = checkout_idxentry_cmp(ca->theirs, cb->theirs);
+
+	return diff;
+}
+
+int checkout_conflictdata_empty(const git_vector *conflicts, size_t idx)
+{
+	checkout_conflictdata *conflict;
+
+	if ((conflict = git_vector_get(conflicts, idx)) == NULL)
+		return -1;
+
+	if (conflict->ancestor || conflict->ours || conflict->theirs)
+		return 0;
+
+	git__free(conflict);
+	return 1;
+}
+
+GIT_INLINE(bool) conflict_pathspec_match(
+	checkout_data *data,
+	git_iterator *workdir,
+	git_vector *pathspec,
+	const git_index_entry *ancestor,
+	const git_index_entry *ours,
+	const git_index_entry *theirs)
+{
+	/* if the pathspec matches ours *or* theirs, proceed */
+	if (ours && git_pathspec__match(pathspec, ours->path,
+		(data->strategy & GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH) != 0,
+		git_iterator_ignore_case(workdir), NULL, NULL))
+		return true;
+
+	if (theirs && git_pathspec__match(pathspec, theirs->path,
+		(data->strategy & GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH) != 0,
+		git_iterator_ignore_case(workdir), NULL, NULL))
+		return true;
+
+	if (ancestor && git_pathspec__match(pathspec, ancestor->path,
+		(data->strategy & GIT_CHECKOUT_DISABLE_PATHSPEC_MATCH) != 0,
+		git_iterator_ignore_case(workdir), NULL, NULL))
+		return true;
+
+	return false;
+}
+
+static int checkout_conflicts_load(checkout_data *data, git_iterator *workdir, git_vector *pathspec)
+{
+	git_index_conflict_iterator *iterator = NULL;
+	const git_index_entry *ancestor, *ours, *theirs;
+	checkout_conflictdata *conflict;
+	int error = 0;
+
+	if ((error = git_index_conflict_iterator_new(&iterator, data->index)) < 0)
+		goto done;
+
+	data->conflicts._cmp = checkout_conflictdata_cmp;
+
+	/* Collect the conflicts */
+	while ((error = git_index_conflict_next(&ancestor, &ours, &theirs, iterator)) == 0) {
+		if (!conflict_pathspec_match(data, workdir, pathspec, ancestor, ours, theirs))
+			continue;
+
+		conflict = git__calloc(1, sizeof(checkout_conflictdata));
+		GITERR_CHECK_ALLOC(conflict);
+
+		conflict->ancestor = ancestor;
+		conflict->ours = ours;
+		conflict->theirs = theirs;
+
+		git_vector_insert(&data->conflicts, conflict);
+	}
+
+	if (error == GIT_ITEROVER)
+		error = 0;
+
+done:
+	git_index_conflict_iterator_free(iterator);
+
+	return error;
+}
+
+GIT_INLINE(int) checkout_conflicts_cmp_entry(
+	const char *path,
+	const git_index_entry *entry)
+{
+	return strcmp((const char *)path, entry->path);
+}
+
+static int checkout_conflicts_cmp_ancestor(const void *p, const void *c)
+{
+	const char *path = p;
+	const checkout_conflictdata *conflict = c;
+
+	if (!conflict->ancestor)
+		return 1;
+
+	return checkout_conflicts_cmp_entry(path, conflict->ancestor);
+}
+
+static checkout_conflictdata *checkout_conflicts_search_ancestor(
+	checkout_data *data,
+	const char *path)
+{
+	size_t pos;
+
+	if (git_vector_bsearch2(&pos, &data->conflicts, checkout_conflicts_cmp_ancestor, path) < 0)
+		return NULL;
+
+	return git_vector_get(&data->conflicts, pos);
+}
+
+static checkout_conflictdata *checkout_conflicts_search_branch(
+	checkout_data *data,
+	const char *path)
+{
+	checkout_conflictdata *conflict;
+	size_t i;
+
+	git_vector_foreach(&data->conflicts, i, conflict) {
+		int cmp = -1;
+
+		if (conflict->ancestor)
+			break;
+
+		if (conflict->ours)
+			cmp = checkout_conflicts_cmp_entry(path, conflict->ours);
+		else if (conflict->theirs)
+			cmp = checkout_conflicts_cmp_entry(path, conflict->theirs);
+
+		if (cmp == 0)
+			return conflict;
+	}
+
+	return NULL;
+}
+
+static int checkout_conflicts_load_byname_entry(
+	checkout_conflictdata **ancestor_out,
+	checkout_conflictdata **ours_out,
+	checkout_conflictdata **theirs_out,
+	checkout_data *data,
+	const git_index_name_entry *name_entry)
+{
+	checkout_conflictdata *ancestor, *ours = NULL, *theirs = NULL;
+	int error = 0;
+
+	*ancestor_out = NULL;
+	*ours_out = NULL;
+	*theirs_out = NULL;
+
+	if (!name_entry->ancestor) {
+		giterr_set(GITERR_INDEX, "A NAME entry exists without an ancestor");
+		error = -1;
+		goto done;
+	}
+
+	if (!name_entry->ours && !name_entry->theirs) {
+		giterr_set(GITERR_INDEX, "A NAME entry exists without an ours or theirs");
+		error = -1;
+		goto done;
+	}
+
+	if ((ancestor = checkout_conflicts_search_ancestor(data,
+		name_entry->ancestor)) == NULL) {
+		giterr_set(GITERR_INDEX,
+			"A NAME entry referenced ancestor entry '%s' which does not exist in the main index",
+			name_entry->ancestor);
+		error = -1;
+		goto done;
+	}
+
+	if (name_entry->ours) {
+		if (strcmp(name_entry->ancestor, name_entry->ours) == 0)
+			ours = ancestor;
+		else if ((ours = checkout_conflicts_search_branch(data, name_entry->ours)) == NULL ||
+			ours->ours == NULL) {
+			giterr_set(GITERR_INDEX,
+				"A NAME entry referenced our entry '%s' which does not exist in the main index",
+				name_entry->ours);
+			error = -1;
+			goto done;
+		}
+	}
+
+	if (name_entry->theirs) {
+		if (strcmp(name_entry->ancestor, name_entry->theirs) == 0)
+			theirs = ancestor;
+		else if (name_entry->ours && strcmp(name_entry->ours, name_entry->theirs) == 0)
+			theirs = ours;
+		else if ((theirs = checkout_conflicts_search_branch(data, name_entry->theirs)) == NULL ||
+			theirs->theirs == NULL) {
+			giterr_set(GITERR_INDEX,
+				"A NAME entry referenced their entry '%s' which does not exist in the main index",
+				name_entry->theirs);
+			error = -1;
+			goto done;
+		}
+	}
+
+	*ancestor_out = ancestor;
+	*ours_out = ours;
+	*theirs_out = theirs;
+
+done:
+	return error;
+}
+
+static int checkout_conflicts_coalesce_renames(
+	checkout_data *data)
+{
+	const git_index_name_entry *name_entry;
+	checkout_conflictdata *ancestor_conflict, *our_conflict, *their_conflict;
+	size_t i, names;
+	int error = 0;
+
+	/* Juggle entries based on renames */
+	names = git_index_name_entrycount(data->index);
+	
+	for (i = 0; i < names; i++) {
+		name_entry = git_index_name_get_byindex(data->index, i);
+
+		if ((error = checkout_conflicts_load_byname_entry(
+			&ancestor_conflict, &our_conflict, &their_conflict,
+			data, name_entry)) < 0)
+			goto done;
+
+		if (our_conflict && our_conflict != ancestor_conflict) {
+			ancestor_conflict->ours = our_conflict->ours;
+			our_conflict->ours = NULL;
+
+			if (our_conflict->theirs)
+				our_conflict->name_collision = 1;
+
+			if (our_conflict->name_collision)
+				ancestor_conflict->name_collision = 1;
+		}
+
+		if (their_conflict && their_conflict != ancestor_conflict) {
+			ancestor_conflict->theirs = their_conflict->theirs;
+			their_conflict->theirs = NULL;
+
+			if (their_conflict->ours)
+				their_conflict->name_collision = 1;
+
+			if (their_conflict->name_collision)
+				ancestor_conflict->name_collision = 1;
+		}
+
+		if (our_conflict && our_conflict != ancestor_conflict &&
+			their_conflict && their_conflict != ancestor_conflict)
+			ancestor_conflict->one_to_two = 1;
+	}
+
+	git_vector_remove_matching(&data->conflicts, checkout_conflictdata_empty);
+
+done:
+	return error;
+}
+
+static int checkout_conflicts_mark_directoryfile(
+	checkout_data *data)
+{
+	checkout_conflictdata *conflict;
+	const git_index_entry *entry;
+	size_t i, j, len;
+	const char *path;
+	int prefixed, error = 0;
+
+	len = git_index_entrycount(data->index);
+
+	/* Find d/f conflicts */
+	git_vector_foreach(&data->conflicts, i, conflict) {
+		if ((conflict->ours && conflict->theirs) ||
+			(!conflict->ours && !conflict->theirs))
+			continue;
+
+		path = conflict->ours ?
+			conflict->ours->path : conflict->theirs->path;
+
+		if ((error = git_index_find(&j, data->index, path)) < 0) {
+			if (error == GIT_ENOTFOUND)
+				giterr_set(GITERR_INDEX,
+					"Index inconsistency, could not find entry for expected conflict '%s'", path);
+
+			goto done;
+		}
+
+		for (; j < len; j++) {
+			if ((entry = git_index_get_byindex(data->index, j)) == NULL) {
+				giterr_set(GITERR_INDEX,
+					"Index inconsistency, truncated index while loading expected conflict '%s'", path);
+				error = -1;
+				goto done;
+			}
+
+			prefixed = git_path_equal_or_prefixed(path, entry->path);
+
+			if (prefixed == GIT_PATH_EQUAL)
+				continue;
+
+			if (prefixed == GIT_PATH_PREFIX)
+				conflict->directoryfile = 1;
+
+			break;
+		}
+	}
+
+done:
+	return error;
+}
+
+static int checkout_get_conflicts(
+	checkout_data *data,
+	git_iterator *workdir,
+	git_vector *pathspec)
+{
+	int error = 0;
+
+	if (data->strategy & GIT_CHECKOUT_SKIP_UNMERGED)
+		return 0;
+
+	if ((error = checkout_conflicts_load(data, workdir, pathspec)) < 0 ||
+		(error = checkout_conflicts_coalesce_renames(data)) < 0 ||
+		(error = checkout_conflicts_mark_directoryfile(data)) < 0)
+		goto done;
+
+done:
+	return error;
+}
+
 static int checkout_get_actions(
 	uint32_t **actions_ptr,
 	size_t **counts_ptr,
@@ -607,7 +974,7 @@ static int checkout_get_actions(
 	uint32_t *actions = NULL;
 
 	if (data->opts.paths.count > 0 &&
-		git_pathspec_init(&pathspec, &data->opts.paths, &pathpool) < 0)
+		git_pathspec__vinit(&pathspec, &data->opts.paths, &pathpool) < 0)
 		return -1;
 
 	if ((error = git_iterator_current(&wditem, workdir)) < 0 &&
@@ -659,7 +1026,13 @@ static int checkout_get_actions(
 		goto fail;
 	}
 
-	git_pathspec_free(&pathspec);
+
+	if ((error = checkout_get_conflicts(data, workdir, &pathspec)) < 0)
+		goto fail;
+
+	counts[CHECKOUT_ACTION__UPDATE_CONFLICT] = git_vector_length(&data->conflicts);
+
+	git_pathspec__vfree(&pathspec);
 	git_pool_clear(&pathpool);
 
 	return 0;
@@ -670,7 +1043,7 @@ fail:
 	*actions_ptr = NULL;
 	git__free(actions);
 
-	git_pathspec_free(&pathspec);
+	git_pathspec__vfree(&pathspec);
 	git_pool_clear(&pathpool);
 
 	return error;
@@ -678,7 +1051,7 @@ fail:
 
 static int buffer_to_file(
 	struct stat *st,
-	git_buf *buffer,
+	git_buf *buf,
 	const char *path,
 	mode_t dir_mode,
 	int file_open_flags,
@@ -690,80 +1063,52 @@ static int buffer_to_file(
 		return error;
 
 	if ((error = git_futils_writebuffer(
-			buffer, path, file_open_flags, file_mode)) < 0)
+			buf, path, file_open_flags, file_mode)) < 0)
 		return error;
 
-	if (st != NULL && (error = p_stat(path, st)) < 0) {
-		giterr_set(GITERR_OS, "Error while statting '%s'", path);
-		return error;
-	}
+	if (st != NULL && (error = p_stat(path, st)) < 0)
+		giterr_set(GITERR_OS, "Error statting '%s'", path);
 
-	if ((file_mode & 0100) != 0 && (error = p_chmod(path, file_mode)) < 0) {
+	else if (GIT_PERMS_IS_EXEC(file_mode) &&
+			(error = p_chmod(path, file_mode)) < 0)
 		giterr_set(GITERR_OS, "Failed to set permissions on '%s'", path);
-		return error;
-	}
 
-	return 0;
+	return error;
 }
 
 static int blob_content_to_file(
 	struct stat *st,
 	git_blob *blob,
 	const char *path,
+	const char * hint_path,
 	mode_t entry_filemode,
 	git_checkout_opts *opts)
 {
-	int error = -1, nb_filters = 0;
-	mode_t file_mode = opts->file_mode;
-	bool dont_free_filtered;
-	git_buf unfiltered = GIT_BUF_INIT, filtered = GIT_BUF_INIT;
-	git_vector filters = GIT_VECTOR_INIT;
-
-	/* Create a fake git_buf from the blob raw data... */
-	filtered.ptr  = (void *)git_blob_rawcontent(blob);
-	filtered.size = (size_t)git_blob_rawsize(blob);
-	/* ... and make sure it doesn't get unexpectedly freed */
-	dont_free_filtered = true;
-
-	if (!opts->disable_filters &&
-		!git_buf_text_is_binary(&filtered) &&
-		(nb_filters = git_filters_load(
-			&filters,
-			git_object_owner((git_object *)blob),
-			path,
-			GIT_FILTER_TO_WORKTREE)) > 0)
-	{
-		/* reset 'filtered' so it can be a filter target */
-		git_buf_init(&filtered, 0);
-		dont_free_filtered = false;
-	}
+	int error = 0;
+	mode_t file_mode = opts->file_mode ? opts->file_mode : entry_filemode;
+	git_buf out = GIT_BUF_INIT;
+	git_filter_list *fl = NULL;
 
-	if (nb_filters < 0)
-		return nb_filters;
+	if (hint_path == NULL)
+		hint_path = path;
 
-	if (nb_filters > 0)	 {
-		if ((error = git_blob__getbuf(&unfiltered, blob)) < 0)
-			goto cleanup;
+	if (!opts->disable_filters)
+		error = git_filter_list_load(
+			&fl, git_blob_owner(blob), blob, hint_path, GIT_FILTER_TO_WORKTREE);
 
-		if ((error = git_filters_apply(&filtered, &unfiltered, &filters)) < 0)
-			goto cleanup;
-	}
+	if (!error)
+		error = git_filter_list_apply_to_blob(&out, fl, blob);
 
-	/* Allow overriding of file mode */
-	if (!file_mode)
-		file_mode = entry_filemode;
+	git_filter_list_free(fl);
 
-	error = buffer_to_file(
-		st, &filtered, path, opts->dir_mode, opts->file_open_flags, file_mode);
+	if (!error) {
+		error = buffer_to_file(
+			st, &out, path, opts->dir_mode, opts->file_open_flags, file_mode);
 
-	if (!error)
 		st->st_mode = entry_filemode;
 
-cleanup:
-	git_filters_free(&filters);
-	git_buf_free(&unfiltered);
-	if (!dont_free_filtered)
-		git_buf_free(&filtered);
+		git_buf_free(&out);
+	}
 
 	return error;
 }
@@ -815,7 +1160,8 @@ static int checkout_update_index(
 
 	memset(&entry, 0, sizeof(entry));
 	entry.path = (char *)file->path; /* cast to prevent warning */
-	git_index_entry__init_from_stat(&entry, st);
+	git_index_entry__init_from_stat(
+		&entry, st, !(git_index_caps(data->index) & GIT_INDEXCAP_NO_FILEMODE));
 	git_oid_cpy(&entry.oid, &file->oid);
 
 	return git_index_add(data->index, &entry);
@@ -917,34 +1263,26 @@ static int checkout_safe_for_update_only(const char *path, mode_t expected_mode)
 	return 0;
 }
 
-static int checkout_blob(
+static int checkout_write_content(
 	checkout_data *data,
-	const git_diff_file *file)
+	const git_oid *oid,
+	const char *full_path,
+	const char *hint_path,
+	unsigned int mode,
+	struct stat *st)
 {
 	int error = 0;
 	git_blob *blob;
-	struct stat st;
 
-	git_buf_truncate(&data->path, data->workdir_len);
-	if (git_buf_puts(&data->path, file->path) < 0)
-		return -1;
-
-	if ((data->strategy & GIT_CHECKOUT_UPDATE_ONLY) != 0) {
-		int rval = checkout_safe_for_update_only(
-			git_buf_cstr(&data->path), file->mode);
-		if (rval <= 0)
-			return rval;
-	}
-
-	if ((error = git_blob_lookup(&blob, data->repo, &file->oid)) < 0)
+	if ((error = git_blob_lookup(&blob, data->repo, oid)) < 0)
 		return error;
 
-	if (S_ISLNK(file->mode))
+	if (S_ISLNK(mode))
 		error = blob_content_to_link(
-			&st, blob, git_buf_cstr(&data->path), data->opts.dir_mode, data->can_symlink);
+			st, blob, full_path, data->opts.dir_mode, data->can_symlink);
 	else
 		error = blob_content_to_file(
-			&st, blob, git_buf_cstr(&data->path), file->mode, &data->opts);
+			st, blob, full_path, hint_path, mode, &data->opts);
 
 	git_blob_free(blob);
 
@@ -959,6 +1297,30 @@ static int checkout_blob(
 		error = 0;
 	}
 
+	return error;
+}
+
+static int checkout_blob(
+	checkout_data *data,
+	const git_diff_file *file)
+{
+	int error = 0;
+	struct stat st;
+
+	git_buf_truncate(&data->path, data->workdir_len);
+	if (git_buf_puts(&data->path, file->path) < 0)
+		return -1;
+
+	if ((data->strategy & GIT_CHECKOUT_UPDATE_ONLY) != 0) {
+		int rval = checkout_safe_for_update_only(
+			git_buf_cstr(&data->path), file->mode);
+		if (rval <= 0)
+			return rval;
+	}
+
+	error = checkout_write_content(
+		data, &file->oid, git_buf_cstr(&data->path), NULL, file->mode, &st);
+
 	/* update the index unless prevented */
 	if (!error && (data->strategy & GIT_CHECKOUT_DONT_UPDATE_INDEX) == 0)
 		error = checkout_update_index(data, file, &st);
@@ -1129,8 +1491,278 @@ static int checkout_lookup_head_tree(git_tree **out, git_repository *repo)
 	return error;
 }
 
+
+static int conflict_entry_name(
+	git_buf *out,
+	const char *side_name,
+	const char *filename)
+{
+	if (git_buf_puts(out, side_name) < 0 ||
+		git_buf_putc(out, ':') < 0 ||
+		git_buf_puts(out, filename) < 0)
+		return -1;
+
+	return 0;
+}
+
+static int checkout_path_suffixed(git_buf *path, const char *suffix)
+{
+	size_t path_len;
+	int i = 0, error = 0;
+
+	if ((error = git_buf_putc(path, '~')) < 0 || (error = git_buf_puts(path, suffix)) < 0)
+		return -1;
+
+	path_len = git_buf_len(path);
+
+	while (git_path_exists(git_buf_cstr(path)) && i < INT_MAX) {
+		git_buf_truncate(path, path_len);
+
+		if ((error = git_buf_putc(path, '_')) < 0 ||
+			(error = git_buf_printf(path, "%d", i)) < 0)
+			return error;
+
+		i++;
+	}
+
+	if (i == INT_MAX) {
+		git_buf_truncate(path, path_len);
+
+		giterr_set(GITERR_CHECKOUT, "Could not write '%s': working directory file exists", path);
+		return GIT_EEXISTS;
+	}
+
+	return 0;
+}
+
+static int checkout_write_entry(
+	checkout_data *data,
+	checkout_conflictdata *conflict,
+	const git_index_entry *side)
+{
+	const char *hint_path = NULL, *suffix;
+	struct stat st;
+	int error;
+
+	assert (side == conflict->ours || side == conflict->theirs);
+
+	git_buf_truncate(&data->path, data->workdir_len);
+	if (git_buf_puts(&data->path, side->path) < 0)
+		return -1;
+
+	if ((conflict->name_collision || conflict->directoryfile) &&
+		(data->strategy & GIT_CHECKOUT_USE_OURS) == 0 &&
+		(data->strategy & GIT_CHECKOUT_USE_THEIRS) == 0) {
+
+		if (side == conflict->ours)
+			suffix = data->opts.our_label ? data->opts.our_label :
+				"ours";
+		else
+			suffix = data->opts.their_label ? data->opts.their_label :
+				"theirs";
+
+		if (checkout_path_suffixed(&data->path, suffix) < 0)
+			return -1;
+
+		hint_path = side->path;
+	}
+
+	if ((data->strategy & GIT_CHECKOUT_UPDATE_ONLY) != 0 &&
+		(error = checkout_safe_for_update_only(git_buf_cstr(&data->path), side->mode)) <= 0)
+		return error;
+
+	return checkout_write_content(data,
+		&side->oid, git_buf_cstr(&data->path), hint_path, side->mode, &st);
+}
+
+static int checkout_write_entries(
+	checkout_data *data,
+	checkout_conflictdata *conflict)
+{
+	int error = 0;
+
+	if ((error = checkout_write_entry(data, conflict, conflict->ours)) >= 0)
+		error = checkout_write_entry(data, conflict, conflict->theirs);
+
+	return error;
+}
+
+static int checkout_merge_path(
+	git_buf *out,
+	checkout_data *data,
+	checkout_conflictdata *conflict,
+	git_merge_file_result *result)
+{
+	const char *our_label_raw, *their_label_raw, *suffix;
+	int error = 0;
+
+	if ((error = git_buf_joinpath(out, git_repository_workdir(data->repo), result->path)) < 0)
+		return error;
+
+	/* Most conflicts simply use the filename in the index */
+	if (!conflict->name_collision)
+		return 0;
+
+	/* Rename 2->1 conflicts need the branch name appended */
+	our_label_raw = data->opts.our_label ? data->opts.our_label : "ours";
+	their_label_raw = data->opts.their_label ? data->opts.their_label : "theirs";
+	suffix = strcmp(result->path, conflict->ours->path) == 0 ? our_label_raw : their_label_raw;
+
+	if ((error = checkout_path_suffixed(out, suffix)) < 0)
+		return error;
+
+	return 0;
+}
+
+static int checkout_write_merge(
+	checkout_data *data,
+	checkout_conflictdata *conflict)
+{
+	git_buf our_label = GIT_BUF_INIT, their_label = GIT_BUF_INIT,
+		path_suffixed = GIT_BUF_INIT, path_workdir = GIT_BUF_INIT;
+	git_merge_file_input ancestor = GIT_MERGE_FILE_INPUT_INIT,
+		ours = GIT_MERGE_FILE_INPUT_INIT,
+		theirs = GIT_MERGE_FILE_INPUT_INIT;
+	git_merge_file_result result = GIT_MERGE_FILE_RESULT_INIT;
+	git_filebuf output = GIT_FILEBUF_INIT;
+	int error = 0;
+
+	if ((conflict->ancestor &&
+		(error = git_merge_file_input_from_index_entry(
+		&ancestor, data->repo, conflict->ancestor)) < 0) ||
+		(error = git_merge_file_input_from_index_entry(
+		&ours, data->repo, conflict->ours)) < 0 ||
+		(error = git_merge_file_input_from_index_entry(
+		&theirs, data->repo, conflict->theirs)) < 0)
+		goto done;
+
+	ancestor.label = NULL;
+	ours.label = data->opts.our_label ? data->opts.our_label : "ours";
+	theirs.label = data->opts.their_label ? data->opts.their_label : "theirs";
+
+	/* If all the paths are identical, decorate the diff3 file with the branch
+	 * names.  Otherwise, append branch_name:path.
+	 */
+	if (conflict->ours && conflict->theirs &&
+		strcmp(conflict->ours->path, conflict->theirs->path) != 0) {
+
+		if ((error = conflict_entry_name(
+			&our_label, ours.label, conflict->ours->path)) < 0 ||
+			(error = conflict_entry_name(
+			&their_label, theirs.label, conflict->theirs->path)) < 0)
+			goto done;
+
+		ours.label = git_buf_cstr(&our_label);
+		theirs.label = git_buf_cstr(&their_label);
+	}
+
+	if ((error = git_merge_files(&result, &ancestor, &ours, &theirs, 0)) < 0)
+		goto done;
+
+	if (result.path == NULL || result.mode == 0) {
+		giterr_set(GITERR_CHECKOUT, "Could not merge contents of file");
+		error = GIT_EMERGECONFLICT;
+		goto done;
+	}
+
+	if ((error = checkout_merge_path(&path_workdir, data, conflict, &result)) < 0)
+		goto done;
+
+	if ((data->strategy & GIT_CHECKOUT_UPDATE_ONLY) != 0 &&
+		(error = checkout_safe_for_update_only(git_buf_cstr(&path_workdir), result.mode)) <= 0)
+		goto done;
+
+	if ((error = git_futils_mkpath2file(path_workdir.ptr, 0755)) < 0 ||
+		(error = git_filebuf_open(&output, path_workdir.ptr, GIT_FILEBUF_DO_NOT_BUFFER, result.mode)) < 0 ||
+		(error = git_filebuf_write(&output, result.data, result.len)) < 0 ||
+		(error = git_filebuf_commit(&output)) < 0)
+		goto done;
+
+done:
+	git_buf_free(&our_label);
+	git_buf_free(&their_label);
+
+	git_merge_file_input_free(&ancestor);
+	git_merge_file_input_free(&ours);
+	git_merge_file_input_free(&theirs);
+	git_merge_file_result_free(&result);
+	git_buf_free(&path_workdir);
+	git_buf_free(&path_suffixed);
+
+	return error;
+}
+
+static int checkout_create_conflicts(checkout_data *data)
+{
+	checkout_conflictdata *conflict;
+	size_t i;
+	int error = 0;
+
+	git_vector_foreach(&data->conflicts, i, conflict) {
+		/* Both deleted: nothing to do */
+		if (conflict->ours == NULL && conflict->theirs == NULL)
+			error = 0;
+
+		else if ((data->strategy & GIT_CHECKOUT_USE_OURS) &&
+			conflict->ours)
+			error = checkout_write_entry(data, conflict, conflict->ours);
+		else if ((data->strategy & GIT_CHECKOUT_USE_THEIRS) &&
+			conflict->theirs)
+			error = checkout_write_entry(data, conflict, conflict->theirs);
+
+		/* Ignore the other side of name collisions. */
+		else if ((data->strategy & GIT_CHECKOUT_USE_OURS) &&
+			!conflict->ours && conflict->name_collision)
+			error = 0;
+		else if ((data->strategy & GIT_CHECKOUT_USE_THEIRS) &&
+			!conflict->theirs && conflict->name_collision)
+			error = 0;
+
+		/* For modify/delete, name collisions and d/f conflicts, write
+		 * the file (potentially with the name mangled.
+		 */
+		else if (conflict->ours != NULL && conflict->theirs == NULL)
+			error = checkout_write_entry(data, conflict, conflict->ours);
+		else if (conflict->ours == NULL && conflict->theirs != NULL)
+			error = checkout_write_entry(data, conflict, conflict->theirs);
+
+		/* Add/add conflicts and rename 1->2 conflicts, write the
+		 * ours/theirs sides (potentially name mangled).
+		 */
+		else if (conflict->one_to_two)
+			error = checkout_write_entries(data, conflict);
+
+		/* If all sides are links, write the ours side */
+		else if (S_ISLNK(conflict->ours->mode) &&
+			S_ISLNK(conflict->theirs->mode))
+			error = checkout_write_entry(data, conflict, conflict->ours);
+		/* Link/file conflicts, write the file side */
+		else if (S_ISLNK(conflict->ours->mode))
+			error = checkout_write_entry(data, conflict, conflict->theirs);
+		else if (S_ISLNK(conflict->theirs->mode))
+			error = checkout_write_entry(data, conflict, conflict->ours);
+
+		else
+			error = checkout_write_merge(data, conflict);
+
+		if (error)
+			break;
+
+		data->completed_steps++;
+		report_progress(data,
+			conflict->ours ? conflict->ours->path :
+			(conflict->theirs ? conflict->theirs->path : conflict->ancestor->path));
+	}
+
+	return error;
+}
+
+
 static void checkout_data_clear(checkout_data *data)
 {
+	checkout_conflictdata *conflict;
+	size_t i;
+
 	if (data->opts_free_baseline) {
 		git_tree_free(data->opts.baseline);
 		data->opts.baseline = NULL;
@@ -1139,6 +1771,11 @@ static void checkout_data_clear(checkout_data *data)
 	git_vector_free(&data->removes);
 	git_pool_clear(&data->pool);
 
+	git_vector_foreach(&data->conflicts, i, conflict)
+		git__free(conflict);
+
+	git_vector_free(&data->conflicts);
+
 	git__free(data->pfx);
 	data->pfx = NULL;
 
@@ -1151,7 +1788,7 @@ static void checkout_data_clear(checkout_data *data)
 static int checkout_data_init(
 	checkout_data *data,
 	git_iterator *target,
-	git_checkout_opts *proposed)
+	const git_checkout_opts *proposed)
 {
 	int error = 0;
 	git_repository *repo = git_iterator_owner(target);
@@ -1200,10 +1837,20 @@ static int checkout_data_init(
 		} else {
 			/* otherwise, grab and reload the index */
 			if ((error = git_repository_index(&data->index, data->repo)) < 0 ||
-				(error = git_index_read(data->index)) < 0)
+				(error = git_index_read(data->index, true)) < 0)
+				goto cleanup;
+
+			/* cannot checkout if unresolved conflicts exist */
+			if ((data->opts.checkout_strategy & GIT_CHECKOUT_FORCE) == 0 &&
+				git_index_has_conflicts(data->index)) {
+				error = GIT_EMERGECONFLICT;
+				giterr_set(GITERR_CHECKOUT,
+					"unresolved conflicts exist in the index");
 				goto cleanup;
+			}
 
-			/* clear the REUC when doing a tree or commit checkout */
+			/* clean conflict data when doing a tree or commit checkout */
+			git_index_name_clear(data->index);
 			git_index_reuc_clear(data->index);
 		}
 	}
@@ -1235,7 +1882,7 @@ static int checkout_data_init(
 
 		error = checkout_lookup_head_tree(&data->opts.baseline, repo);
 
-		if (error == GIT_EORPHANEDHEAD) {
+		if (error == GIT_EUNBORNBRANCH) {
 			error = 0;
 			giterr_clear();
 		}
@@ -1245,6 +1892,7 @@ static int checkout_data_init(
 	}
 
 	if ((error = git_vector_init(&data->removes, 0, git__strcmp_cb)) < 0 ||
+		(error = git_vector_init(&data->conflicts, 0, NULL)) < 0 ||
 		(error = git_pool_init(&data->pool, 1, 0)) < 0 ||
 		(error = git_buf_puts(&data->path, data->opts.target_directory)) < 0 ||
 		(error = git_path_to_dir(&data->path)) < 0)
@@ -1261,7 +1909,7 @@ cleanup:
 
 int git_checkout_iterator(
 	git_iterator *target,
-	git_checkout_opts *opts)
+	const git_checkout_opts *opts)
 {
 	int error = 0;
 	git_iterator *baseline = NULL, *workdir = NULL;
@@ -1315,14 +1963,16 @@ int git_checkout_iterator(
 		goto cleanup;
 
 	/* Loop through diff (and working directory iterator) building a list of
-	 * actions to be taken, plus look for conflicts and send notifications.
+	 * actions to be taken, plus look for conflicts and send notifications,
+	 * then loop through conflicts.
 	 */
 	if ((error = checkout_get_actions(&actions, &counts, &data, workdir)) < 0)
 		goto cleanup;
 
 	data.total_steps = counts[CHECKOUT_ACTION__REMOVE] +
 		counts[CHECKOUT_ACTION__UPDATE_BLOB] +
-		counts[CHECKOUT_ACTION__UPDATE_SUBMODULE];
+		counts[CHECKOUT_ACTION__UPDATE_SUBMODULE] +
+		counts[CHECKOUT_ACTION__UPDATE_CONFLICT];
 
 	report_progress(&data, NULL); /* establish 0 baseline */
 
@@ -1341,6 +1991,10 @@ int git_checkout_iterator(
 		(error = checkout_create_submodules(actions, &data)) < 0)
 		goto cleanup;
 
+	if (counts[CHECKOUT_ACTION__UPDATE_CONFLICT] > 0 &&
+		(error = checkout_create_conflicts(&data)) < 0)
+		goto cleanup;
+
 	assert(data.completed_steps == data.total_steps);
 
 cleanup:
@@ -1351,7 +2005,7 @@ cleanup:
 		(data.strategy & GIT_CHECKOUT_DONT_UPDATE_INDEX) == 0)
 		error = git_index_write(data.index);
 
-	git_diff_list_free(data.diff);
+	git_diff_free(data.diff);
 	git_iterator_free(workdir);
 	git_iterator_free(baseline);
 	git__free(actions);
@@ -1364,7 +2018,7 @@ cleanup:
 int git_checkout_index(
 	git_repository *repo,
 	git_index *index,
-	git_checkout_opts *opts)
+	const git_checkout_opts *opts)
 {
 	int error;
 	git_iterator *index_i;
@@ -1399,7 +2053,7 @@ int git_checkout_index(
 int git_checkout_tree(
 	git_repository *repo,
 	const git_object *treeish,
-	git_checkout_opts *opts)
+	const git_checkout_opts *opts)
 {
 	int error;
 	git_tree *tree = NULL;
@@ -1419,10 +2073,21 @@ int git_checkout_tree(
 	if (!repo)
 		repo = git_object_owner(treeish);
 
-	if (git_object_peel((git_object **)&tree, treeish, GIT_OBJ_TREE) < 0) {
-		giterr_set(
-			GITERR_CHECKOUT, "Provided object cannot be peeled to a tree");
-		return -1;
+	if (treeish) {
+		if (git_object_peel((git_object **)&tree, treeish, GIT_OBJ_TREE) < 0) {
+			giterr_set(
+				GITERR_CHECKOUT, "Provided object cannot be peeled to a tree");
+			return -1;
+		}
+	}
+	else {
+		if ((error = checkout_lookup_head_tree(&tree, repo)) < 0) {
+			if (error != GIT_EUNBORNBRANCH)
+				giterr_set(
+					GITERR_CHECKOUT,
+					"HEAD could not be peeled to a tree and no treeish given");
+			return error;
+		}
 	}
 
 	if (!(error = git_iterator_for_tree(&tree_i, tree, 0, NULL, NULL)))
@@ -1436,20 +2101,8 @@ int git_checkout_tree(
 
 int git_checkout_head(
 	git_repository *repo,
-	git_checkout_opts *opts)
+	const git_checkout_opts *opts)
 {
-	int error;
-	git_tree *head = NULL;
-	git_iterator *head_i = NULL;
-
 	assert(repo);
-
-	if (!(error = checkout_lookup_head_tree(&head, repo)) &&
-		!(error = git_iterator_for_tree(&head_i, head, 0, NULL, NULL)))
-		error = git_checkout_iterator(head_i, opts);
-
-	git_iterator_free(head_i);
-	git_tree_free(head);
-
-	return error;
+	return git_checkout_tree(repo, NULL, opts);
 }
diff --git a/src/checkout.h b/src/checkout.h
index b1dc80c38..6d7186860 100644
--- a/src/checkout.h
+++ b/src/checkout.h
@@ -19,6 +19,6 @@
  */
 extern int git_checkout_iterator(
 	git_iterator *target,
-	git_checkout_opts *opts);
+	const git_checkout_opts *opts);
 
 #endif
diff --git a/src/clone.c b/src/clone.c
index 5b6c6f77d..23aacd478 100644
--- a/src/clone.c
+++ b/src/clone.c
@@ -176,25 +176,20 @@ static int update_head_to_new_branch(
 	return error;
 }
 
-static int get_head_callback(git_remote_head *head, void *payload)
-{
-	git_remote_head **destination = (git_remote_head **)payload;
-
-	/* Save the first entry, and terminate the enumeration */
-	*destination = head;
-	return 1;
-}
-
 static int update_head_to_remote(git_repository *repo, git_remote *remote)
 {
 	int retcode = -1;
+	size_t refs_len;
 	git_refspec dummy_spec;
-	git_remote_head *remote_head;
+	const git_remote_head *remote_head, **refs;
 	struct head_info head_info;
 	git_buf remote_master_name = GIT_BUF_INIT;
 
+	if (git_remote_ls(&refs, &refs_len, remote) < 0)
+		return -1;
+
 	/* Did we just clone an empty repository? */
-	if (remote->refs.length == 0) {
+	if (refs_len == 0) {
 		return setup_tracking_config(
 			repo,
 			"master",
@@ -202,12 +197,8 @@ static int update_head_to_remote(git_repository *repo, git_remote *remote)
 			GIT_REFS_HEADS_MASTER_FILE);
 	}
 
-	/* Get the remote's HEAD. This is always the first ref in remote->refs. */
-	remote_head = NULL;
-	
-	if (!remote->transport->ls(remote->transport, get_head_callback, &remote_head))
-		return -1;
-
+	/* Get the remote's HEAD. This is always the first ref in the list. */
+	remote_head = refs[0];
 	assert(remote_head);
 
 	git_oid_cpy(&head_info.remote_head_oid, &remote_head->oid);
@@ -220,7 +211,7 @@ static int update_head_to_remote(git_repository *repo, git_remote *remote)
 		memset(&dummy_spec, 0, sizeof(git_refspec));
 		head_info.refspec = &dummy_spec;
 	}
-	
+
 	/* Determine the remote tracking reference name from the local master */
 	if (git_refspec_transform_r(
 		&remote_master_name,
@@ -270,23 +261,23 @@ cleanup:
 
 static int update_head_to_branch(
 		git_repository *repo,
-		const git_clone_options *options)
+		const char *remote_name,
+		const char *branch)
 {
 	int retcode;
 	git_buf remote_branch_name = GIT_BUF_INIT;
 	git_reference* remote_ref = NULL;
 
-	assert(options->checkout_branch);
+	assert(remote_name && branch);
 
 	if ((retcode = git_buf_printf(&remote_branch_name, GIT_REFS_REMOTES_DIR "%s/%s",
-		options->remote_name, options->checkout_branch)) < 0 )
+		remote_name, branch)) < 0 )
 		goto cleanup;
 
 	if ((retcode = git_reference_lookup(&remote_ref, repo, git_buf_cstr(&remote_branch_name))) < 0)
 		goto cleanup;
 
-	retcode = update_head_to_new_branch(repo, git_reference_target(remote_ref),
-		options->checkout_branch);
+	retcode = update_head_to_new_branch(repo, git_reference_target(remote_ref), branch);
 
 cleanup:
 	git_reference_free(remote_ref);
@@ -306,41 +297,18 @@ static int create_and_configure_origin(
 {
 	int error;
 	git_remote *origin = NULL;
+	const char *name;
 
-	if ((error = git_remote_create(&origin, repo, options->remote_name, url)) < 0)
+	name = options->remote_name ? options->remote_name : "origin";
+	if ((error = git_remote_create(&origin, repo, name, url)) < 0)
 		goto on_error;
 
-	git_remote_set_cred_acquire_cb(origin, options->cred_acquire_cb,
-			options->cred_acquire_payload);
-	git_remote_set_autotag(origin, options->remote_autotag);
-	/*
-	 * Don't write FETCH_HEAD, we'll check out the remote tracking
-	 * branch ourselves based on the server's default.
-	 */
-	git_remote_set_update_fetchhead(origin, 0);
+	if (options->ignore_cert_errors)
+		git_remote_check_cert(origin, 0);
 
-	if (options->remote_callbacks &&
-	    (error = git_remote_set_callbacks(origin, options->remote_callbacks)) < 0)
+	if ((error = git_remote_set_callbacks(origin, &options->remote_callbacks)) < 0)
 		goto on_error;
 
-	if (options->fetch_spec) {
-		git_remote_clear_refspecs(origin);
-		if ((error = git_remote_add_fetch(origin, options->fetch_spec)) < 0)
-			goto on_error;
-	}
-
-	if (options->push_spec &&
-	    (error = git_remote_add_push(origin, options->push_spec)) < 0)
-		goto on_error;
-
-	if (options->pushurl &&
-	    (error = git_remote_set_pushurl(origin, options->pushurl)) < 0)
-		goto on_error;
-
-	if (options->transport_flags == GIT_TRANSPORTFLAGS_NO_CHECK_CERT) {
-        git_remote_check_cert(origin, 0);
-    }
-
 	if ((error = git_remote_save(origin)) < 0)
 		goto on_error;
 
@@ -352,59 +320,10 @@ on_error:
 	return error;
 }
 
-
-static int setup_remotes_and_fetch(
-		git_repository *repo,
-		const char *url,
-		const git_clone_options *options)
-{
-	int retcode = GIT_ERROR;
-	git_remote *origin = NULL;
-
-	/* Construct an origin remote */
-	if ((retcode = create_and_configure_origin(&origin, repo, url, options)) < 0)
-		goto on_error;
-
-	git_remote_set_update_fetchhead(origin, 0);
-
-	/* If the download_tags value has not been specified, then make sure to
-		* download tags as well. It is set here because we want to download tags
-		* on the initial clone, but do not want to persist the value in the
-		* configuration file.
-		*/
-	if (origin->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_AUTO &&
-		((retcode = git_remote_add_fetch(origin, "refs/tags/*:refs/tags/*")) < 0))
-		goto on_error;
-
-	/* Connect and download everything */
-	if ((retcode = git_remote_connect(origin, GIT_DIRECTION_FETCH)) < 0)
-		goto on_error;
-
-	if ((retcode = git_remote_download(origin, options->fetch_progress_cb,
-		options->fetch_progress_payload)) < 0)
-		goto on_error;
-
-	/* Create "origin/foo" branches for all remote branches */
-	if ((retcode = git_remote_update_tips(origin)) < 0)
-		goto on_error;
-
-	/* Point HEAD to the requested branch */
-	if (options->checkout_branch)
-		retcode = update_head_to_branch(repo, options);
-	/* Point HEAD to the same ref as the remote's head */
-	else
-		retcode = update_head_to_remote(repo, origin);
-
-on_error:
-	git_remote_free(origin);
-	return retcode;
-}
-
-
 static bool should_checkout(
 	git_repository *repo,
 	bool is_bare,
-	git_checkout_opts *opts)
+	const git_checkout_opts *opts)
 {
 	if (is_bare)
 		return false;
@@ -415,64 +334,102 @@ static bool should_checkout(
 	if (opts->checkout_strategy == GIT_CHECKOUT_NONE)
 		return false;
 
-	return !git_repository_head_orphan(repo);
+	return !git_repository_head_unborn(repo);
 }
 
-static void normalize_options(git_clone_options *dst, const git_clone_options *src)
+int git_clone_into(git_repository *repo, git_remote *remote, const git_checkout_opts *co_opts, const char *branch)
 {
-	git_clone_options default_options = GIT_CLONE_OPTIONS_INIT;
-	if (!src) src = &default_options;
+	int error = 0, old_fetchhead;
+	git_strarray refspecs;
+
+	assert(repo && remote);
+
+	if (!git_repository_is_empty(repo)) {
+		giterr_set(GITERR_INVALID, "the repository is not empty");
+		return -1;
+	}
+
+
+	if ((error = git_remote_get_fetch_refspecs(&refspecs, remote)) < 0)
+		return error;
+
+	if ((error = git_remote_add_fetch(remote, "refs/tags/*:refs/tags/*")) < 0)
+		return error;
 
-	*dst = *src;
+	old_fetchhead = git_remote_update_fetchhead(remote);
+	git_remote_set_update_fetchhead(remote, 0);
 
-	/* Provide defaults for null pointers */
-	if (!dst->remote_name) dst->remote_name = "origin";
+	if ((error = git_remote_fetch(remote)) < 0)
+		goto cleanup;
+
+	if (branch)
+		error = update_head_to_branch(repo, git_remote_name(remote), branch);
+	/* Point HEAD to the same ref as the remote's head */
+	else
+		error = update_head_to_remote(repo, remote);
+
+	if (!error && should_checkout(repo, git_repository_is_bare(repo), co_opts))
+		error = git_checkout_head(repo, co_opts);
+
+cleanup:
+	git_remote_set_update_fetchhead(remote, old_fetchhead);
+	/* Go back to the original refspecs */
+	if (git_remote_set_fetch_refspecs(remote, &refspecs) < 0) {
+		git_strarray_free(&refspecs);
+		return -1;
+	}
+
+	git_strarray_free(&refspecs);
+
+	return error;
 }
 
 int git_clone(
 	git_repository **out,
 	const char *url,
 	const char *local_path,
-	const git_clone_options *options)
+	const git_clone_options *_options)
 {
-	int retcode = GIT_ERROR;
+	int error = 0;
 	git_repository *repo = NULL;
-	git_clone_options normOptions;
-	int remove_directory_on_failure = 0;
+	git_remote *origin;
+	git_clone_options options = GIT_CLONE_OPTIONS_INIT;
+	uint32_t rmdir_flags = GIT_RMDIR_REMOVE_FILES;
 
 	assert(out && url && local_path);
 
-	normalize_options(&normOptions, options);
-	GITERR_CHECK_VERSION(&normOptions, GIT_CLONE_OPTIONS_VERSION, "git_clone_options");
+	if (_options)
+		memcpy(&options, _options, sizeof(git_clone_options));
+
+	GITERR_CHECK_VERSION(&options, GIT_CLONE_OPTIONS_VERSION, "git_clone_options");
 
 	/* Only clone to a new directory or an empty directory */
 	if (git_path_exists(local_path) && !git_path_is_empty_dir(local_path)) {
 		giterr_set(GITERR_INVALID,
 			"'%s' exists and is not an empty directory", local_path);
-		return GIT_ERROR;
+		return GIT_EEXISTS;
 	}
 
-	/* Only remove the directory on failure if we create it */
-	remove_directory_on_failure = !git_path_exists(local_path);
+	/* Only remove the root directory on failure if we create it */
+	if (git_path_exists(local_path))
+		rmdir_flags |= GIT_RMDIR_SKIP_ROOT;
 
-	if (!(retcode = git_repository_init(&repo, local_path, normOptions.bare))) {
-		if ((retcode = setup_remotes_and_fetch(repo, url, &normOptions)) < 0) {
-			/* Failed to fetch; clean up */
-			git_repository_free(repo);
+	if ((error = git_repository_init(&repo, local_path, options.bare)) < 0)
+		return error;
 
-			if (remove_directory_on_failure)
-				git_futils_rmdir_r(local_path, NULL, GIT_RMDIR_REMOVE_FILES);
-			else
-				git_futils_cleanupdir_r(local_path);
+	if (!(error = create_and_configure_origin(&origin, repo, url, &options))) {
+		error = git_clone_into(
+			repo, origin, &options.checkout_opts, options.checkout_branch);
 
-		} else {
-			*out = repo;
-			retcode = 0;
-		}
+		git_remote_free(origin);
 	}
 
-	if (!retcode && should_checkout(repo, normOptions.bare, &normOptions.checkout_opts))
-		retcode = git_checkout_head(*out, &normOptions.checkout_opts);
+	if (error < 0) {
+		git_repository_free(repo);
+		repo = NULL;
+		(void)git_futils_rmdir_r(local_path, NULL, rmdir_flags);
+	}
 
-	return retcode;
+	*out = repo;
+	return error;
 }
diff --git a/src/commit.c b/src/commit.c
index 1ab9b34f7..91b60bbb2 100644
--- a/src/commit.c
+++ b/src/commit.c
@@ -19,30 +19,19 @@
 
 #include <stdarg.h>
 
-static void clear_parents(git_commit *commit)
-{
-	size_t i;
-
-	for (i = 0; i < commit->parent_ids.length; ++i) {
-		git_oid *parent = git_vector_get(&commit->parent_ids, i);
-		git__free(parent);
-	}
-
-	git_vector_clear(&commit->parent_ids);
-}
-
 void git_commit__free(void *_commit)
 {
 	git_commit *commit = _commit;
 
-	clear_parents(commit);
-	git_vector_free(&commit->parent_ids);
+	git_array_clear(commit->parent_ids);
 
 	git_signature_free(commit->author);
 	git_signature_free(commit->committer);
 
-	git__free(commit->message);
+	git__free(commit->raw_header);
+	git__free(commit->raw_message);
 	git__free(commit->message_encoding);
+
 	git__free(commit);
 }
 
@@ -171,12 +160,35 @@ int git_commit_create(
 int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 {
 	git_commit *commit = _commit;
-	const char *buffer = git_odb_object_data(odb_obj);
-	const char *buffer_end = buffer + git_odb_object_size(odb_obj);
+	const char *buffer_start = git_odb_object_data(odb_obj), *buffer;
+	const char *buffer_end = buffer_start + git_odb_object_size(odb_obj);
 	git_oid parent_id;
+	uint32_t parent_count = 0;
+	size_t header_len;
+
+	/* find end-of-header (counting parents as we go) */
+	for (buffer = buffer_start; buffer < buffer_end; ++buffer) {
+		if (!strncmp("\n\n", buffer, 2)) {
+			++buffer;
+			break;
+		}
+		if (!strncmp("\nparent ", buffer, strlen("\nparent ")))
+			++parent_count;
+	}
 
-	if (git_vector_init(&commit->parent_ids, 4, NULL) < 0)
-		return -1;
+	header_len = buffer - buffer_start;
+	commit->raw_header = git__strndup(buffer_start, header_len);
+	GITERR_CHECK_ALLOC(commit->raw_header);
+
+	/* point "buffer" to header data */
+	buffer = commit->raw_header;
+	buffer_end = commit->raw_header + header_len;
+
+	if (parent_count < 1)
+		parent_count = 1;
+
+	git_array_init_to_size(commit->parent_ids, parent_count);
+	GITERR_CHECK_ARRAY(commit->parent_ids);
 
 	if (git_oid__parse(&commit->tree_id, &buffer, buffer_end, "tree ") < 0)
 		goto bad_buffer;
@@ -186,13 +198,10 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 	 */
 
 	while (git_oid__parse(&parent_id, &buffer, buffer_end, "parent ") == 0) {
-		git_oid *new_id = git__malloc(sizeof(git_oid));
+		git_oid *new_id = git_array_alloc(commit->parent_ids);
 		GITERR_CHECK_ALLOC(new_id);
 
 		git_oid_cpy(new_id, &parent_id);
-
-		if (git_vector_insert(&commit->parent_ids, new_id) < 0)
-			return -1;
 	}
 
 	commit->author = git__malloc(sizeof(git_signature));
@@ -208,8 +217,8 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 	if (git_signature__parse(commit->committer, &buffer, buffer_end, "committer ", '\n') < 0)
 		return -1;
 
-	/* Parse add'l header entries until blank line found */
-	while (buffer < buffer_end && *buffer != '\n') {
+	/* Parse add'l header entries */
+	while (buffer < buffer_end) {
 		const char *eoln = buffer;
 		while (eoln < buffer_end && *eoln != '\n')
 			++eoln;
@@ -223,18 +232,21 @@ int git_commit__parse(void *_commit, git_odb_object *odb_obj)
 
 		if (eoln < buffer_end && *eoln == '\n')
 			++eoln;
-
 		buffer = eoln;
 	}
 
-	/* buffer is now at the end of the header, double-check and move forward into the message */
-	if (buffer < buffer_end && *buffer == '\n')
-		buffer++;
+	/* point "buffer" to data after header */
+	buffer = git_odb_object_data(odb_obj);
+	buffer_end = buffer + git_odb_object_size(odb_obj);
 
-	/* parse commit message */
+	buffer += header_len;
+	if (*buffer == '\n')
+		++buffer;
+
+	/* extract commit message */
 	if (buffer <= buffer_end) {
-		commit->message = git__strndup(buffer, buffer_end - buffer);
-		GITERR_CHECK_ALLOC(commit->message);
+		commit->raw_message = git__strndup(buffer, buffer_end - buffer);
+		GITERR_CHECK_ALLOC(commit->raw_message);
 	}
 
 	return 0;
@@ -253,13 +265,27 @@ bad_buffer:
 
 GIT_COMMIT_GETTER(const git_signature *, author, commit->author)
 GIT_COMMIT_GETTER(const git_signature *, committer, commit->committer)
-GIT_COMMIT_GETTER(const char *, message, commit->message)
+GIT_COMMIT_GETTER(const char *, message_raw, commit->raw_message)
 GIT_COMMIT_GETTER(const char *, message_encoding, commit->message_encoding)
+GIT_COMMIT_GETTER(const char *, raw_header, commit->raw_header)
 GIT_COMMIT_GETTER(git_time_t, time, commit->committer->when.time)
 GIT_COMMIT_GETTER(int, time_offset, commit->committer->when.offset)
-GIT_COMMIT_GETTER(unsigned int, parentcount, (unsigned int)commit->parent_ids.length)
+GIT_COMMIT_GETTER(unsigned int, parentcount, (unsigned int)git_array_size(commit->parent_ids))
 GIT_COMMIT_GETTER(const git_oid *, tree_id, &commit->tree_id);
 
+const char *git_commit_message(const git_commit *commit)
+{
+	const char *message = commit->raw_message;
+
+	assert(commit);
+
+	/* trim leading newlines from raw message */
+	while (*message && *message == '\n')
+		++message;
+
+	return message;
+}
+
 int git_commit_tree(git_tree **tree_out, const git_commit *commit)
 {
 	assert(commit);
@@ -271,7 +297,7 @@ const git_oid *git_commit_parent_id(
 {
 	assert(commit);
 
-	return git_vector_get(&commit->parent_ids, n);
+	return git_array_get(commit->parent_ids, n);
 }
 
 int git_commit_parent(
diff --git a/src/commit.h b/src/commit.h
index d0981b125..d452e2975 100644
--- a/src/commit.h
+++ b/src/commit.h
@@ -10,21 +10,22 @@
 #include "git2/commit.h"
 #include "tree.h"
 #include "repository.h"
-#include "vector.h"
+#include "array.h"
 
 #include <time.h>
 
 struct git_commit {
 	git_object object;
 
-	git_vector parent_ids;
+	git_array_t(git_oid) parent_ids;
 	git_oid tree_id;
 
 	git_signature *author;
 	git_signature *committer;
 
 	char *message_encoding;
-	char *message;
+	char *raw_message;
+	char *raw_header;
 };
 
 void git_commit__free(void *commit);
diff --git a/src/commit_list.c b/src/commit_list.c
index bd5b5201a..64416e54d 100644
--- a/src/commit_list.c
+++ b/src/commit_list.c
@@ -36,7 +36,7 @@ git_commit_list *git_commit_list_insert_by_date(git_commit_list_node *item, git_
 	git_commit_list *p;
 
 	while ((p = *pp) != NULL) {
-		if (git_commit_list_time_cmp(p->item, item) < 0)
+		if (git_commit_list_time_cmp(p->item, item) > 0)
 			break;
 
 		pp = &p->next;
diff --git a/src/common.h b/src/common.h
index 02d9ce9b6..159d31b2e 100644
--- a/src/common.h
+++ b/src/common.h
@@ -74,6 +74,30 @@ void giterr_set(int error_class, const char *string, ...);
 int giterr_set_regex(const regex_t *regex, int error_code);
 
 /**
+ * Gets the system error code for this thread.
+ */
+GIT_INLINE(int) giterr_system_last(void)
+{
+#ifdef GIT_WIN32
+	return GetLastError();
+#else
+	return errno;
+#endif
+}
+
+/**
+ * Sets the system error code for this thread.
+ */
+GIT_INLINE(void) giterr_system_set(int code)
+{
+#ifdef GIT_WIN32
+	SetLastError(code);
+#else
+	errno = code;
+#endif
+}
+
+/**
  * Check a versioned structure for validity
  */
 GIT_INLINE(int) giterr__check_version(const void *structure, unsigned int expected_max, const char *name)
diff --git a/src/config.c b/src/config.c
index 068c40260..0d9471383 100644
--- a/src/config.c
+++ b/src/config.c
@@ -315,30 +315,241 @@ int git_config_refresh(git_config *cfg)
  * Loop over all the variables
  */
 
+typedef struct {
+	git_config_iterator parent;
+	git_config_iterator *current;
+	const git_config *cfg;
+	regex_t regex;
+	int has_regex;
+	size_t i;
+} all_iter;
+
+static int find_next_backend(size_t *out, const git_config *cfg, size_t i)
+{
+	file_internal *internal;
+
+	for (; i > 0; --i) {
+		internal = git_vector_get(&cfg->files, i - 1);
+		if (!internal || !internal->file)
+			continue;
+
+		*out = i;
+		return 0;
+	}
+
+	return -1;
+}
+
+static int all_iter_next(git_config_entry **entry, git_config_iterator *_iter)
+{
+	all_iter *iter = (all_iter *) _iter;
+	file_internal *internal;
+	git_config_backend *backend;
+	size_t i;
+	int error = 0;
+
+	if (iter->current != NULL &&
+	    (error = iter->current->next(entry, iter->current)) == 0) {
+		return 0;
+	}
+
+	if (error < 0 && error != GIT_ITEROVER)
+		return error;
+
+	do {
+		if (find_next_backend(&i, iter->cfg, iter->i) < 0)
+			return GIT_ITEROVER;
+
+		internal = git_vector_get(&iter->cfg->files, i - 1);
+		backend = internal->file;
+		iter->i = i - 1;
+
+		if (iter->current)
+			iter->current->free(iter->current);
+
+		iter->current = NULL;
+		error = backend->iterator(&iter->current, backend);
+		if (error == GIT_ENOTFOUND)
+			continue;
+
+		if (error < 0)
+			return error;
+
+		error = iter->current->next(entry, iter->current);
+		/* If this backend is empty, then keep going */
+		if (error == GIT_ITEROVER)
+			continue;
+
+		return error;
+
+	} while(1);
+
+	return GIT_ITEROVER;
+}
+
+static int all_iter_glob_next(git_config_entry **entry, git_config_iterator *_iter)
+{
+	int error;
+	all_iter *iter = (all_iter *) _iter;
+
+	/*
+	 * We use the "normal" function to grab the next one across
+	 * backends and then apply the regex
+	 */
+	while ((error = all_iter_next(entry, _iter)) == 0) {
+		/* skip non-matching keys if regexp was provided */
+		if (regexec(&iter->regex, (*entry)->name, 0, NULL, 0) != 0)
+			continue;
+
+		/* and simply return if we like the entry's name */
+		return 0;
+	}
+
+	return error;
+}
+
+static void all_iter_free(git_config_iterator *_iter)
+{
+	all_iter *iter = (all_iter *) _iter;
+
+	if (iter->current)
+		iter->current->free(iter->current);
+
+	git__free(iter);
+}
+
+static void all_iter_glob_free(git_config_iterator *_iter)
+{
+	all_iter *iter = (all_iter *) _iter;
+
+	regfree(&iter->regex);
+	all_iter_free(_iter);
+}
+
+int git_config_iterator_new(git_config_iterator **out, const git_config *cfg)
+{
+	all_iter *iter;
+
+	iter = git__calloc(1, sizeof(all_iter));
+	GITERR_CHECK_ALLOC(iter);
+
+	iter->parent.free = all_iter_free;
+	iter->parent.next = all_iter_next;
+
+	iter->i = cfg->files.length;
+	iter->cfg = cfg;
+
+	*out = (git_config_iterator *) iter;
+
+	return 0;
+}
+
+int git_config_iterator_glob_new(git_config_iterator **out, const git_config *cfg, const char *regexp)
+{
+	all_iter *iter;
+	int result;
+
+	if (regexp == NULL)
+		return git_config_iterator_new(out, cfg);
+
+	iter = git__calloc(1, sizeof(all_iter));
+	GITERR_CHECK_ALLOC(iter);
+
+	if ((result = regcomp(&iter->regex, regexp, REG_EXTENDED)) < 0) {
+		giterr_set_regex(&iter->regex, result);
+		regfree(&iter->regex);
+		return -1;
+	}
+
+	iter->parent.next = all_iter_glob_next;
+	iter->parent.free = all_iter_glob_free;
+	iter->i = cfg->files.length;
+	iter->cfg = cfg;
+
+	*out = (git_config_iterator *) iter;
+
+	return 0;
+}
+
 int git_config_foreach(
 	const git_config *cfg, git_config_foreach_cb cb, void *payload)
 {
 	return git_config_foreach_match(cfg, NULL, cb, payload);
 }
 
+int git_config_backend_foreach_match(
+	git_config_backend *backend,
+	const char *regexp,
+	int (*fn)(const git_config_entry *, void *),
+	void *data)
+{
+	git_config_entry *entry;
+	git_config_iterator* iter;
+	regex_t regex;
+	int result = 0;
+
+	if (regexp != NULL) {
+		if ((result = regcomp(&regex, regexp, REG_EXTENDED)) < 0) {
+			giterr_set_regex(&regex, result);
+			regfree(&regex);
+			return -1;
+		}
+	}
+
+	if ((result = backend->iterator(&iter, backend)) < 0) {
+		iter = NULL;
+		return -1;
+	}
+
+	while(!(iter->next(&entry, iter) < 0)) {
+		/* skip non-matching keys if regexp was provided */
+		if (regexp && regexec(&regex, entry->name, 0, NULL, 0) != 0)
+			continue;
+
+		/* abort iterator on non-zero return value */
+		if (fn(entry, data)) {
+			giterr_clear();
+			result = GIT_EUSER;
+			goto cleanup;
+		}
+	}
+
+cleanup:
+	if (regexp != NULL)
+		regfree(&regex);
+
+	iter->free(iter);
+
+	return result;
+}
+
 int git_config_foreach_match(
 	const git_config *cfg,
 	const char *regexp,
 	git_config_foreach_cb cb,
 	void *payload)
 {
-	int ret = 0;
-	size_t i;
-	file_internal *internal;
-	git_config_backend *file;
+	int error;
+	git_config_iterator *iter;
+	git_config_entry *entry;
 
-	for (i = 0; i < cfg->files.length && ret == 0; ++i) {
-		internal = git_vector_get(&cfg->files, i);
-		file = internal->file;
-		ret = file->foreach(file, regexp, cb, payload);
+	if ((error = git_config_iterator_glob_new(&iter, cfg, regexp)) < 0)
+		return error;
+
+	while ((error = git_config_next(&entry, iter)) == 0) {
+		if(cb(entry, payload)) {
+			giterr_clear();
+			error = GIT_EUSER;
+			break;
+		}
 	}
 
-	return ret;
+	git_config_iterator_free(iter);
+
+	if (error == GIT_ITEROVER)
+		error = 0;
+
+	return error;
 }
 
 /**************
@@ -528,31 +739,114 @@ int git_config_get_entry(const git_config_entry **out, const git_config *cfg, co
 	return config_error_notfound(name);
 }
 
-int git_config_get_multivar(
+int git_config_get_multivar_foreach(
 	const git_config *cfg, const char *name, const char *regexp,
 	git_config_foreach_cb cb, void *payload)
 {
-	file_internal *internal;
-	git_config_backend *file;
-	int ret = GIT_ENOTFOUND;
-	size_t i;
+	int err, found;
+	git_config_iterator *iter;
+	git_config_entry *entry;
+
+	if ((err = git_config_multivar_iterator_new(&iter, cfg, name, regexp)) < 0)
+		return err;
+
+	found = 0;
+	while ((err = iter->next(&entry, iter)) == 0) {
+		found = 1;
+		if(cb(entry, payload)) {
+			iter->free(iter);
+			return GIT_EUSER;
+		}
+	}
 
-	/*
-	 * This loop runs the "wrong" way 'round because we need to
-	 * look at every value from the most general to most specific
-	 */
-	for (i = cfg->files.length; i > 0; --i) {
-		internal = git_vector_get(&cfg->files, i - 1);
-		if (!internal || !internal->file)
+	iter->free(iter);
+	if (err == GIT_ITEROVER)
+		err = 0;
+
+	if (found == 0 && err == 0)
+		err = config_error_notfound(name);
+
+	return err;
+}
+
+typedef struct {
+	git_config_iterator parent;
+	git_config_iterator *iter;
+	char *name;
+	regex_t regex;
+	int have_regex;
+} multivar_iter;
+
+static int multivar_iter_next(git_config_entry **entry, git_config_iterator *_iter)
+{
+	multivar_iter *iter = (multivar_iter *) _iter;
+	int error = 0;
+
+	while ((error = iter->iter->next(entry, iter->iter)) == 0) {
+		if (git__strcmp(iter->name, (*entry)->name))
 			continue;
-		file = internal->file;
 
-		ret = file->get_multivar(file, name, regexp, cb, payload);
-		if (ret < 0 && ret != GIT_ENOTFOUND)
-			return ret;
+		if (!iter->have_regex)
+			return 0;
+
+		if (regexec(&iter->regex, (*entry)->value, 0, NULL, 0) == 0)
+			return 0;
+	}
+
+	return error;
+}
+
+void multivar_iter_free(git_config_iterator *_iter)
+{
+	multivar_iter *iter = (multivar_iter *) _iter;
+
+	iter->iter->free(iter->iter);
+
+	git__free(iter->name);
+	regfree(&iter->regex);
+	git__free(iter);
+}
+
+int git_config_multivar_iterator_new(git_config_iterator **out, const git_config *cfg, const char *name, const char *regexp)
+{
+	multivar_iter *iter = NULL;
+	git_config_iterator *inner = NULL;
+	int error;
+
+	if ((error = git_config_iterator_new(&inner, cfg)) < 0)
+		return error;
+
+	iter = git__calloc(1, sizeof(multivar_iter));
+	GITERR_CHECK_ALLOC(iter);
+
+	if ((error = git_config__normalize_name(name, &iter->name)) < 0)
+		goto on_error;
+
+	if (regexp != NULL) {
+		error = regcomp(&iter->regex, regexp, REG_EXTENDED);
+		if (error < 0) {
+			giterr_set_regex(&iter->regex, error);
+			error = -1;
+			regfree(&iter->regex);
+			goto on_error;
+		}
+
+		iter->have_regex = 1;
 	}
 
-	return (ret == GIT_ENOTFOUND) ? config_error_notfound(name) : 0;
+	iter->iter = inner;
+	iter->parent.free = multivar_iter_free;
+	iter->parent.next = multivar_iter_next;
+
+	*out = (git_config_iterator *) iter;
+
+	return 0;
+
+on_error:
+
+	inner->free(inner);
+	git__free(iter);
+	return error;
 }
 
 int git_config_set_multivar(git_config *cfg, const char *name, const char *regexp, const char *value)
@@ -568,6 +862,29 @@ int git_config_set_multivar(git_config *cfg, const char *name, const char *regex
 	return file->set_multivar(file, name, regexp, value);
 }
 
+int git_config_delete_multivar(git_config *cfg, const char *name, const char *regexp)
+{
+	git_config_backend *file;
+	file_internal *internal;
+
+	internal = git_vector_get(&cfg->files, 0);
+	if (!internal || !internal->file)
+		return config_error_nofiles(name);
+	file = internal->file;
+
+	return file->del_multivar(file, name, regexp);
+}
+
+int git_config_next(git_config_entry **entry, git_config_iterator *iter)
+{
+	return iter->next(entry, iter);
+}
+
+void git_config_iterator_free(git_config_iterator *iter)
+{
+	iter->free(iter);
+}
+
 static int git_config__find_file_to_path(
 	char *out, size_t outlen, int (*find)(git_buf *buf))
 {
@@ -811,6 +1128,41 @@ fail_parse:
 	return -1;
 }
 
+/* Take something the user gave us and make it nice for our hash function */
+int git_config__normalize_name(const char *in, char **out)
+{
+	char *name, *fdot, *ldot;
+
+	assert(in && out);
+
+	name = git__strdup(in);
+	GITERR_CHECK_ALLOC(name);
+
+	fdot = strchr(name, '.');
+	ldot = strrchr(name, '.');
+
+	if (fdot == NULL || fdot == name || ldot == NULL || !ldot[1])
+		goto invalid;
+
+	/* Validate and downcase up to first dot and after last dot */
+	if (git_config_file_normalize_section(name, fdot) < 0 ||
+		git_config_file_normalize_section(ldot + 1, NULL) < 0)
+		goto invalid;
+
+	/* If there is a middle range, make sure it doesn't have newlines */
+	while (fdot < ldot)
+		if (*fdot++ == '\n')
+			goto invalid;
+
+	*out = name;
+	return 0;
+
+invalid:
+	git__free(name);
+	giterr_set(GITERR_CONFIG, "Invalid config item name '%s'", in);
+	return GIT_EINVALIDSPEC;
+}
+
 struct rename_data {
 	git_config *config;
 	git_buf *name;
diff --git a/src/config.h b/src/config.h
index c5c11ae14..01e8465cc 100644
--- a/src/config.h
+++ b/src/config.h
@@ -47,6 +47,9 @@ extern int git_config_rename_section(
  * @param out the new backend
  * @param path where the config file is located
  */
-extern int git_config_file__ondisk(struct git_config_backend **out, const char *path);
+extern int git_config_file__ondisk(git_config_backend **out, const char *path);
+
+extern int git_config__normalize_name(const char *in, char **out);
+
 
 #endif
diff --git a/src/config_cache.c b/src/config_cache.c
index 84de3a5ed..6808521a3 100644
--- a/src/config_cache.c
+++ b/src/config_cache.c
@@ -67,6 +67,7 @@ static struct map_data _cvar_maps[] = {
 	{"core.ignorestat", NULL, 0, GIT_IGNORESTAT_DEFAULT },
 	{"core.trustctime", NULL, 0, GIT_TRUSTCTIME_DEFAULT },
 	{"core.abbrev", _cvar_map_int, 1, GIT_ABBREV_DEFAULT },
+	{"core.precomposeunicode", NULL, 0, GIT_PRECOMPOSE_DEFAULT },
 };
 
 int git_repository__cvar(int *out, git_repository *repo, git_cvar_cached cvar)
diff --git a/src/config_file.c b/src/config_file.c
index dec952115..15c8de49c 100644
--- a/src/config_file.c
+++ b/src/config_file.c
@@ -15,6 +15,7 @@
 #include "git2/sys/config.h"
 #include "git2/types.h"
 #include "strmap.h"
+#include "array.h"
 
 #include <ctype.h>
 #include <sys/types.h>
@@ -25,8 +26,18 @@ GIT__USE_STRMAP;
 typedef struct cvar_t {
 	struct cvar_t *next;
 	git_config_entry *entry;
+	int included; /* whether this is part of [include] */
 } cvar_t;
 
+typedef struct git_config_file_iter {
+	git_config_iterator parent;
+	git_strmap_iter iter;
+	cvar_t* next_var;
+} git_config_file_iter;
+
+/* Max depth for [include] directives */
+#define MAX_INCLUDE_DEPTH 10
+
 #define CVAR_LIST_HEAD(list) ((list)->head)
 
 #define CVAR_LIST_TAIL(list) ((list)->tail)
@@ -65,34 +76,37 @@ typedef struct cvar_t {
 		 (iter) && (((tmp) = CVAR_LIST_NEXT(iter) || 1));\
 		 (iter) = (tmp))
 
+struct reader {
+	time_t file_mtime;
+	size_t file_size;
+	char *file_path;
+	git_buf buffer;
+	char *read_ptr;
+	int line_number;
+	int eof;
+};
+
 typedef struct {
 	git_config_backend parent;
 
 	git_strmap *values;
 
-	struct {
-		git_buf buffer;
-		char *read_ptr;
-		int line_number;
-		int eof;
-	} reader;
+	git_array_t(struct reader) readers;
 
 	char  *file_path;
-	time_t file_mtime;
-	size_t file_size;
 
 	git_config_level_t level;
 } diskfile_backend;
 
-static int config_parse(diskfile_backend *cfg_file, git_config_level_t level);
-static int parse_variable(diskfile_backend *cfg, char **var_name, char **var_value);
+static int config_parse(diskfile_backend *cfg_file, struct reader *reader, git_config_level_t level, int depth);
+static int parse_variable(struct reader *reader, char **var_name, char **var_value);
 static int config_write(diskfile_backend *cfg, const char *key, const regex_t *preg, const char *value);
 static char *escape_value(const char *ptr);
 
-static void set_parse_error(diskfile_backend *backend, int col, const char *error_str)
+static void set_parse_error(struct reader *reader, int col, const char *error_str)
 {
 	giterr_set(GITERR_CONFIG, "Failed to parse config file: %s (in %s:%d, column %d)",
-		error_str, backend->file_path, backend->reader.line_number, col);
+		error_str, reader->file_path, reader->line_number, col);
 }
 
 static void cvar_free(cvar_t *var)
@@ -106,6 +120,18 @@ static void cvar_free(cvar_t *var)
 	git__free(var);
 }
 
+static int cvar_length(cvar_t *var)
+{
+	int length = 0;
+
+	while (var) {
+		length++;
+		var = var->next;
+	}
+
+	return length;
+}
+
 int git_config_file_normalize_section(char *start, char *end)
 {
 	char *scan;
@@ -118,7 +144,7 @@ int git_config_file_normalize_section(char *start, char *end)
 		if (end && scan >= end)
 			break;
 		if (isalnum(*scan))
-			*scan = tolower(*scan);
+			*scan = (char)tolower(*scan);
 		else if (*scan != '-' || scan == start)
 			return GIT_EINVALIDSPEC;
 	}
@@ -129,41 +155,6 @@ int git_config_file_normalize_section(char *start, char *end)
 	return 0;
 }
 
-/* Take something the user gave us and make it nice for our hash function */
-static int normalize_name(const char *in, char **out)
-{
-	char *name, *fdot, *ldot;
-
-	assert(in && out);
-
-	name = git__strdup(in);
-	GITERR_CHECK_ALLOC(name);
-
-	fdot = strchr(name, '.');
-	ldot = strrchr(name, '.');
-
-	if (fdot == NULL || fdot == name || ldot == NULL || !ldot[1])
-		goto invalid;
-
-	/* Validate and downcase up to first dot and after last dot */
-	if (git_config_file_normalize_section(name, fdot) < 0 ||
-		git_config_file_normalize_section(ldot + 1, NULL) < 0)
-		goto invalid;
-
-	/* If there is a middle range, make sure it doesn't have newlines */
-	while (fdot < ldot)
-		if (*fdot++ == '\n')
-			goto invalid;
-
-	*out = name;
-	return 0;
-
-invalid:
-	git__free(name);
-	giterr_set(GITERR_CONFIG, "Invalid config item name '%s'", in);
-	return GIT_EINVALIDSPEC;
-}
-
 static void free_vars(git_strmap *values)
 {
 	cvar_t *var = NULL;
@@ -184,6 +175,7 @@ static void free_vars(git_strmap *values)
 static int config_open(git_config_backend *cfg, git_config_level_t level)
 {
 	int res;
+	struct reader *reader;
 	diskfile_backend *b = (diskfile_backend *)cfg;
 
 	b->level = level;
@@ -191,32 +183,52 @@ static int config_open(git_config_backend *cfg, git_config_level_t level)
 	b->values = git_strmap_alloc();
 	GITERR_CHECK_ALLOC(b->values);
 
-	git_buf_init(&b->reader.buffer, 0);
+	git_array_init(b->readers);
+	reader = git_array_alloc(b->readers);
+	memset(reader, 0, sizeof(struct reader));
+
+	reader->file_path = git__strdup(b->file_path);
+	GITERR_CHECK_ALLOC(reader->file_path);
+
+	git_buf_init(&reader->buffer, 0);
 	res = git_futils_readbuffer_updated(
-		&b->reader.buffer, b->file_path, &b->file_mtime, &b->file_size, NULL);
+		&reader->buffer, b->file_path, &reader->file_mtime, &reader->file_size, NULL);
 
 	/* It's fine if the file doesn't exist */
 	if (res == GIT_ENOTFOUND)
 		return 0;
 
-	if (res < 0 || (res = config_parse(b, level)) < 0) {
+	if (res < 0 || (res = config_parse(b, reader, level, 0)) < 0) {
 		free_vars(b->values);
 		b->values = NULL;
 	}
 
-	git_buf_free(&b->reader.buffer);
+	reader = git_array_get(b->readers, 0);
+	git_buf_free(&reader->buffer);
 	return res;
 }
 
 static int config_refresh(git_config_backend *cfg)
 {
-	int res, updated = 0;
+	int res = 0, updated = 0, any_updated = 0;
 	diskfile_backend *b = (diskfile_backend *)cfg;
 	git_strmap *old_values;
+	struct reader *reader;
+	uint32_t i;
 
-	res = git_futils_readbuffer_updated(
-		&b->reader.buffer, b->file_path, &b->file_mtime, &b->file_size, &updated);
-	if (res < 0 || !updated)
+	for (i = 0; i < git_array_size(b->readers); i++) {
+		reader = git_array_get(b->readers, i);
+		res = git_futils_readbuffer_updated(
+			&reader->buffer, reader->file_path, &reader->file_mtime, &reader->file_size, &updated);
+
+		if (res < 0)
+			return (res == GIT_ENOTFOUND) ? 0 : res;
+
+		if (updated)
+			any_updated = 1;
+	}
+
+	if (!any_updated)
 		return (res == GIT_ENOTFOUND) ? 0 : res;
 
 	/* need to reload - store old values and prep for reload */
@@ -224,74 +236,88 @@ static int config_refresh(git_config_backend *cfg)
 	b->values = git_strmap_alloc();
 	GITERR_CHECK_ALLOC(b->values);
 
-	if ((res = config_parse(b, b->level)) < 0) {
+	if ((res = config_parse(b, reader, b->level, 0)) < 0) {
 		free_vars(b->values);
 		b->values = old_values;
 	} else {
 		free_vars(old_values);
 	}
 
-	git_buf_free(&b->reader.buffer);
+	git_buf_free(&reader->buffer);
 	return res;
 }
 
 static void backend_free(git_config_backend *_backend)
 {
 	diskfile_backend *backend = (diskfile_backend *)_backend;
+	uint32_t i;
 
 	if (backend == NULL)
 		return;
 
+	for (i = 0; i < git_array_size(backend->readers); i++) {
+		struct reader *r = git_array_get(backend->readers, i);
+		git__free(r->file_path);
+	}
+	git_array_clear(backend->readers);
+
 	git__free(backend->file_path);
 	free_vars(backend->values);
 	git__free(backend);
 }
 
-static int file_foreach(
-	git_config_backend *backend,
-	const char *regexp,
-	int (*fn)(const git_config_entry *, void *),
-	void *data)
+static void config_iterator_free(
+	git_config_iterator* iter)
 {
-	diskfile_backend *b = (diskfile_backend *)backend;
-	cvar_t *var, *next_var;
-	const char *key;
-	regex_t regex;
-	int result = 0;
+	git__free(iter);
+}
 
-	if (!b->values)
-		return 0;
+static int config_iterator_next(
+	git_config_entry **entry,
+	git_config_iterator *iter)
+{
+	git_config_file_iter *it = (git_config_file_iter *) iter;
+	diskfile_backend *b = (diskfile_backend *) it->parent.backend;
+	int err = 0;
+	cvar_t * var;
 
-	if (regexp != NULL) {
-		if ((result = regcomp(&regex, regexp, REG_EXTENDED)) < 0) {
-			giterr_set_regex(&regex, result);
-			regfree(&regex);
-			return -1;
-		}
+	if (it->next_var == NULL) {
+		err = git_strmap_next((void**) &var, &(it->iter), b->values);
+	} else {
+		var = it->next_var;
 	}
 
-	git_strmap_foreach(b->values, key, var,
-		for (; var != NULL; var = next_var) {
-			next_var = CVAR_LIST_NEXT(var);
+	if (err < 0) {
+		it->next_var = NULL;
+		return err;
+	}
 
-			/* skip non-matching keys if regexp was provided */
-			if (regexp && regexec(&regex, key, 0, NULL, 0) != 0)
-				continue;
+	*entry = var->entry;
+	it->next_var = CVAR_LIST_NEXT(var);
 
-			/* abort iterator on non-zero return value */
-			if (fn(var->entry, data)) {
-				giterr_clear();
-				result = GIT_EUSER;
-				goto cleanup;
-			}
-		}
-	);
+	return 0;
+}
 
-cleanup:
-	if (regexp != NULL)
-		regfree(&regex);
+static int config_iterator_new(
+	git_config_iterator **iter,
+	struct git_config_backend* backend)
+{
+	diskfile_backend *b = (diskfile_backend *)backend;
+	git_config_file_iter *it = git__calloc(1, sizeof(git_config_file_iter));
 
-	return result;
+	GIT_UNUSED(b);
+
+	GITERR_CHECK_ALLOC(it);
+
+	it->parent.backend = backend;
+	it->iter = git_strmap_begin(b->values);
+	it->next_var = NULL;
+
+	it->parent.next = config_iterator_next;
+	it->parent.free = config_iterator_free;
+	*iter = (git_config_iterator *) it;
+
+	return 0;
 }
 
 static int config_set(git_config_backend *cfg, const char *name, const char *value)
@@ -302,7 +328,7 @@ static int config_set(git_config_backend *cfg, const char *name, const char *val
 	khiter_t pos;
 	int rval, ret;
 
-	if ((rval = normalize_name(name, &key)) < 0)
+	if ((rval = git_config__normalize_name(name, &key)) < 0)
 		return rval;
 
 	/*
@@ -359,10 +385,10 @@ static int config_set(git_config_backend *cfg, const char *name, const char *val
 		GITERR_CHECK_ALLOC(esc_value);
 	}
 
-	if (config_write(b, key, NULL, esc_value) < 0) {
+	if ((ret = config_write(b, key, NULL, esc_value)) < 0) {
 		git__free(esc_value);
 		cvar_free(var);
-		return -1;
+		return ret;
 	}
 
 	git__free(esc_value);
@@ -384,82 +410,23 @@ static int config_get(const git_config_backend *cfg, const char *name, const git
 	char *key;
 	khiter_t pos;
 	int error;
-
-	if ((error = normalize_name(name, &key)) < 0)
-		return error;
-
-	pos = git_strmap_lookup_index(b->values, key);
-	git__free(key);
-
-	/* no error message; the config system will write one */
-	if (!git_strmap_valid_index(b->values, pos))
-		return GIT_ENOTFOUND;
-
-	*out = ((cvar_t *)git_strmap_value_at(b->values, pos))->entry;
-
-	return 0;
-}
-
-static int config_get_multivar(
-	git_config_backend *cfg,
-	const char *name,
-	const char *regex_str,
-	int (*fn)(const git_config_entry *, void *),
-	void *data)
-{
 	cvar_t *var;
-	diskfile_backend *b = (diskfile_backend *)cfg;
-	char *key;
-	khiter_t pos;
-	int error;
 
-	if ((error = normalize_name(name, &key)) < 0)
+	if ((error = git_config__normalize_name(name, &key)) < 0)
 		return error;
 
 	pos = git_strmap_lookup_index(b->values, key);
 	git__free(key);
 
+	/* no error message; the config system will write one */
 	if (!git_strmap_valid_index(b->values, pos))
 		return GIT_ENOTFOUND;
 
 	var = git_strmap_value_at(b->values, pos);
+	while (var->next)
+		var = var->next;
 
-	if (regex_str != NULL) {
-		regex_t regex;
-		int result;
-
-		/* regex matching; build the regex */
-		result = regcomp(&regex, regex_str, REG_EXTENDED);
-		if (result < 0) {
-			giterr_set_regex(&regex, result);
-			regfree(&regex);
-			return -1;
-		}
-
-		/* and throw the callback only on the variables that
-		 * match the regex */
-		do {
-			if (regexec(&regex, var->entry->value, 0, NULL, 0) == 0) {
-				/* early termination by the user is not an error;
-				 * just break and return successfully */
-				if (fn(var->entry, data) < 0)
-					break;
-			}
-
-			var = var->next;
-		} while (var != NULL);
-		regfree(&regex);
-	} else {
-		/* no regex; go through all the variables */
-		do {
-			/* early termination by the user is not an error;
-			 * just break and return successfully */
-			if (fn(var->entry, data) < 0)
-				break;
-
-			var = var->next;
-		} while (var != NULL);
-	}
+	*out = var->entry;
 
 	return 0;
 }
@@ -477,7 +444,7 @@ static int config_set_multivar(
 
 	assert(regexp);
 
-	if ((result = normalize_name(name, &key)) < 0)
+	if ((result = git_config__normalize_name(name, &key)) < 0)
 		return result;
 
 	pos = git_strmap_lookup_index(b->values, key);
@@ -550,7 +517,7 @@ static int config_delete(git_config_backend *cfg, const char *name)
 	int result;
 	khiter_t pos;
 
-	if ((result = normalize_name(name, &key)) < 0)
+	if ((result = git_config__normalize_name(name, &key)) < 0)
 		return result;
 
 	pos = git_strmap_lookup_index(b->values, key);
@@ -576,6 +543,80 @@ static int config_delete(git_config_backend *cfg, const char *name)
 	return result;
 }
 
+static int config_delete_multivar(git_config_backend *cfg, const char *name, const char *regexp)
+{
+	cvar_t *var, *prev = NULL, *new_head = NULL;
+	cvar_t **to_delete;
+	int to_delete_idx;
+	diskfile_backend *b = (diskfile_backend *)cfg;
+	char *key;
+	regex_t preg;
+	int result;
+	khiter_t pos;
+
+	if ((result = git_config__normalize_name(name, &key)) < 0)
+		return result;
+
+	pos = git_strmap_lookup_index(b->values, key);
+
+	if (!git_strmap_valid_index(b->values, pos)) {
+		giterr_set(GITERR_CONFIG, "Could not find key '%s' to delete", name);
+		git__free(key);
+		return GIT_ENOTFOUND;
+	}
+
+	var = git_strmap_value_at(b->values, pos);
+
+	result = regcomp(&preg, regexp, REG_EXTENDED);
+	if (result < 0) {
+		git__free(key);
+		giterr_set_regex(&preg, result);
+		regfree(&preg);
+		return -1;
+	}
+
+	to_delete = git__calloc(cvar_length(var), sizeof(cvar_t *));
+	GITERR_CHECK_ALLOC(to_delete);
+	to_delete_idx = 0;
+
+	while (var != NULL) {
+		cvar_t *next = var->next;
+
+		if (regexec(&preg, var->entry->value, 0, NULL, 0) == 0) {
+			// If we are past the head, reattach previous node to next one,
+			// otherwise set the new head for the strmap.
+			if (prev != NULL) {
+				prev->next = next;
+			} else {
+				new_head = next;
+			}
+
+			to_delete[to_delete_idx++] = var;
+		} else {
+			prev = var;
+		}
+
+		var = next;
+	}
+
+	if (new_head != NULL) {
+		git_strmap_set_value_at(b->values, pos, new_head);
+	} else {
+		git_strmap_delete_at(b->values, pos);
+	}
+
+	if (to_delete_idx > 0)
+		result = config_write(b, key, &preg, NULL);
+
+	while (to_delete_idx-- > 0)
+		cvar_free(to_delete[to_delete_idx]);
+
+	git__free(key);
+	git__free(to_delete);
+	regfree(&preg);
+	return result;
+}
+
 int git_config_file__ondisk(git_config_backend **out, const char *path)
 {
 	diskfile_backend *backend;
@@ -590,11 +631,11 @@ int git_config_file__ondisk(git_config_backend **out, const char *path)
 
 	backend->parent.open = config_open;
 	backend->parent.get = config_get;
-	backend->parent.get_multivar = config_get_multivar;
 	backend->parent.set = config_set;
 	backend->parent.set_multivar = config_set_multivar;
 	backend->parent.del = config_delete;
-	backend->parent.foreach = file_foreach;
+	backend->parent.del_multivar = config_delete_multivar;
+	backend->parent.iterator = config_iterator_new;
 	backend->parent.refresh = config_refresh;
 	backend->parent.free = backend_free;
 
@@ -603,26 +644,26 @@ int git_config_file__ondisk(git_config_backend **out, const char *path)
 	return 0;
 }
 
-static int cfg_getchar_raw(diskfile_backend *cfg)
+static int reader_getchar_raw(struct reader *reader)
 {
 	int c;
 
-	c = *cfg->reader.read_ptr++;
+	c = *reader->read_ptr++;
 
 	/*
 	Win 32 line breaks: if we find a \r\n sequence,
 	return only the \n as a newline
 	*/
-	if (c == '\r' && *cfg->reader.read_ptr == '\n') {
-		cfg->reader.read_ptr++;
+	if (c == '\r' && *reader->read_ptr == '\n') {
+		reader->read_ptr++;
 		c = '\n';
 	}
 
 	if (c == '\n')
-		cfg->reader.line_number++;
+		reader->line_number++;
 
 	if (c == 0) {
-		cfg->reader.eof = 1;
+		reader->eof = 1;
 		c = '\n';
 	}
 
@@ -632,21 +673,23 @@ static int cfg_getchar_raw(diskfile_backend *cfg)
 #define SKIP_WHITESPACE (1 << 1)
 #define SKIP_COMMENTS (1 << 2)
 
-static int cfg_getchar(diskfile_backend *cfg_file, int flags)
+static int reader_getchar(struct reader *reader, int flags)
 {
 	const int skip_whitespace = (flags & SKIP_WHITESPACE);
 	const int skip_comments = (flags & SKIP_COMMENTS);
 	int c;
 
-	assert(cfg_file->reader.read_ptr);
+	assert(reader->read_ptr);
 
-	do c = cfg_getchar_raw(cfg_file);
-	while (skip_whitespace && git__isspace(c) &&
-	       !cfg_file->reader.eof);
+	do {
+		c = reader_getchar_raw(reader);
+	} while (skip_whitespace && git__isspace(c) &&
+	       !reader->eof);
 
 	if (skip_comments && (c == '#' || c == ';')) {
-		do c = cfg_getchar_raw(cfg_file);
-		while (c != '\n');
+		do {
+			c = reader_getchar_raw(reader);
+		} while (c != '\n');
 	}
 
 	return c;
@@ -655,23 +698,23 @@ static int cfg_getchar(diskfile_backend *cfg_file, int flags)
 /*
  * Read the next char, but don't move the reading pointer.
  */
-static int cfg_peek(diskfile_backend *cfg, int flags)
+static int reader_peek(struct reader *reader, int flags)
 {
 	void *old_read_ptr;
 	int old_lineno, old_eof;
 	int ret;
 
-	assert(cfg->reader.read_ptr);
+	assert(reader->read_ptr);
 
-	old_read_ptr = cfg->reader.read_ptr;
-	old_lineno = cfg->reader.line_number;
-	old_eof = cfg->reader.eof;
+	old_read_ptr = reader->read_ptr;
+	old_lineno = reader->line_number;
+	old_eof = reader->eof;
 
-	ret = cfg_getchar(cfg, flags);
+	ret = reader_getchar(reader, flags);
 
-	cfg->reader.read_ptr = old_read_ptr;
-	cfg->reader.line_number = old_lineno;
-	cfg->reader.eof = old_eof;
+	reader->read_ptr = old_read_ptr;
+	reader->line_number = old_lineno;
+	reader->eof = old_eof;
 
 	return ret;
 }
@@ -679,13 +722,13 @@ static int cfg_peek(diskfile_backend *cfg, int flags)
 /*
  * Read and consume a line, returning it in newly-allocated memory.
  */
-static char *cfg_readline(diskfile_backend *cfg, bool skip_whitespace)
+static char *reader_readline(struct reader *reader, bool skip_whitespace)
 {
 	char *line = NULL;
 	char *line_src, *line_end;
 	size_t line_len;
 
-	line_src = cfg->reader.read_ptr;
+	line_src = reader->read_ptr;
 
 	if (skip_whitespace) {
 		/* Skip empty empty lines */
@@ -714,10 +757,10 @@ static char *cfg_readline(diskfile_backend *cfg, bool skip_whitespace)
 		line_end++;
 
 	if (*line_end == '\0')
-		cfg->reader.eof = 1;
+		reader->eof = 1;
 
-	cfg->reader.line_number++;
-	cfg->reader.read_ptr = line_end;
+	reader->line_number++;
+	reader->read_ptr = line_end;
 
 	return line;
 }
@@ -725,11 +768,11 @@ static char *cfg_readline(diskfile_backend *cfg, bool skip_whitespace)
 /*
  * Consume a line, without storing it anywhere
  */
-static void cfg_consume_line(diskfile_backend *cfg)
+static void reader_consume_line(struct reader *reader)
 {
 	char *line_start, *line_end;
 
-	line_start = cfg->reader.read_ptr;
+	line_start = reader->read_ptr;
 	line_end = strchr(line_start, '\n');
 	/* No newline at EOF */
 	if(line_end == NULL){
@@ -740,10 +783,10 @@ static void cfg_consume_line(diskfile_backend *cfg)
 		line_end++;
 
 	if (*line_end == '\0')
-		cfg->reader.eof = 1;
+		reader->eof = 1;
 
-	cfg->reader.line_number++;
-	cfg->reader.read_ptr = line_end;
+	reader->line_number++;
+	reader->read_ptr = line_end;
 }
 
 GIT_INLINE(int) config_keychar(int c)
@@ -751,12 +794,11 @@ GIT_INLINE(int) config_keychar(int c)
 	return isalnum(c) || c == '-';
 }
 
-static int parse_section_header_ext(diskfile_backend *cfg, const char *line, const char *base_name, char **section_name)
+static int parse_section_header_ext(struct reader *reader, const char *line, const char *base_name, char **section_name)
 {
 	int c, rpos;
 	char *first_quote, *last_quote;
 	git_buf buf = GIT_BUF_INIT;
-	int quote_marks;
 	/*
 	 * base_name is what came before the space. We should be at the
 	 * first quotation mark, except for now, line isn't being kept in
@@ -767,7 +809,7 @@ static int parse_section_header_ext(diskfile_backend *cfg, const char *line, con
 	last_quote = strrchr(line, '"');
 
 	if (last_quote - first_quote == 0) {
-		set_parse_error(cfg, 0, "Missing closing quotation mark in section header");
+		set_parse_error(reader, 0, "Missing closing quotation mark in section header");
 		return -1;
 	}
 
@@ -775,37 +817,30 @@ static int parse_section_header_ext(diskfile_backend *cfg, const char *line, con
 	git_buf_printf(&buf, "%s.", base_name);
 
 	rpos = 0;
-	quote_marks = 0;
 
 	line = first_quote;
-	c = line[rpos++];
+	c = line[++rpos];
 
 	/*
 	 * At the end of each iteration, whatever is stored in c will be
 	 * added to the string. In case of error, jump to out
 	 */
 	do {
-		if (quote_marks == 2) {
-			set_parse_error(cfg, rpos, "Unexpected text after closing quotes");
+
+		switch (c) {
+		case 0:
+			set_parse_error(reader, 0, "Unexpected end-of-line in section header");
 			git_buf_free(&buf);
 			return -1;
-		}
 
-		switch (c) {
 		case '"':
-			++quote_marks;
-			continue;
+			goto end_parse;
 
 		case '\\':
-			c = line[rpos++];
-
-			switch (c) {
-			case '"':
-			case '\\':
-				break;
+			c = line[++rpos];
 
-			default:
-				set_parse_error(cfg, rpos, "Unsupported escape sequence");
+			if (c == 0) {
+				set_parse_error(reader, rpos, "Unexpected end-of-line in section header");
 				git_buf_free(&buf);
 				return -1;
 			}
@@ -814,29 +849,37 @@ static int parse_section_header_ext(diskfile_backend *cfg, const char *line, con
 			break;
 		}
 
-		git_buf_putc(&buf, c);
-	} while ((c = line[rpos++]) != ']');
+		git_buf_putc(&buf, (char)c);
+		c = line[++rpos];
+	} while (line + rpos < last_quote);
+
+end_parse:
+	if (line[rpos] != '"' || line[rpos + 1] != ']') {
+		set_parse_error(reader, rpos, "Unexpected text after closing quotes");
+		git_buf_free(&buf);
+		return -1;
+	}
 
 	*section_name = git_buf_detach(&buf);
 	return 0;
 }
 
-static int parse_section_header(diskfile_backend *cfg, char **section_out)
+static int parse_section_header(struct reader *reader, char **section_out)
 {
 	char *name, *name_end;
 	int name_length, c, pos;
 	int result;
 	char *line;
 
-	line = cfg_readline(cfg, true);
+	line = reader_readline(reader, true);
 	if (line == NULL)
 		return -1;
 
 	/* find the end of the variable's name */
-	name_end = strchr(line, ']');
+	name_end = strrchr(line, ']');
 	if (name_end == NULL) {
 		git__free(line);
-		set_parse_error(cfg, 0, "Missing ']' in section header");
+		set_parse_error(reader, 0, "Missing ']' in section header");
 		return -1;
 	}
 
@@ -855,14 +898,14 @@ static int parse_section_header(diskfile_backend *cfg, char **section_out)
 	do {
 		if (git__isspace(c)){
 			name[name_length] = '\0';
-			result = parse_section_header_ext(cfg, line, name, section_out);
+			result = parse_section_header_ext(reader, line, name, section_out);
 			git__free(line);
 			git__free(name);
 			return result;
 		}
 
 		if (!config_keychar(c) && c != '.') {
-			set_parse_error(cfg, pos, "Unexpected character in header");
+			set_parse_error(reader, pos, "Unexpected character in header");
 			goto fail_parse;
 		}
 
@@ -871,7 +914,7 @@ static int parse_section_header(diskfile_backend *cfg, char **section_out)
 	} while ((c = line[pos++]) != ']');
 
 	if (line[pos - 1] != ']') {
-		set_parse_error(cfg, pos, "Unexpected end of file");
+		set_parse_error(reader, pos, "Unexpected end of file");
 		goto fail_parse;
 	}
 
@@ -888,14 +931,14 @@ fail_parse:
 	return -1;
 }
 
-static int skip_bom(diskfile_backend *cfg)
+static int skip_bom(struct reader *reader)
 {
 	git_bom_t bom;
 	int bom_offset = git_buf_text_detect_bom(&bom,
-		&cfg->reader.buffer, cfg->reader.read_ptr - cfg->reader.buffer.ptr);
+		&reader->buffer, reader->read_ptr - reader->buffer.ptr);
 
 	if (bom == GIT_BOM_UTF8)
-		cfg->reader.read_ptr += bom_offset;
+		reader->read_ptr += bom_offset;
 
 	/* TODO: reference implementation is pretty stupid with BoM */
 
@@ -965,7 +1008,16 @@ static int strip_comments(char *line, int in_quotes)
 	return quote_count;
 }
 
-static int config_parse(diskfile_backend *cfg_file, git_config_level_t level)
+static int included_path(git_buf *out, const char *dir, const char *path)
+{
+	/* From the user's home */
+	if (path[0] == '~' && path[1] == '/')
+		return git_futils_find_global_file(out, &path[1]);
+
+	return git_path_join_unrooted(out, path, dir, NULL);
+}
+
+static int config_parse(diskfile_backend *cfg_file, struct reader *reader, git_config_level_t level, int depth)
 {
 	int c;
 	char *current_section = NULL;
@@ -975,39 +1027,46 @@ static int config_parse(diskfile_backend *cfg_file, git_config_level_t level)
 	git_buf buf = GIT_BUF_INIT;
 	int result = 0;
 	khiter_t pos;
+	uint32_t reader_idx;
+
+	if (depth >= MAX_INCLUDE_DEPTH) {
+		giterr_set(GITERR_CONFIG, "Maximum config include depth reached");
+		return -1;
+	}
 
+	reader_idx = git_array_size(cfg_file->readers) - 1;
 	/* Initialize the reading position */
-	cfg_file->reader.read_ptr = cfg_file->reader.buffer.ptr;
-	cfg_file->reader.eof = 0;
+	reader->read_ptr = reader->buffer.ptr;
+	reader->eof = 0;
 
 	/* If the file is empty, there's nothing for us to do */
-	if (*cfg_file->reader.read_ptr == '\0')
+	if (*reader->read_ptr == '\0')
 		return 0;
 
-	skip_bom(cfg_file);
+	skip_bom(reader);
 
-	while (result == 0 && !cfg_file->reader.eof) {
+	while (result == 0 && !reader->eof) {
 
-		c = cfg_peek(cfg_file, SKIP_WHITESPACE);
+		c = reader_peek(reader, SKIP_WHITESPACE);
 
 		switch (c) {
 		case '\n': /* EOF when peeking, set EOF in the reader to exit the loop */
-			cfg_file->reader.eof = 1;
+			reader->eof = 1;
 			break;
 
 		case '[': /* section header, new section begins */
 			git__free(current_section);
 			current_section = NULL;
-			result = parse_section_header(cfg_file, &current_section);
+			result = parse_section_header(reader, &current_section);
 			break;
 
 		case ';':
 		case '#':
-			cfg_consume_line(cfg_file);
+			reader_consume_line(reader);
 			break;
 
 		default: /* assume variable declaration */
-			result = parse_variable(cfg_file, &var_name, &var_value);
+			result = parse_variable(reader, &var_name, &var_value);
 			if (result < 0)
 				break;
 
@@ -1028,6 +1087,7 @@ static int config_parse(diskfile_backend *cfg_file, git_config_level_t level)
 			var->entry->name = git_buf_detach(&buf);
 			var->entry->value = var_value;
 			var->entry->level = level;
+			var->included = !!depth;
 
 			/* Add or append the new config option */
 			pos = git_strmap_lookup_index(cfg_file->values, var->entry->name);
@@ -1044,6 +1104,42 @@ static int config_parse(diskfile_backend *cfg_file, git_config_level_t level)
 				existing->next = var;
 			}
 
+			if (!git__strcmp(var->entry->name, "include.path")) {
+				struct reader *r;
+				git_buf path = GIT_BUF_INIT;
+				char *dir;
+				uint32_t index;
+
+				r = git_array_alloc(cfg_file->readers);
+				/* The reader may have been reallocated */
+				reader = git_array_get(cfg_file->readers, reader_idx);
+				memset(r, 0, sizeof(struct reader));
+				if ((result = git_path_dirname_r(&path, reader->file_path)) < 0)
+					break;
+
+				/* We need to know out index in the array, as the next config_parse call may realloc */
+				index = git_array_size(cfg_file->readers) - 1;
+				dir = git_buf_detach(&path);
+				result = included_path(&path, dir, var->entry->value);
+				git__free(dir);
+
+				if (result < 0)
+					break;
+
+				r->file_path = git_buf_detach(&path);
+				git_buf_init(&r->buffer, 0);
+				if ((result = git_futils_readbuffer_updated(&r->buffer, r->file_path, &r->file_mtime,
+									    &r->file_size, NULL)) < 0)
+					break;
+
+				result = config_parse(cfg_file, r, level, depth+1);
+				r = git_array_get(cfg_file->readers, index);
+				git_buf_free(&r->buffer);
+
+				if (result < 0)
+					break;
+			}
+
 			break;
 		}
 	}
@@ -1082,6 +1178,24 @@ static int write_section(git_filebuf *file, const char *key)
 	return result;
 }
 
+static const char *quotes_for_value(const char *value)
+{
+	const char *ptr;
+
+	if (value[0] == ' ' || value[0] == '\0')
+		return "\"";
+
+	for (ptr = value; *ptr; ++ptr) {
+		if (*ptr == ';' || *ptr == '#')
+			return "\"";
+	}
+
+	if (ptr[-1] == ' ')
+		return "\"";
+
+	return "";
+}
+
 /*
  * This is pretty much the parsing, except we write out anything we don't have
  */
@@ -1089,38 +1203,44 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 {
 	int result, c;
 	int section_matches = 0, last_section_matched = 0, preg_replaced = 0, write_trailer = 0;
-	const char *pre_end = NULL, *post_start = NULL, *data_start;
+	const char *pre_end = NULL, *post_start = NULL, *data_start, *write_start;
 	char *current_section = NULL, *section, *name, *ldot;
 	git_filebuf file = GIT_FILEBUF_INIT;
+	struct reader *reader = git_array_get(cfg->readers, 0);
 
 	/* We need to read in our own config file */
-	result = git_futils_readbuffer(&cfg->reader.buffer, cfg->file_path);
+	result = git_futils_readbuffer(&reader->buffer, cfg->file_path);
 
 	/* Initialise the reading position */
 	if (result == GIT_ENOTFOUND) {
-		cfg->reader.read_ptr = NULL;
-		cfg->reader.eof = 1;
+		reader->read_ptr = NULL;
+		reader->eof = 1;
 		data_start = NULL;
-		git_buf_clear(&cfg->reader.buffer);
+		git_buf_clear(&reader->buffer);
 	} else if (result == 0) {
-		cfg->reader.read_ptr = cfg->reader.buffer.ptr;
-		cfg->reader.eof = 0;
-		data_start = cfg->reader.read_ptr;
+		reader->read_ptr = reader->buffer.ptr;
+		reader->eof = 0;
+		data_start = reader->read_ptr;
 	} else {
 		return -1; /* OS error when reading the file */
 	}
 
+	write_start = data_start;
+
 	/* Lock the file */
-	if (git_filebuf_open(&file, cfg->file_path, 0) < 0)
-		return -1;
+	if ((result = git_filebuf_open(
+		&file, cfg->file_path, 0, GIT_CONFIG_FILE_MODE)) < 0) {
+			git_buf_free(&reader->buffer);
+			return result;
+	}
 
-	skip_bom(cfg);
+	skip_bom(reader);
 	ldot = strrchr(key, '.');
 	name = ldot + 1;
 	section = git__strndup(key, ldot - key);
 
-	while (!cfg->reader.eof) {
-		c = cfg_peek(cfg, SKIP_WHITESPACE);
+	while (!reader->eof) {
+		c = reader_peek(reader, SKIP_WHITESPACE);
 
 		if (c == '\0') { /* We've arrived at the end of the file */
 			break;
@@ -1133,11 +1253,11 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 			 * new section. If we actually want to replace it, the
 			 * default case will take care of updating them.
 			 */
-			pre_end = post_start = cfg->reader.read_ptr;
+			pre_end = post_start = reader->read_ptr;
 
 			git__free(current_section);
 			current_section = NULL;
-			if (parse_section_header(cfg, &current_section) < 0)
+			if (parse_section_header(reader, &current_section) < 0)
 				goto rewrite_fail;
 
 			/* Keep track of when it stops matching */
@@ -1146,7 +1266,7 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 		}
 
 		else if (c == ';' || c == '#') {
-			cfg_consume_line(cfg);
+			reader_consume_line(reader);
 		}
 
 		else {
@@ -1162,15 +1282,15 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 			 */
 			if (!section_matches) {
 				if (!last_section_matched) {
-					cfg_consume_line(cfg);
+					reader_consume_line(reader);
 					continue;
 				}
 			} else {
 				int has_matched = 0;
 				char *var_name, *var_value;
 
-				pre_end = cfg->reader.read_ptr;
-				if (parse_variable(cfg, &var_name, &var_value) < 0)
+				pre_end = reader->read_ptr;
+				if (parse_variable(reader, &var_name, &var_value) < 0)
 					goto rewrite_fail;
 
 				/* First try to match the name of the variable */
@@ -1189,23 +1309,28 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 				if (!has_matched)
 					continue;
 
-				post_start = cfg->reader.read_ptr;
+				post_start = reader->read_ptr;
 			}
 
 			/* We've found the variable we wanted to change, so
 			 * write anything up to it */
-			git_filebuf_write(&file, data_start, pre_end - data_start);
+			git_filebuf_write(&file, write_start, pre_end - write_start);
 			preg_replaced = 1;
 
 			/* Then replace the variable. If the value is NULL, it
 			 * means we want to delete it, so don't write anything. */
 			if (value != NULL) {
-				git_filebuf_printf(&file, "\t%s = %s\n", name, value);
+				const char *q = quotes_for_value(value);
+				git_filebuf_printf(&file, "\t%s = %s%s%s\n", name, q, value, q);
 			}
 
-			/* multiline variable? we need to keep reading lines to match */
-			if (preg != NULL) {
-				data_start = post_start;
+			/*
+			 * If we have a multivar, we should keep looking for entries,
+			 * but only if we're in the right section. Otherwise we'll end up
+			 * looping on the edge of a matching and a non-matching section.
+			 */
+			if (section_matches && preg != NULL) {
+				write_start = post_start;
 				continue;
 			}
 
@@ -1232,12 +1357,14 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 	 */
 	if (write_trailer) {
 		/* Write out rest of the file */
-		git_filebuf_write(&file, post_start, cfg->reader.buffer.size - (post_start - data_start));
+		git_filebuf_write(&file, post_start, reader->buffer.size - (post_start - data_start));
 	} else {
 		if (preg_replaced) {
-			git_filebuf_printf(&file, "\n%s", data_start);
+			git_filebuf_printf(&file, "\n%s", write_start);
 		} else {
-			git_filebuf_write(&file, cfg->reader.buffer.ptr, cfg->reader.buffer.size);
+			const char *q;
+
+			git_filebuf_write(&file, reader->buffer.ptr, reader->buffer.size);
 
 			/* And now if we just need to add a variable */
 			if (!section_matches && write_section(&file, section) < 0)
@@ -1253,10 +1380,11 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 			}
 
 			/* If we are here, there is at least a section line */
-			if (cfg->reader.buffer.size > 0 && *(cfg->reader.buffer.ptr + cfg->reader.buffer.size - 1) != '\n')
+			if (reader->buffer.size > 0 && *(reader->buffer.ptr + reader->buffer.size - 1) != '\n')
 				git_filebuf_write(&file, "\n", 1);
 
-			git_filebuf_printf(&file, "\t%s = %s\n", name, value);
+			q = quotes_for_value(value);
+			git_filebuf_printf(&file, "\t%s = %s%s%s\n", name, q, value, q);
 		}
 	}
 
@@ -1264,10 +1392,10 @@ static int config_write(diskfile_backend *cfg, const char *key, const regex_t *p
 	git__free(current_section);
 
 	/* refresh stats - if this errors, then commit will error too */
-	(void)git_filebuf_stats(&cfg->file_mtime, &cfg->file_size, &file);
+	(void)git_filebuf_stats(&reader->file_mtime, &reader->file_size, &file);
 
-	result = git_filebuf_commit(&file, GIT_CONFIG_FILE_MODE);
-	git_buf_free(&cfg->reader.buffer);
+	result = git_filebuf_commit(&file);
+	git_buf_free(&reader->buffer);
 
 	return result;
 
@@ -1276,7 +1404,7 @@ rewrite_fail:
 	git__free(current_section);
 
 	git_filebuf_cleanup(&file);
-	git_buf_free(&cfg->reader.buffer);
+	git_buf_free(&reader->buffer);
 	return -1;
 }
 
@@ -1293,6 +1421,9 @@ static char *escape_value(const char *ptr)
 	assert(ptr);
 
 	len = strlen(ptr);
+	if (!len)
+		return git__calloc(1, sizeof(char));
+
 	git_buf_grow(&buf, len);
 
 	while (*ptr != '\0') {
@@ -1365,19 +1496,19 @@ static int is_multiline_var(const char *str)
 	return (end > str) && (count & 1);
 }
 
-static int parse_multiline_variable(diskfile_backend *cfg, git_buf *value, int in_quotes)
+static int parse_multiline_variable(struct reader *reader, git_buf *value, int in_quotes)
 {
 	char *line = NULL, *proc_line = NULL;
 	int quote_count;
 
 	/* Check that the next line exists */
-	line = cfg_readline(cfg, false);
+	line = reader_readline(reader, false);
 	if (line == NULL)
 		return -1;
 
 	/* We've reached the end of the file, there is input missing */
 	if (line[0] == '\0') {
-		set_parse_error(cfg, 0, "Unexpected end of file while parsing multine var");
+		set_parse_error(reader, 0, "Unexpected end of file while parsing multine var");
 		git__free(line);
 		return -1;
 	}
@@ -1387,7 +1518,7 @@ static int parse_multiline_variable(diskfile_backend *cfg, git_buf *value, int i
 	/* If it was just a comment, pretend it didn't exist */
 	if (line[0] == '\0') {
 		git__free(line);
-		return parse_multiline_variable(cfg, value, quote_count);
+		return parse_multiline_variable(reader, value, quote_count);
 		/* TODO: unbounded recursion. This **could** be exploitable */
 	}
 
@@ -1395,7 +1526,7 @@ static int parse_multiline_variable(diskfile_backend *cfg, git_buf *value, int i
 	 * standard, this character **has** to be last one in the buf, with
 	 * no whitespace after it */
 	assert(is_multiline_var(value->ptr));
-	git_buf_truncate(value, git_buf_len(value) - 1);
+	git_buf_shorten(value, 1);
 
 	proc_line = fixup_line(line, in_quotes);
 	if (proc_line == NULL) {
@@ -1412,19 +1543,19 @@ static int parse_multiline_variable(diskfile_backend *cfg, git_buf *value, int i
 	 * keep putting stuff in the buffer
 	 */
 	if (is_multiline_var(value->ptr))
-		return parse_multiline_variable(cfg, value, quote_count);
+		return parse_multiline_variable(reader, value, quote_count);
 
 	return 0;
 }
 
-static int parse_variable(diskfile_backend *cfg, char **var_name, char **var_value)
+static int parse_variable(struct reader *reader, char **var_name, char **var_value)
 {
 	const char *var_end = NULL;
 	const char *value_start = NULL;
 	char *line;
 	int quote_count;
 
-	line = cfg_readline(cfg, true);
+	line = reader_readline(reader, true);
 	if (line == NULL)
 		return -1;
 
@@ -1459,7 +1590,7 @@ static int parse_variable(diskfile_backend *cfg, char **var_name, char **var_val
 			GITERR_CHECK_ALLOC(proc_line);
 			git_buf_puts(&multi_value, proc_line);
 			git__free(proc_line);
-			if (parse_multiline_variable(cfg, &multi_value, quote_count) < 0 || git_buf_oom(&multi_value)) {
+			if (parse_multiline_variable(reader, &multi_value, quote_count) < 0 || git_buf_oom(&multi_value)) {
 				git__free(*var_name);
 				git__free(line);
 				git_buf_free(&multi_value);
diff --git a/src/config_file.h b/src/config_file.h
index 7445859c4..d4a1a4061 100644
--- a/src/config_file.h
+++ b/src/config_file.h
@@ -42,7 +42,7 @@ GIT_INLINE(int) git_config_file_foreach(
 	int (*fn)(const git_config_entry *entry, void *data),
 	void *data)
 {
-	return cfg->foreach(cfg, NULL, fn, data);
+	return git_config_backend_foreach_match(cfg, NULL, fn, data);
 }
 
 GIT_INLINE(int) git_config_file_foreach_match(
@@ -51,7 +51,7 @@ GIT_INLINE(int) git_config_file_foreach_match(
 	int (*fn)(const git_config_entry *entry, void *data),
 	void *data)
 {
-	return cfg->foreach(cfg, regexp, fn, data);
+	return git_config_backend_foreach_match(cfg, regexp, fn, data);
 }
 
 extern int git_config_file_normalize_section(char *start, char *end);
diff --git a/src/crlf.c b/src/crlf.c
index 65039f9cc..b25c02cce 100644
--- a/src/crlf.c
+++ b/src/crlf.c
@@ -8,6 +8,7 @@
 #include "git2/attr.h"
 #include "git2/blob.h"
 #include "git2/index.h"
+#include "git2/sys/filter.h"
 
 #include "common.h"
 #include "fileops.h"
@@ -19,13 +20,11 @@
 struct crlf_attrs {
 	int crlf_action;
 	int eol;
+	int auto_crlf;
 };
 
 struct crlf_filter {
 	git_filter f;
-	struct crlf_attrs attrs;
-	git_repository *repo;
-	char path[GIT_FLEX_ARRAY];
 };
 
 static int check_crlf(const char *value)
@@ -76,41 +75,10 @@ static int crlf_input_action(struct crlf_attrs *ca)
 	return ca->crlf_action;
 }
 
-static int crlf_load_attributes(struct crlf_attrs *ca, git_repository *repo, const char *path)
+static int has_cr_in_index(const git_filter_source *src)
 {
-#define NUM_CONV_ATTRS 3
-
-	static const char *attr_names[NUM_CONV_ATTRS] = {
-		"crlf", "eol", "text",
-	};
-
-	const char *attr_vals[NUM_CONV_ATTRS];
-	int error;
-
-	error = git_attr_get_many(attr_vals,
-		repo, 0, path, NUM_CONV_ATTRS, attr_names);
-
-	if (error == GIT_ENOTFOUND) {
-		ca->crlf_action = GIT_CRLF_GUESS;
-		ca->eol = GIT_EOL_UNSET;
-		return 0;
-	}
-
-	if (error == 0) {
-		ca->crlf_action = check_crlf(attr_vals[2]); /* text */
-		if (ca->crlf_action == GIT_CRLF_GUESS)
-			ca->crlf_action = check_crlf(attr_vals[0]); /* clrf */
-
-		ca->eol = check_eol(attr_vals[1]); /* eol */
-		return 0;
-	}
-
-	return -1;
-}
-
-static int has_cr_in_index(git_filter *self)
-{
-	struct crlf_filter *filter = (struct crlf_filter *)self;
+	git_repository *repo = git_filter_source_repo(src);
+	const char *path = git_filter_source_path(src);
 	git_index *index;
 	const git_index_entry *entry;
 	git_blob *blob;
@@ -118,19 +86,22 @@ static int has_cr_in_index(git_filter *self)
 	git_off_t blobsize;
 	bool found_cr;
 
-	if (git_repository_index__weakptr(&index, filter->repo) < 0) {
+	if (!path)
+		return false;
+
+	if (git_repository_index__weakptr(&index, repo) < 0) {
 		giterr_clear();
 		return false;
 	}
 
-	if (!(entry = git_index_get_bypath(index, filter->path, 0)) &&
-		!(entry = git_index_get_bypath(index, filter->path, 1)))
+	if (!(entry = git_index_get_bypath(index, path, 0)) &&
+		!(entry = git_index_get_bypath(index, path, 1)))
 		return false;
 
 	if (!S_ISREG(entry->mode)) /* don't crlf filter non-blobs */
 		return true;
 
-	if (git_blob_lookup(&blob, filter->repo, &entry->oid) < 0)
+	if (git_blob_lookup(&blob, repo, &entry->oid) < 0)
 		return false;
 
 	blobcontent = git_blob_rawcontent(blob);
@@ -147,27 +118,24 @@ static int has_cr_in_index(git_filter *self)
 }
 
 static int crlf_apply_to_odb(
-	git_filter *self, git_buf *dest, const git_buf *source)
+	struct crlf_attrs *ca,
+	git_buf *to,
+	const git_buf *from,
+	const git_filter_source *src)
 {
-	struct crlf_filter *filter = (struct crlf_filter *)self;
-
-	assert(self && dest && source);
-
 	/* Empty file? Nothing to do */
-	if (git_buf_len(source) == 0)
+	if (!git_buf_len(from))
 		return 0;
 
 	/* Heuristics to see if we can skip the conversion.
 	 * Straight from Core Git.
 	 */
-	if (filter->attrs.crlf_action == GIT_CRLF_AUTO ||
-		filter->attrs.crlf_action == GIT_CRLF_GUESS) {
-
+	if (ca->crlf_action == GIT_CRLF_AUTO || ca->crlf_action == GIT_CRLF_GUESS) {
 		git_buf_text_stats stats;
 
-		/* Check heuristics for binary vs text... */
-		if (git_buf_text_gather_stats(&stats, source, false))
-			return -1;
+		/* Check heuristics for binary vs text - returns true if binary */
+		if (git_buf_text_gather_stats(&stats, from, false))
+			return GIT_PASSTHROUGH;
 
 		/*
 		 * We're currently not going to even try to convert stuff
@@ -175,28 +143,28 @@ static int crlf_apply_to_odb(
 		 * stuff?
 		 */
 		if (stats.cr != stats.crlf)
-			return -1;
+			return GIT_PASSTHROUGH;
 
-		if (filter->attrs.crlf_action == GIT_CRLF_GUESS) {
+		if (ca->crlf_action == GIT_CRLF_GUESS) {
 			/*
 			 * If the file in the index has any CR in it, do not convert.
 			 * This is the new safer autocrlf handling.
 			 */
-			if (has_cr_in_index(self))
-				return -1;
+			if (has_cr_in_index(src))
+				return GIT_PASSTHROUGH;
 		}
 
 		if (!stats.cr)
-			return -1;
+			return GIT_PASSTHROUGH;
 	}
 
 	/* Actually drop the carriage returns */
-	return git_buf_text_crlf_to_lf(dest, source);
+	return git_buf_text_crlf_to_lf(to, from);
 }
 
-static const char *line_ending(struct crlf_filter *filter)
+static const char *line_ending(struct crlf_attrs *ca)
 {
-	switch (filter->attrs.crlf_action) {
+	switch (ca->crlf_action) {
 	case GIT_CRLF_BINARY:
 	case GIT_CRLF_INPUT:
 		return "\n";
@@ -213,11 +181,9 @@ static const char *line_ending(struct crlf_filter *filter)
 		goto line_ending_error;
 	}
 
-	switch (filter->attrs.eol) {
+	switch (ca->eol) {
 	case GIT_EOL_UNSET:
-		return GIT_EOL_NATIVE == GIT_EOL_CRLF
-			? "\r\n"
-			: "\n";
+		return GIT_EOL_NATIVE == GIT_EOL_CRLF ? "\r\n" : "\n";
 
 	case GIT_EOL_CRLF:
 		return "\r\n";
@@ -235,41 +201,64 @@ line_ending_error:
 }
 
 static int crlf_apply_to_workdir(
-	git_filter *self, git_buf *dest, const git_buf *source)
+	struct crlf_attrs *ca, git_buf *to, const git_buf *from)
 {
-	struct crlf_filter *filter = (struct crlf_filter *)self;
 	const char *workdir_ending = NULL;
 
-	assert(self && dest && source);
-
 	/* Empty file? Nothing to do. */
-	if (git_buf_len(source) == 0)
-		return -1;
+	if (git_buf_len(from) == 0)
+		return 0;
+
+	/* Don't filter binary files */
+	if (git_buf_text_is_binary(from))
+		return GIT_PASSTHROUGH;
 
 	/* Determine proper line ending */
-	workdir_ending = line_ending(filter);
+	workdir_ending = line_ending(ca);
 	if (!workdir_ending)
 		return -1;
-	if (!strcmp("\n", workdir_ending)) /* do nothing for \n ending */
-		return -1;
 
-	/* for now, only lf->crlf conversion is supported here */
-	assert(!strcmp("\r\n", workdir_ending));
-	return git_buf_text_lf_to_crlf(dest, source);
+	if (!strcmp("\n", workdir_ending)) {
+		if (ca->crlf_action == GIT_CRLF_GUESS && ca->auto_crlf)
+			return GIT_PASSTHROUGH;
+
+		if (git_buf_find(from, '\r') < 0)
+			return GIT_PASSTHROUGH;
+
+		if (git_buf_text_crlf_to_lf(to, from) < 0)
+			return -1;
+	} else {
+		/* only other supported option is lf->crlf conversion */
+		assert(!strcmp("\r\n", workdir_ending));
+
+		if (git_buf_text_lf_to_crlf(to, from) < 0)
+			return -1;
+	}
+
+	return 0;
 }
 
-static int find_and_add_filter(
-	git_vector *filters, git_repository *repo, const char *path,
-	int (*apply)(struct git_filter *self, git_buf *dest, const git_buf *source))
+static int crlf_check(
+	git_filter        *self,
+	void              **payload, /* points to NULL ptr on entry, may be set */
+	const git_filter_source *src,
+	const char **attr_values)
 {
-	struct crlf_attrs ca;
-	struct crlf_filter *filter;
-	size_t pathlen;
 	int error;
+	struct crlf_attrs ca;
+
+	GIT_UNUSED(self);
 
-	/* Load gitattributes for the path */
-	if ((error = crlf_load_attributes(&ca, repo, path)) < 0)
-		return error;
+	if (!attr_values) {
+		ca.crlf_action = GIT_CRLF_GUESS;
+		ca.eol = GIT_EOL_UNSET;
+	} else {
+		ca.crlf_action = check_crlf(attr_values[2]); /* text */
+		if (ca.crlf_action == GIT_CRLF_GUESS)
+			ca.crlf_action = check_crlf(attr_values[0]); /* clrf */
+		ca.eol = check_eol(attr_values[1]); /* eol */
+	}
+	ca.auto_crlf = GIT_AUTO_CRLF_DEFAULT;
 
 	/*
 	 * Use the core Git logic to see if we should perform CRLF for this file
@@ -278,41 +267,64 @@ static int find_and_add_filter(
 	ca.crlf_action = crlf_input_action(&ca);
 
 	if (ca.crlf_action == GIT_CRLF_BINARY)
-		return 0;
+		return GIT_PASSTHROUGH;
 
 	if (ca.crlf_action == GIT_CRLF_GUESS) {
-		int auto_crlf;
-
-		if ((error = git_repository__cvar(&auto_crlf, repo, GIT_CVAR_AUTO_CRLF)) < 0)
+		error = git_repository__cvar(
+			&ca.auto_crlf, git_filter_source_repo(src), GIT_CVAR_AUTO_CRLF);
+		if (error < 0)
 			return error;
 
-		if (auto_crlf == GIT_AUTO_CRLF_FALSE)
-			return 0;
+		if (ca.auto_crlf == GIT_AUTO_CRLF_FALSE)
+			return GIT_PASSTHROUGH;
 	}
 
-	/* If we're good, we create a new filter object and push it
-	 * into the filters array */
-	pathlen = strlen(path);
-	filter = git__malloc(sizeof(struct crlf_filter) + pathlen + 1);
-	GITERR_CHECK_ALLOC(filter);
+	*payload = git__malloc(sizeof(ca));
+	GITERR_CHECK_ALLOC(*payload);
+	memcpy(*payload, &ca, sizeof(ca));
+
+	return 0;
+}
 
-	filter->f.apply = apply;
-	filter->f.do_free = NULL;
-	memcpy(&filter->attrs, &ca, sizeof(struct crlf_attrs));
-	filter->repo = repo;
-	memcpy(filter->path, path, pathlen + 1);
+static int crlf_apply(
+	git_filter    *self,
+	void         **payload, /* may be read and/or set */
+	git_buf       *to,
+	const git_buf *from,
+	const git_filter_source *src)
+{
+	/* initialize payload in case `check` was bypassed */
+	if (!*payload) {
+		int error = crlf_check(self, payload, src, NULL);
+		if (error < 0 && error != GIT_PASSTHROUGH)
+			return error;
+	}
 
-	return git_vector_insert(filters, filter);
+	if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE)
+		return crlf_apply_to_workdir(*payload, to, from);
+	else
+		return crlf_apply_to_odb(*payload, to, from, src);
 }
 
-int git_filter_add__crlf_to_odb(
-	git_vector *filters, git_repository *repo, const char *path)
+static void crlf_cleanup(
+	git_filter *self,
+	void       *payload)
 {
-	return find_and_add_filter(filters, repo, path, &crlf_apply_to_odb);
+	GIT_UNUSED(self);
+	git__free(payload);
 }
 
-int git_filter_add__crlf_to_workdir(
-	git_vector *filters, git_repository *repo, const char *path)
+git_filter *git_crlf_filter_new(void)
 {
-	return find_and_add_filter(filters, repo, path, &crlf_apply_to_workdir);
+	struct crlf_filter *f = git__calloc(1, sizeof(struct crlf_filter));
+
+	f->f.version = GIT_FILTER_VERSION;
+	f->f.attributes = "crlf eol text";
+	f->f.initialize = NULL;
+	f->f.shutdown = git_filter_free;
+	f->f.check    = crlf_check;
+	f->f.apply    = crlf_apply;
+	f->f.cleanup  = crlf_cleanup;
+
+	return (git_filter *)f;
 }
diff --git a/src/date.c b/src/date.c
index 48841e4f9..7849c2f02 100644
--- a/src/date.c
+++ b/src/date.c
@@ -823,15 +823,13 @@ static void pending_number(struct tm *tm, int *num)
 }
 
 static git_time_t approxidate_str(const char *date,
-									const struct timeval *tv,
-									int *error_ret)
+	time_t time_sec,
+	int *error_ret)
 {
 	int number = 0;
 	int touched = 0;
 	struct tm tm = {0}, now;
-	time_t time_sec;
 
-	time_sec = tv->tv_sec;
 	p_localtime_r(&time_sec, &tm);
 	now = tm;
 
@@ -861,7 +859,7 @@ static git_time_t approxidate_str(const char *date,
 
 int git__date_parse(git_time_t *out, const char *date)
 {
-	struct timeval tv;
+	time_t time_sec;
 	git_time_t timestamp;
 	int offset, error_ret=0;
 
@@ -870,7 +868,9 @@ int git__date_parse(git_time_t *out, const char *date)
 		return 0;
 	}
 
-	p_gettimeofday(&tv, NULL);
-	*out = approxidate_str(date, &tv, &error_ret);
+	if (time(&time_sec) == -1)
+		return -1;
+
+	*out = approxidate_str(date, time_sec, &error_ret);
    return error_ret;
 }
diff --git a/src/diff.c b/src/diff.c
index 26e117402..4c33a0213 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -13,6 +13,7 @@
 #include "pathspec.h"
 #include "index.h"
 #include "odb.h"
+#include "submodule.h"
 
 #define DIFF_FLAG_IS_SET(DIFF,FLAG) (((DIFF)->opts.flags & (FLAG)) != 0)
 #define DIFF_FLAG_ISNT_SET(DIFF,FLAG) (((DIFF)->opts.flags & (FLAG)) == 0)
@@ -20,7 +21,7 @@
 	(VAL) ? ((DIFF)->opts.flags | (FLAG)) : ((DIFF)->opts.flags & ~(VAL))
 
 static git_diff_delta *diff_delta__alloc(
-	git_diff_list *diff,
+	git_diff *diff,
 	git_delta_t status,
 	const char *path)
 {
@@ -49,7 +50,7 @@ static git_diff_delta *diff_delta__alloc(
 }
 
 static int diff_notify(
-	const git_diff_list *diff,
+	const git_diff *diff,
 	const git_diff_delta *delta,
 	const char *matched_pathspec)
 {
@@ -61,14 +62,17 @@ static int diff_notify(
 }
 
 static int diff_delta__from_one(
-	git_diff_list *diff,
-	git_delta_t   status,
+	git_diff *diff,
+	git_delta_t status,
 	const git_index_entry *entry)
 {
 	git_diff_delta *delta;
 	const char *matched_pathspec;
 	int notify_res;
 
+	if ((entry->flags & GIT_IDXENTRY_VALID) != 0)
+		return 0;
+
 	if (status == GIT_DELTA_IGNORED &&
 		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_IGNORED))
 		return 0;
@@ -77,15 +81,11 @@ static int diff_delta__from_one(
 		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_UNTRACKED))
 		return 0;
 
-	if (entry->mode == GIT_FILEMODE_COMMIT &&
-		DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES))
-		return 0;
-
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			&diff->pathspec, entry->path,
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
-			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE),
-			&matched_pathspec))
+			DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_CASE),
+			&matched_pathspec, NULL))
 		return 0;
 
 	delta = diff_delta__alloc(diff, status, entry->path);
@@ -93,6 +93,7 @@ static int diff_delta__from_one(
 
 	/* This fn is just for single-sided diffs */
 	assert(status != GIT_DELTA_MODIFIED);
+	delta->nfiles = 1;
 
 	if (delta->status == GIT_DELTA_DELETED) {
 		delta->old_file.mode = entry->mode;
@@ -123,7 +124,7 @@ static int diff_delta__from_one(
 }
 
 static int diff_delta__from_two(
-	git_diff_list *diff,
+	git_diff *diff,
 	git_delta_t   status,
 	const git_index_entry *old_entry,
 	uint32_t old_mode,
@@ -140,11 +141,6 @@ static int diff_delta__from_two(
 		DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_INCLUDE_UNMODIFIED))
 		return 0;
 
-	if (old_entry->mode == GIT_FILEMODE_COMMIT &&
-		new_entry->mode == GIT_FILEMODE_COMMIT &&
-		DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES))
-		return 0;
-
 	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE)) {
 		uint32_t temp_mode = old_mode;
 		const git_index_entry *temp_entry = old_entry;
@@ -156,6 +152,7 @@ static int diff_delta__from_two(
 
 	delta = diff_delta__alloc(diff, status, canonical_path);
 	GITERR_CHECK_ALLOC(delta);
+	delta->nfiles = 2;
 
 	git_oid_cpy(&delta->old_file.oid, &old_entry->oid);
 	delta->old_file.size = old_entry->file_size;
@@ -189,7 +186,7 @@ static int diff_delta__from_two(
 }
 
 static git_diff_delta *diff_delta__last_for_item(
-	git_diff_list *diff,
+	git_diff *diff,
 	const git_index_entry *item)
 {
 	git_diff_delta *delta = git_vector_last(&diff->deltas);
@@ -247,6 +244,11 @@ GIT_INLINE(const char *) diff_delta__path(const git_diff_delta *delta)
 	return str;
 }
 
+const char *git_diff_delta__path(const git_diff_delta *delta)
+{
+	return diff_delta__path(delta);
+}
+
 int git_diff_delta__cmp(const void *a, const void *b)
 {
 	const git_diff_delta *da = a, *db = b;
@@ -261,6 +263,26 @@ int git_diff_delta__casecmp(const void *a, const void *b)
 	return val ? val : ((int)da->status - (int)db->status);
 }
 
+GIT_INLINE(const char *) diff_delta__i2w_path(const git_diff_delta *delta)
+{
+	return delta->old_file.path ?
+		delta->old_file.path : delta->new_file.path;
+}
+
+int git_diff_delta__i2w_cmp(const void *a, const void *b)
+{
+	const git_diff_delta *da = a, *db = b;
+	int val = strcmp(diff_delta__i2w_path(da), diff_delta__i2w_path(db));
+	return val ? val : ((int)da->status - (int)db->status);
+}
+
+int git_diff_delta__i2w_casecmp(const void *a, const void *b)
+{
+	const git_diff_delta *da = a, *db = b;
+	int val = strcasecmp(diff_delta__i2w_path(da), diff_delta__i2w_path(db));
+	return val ? val : ((int)da->status - (int)db->status);
+}
+
 bool git_diff_delta__should_skip(
 	const git_diff_options *opts, const git_diff_delta *delta)
 {
@@ -323,13 +345,13 @@ static const char *diff_mnemonic_prefix(
 	return pfx;
 }
 
-static git_diff_list *diff_list_alloc(
+static git_diff *diff_list_alloc(
 	git_repository *repo,
 	git_iterator *old_iter,
 	git_iterator *new_iter)
 {
 	git_diff_options dflt = GIT_DIFF_OPTIONS_INIT;
-	git_diff_list *diff = git__calloc(1, sizeof(git_diff_list));
+	git_diff *diff = git__calloc(1, sizeof(git_diff));
 	if (!diff)
 		return NULL;
 
@@ -343,7 +365,7 @@ static git_diff_list *diff_list_alloc(
 
 	if (git_vector_init(&diff->deltas, 0, git_diff_delta__cmp) < 0 ||
 		git_pool_init(&diff->pool, 1, 0) < 0) {
-		git_diff_list_free(diff);
+		git_diff_free(diff);
 		return NULL;
 	}
 
@@ -351,14 +373,14 @@ static git_diff_list *diff_list_alloc(
 	 * the ignore_case bit set */
 	if (!git_iterator_ignore_case(old_iter) &&
 		!git_iterator_ignore_case(new_iter)) {
-		diff->opts.flags &= ~GIT_DIFF_DELTAS_ARE_ICASE;
+		diff->opts.flags &= ~GIT_DIFF_IGNORE_CASE;
 
 		diff->strcomp    = git__strcmp;
 		diff->strncomp   = git__strncmp;
 		diff->pfxcomp    = git__prefixcmp;
 		diff->entrycomp  = git_index_entry__cmp;
 	} else {
-		diff->opts.flags |= GIT_DIFF_DELTAS_ARE_ICASE;
+		diff->opts.flags |= GIT_DIFF_IGNORE_CASE;
 
 		diff->strcomp    = git__strcasecmp;
 		diff->strncomp   = git__strncasecmp;
@@ -372,7 +394,7 @@ static git_diff_list *diff_list_alloc(
 }
 
 static int diff_list_apply_options(
-	git_diff_list *diff,
+	git_diff *diff,
 	const git_diff_options *opts)
 {
 	git_config *cfg;
@@ -382,12 +404,12 @@ static int diff_list_apply_options(
 
 	if (opts) {
 		/* copy user options (except case sensitivity info from iterators) */
-		bool icase = DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE);
+		bool icase = DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_CASE);
 		memcpy(&diff->opts, opts, sizeof(diff->opts));
-		DIFF_FLAG_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE, icase);
+		DIFF_FLAG_SET(diff, GIT_DIFF_IGNORE_CASE, icase);
 
 		/* initialize pathspec from options */
-		if (git_pathspec_init(&diff->pathspec, &opts->pathspec, pool) < 0)
+		if (git_pathspec__vinit(&diff->pathspec, &opts->pathspec, pool) < 0)
 			return -1;
 	}
 
@@ -396,7 +418,7 @@ static int diff_list_apply_options(
 		diff->opts.flags |= GIT_DIFF_INCLUDE_TYPECHANGE;
 
 	/* flag INCLUDE_UNTRACKED_CONTENT implies INCLUDE_UNTRACKED */
-	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_INCLUDE_UNTRACKED_CONTENT))
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_SHOW_UNTRACKED_CONTENT))
 		diff->opts.flags |= GIT_DIFF_INCLUDE_UNTRACKED;
 
 	/* load config values that affect diff behavior */
@@ -407,7 +429,7 @@ static int diff_list_apply_options(
 		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_HAS_SYMLINKS;
 
 	if (!git_repository__cvar(&val, repo, GIT_CVAR_IGNORESTAT) && val)
-		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_ASSUME_UNCHANGED;
+		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_IGNORE_STAT;
 
 	if ((diff->opts.flags & GIT_DIFF_IGNORE_FILEMODE) == 0 &&
 		!git_repository__cvar(&val, repo, GIT_CVAR_FILEMODE) && val)
@@ -423,10 +445,28 @@ static int diff_list_apply_options(
 
 	/* If not given explicit `opts`, check `diff.xyz` configs */
 	if (!opts) {
-		diff->opts.context_lines = config_int(cfg, "diff.context", 3);
+		int context = config_int(cfg, "diff.context", 3);
+		diff->opts.context_lines = context >= 0 ? (uint16_t)context : 3;
+
+		/* add other defaults here */
+	}
+
+	/* Reverse src info if diff is reversed */
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE)) {
+		git_iterator_type_t tmp_src = diff->old_src;
+		diff->old_src = diff->new_src;
+		diff->new_src = tmp_src;
+	}
 
-		if (config_bool(cfg, "diff.ignoreSubmodules", 0))
-			diff->opts.flags |= GIT_DIFF_IGNORE_SUBMODULES;
+	/* if ignore_submodules not explicitly set, check diff config */
+	if (diff->opts.ignore_submodules <= 0) {
+		const char *str;
+
+		if (git_config_get_string(&str , cfg, "diff.ignoreSubmodules") < 0)
+			giterr_clear();
+		else if (str != NULL &&
+			git_submodule_parse_ignore(&diff->opts.ignore_submodules, str) < 0)
+			giterr_clear();
 	}
 
 	/* if either prefix is not set, figure out appropriate value */
@@ -454,15 +494,15 @@ static int diff_list_apply_options(
 		return -1;
 
 	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_REVERSE)) {
-		const char *swap = diff->opts.old_prefix;
-		diff->opts.old_prefix = diff->opts.new_prefix;
-		diff->opts.new_prefix = swap;
+		const char *tmp_prefix = diff->opts.old_prefix;
+		diff->opts.old_prefix  = diff->opts.new_prefix;
+		diff->opts.new_prefix  = tmp_prefix;
 	}
 
 	return 0;
 }
 
-static void diff_list_free(git_diff_list *diff)
+static void diff_list_free(git_diff *diff)
 {
 	git_diff_delta *delta;
 	unsigned int i;
@@ -473,14 +513,14 @@ static void diff_list_free(git_diff_list *diff)
 	}
 	git_vector_free(&diff->deltas);
 
-	git_pathspec_free(&diff->pathspec);
+	git_pathspec__vfree(&diff->pathspec);
 	git_pool_clear(&diff->pool);
 
 	git__memzero(diff, sizeof(*diff));
 	git__free(diff);
 }
 
-void git_diff_list_free(git_diff_list *diff)
+void git_diff_free(git_diff *diff)
 {
 	if (!diff)
 		return;
@@ -488,7 +528,7 @@ void git_diff_list_free(git_diff_list *diff)
 	GIT_REFCOUNT_DEC(diff, diff_list_free);
 }
 
-void git_diff_list_addref(git_diff_list *diff)
+void git_diff_addref(git_diff *diff)
 {
 	GIT_REFCOUNT_INC(diff);
 }
@@ -541,21 +581,21 @@ int git_diff__oid_for_file(
 		giterr_set(GITERR_OS, "File size overflow (for 32-bits) on '%s'", path);
 		result = -1;
 	} else {
-		git_vector filters = GIT_VECTOR_INIT;
+		git_filter_list *fl = NULL;
 
-		result = git_filters_load(&filters, repo, path, GIT_FILTER_TO_ODB);
-		if (result >= 0) {
+		result = git_filter_list_load(&fl, repo, NULL, path, GIT_FILTER_TO_ODB);
+		if (!result) {
 			int fd = git_futils_open_ro(full_path.ptr);
 			if (fd < 0)
 				result = fd;
 			else {
 				result = git_odb__hashfd_filtered(
-					oid, fd, (size_t)size, GIT_OBJ_BLOB, &filters);
+					oid, fd, (size_t)size, GIT_OBJ_BLOB, fl);
 				p_close(fd);
 			}
-		}
 
-		git_filters_free(&filters);
+			git_filter_list_free(fl);
+		}
 	}
 
 cleanup:
@@ -584,45 +624,54 @@ typedef struct {
 static int maybe_modified_submodule(
 	git_delta_t *status,
 	git_oid *found_oid,
-	git_diff_list *diff,
+	git_diff *diff,
 	diff_in_progress *info)
 {
 	int error = 0;
 	git_submodule *sub;
 	unsigned int sm_status = 0;
-	const git_oid *sm_oid;
+	git_submodule_ignore_t ign = diff->opts.ignore_submodules;
 
 	*status = GIT_DELTA_UNMODIFIED;
 
-	if (!DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES) &&
-		!(error = git_submodule_lookup(
-			  &sub, diff->repo, info->nitem->path)) &&
-		git_submodule_ignore(sub) != GIT_SUBMODULE_IGNORE_ALL &&
-		!(error = git_submodule_status(&sm_status, sub)))
-	{
-		/* check IS_WD_UNMODIFIED because this case is only used
-		 * when the new side of the diff is the working directory
-		 */
-		if (!GIT_SUBMODULE_STATUS_IS_WD_UNMODIFIED(sm_status))
-			*status = GIT_DELTA_MODIFIED;
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_SUBMODULES) ||
+		ign == GIT_SUBMODULE_IGNORE_ALL)
+		return 0;
 
-		/* grab OID while we are here */
-		if (git_oid_iszero(&info->nitem->oid) &&
-			(sm_oid = git_submodule_wd_id(sub)) != NULL)
-			git_oid_cpy(found_oid, sm_oid);
-	}
+	if ((error = git_submodule_lookup(
+			&sub, diff->repo, info->nitem->path)) < 0) {
 
-	/* GIT_EEXISTS means a dir with .git in it was found - ignore it */
-	if (error == GIT_EEXISTS) {
-		giterr_clear();
-		error = 0;
+		/* GIT_EEXISTS means dir with .git in it was found - ignore it */
+		if (error == GIT_EEXISTS) {
+			giterr_clear();
+			error = 0;
+		}
+		return error;
 	}
 
-	return error;
+	if (ign <= 0 && git_submodule_ignore(sub) == GIT_SUBMODULE_IGNORE_ALL)
+		return 0;
+
+	if ((error = git_submodule__status(
+			&sm_status, NULL, NULL, found_oid, sub, ign)) < 0)
+		return error;
+
+	/* check IS_WD_UNMODIFIED because this case is only used
+	 * when the new side of the diff is the working directory
+	 */
+	if (!GIT_SUBMODULE_STATUS_IS_WD_UNMODIFIED(sm_status))
+		*status = GIT_DELTA_MODIFIED;
+
+	/* now that we have a HEAD OID, check if HEAD moved */
+	if ((sm_status & GIT_SUBMODULE_STATUS_IN_WD) != 0 &&
+		!git_oid_equal(&info->oitem->oid, found_oid))
+		*status = GIT_DELTA_MODIFIED;
+
+	return 0;
 }
 
 static int maybe_modified(
-	git_diff_list *diff,
+	git_diff *diff,
 	diff_in_progress *info)
 {
 	git_oid noid;
@@ -634,11 +683,11 @@ static int maybe_modified(
 	bool new_is_workdir = (info->new_iter->type == GIT_ITERATOR_TYPE_WORKDIR);
 	const char *matched_pathspec;
 
-	if (!git_pathspec_match_path(
+	if (!git_pathspec__match(
 			&diff->pathspec, oitem->path,
 			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DISABLE_PATHSPEC_MATCH),
-			DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE),
-			&matched_pathspec))
+			DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_CASE),
+			&matched_pathspec, NULL))
 		return 0;
 
 	memset(&noid, 0, sizeof(noid));
@@ -655,9 +704,8 @@ static int maybe_modified(
 		nmode = (nmode & ~MODE_BITS_MASK) | (omode & MODE_BITS_MASK);
 
 	/* support "assume unchanged" (poorly, b/c we still stat everything) */
-	if ((diff->diffcaps & GIT_DIFFCAPS_ASSUME_UNCHANGED) != 0)
-		status = (oitem->flags_extended & GIT_IDXENTRY_INTENT_TO_ADD) ?
-			GIT_DELTA_MODIFIED : GIT_DELTA_UNMODIFIED;
+	if ((oitem->flags & GIT_IDXENTRY_VALID) != 0)
+		status = GIT_DELTA_UNMODIFIED;
 
 	/* support "skip worktree" index bit */
 	else if ((oitem->flags_extended & GIT_IDXENTRY_SKIP_WORKTREE) != 0)
@@ -719,7 +767,7 @@ static int maybe_modified(
 	/* if we got here and decided that the files are modified, but we
 	 * haven't calculated the OID of the new item, then calculate it now
 	 */
-	if (status != GIT_DELTA_UNMODIFIED && git_oid_iszero(&nitem->oid)) {
+	if (status == GIT_DELTA_MODIFIED && git_oid_iszero(&nitem->oid)) {
 		if (git_oid_iszero(&noid)) {
 			if (git_diff__oid_for_file(diff->repo,
 					nitem->path, nitem->mode, nitem->file_size, &noid) < 0)
@@ -741,7 +789,7 @@ static int maybe_modified(
 }
 
 static bool entry_is_prefixed(
-	git_diff_list *diff,
+	git_diff *diff,
 	const git_index_entry *item,
 	const git_index_entry *prefix_item)
 {
@@ -758,7 +806,7 @@ static bool entry_is_prefixed(
 }
 
 static int diff_scan_inside_untracked_dir(
-	git_diff_list *diff, diff_in_progress *info, git_delta_t *delta_type)
+	git_diff *diff, diff_in_progress *info, git_delta_t *delta_type)
 {
 	int error = 0;
 	git_buf base = GIT_BUF_INIT;
@@ -824,7 +872,7 @@ done:
 }
 
 static int handle_unmatched_new_item(
-	git_diff_list *diff, diff_in_progress *info)
+	git_diff *diff, diff_in_progress *info)
 {
 	int error = 0;
 	const git_index_entry *nitem = info->nitem;
@@ -842,7 +890,7 @@ static int handle_unmatched_new_item(
 			git_buf_clear(&info->ignore_prefix);
 	}
 
-	if (S_ISDIR(nitem->mode)) {
+	if (nitem->mode == GIT_FILEMODE_TREE) {
 		bool recurse_into_dir = contains_oitem;
 
 		/* if not already inside an ignored dir, check if this is ignored */
@@ -873,7 +921,7 @@ static int handle_unmatched_new_item(
 		 */
 		if (!recurse_into_dir &&
 			delta_type == GIT_DELTA_UNTRACKED &&
-			DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_FAST_UNTRACKED_DIRS))
+			DIFF_FLAG_ISNT_SET(diff, GIT_DIFF_ENABLE_FAST_UNTRACKED_DIRS))
 		{
 			git_diff_delta *last;
 
@@ -946,6 +994,16 @@ static int handle_unmatched_new_item(
 	else if (info->new_iter->type != GIT_ITERATOR_TYPE_WORKDIR)
 		delta_type = GIT_DELTA_ADDED;
 
+	else if (nitem->mode == GIT_FILEMODE_COMMIT) {
+		git_submodule *sm;
+
+		/* ignore things that are not actual submodules */
+		if (git_submodule_lookup(&sm, info->repo, nitem->path) != 0) {
+			giterr_clear();
+			delta_type = GIT_DELTA_IGNORED;
+		}
+	}
+
 	/* Actually create the record for this item if necessary */
 	if ((error = diff_delta__from_one(diff, delta_type, nitem)) < 0)
 		return error;
@@ -969,7 +1027,7 @@ static int handle_unmatched_new_item(
 }
 
 static int handle_unmatched_old_item(
-	git_diff_list *diff, diff_in_progress *info)
+	git_diff *diff, diff_in_progress *info)
 {
 	int error = diff_delta__from_one(diff, GIT_DELTA_DELETED, info->oitem);
 	if (error < 0)
@@ -1001,7 +1059,7 @@ static int handle_unmatched_old_item(
 }
 
 static int handle_matched_item(
-	git_diff_list *diff, diff_in_progress *info)
+	git_diff *diff, diff_in_progress *info)
 {
 	int error = 0;
 
@@ -1016,7 +1074,7 @@ static int handle_matched_item(
 }
 
 int git_diff__from_iterators(
-	git_diff_list **diff_ptr,
+	git_diff **diff_ptr,
 	git_repository *repo,
 	git_iterator *old_iter,
 	git_iterator *new_iter,
@@ -1024,7 +1082,7 @@ int git_diff__from_iterators(
 {
 	int error = 0;
 	diff_in_progress info;
-	git_diff_list *diff;
+	git_diff *diff;
 
 	*diff_ptr = NULL;
 
@@ -1037,7 +1095,7 @@ int git_diff__from_iterators(
 	git_buf_init(&info.ignore_prefix, 0);
 
 	/* make iterators have matching icase behavior */
-	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_DELTAS_ARE_ICASE)) {
+	if (DIFF_FLAG_IS_SET(diff, GIT_DIFF_IGNORE_CASE)) {
 		if ((error = git_iterator_set_ignore_case(old_iter, true)) < 0 ||
 			(error = git_iterator_set_ignore_case(new_iter, true)) < 0)
 			goto cleanup;
@@ -1085,7 +1143,7 @@ cleanup:
 	if (!error)
 		*diff_ptr = diff;
 	else
-		git_diff_list_free(diff);
+		git_diff_free(diff);
 
 	git_buf_free(&info.ignore_prefix);
 
@@ -1102,7 +1160,7 @@ cleanup:
 } while (0)
 
 int git_diff_tree_to_tree(
-	git_diff_list **diff,
+	git_diff **diff,
 	git_repository *repo,
 	git_tree *old_tree,
 	git_tree *new_tree,
@@ -1117,7 +1175,7 @@ int git_diff_tree_to_tree(
 	 * currently case insensitive, unless the user explicitly asked
 	 * for case insensitivity
 	 */
-	if (opts && (opts->flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0)
+	if (opts && (opts->flags & GIT_DIFF_IGNORE_CASE) != 0)
 		iflag = GIT_ITERATOR_IGNORE_CASE;
 
 	DIFF_FROM_ITERATORS(
@@ -1128,8 +1186,19 @@ int git_diff_tree_to_tree(
 	return error;
 }
 
+static int diff_load_index(git_index **index, git_repository *repo)
+{
+	int error = git_repository_index__weakptr(index, repo);
+
+	/* reload the repository index when user did not pass one in */
+	if (!error && git_index_read(*index, false) < 0)
+		giterr_clear();
+
+	return error;
+}
+
 int git_diff_tree_to_index(
-	git_diff_list **diff,
+	git_diff **diff,
 	git_repository *repo,
 	git_tree *old_tree,
 	git_index *index,
@@ -1140,7 +1209,7 @@ int git_diff_tree_to_index(
 
 	assert(diff && repo);
 
-	if (!index && (error = git_repository_index__weakptr(&index, repo)) < 0)
+	if (!index && (error = diff_load_index(&index, repo)) < 0)
 		return error;
 
 	if (index->ignore_case) {
@@ -1157,9 +1226,9 @@ int git_diff_tree_to_index(
 		git_index__set_ignore_case(index, true);
 
 		if (!error) {
-			git_diff_list *d = *diff;
+			git_diff *d = *diff;
 
-			d->opts.flags |= GIT_DIFF_DELTAS_ARE_ICASE;
+			d->opts.flags |= GIT_DIFF_IGNORE_CASE;
 			d->strcomp    = git__strcasecmp;
 			d->strncomp   = git__strncasecmp;
 			d->pfxcomp    = git__prefixcmp_icase;
@@ -1174,7 +1243,7 @@ int git_diff_tree_to_index(
 }
 
 int git_diff_index_to_workdir(
-	git_diff_list **diff,
+	git_diff **diff,
 	git_repository *repo,
 	git_index *index,
 	const git_diff_options *opts)
@@ -1183,7 +1252,7 @@ int git_diff_index_to_workdir(
 
 	assert(diff && repo);
 
-	if (!index && (error = git_repository_index__weakptr(&index, repo)) < 0)
+	if (!index && (error = diff_load_index(&index, repo)) < 0)
 		return error;
 
 	DIFF_FROM_ITERATORS(
@@ -1195,9 +1264,8 @@ int git_diff_index_to_workdir(
 	return error;
 }
 
-
 int git_diff_tree_to_workdir(
-	git_diff_list **diff,
+	git_diff **diff,
 	git_repository *repo,
 	git_tree *old_tree,
 	const git_diff_options *opts)
@@ -1215,16 +1283,60 @@ int git_diff_tree_to_workdir(
 	return error;
 }
 
-size_t git_diff_num_deltas(git_diff_list *diff)
+int git_diff_tree_to_workdir_with_index(
+	git_diff **diff,
+	git_repository *repo,
+	git_tree *old_tree,
+	const git_diff_options *opts)
+{
+	int error = 0;
+	git_diff *d1 = NULL, *d2 = NULL;
+	git_index *index = NULL;
+
+	assert(diff && repo);
+
+	if ((error = diff_load_index(&index, repo)) < 0)
+		return error;
+
+	if (!(error = git_diff_tree_to_index(&d1, repo, old_tree, index, opts)) &&
+		!(error = git_diff_index_to_workdir(&d2, repo, index, opts)))
+		error = git_diff_merge(d1, d2);
+
+	git_diff_free(d2);
+
+	if (error) {
+		git_diff_free(d1);
+		d1 = NULL;
+	}
+
+	*diff = d1;
+	return error;
+}
+
+int git_diff_options_init(git_diff_options *options, unsigned int version)
+{
+	git_diff_options template = GIT_DIFF_OPTIONS_INIT;
+
+	if (version != template.version) {
+		giterr_set(GITERR_INVALID,
+			"Invalid version %d for git_diff_options", (int)version);
+		return -1;
+	}
+
+	memcpy(options, &template, sizeof(*options));
+	return 0;
+}
+
+size_t git_diff_num_deltas(const git_diff *diff)
 {
 	assert(diff);
-	return (size_t)diff->deltas.length;
+	return diff->deltas.length;
 }
 
-size_t git_diff_num_deltas_of_type(git_diff_list *diff, git_delta_t type)
+size_t git_diff_num_deltas_of_type(const git_diff *diff, git_delta_t type)
 {
 	size_t i, count = 0;
-	git_diff_delta *delta;
+	const git_diff_delta *delta;
 
 	assert(diff);
 
@@ -1235,9 +1347,20 @@ size_t git_diff_num_deltas_of_type(git_diff_list *diff, git_delta_t type)
 	return count;
 }
 
+const git_diff_delta *git_diff_get_delta(const git_diff *diff, size_t idx)
+{
+	assert(diff);
+	return git_vector_get(&diff->deltas, idx);
+}
+
+int git_diff_is_sorted_icase(const git_diff *diff)
+{
+	return (diff->opts.flags & GIT_DIFF_IGNORE_CASE) != 0;
+}
+
 int git_diff__paired_foreach(
-	git_diff_list *head2idx,
-	git_diff_list *idx2wd,
+	git_diff *head2idx,
+	git_diff *idx2wd,
 	int (*cb)(git_diff_delta *h2i, git_diff_delta *i2w, void *payload),
 	void *payload)
 {
@@ -1245,10 +1368,12 @@ int git_diff__paired_foreach(
 	git_diff_delta *h2i, *i2w;
 	size_t i, j, i_max, j_max;
 	int (*strcomp)(const char *, const char *) = git__strcmp;
-	bool icase_mismatch;
+	bool h2i_icase, i2w_icase, icase_mismatch;
 
 	i_max = head2idx ? head2idx->deltas.length : 0;
 	j_max = idx2wd ? idx2wd->deltas.length : 0;
+	if (!i_max && !j_max)
+		return 0;
 
 	/* At some point, tree-to-index diffs will probably never ignore case,
 	 * even if that isn't true now.  Index-to-workdir diffs may or may not
@@ -1258,24 +1383,35 @@ int git_diff__paired_foreach(
 	 * Therefore the main thing we need to do here is make sure the diffs
 	 * are traversed in a compatible order.  To do this, we temporarily
 	 * resort a mismatched diff to get the order correct.
+	 *
+	 * In order to traverse renames in the index->workdir, we need to
+	 * ensure that we compare the index name on both sides, so we
+	 * always sort by the old name in the i2w list.
 	 */
+	h2i_icase = head2idx != NULL &&
+		(head2idx->opts.flags & GIT_DIFF_IGNORE_CASE) != 0;
+
+	i2w_icase = idx2wd != NULL &&
+		(idx2wd->opts.flags & GIT_DIFF_IGNORE_CASE) != 0;
+
 	icase_mismatch =
-		(head2idx != NULL && idx2wd != NULL &&
-		 ((head2idx->opts.flags ^ idx2wd->opts.flags) & GIT_DIFF_DELTAS_ARE_ICASE));
-
-	/* force case-sensitive delta sort */
-	if (icase_mismatch) {
-		if (head2idx->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) {
-			git_vector_set_cmp(&head2idx->deltas, git_diff_delta__cmp);
-			git_vector_sort(&head2idx->deltas);
-		} else {
-			git_vector_set_cmp(&idx2wd->deltas, git_diff_delta__cmp);
-			git_vector_sort(&idx2wd->deltas);
-		}
+		(head2idx != NULL && idx2wd != NULL && h2i_icase != i2w_icase);
+
+	if (icase_mismatch && h2i_icase) {
+		git_vector_set_cmp(&head2idx->deltas, git_diff_delta__cmp);
+		git_vector_sort(&head2idx->deltas);
 	}
-	else if (head2idx->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE)
+
+	if (i2w_icase && !icase_mismatch) {
 		strcomp = git__strcasecmp;
 
+		git_vector_set_cmp(&idx2wd->deltas, git_diff_delta__i2w_casecmp);
+		git_vector_sort(&idx2wd->deltas);
+	} else if (idx2wd != NULL) {
+		git_vector_set_cmp(&idx2wd->deltas, git_diff_delta__i2w_cmp);
+		git_vector_sort(&idx2wd->deltas);
+	}
+
 	for (i = 0, j = 0; i < i_max || j < j_max; ) {
 		h2i = head2idx ? GIT_VECTOR_GET(&head2idx->deltas, i) : NULL;
 		i2w = idx2wd ? GIT_VECTOR_GET(&idx2wd->deltas, j) : NULL;
@@ -1299,14 +1435,16 @@ int git_diff__paired_foreach(
 	}
 
 	/* restore case-insensitive delta sort */
-	if (icase_mismatch) {
-		if (head2idx->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) {
-			git_vector_set_cmp(&head2idx->deltas, git_diff_delta__casecmp);
-			git_vector_sort(&head2idx->deltas);
-		} else {
-			git_vector_set_cmp(&idx2wd->deltas, git_diff_delta__casecmp);
-			git_vector_sort(&idx2wd->deltas);
-		}
+	if (icase_mismatch && h2i_icase) {
+		git_vector_set_cmp(&head2idx->deltas, git_diff_delta__casecmp);
+		git_vector_sort(&head2idx->deltas);
+	}
+
+	/* restore idx2wd sort by new path */
+	if (idx2wd != NULL) {
+		git_vector_set_cmp(&idx2wd->deltas,
+			i2w_icase ? git_diff_delta__casecmp : git_diff_delta__cmp);
+		git_vector_sort(&idx2wd->deltas);
 	}
 
 	return 0;
diff --git a/src/diff.h b/src/diff.h
index 6ef03ee7c..2c9298a5f 100644
--- a/src/diff.h
+++ b/src/diff.h
@@ -16,13 +16,14 @@
 #include "iterator.h"
 #include "repository.h"
 #include "pool.h"
+#include "odb.h"
 
 #define DIFF_OLD_PREFIX_DEFAULT "a/"
 #define DIFF_NEW_PREFIX_DEFAULT "b/"
 
 enum {
 	GIT_DIFFCAPS_HAS_SYMLINKS     = (1 << 0), /* symlinks on platform? */
-	GIT_DIFFCAPS_ASSUME_UNCHANGED = (1 << 1), /* use stat? */
+	GIT_DIFFCAPS_IGNORE_STAT      = (1 << 1), /* use stat? */
 	GIT_DIFFCAPS_TRUST_MODE_BITS  = (1 << 2), /* use st_mode? */
 	GIT_DIFFCAPS_TRUST_CTIME      = (1 << 3), /* use st_ctime? */
 	GIT_DIFFCAPS_USE_DEV          = (1 << 4), /* use st_dev? */
@@ -51,7 +52,7 @@ enum {
 
 #define GIT_DIFF__VERBOSE  (1 << 30)
 
-struct git_diff_list {
+struct git_diff {
 	git_refcount     rc;
 	git_repository   *repo;
 	git_diff_options opts;
@@ -71,27 +72,36 @@ struct git_diff_list {
 extern void git_diff__cleanup_modes(
 	uint32_t diffcaps, uint32_t *omode, uint32_t *nmode);
 
-extern void git_diff_list_addref(git_diff_list *diff);
+extern void git_diff_addref(git_diff *diff);
 
 extern int git_diff_delta__cmp(const void *a, const void *b);
 extern int git_diff_delta__casecmp(const void *a, const void *b);
 
+extern const char *git_diff_delta__path(const git_diff_delta *delta);
+
 extern bool git_diff_delta__should_skip(
 	const git_diff_options *opts, const git_diff_delta *delta);
 
+extern int git_diff_delta__format_file_header(
+	git_buf *out,
+	const git_diff_delta *delta,
+	const char *oldpfx,
+	const char *newpfx,
+	int oid_strlen);
+
 extern int git_diff__oid_for_file(
 	git_repository *, const char *, uint16_t, git_off_t, git_oid *);
 
 extern int git_diff__from_iterators(
-	git_diff_list **diff_ptr,
+	git_diff **diff_ptr,
 	git_repository *repo,
 	git_iterator *old_iter,
 	git_iterator *new_iter,
 	const git_diff_options *opts);
 
 extern int git_diff__paired_foreach(
-	git_diff_list *idx2head,
-	git_diff_list *wd2idx,
+	git_diff *idx2head,
+	git_diff *wd2idx,
 	int (*cb)(git_diff_delta *i2h, git_diff_delta *w2i, void *payload),
 	void *payload);
 
@@ -106,5 +116,33 @@ extern void git_diff_find_similar__hashsig_free(void *sig, void *payload);
 extern int git_diff_find_similar__calc_similarity(
 	int *score, void *siga, void *sigb, void *payload);
 
+/*
+ * Sometimes a git_diff_file will have a zero size; this attempts to
+ * fill in the size without loading the blob if possible.  If that is
+ * not possible, then it will return the git_odb_object that had to be
+ * loaded and the caller can use it or dispose of it as needed.
+ */
+GIT_INLINE(int) git_diff_file__resolve_zero_size(
+	git_diff_file *file, git_odb_object **odb_obj, git_repository *repo)
+{
+	int error;
+	git_odb *odb;
+	size_t len;
+	git_otype type;
+
+	if ((error = git_repository_odb(&odb, repo)) < 0)
+		return error;
+
+	error = git_odb__read_header_or_object(
+		odb_obj, &len, &type, odb, &file->oid);
+
+	git_odb_free(odb);
+
+	if (!error)
+		file->size = (git_off_t)len;
+
+	return error;
+}
+
 #endif
 
diff --git a/src/diff_driver.c b/src/diff_driver.c
index 469be0d14..bd5a8fbd9 100644
--- a/src/diff_driver.c
+++ b/src/diff_driver.c
@@ -187,7 +187,7 @@ static int git_diff_driver_load(
 
 	git_buf_truncate(&name, namelen + strlen("diff.."));
 	git_buf_put(&name, "xfuncname", strlen("xfuncname"));
-	if ((error = git_config_get_multivar(
+	if ((error = git_config_get_multivar_foreach(
 			cfg, name.ptr, NULL, diff_driver_xfuncname, drv)) < 0) {
 		if (error != GIT_ENOTFOUND)
 			goto done;
@@ -196,7 +196,7 @@ static int git_diff_driver_load(
 
 	git_buf_truncate(&name, namelen + strlen("diff.."));
 	git_buf_put(&name, "funcname", strlen("funcname"));
-	if ((error = git_config_get_multivar(
+	if ((error = git_config_get_multivar_foreach(
 			cfg, name.ptr, NULL, diff_driver_funcname, drv)) < 0) {
 		if (error != GIT_ENOTFOUND)
 			goto done;
@@ -373,10 +373,11 @@ static long diff_context_find(
 		!ctxt->match_line(ctxt->driver, ctxt->line.ptr, ctxt->line.size))
 		return -1;
 
-	git_buf_truncate(&ctxt->line, (size_t)out_size);
-	git_buf_copy_cstr(out, (size_t)out_size, &ctxt->line);
+	if (out_size > (long)ctxt->line.size)
+		out_size = (long)ctxt->line.size;
+	memcpy(out, ctxt->line.ptr, (size_t)out_size);
 
-	return (long)ctxt->line.size;
+	return out_size;
 }
 
 void git_diff_find_context_init(
diff --git a/src/diff_file.c b/src/diff_file.c
index 9d06daafa..a4c8641bc 100644
--- a/src/diff_file.c
+++ b/src/diff_file.c
@@ -88,7 +88,7 @@ static int diff_file_content_init_common(
 
 int git_diff_file_content__init_from_diff(
 	git_diff_file_content *fc,
-	git_diff_list *diff,
+	git_diff *diff,
 	size_t delta_index,
 	bool use_old)
 {
@@ -110,7 +110,7 @@ int git_diff_file_content__init_from_diff(
 		has_data = use_old; break;
 	case GIT_DELTA_UNTRACKED:
 		has_data = !use_old &&
-			(diff->opts.flags & GIT_DIFF_INCLUDE_UNTRACKED_CONTENT) != 0;
+			(diff->opts.flags & GIT_DIFF_SHOW_UNTRACKED_CONTENT) != 0;
 		break;
 	case GIT_DELTA_MODIFIED:
 	case GIT_DELTA_COPIED:
@@ -241,19 +241,9 @@ static int diff_file_content_load_blob(git_diff_file_content *fc)
 
 	/* if we don't know size, try to peek at object header first */
 	if (!fc->file->size) {
-		git_odb *odb;
-		size_t len;
-		git_otype type;
-
-		if (!(error = git_repository_odb__weakptr(&odb, fc->repo))) {
-			error = git_odb__read_header_or_object(
-				&odb_obj, &len, &type, odb, &fc->file->oid);
-			git_odb_free(odb);
-		}
-		if (error)
+		if ((error = git_diff_file__resolve_zero_size(
+				fc->file, &odb_obj, fc->repo)) < 0)
 			return error;
-
-		fc->file->size = len;
 	}
 
 	if (diff_file_content_binary_by_size(fc))
@@ -306,9 +296,9 @@ static int diff_file_content_load_workdir_file(
 	git_diff_file_content *fc, git_buf *path)
 {
 	int error = 0;
-	git_vector filters = GIT_VECTOR_INIT;
-	git_buf raw = GIT_BUF_INIT, filtered = GIT_BUF_INIT;
+	git_filter_list *fl = NULL;
 	git_file fd = git_futils_open_ro(git_buf_cstr(path));
+	git_buf raw = GIT_BUF_INIT;
 
 	if (fd < 0)
 		return fd;
@@ -320,41 +310,38 @@ static int diff_file_content_load_workdir_file(
 	if (diff_file_content_binary_by_size(fc))
 		goto cleanup;
 
-	if ((error = git_filters_load(
-			&filters, fc->repo, fc->file->path, GIT_FILTER_TO_ODB)) < 0)
+	if ((error = git_filter_list_load(
+			&fl, fc->repo, NULL, fc->file->path, GIT_FILTER_TO_ODB)) < 0)
 		goto cleanup;
-	/* error >= is a filter count */
 
-	if (error == 0) {
+	/* if there are no filters, try to mmap the file */
+	if (fl == NULL) {
 		if (!(error = git_futils_mmap_ro(
-				&fc->map, fd, 0, (size_t)fc->file->size)))
+				&fc->map, fd, 0, (size_t)fc->file->size))) {
 			fc->flags |= GIT_DIFF_FLAG__UNMAP_DATA;
-		else /* fall through to try readbuffer below */
-			giterr_clear();
+			goto cleanup;
+		}
+
+		/* if mmap failed, fall through to try readbuffer below */
+		giterr_clear();
 	}
 
-	if (error != 0) {
-		error = git_futils_readbuffer_fd(&raw, fd, (size_t)fc->file->size);
-		if (error < 0)
-			goto cleanup;
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, (size_t)fc->file->size))) {
+		git_buf out = GIT_BUF_INIT;
 
-		if (!filters.length)
-			git_buf_swap(&filtered, &raw);
-		else
-			error = git_filters_apply(&filtered, &raw, &filters);
+		error = git_filter_list_apply_to_data(&out, fl, &raw);
+
+		git_buf_free(&raw);
 
 		if (!error) {
-			fc->map.len  = git_buf_len(&filtered);
-			fc->map.data = git_buf_detach(&filtered);
+			fc->map.len  = out.size;
+			fc->map.data = out.ptr;
 			fc->flags |= GIT_DIFF_FLAG__FREE_DATA;
 		}
-
-		git_buf_free(&raw);
-		git_buf_free(&filtered);
 	}
 
 cleanup:
-	git_filters_free(&filters);
+	git_filter_list_free(fl);
 	p_close(fd);
 
 	return error;
@@ -417,6 +404,9 @@ int git_diff_file_content__load(git_diff_file_content *fc)
 
 void git_diff_file_content__unload(git_diff_file_content *fc)
 {
+	if ((fc->flags & GIT_DIFF_FLAG__LOADED) == 0)
+		return;
+
 	if (fc->flags & GIT_DIFF_FLAG__FREE_DATA) {
 		git__free(fc->map.data);
 		fc->map.data = "";
diff --git a/src/diff_file.h b/src/diff_file.h
index fb08cca6a..84bf255aa 100644
--- a/src/diff_file.h
+++ b/src/diff_file.h
@@ -27,7 +27,7 @@ typedef struct {
 
 extern int git_diff_file_content__init_from_diff(
 	git_diff_file_content *fc,
-	git_diff_list *diff,
+	git_diff *diff,
 	size_t delta_index,
 	bool use_old);
 
diff --git a/src/diff_patch.c b/src/diff_patch.c
index 9060d0a24..cc49d68eb 100644
--- a/src/diff_patch.c
+++ b/src/diff_patch.c
@@ -10,38 +10,27 @@
 #include "diff_driver.h"
 #include "diff_patch.h"
 #include "diff_xdiff.h"
-
-/* cached information about a single span in a diff */
-typedef struct diff_patch_line diff_patch_line;
-struct diff_patch_line {
-	const char *ptr;
-	size_t len;
-	size_t lines, oldno, newno;
-	char origin;
-};
+#include "fileops.h"
 
 /* cached information about a hunk in a diff */
 typedef struct diff_patch_hunk diff_patch_hunk;
 struct diff_patch_hunk {
-	git_diff_range range;
-	char   header[128];
-	size_t header_len;
+	git_diff_hunk hunk;
 	size_t line_start;
 	size_t line_count;
 };
 
-struct git_diff_patch {
+struct git_patch {
 	git_refcount rc;
-	git_diff_list *diff; /* for refcount purposes, maybe NULL for blob diffs */
+	git_diff *diff; /* for refcount purposes, maybe NULL for blob diffs */
 	git_diff_delta *delta;
 	size_t delta_index;
 	git_diff_file_content ofile;
 	git_diff_file_content nfile;
 	uint32_t flags;
 	git_array_t(diff_patch_hunk) hunks;
-	git_array_t(diff_patch_line) lines;
-	size_t oldno, newno;
-	size_t content_size;
+	git_array_t(git_diff_line)   lines;
+	size_t content_size, context_size, header_size;
 	git_pool flattened;
 };
 
@@ -54,12 +43,13 @@ enum {
 	GIT_DIFF_PATCH_FLATTENED   = (1 << 5),
 };
 
-static void diff_output_init(git_diff_output*, const git_diff_options*,
-	git_diff_file_cb, git_diff_hunk_cb, git_diff_data_cb, void*);
+static void diff_output_init(
+	git_diff_output*, const git_diff_options*,
+	git_diff_file_cb, git_diff_hunk_cb, git_diff_line_cb, void*);
 
-static void diff_output_to_patch(git_diff_output *, git_diff_patch *);
+static void diff_output_to_patch(git_diff_output *, git_patch *);
 
-static void diff_patch_update_binary(git_diff_patch *patch)
+static void diff_patch_update_binary(git_patch *patch)
 {
 	if ((patch->delta->flags & DIFF_FLAGS_KNOWN_BINARY) != 0)
 		return;
@@ -73,21 +63,21 @@ static void diff_patch_update_binary(git_diff_patch *patch)
 		patch->delta->flags |= GIT_DIFF_FLAG_NOT_BINARY;
 }
 
-static void diff_patch_init_common(git_diff_patch *patch)
+static void diff_patch_init_common(git_patch *patch)
 {
 	diff_patch_update_binary(patch);
 
 	if ((patch->delta->flags & GIT_DIFF_FLAG_BINARY) != 0)
-		patch->flags |= GIT_DIFF_PATCH_LOADED; /* set LOADED but not DIFFABLE */
+		patch->flags |= GIT_DIFF_PATCH_LOADED; /* LOADED but not DIFFABLE */
 
 	patch->flags |= GIT_DIFF_PATCH_INITIALIZED;
 
 	if (patch->diff)
-		git_diff_list_addref(patch->diff);
+		git_diff_addref(patch->diff);
 }
 
 static int diff_patch_init_from_diff(
-	git_diff_patch *patch, git_diff_list *diff, size_t delta_index)
+	git_patch *patch, git_diff *diff, size_t delta_index)
 {
 	int error = 0;
 
@@ -108,12 +98,10 @@ static int diff_patch_init_from_diff(
 }
 
 static int diff_patch_alloc_from_diff(
-	git_diff_patch **out,
-	git_diff_list *diff,
-	size_t delta_index)
+	git_patch **out, git_diff *diff, size_t delta_index)
 {
 	int error;
-	git_diff_patch *patch = git__calloc(1, sizeof(git_diff_patch));
+	git_patch *patch = git__calloc(1, sizeof(git_patch));
 	GITERR_CHECK_ALLOC(patch);
 
 	if (!(error = diff_patch_init_from_diff(patch, diff, delta_index))) {
@@ -128,7 +116,7 @@ static int diff_patch_alloc_from_diff(
 	return error;
 }
 
-static int diff_patch_load(git_diff_patch *patch, git_diff_output *output)
+static int diff_patch_load(git_patch *patch, git_diff_output *output)
 {
 	int error = 0;
 	bool incomplete_data;
@@ -175,9 +163,12 @@ static int diff_patch_load(git_diff_patch *patch, git_diff_output *output)
 			goto cleanup;
 	}
 
-	/* if we were previously missing an oid, update MODIFIED->UNMODIFIED */
+	/* if previously missing an oid, and now that we have it the two sides
+	 * are the same (and not submodules), update MODIFIED -> UNMODIFIED
+	 */
 	if (incomplete_data &&
 		patch->ofile.file->mode == patch->nfile.file->mode &&
+		patch->ofile.file->mode != GIT_FILEMODE_COMMIT &&
 		git_oid_equal(&patch->ofile.file->oid, &patch->nfile.file->oid) &&
 		patch->delta->status == GIT_DELTA_MODIFIED) /* not RENAMED/COPIED! */
 		patch->delta->status = GIT_DELTA_UNMODIFIED;
@@ -203,7 +194,7 @@ cleanup:
 }
 
 static int diff_patch_file_callback(
-	git_diff_patch *patch, git_diff_output *output)
+	git_patch *patch, git_diff_output *output)
 {
 	float progress;
 
@@ -219,13 +210,17 @@ static int diff_patch_file_callback(
 	return output->error;
 }
 
-static int diff_patch_generate(git_diff_patch *patch, git_diff_output *output)
+static int diff_patch_generate(git_patch *patch, git_diff_output *output)
 {
 	int error = 0;
 
 	if ((patch->flags & GIT_DIFF_PATCH_DIFFED) != 0)
 		return 0;
 
+	/* if we are not looking at the hunks and lines, don't do the diff */
+	if (!output->hunk_cb && !output->data_cb)
+		return 0;
+
 	if ((patch->flags & GIT_DIFF_PATCH_LOADED) == 0 &&
 		(error = diff_patch_load(patch, output)) < 0)
 		return error;
@@ -240,7 +235,7 @@ static int diff_patch_generate(git_diff_patch *patch, git_diff_output *output)
 	return error;
 }
 
-static void diff_patch_free(git_diff_patch *patch)
+static void diff_patch_free(git_patch *patch)
 {
 	git_diff_file_content__clear(&patch->ofile);
 	git_diff_file_content__clear(&patch->nfile);
@@ -248,7 +243,7 @@ static void diff_patch_free(git_diff_patch *patch)
 	git_array_clear(patch->lines);
 	git_array_clear(patch->hunks);
 
-	git_diff_list_free(patch->diff); /* decrements refcount */
+	git_diff_free(patch->diff); /* decrements refcount */
 	patch->diff = NULL;
 
 	git_pool_clear(&patch->flattened);
@@ -257,7 +252,7 @@ static void diff_patch_free(git_diff_patch *patch)
 		git__free(patch);
 }
 
-static int diff_required(git_diff_list *diff, const char *action)
+static int diff_required(git_diff *diff, const char *action)
 {
 	if (diff)
 		return 0;
@@ -266,22 +261,22 @@ static int diff_required(git_diff_list *diff, const char *action)
 }
 
 int git_diff_foreach(
-	git_diff_list *diff,
+	git_diff *diff,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
-	git_diff_data_cb data_cb,
+	git_diff_line_cb data_cb,
 	void *payload)
 {
 	int error = 0;
 	git_xdiff_output xo;
 	size_t idx;
-	git_diff_patch patch;
+	git_patch patch;
 
 	if (diff_required(diff, "git_diff_foreach") < 0)
 		return -1;
 
-	diff_output_init((git_diff_output *)&xo,
-		&diff->opts, file_cb, hunk_cb, data_cb, payload);
+	diff_output_init(
+		&xo.output, &diff->opts, file_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, &diff->opts);
 
 	git_vector_foreach(&diff->deltas, idx, patch.delta) {
@@ -292,12 +287,12 @@ int git_diff_foreach(
 
 		if (!(error = diff_patch_init_from_diff(&patch, diff, idx))) {
 
-			error = diff_patch_file_callback(&patch, (git_diff_output *)&xo);
+			error = diff_patch_file_callback(&patch, &xo.output);
 
 			if (!error)
-				error = diff_patch_generate(&patch, (git_diff_output *)&xo);
+				error = diff_patch_generate(&patch, &xo.output);
 
-			git_diff_patch_free(&patch);
+			git_patch_free(&patch);
 		}
 
 		if (error < 0)
@@ -310,7 +305,7 @@ int git_diff_foreach(
 }
 
 typedef struct {
-	git_diff_patch patch;
+	git_patch patch;
 	git_diff_delta delta;
 	char paths[GIT_FLEX_ARRAY];
 } diff_patch_with_delta;
@@ -318,7 +313,7 @@ typedef struct {
 static int diff_single_generate(diff_patch_with_delta *pd, git_xdiff_output *xo)
 {
 	int error = 0;
-	git_diff_patch *patch = &pd->patch;
+	git_patch *patch = &pd->patch;
 	bool has_old = ((patch->ofile.flags & GIT_DIFF_FLAG__NO_DATA) == 0);
 	bool has_new = ((patch->nfile.flags & GIT_DIFF_FLAG__NO_DATA) == 0);
 
@@ -422,7 +417,7 @@ int git_diff_blobs(
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
-	git_diff_data_cb data_cb,
+	git_diff_line_cb data_cb,
 	void *payload)
 {
 	int error = 0;
@@ -433,7 +428,7 @@ int git_diff_blobs(
 	memset(&xo, 0, sizeof(xo));
 
 	diff_output_init(
-		(git_diff_output *)&xo, opts, file_cb, hunk_cb, data_cb, payload);
+		&xo.output, opts, file_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, opts);
 
 	if (!old_path && new_path)
@@ -444,13 +439,13 @@ int git_diff_blobs(
 	error = diff_patch_from_blobs(
 		&pd, &xo, old_blob, old_path, new_blob, new_path, opts);
 
-	git_diff_patch_free((git_diff_patch *)&pd);
+	git_patch_free(&pd.patch);
 
 	return error;
 }
 
-int git_diff_patch_from_blobs(
-	git_diff_patch **out,
+int git_patch_from_blobs(
+	git_patch **out,
 	const git_blob *old_blob,
 	const char *old_path,
 	const git_blob *new_blob,
@@ -469,16 +464,16 @@ int git_diff_patch_from_blobs(
 
 	memset(&xo, 0, sizeof(xo));
 
-	diff_output_to_patch((git_diff_output *)&xo, &pd->patch);
+	diff_output_to_patch(&xo.output, &pd->patch);
 	git_xdiff_init(&xo, opts);
 
 	error = diff_patch_from_blobs(
 		pd, &xo, old_blob, old_path, new_blob, new_path, opts);
 
 	if (!error)
-		*out = (git_diff_patch *)pd;
+		*out = (git_patch *)pd;
 	else
-		git_diff_patch_free((git_diff_patch *)pd);
+		git_patch_free((git_patch *)pd);
 
 	return error;
 }
@@ -534,7 +529,7 @@ int git_diff_blob_to_buffer(
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
-	git_diff_data_cb data_cb,
+	git_diff_line_cb data_cb,
 	void *payload)
 {
 	int error = 0;
@@ -545,7 +540,7 @@ int git_diff_blob_to_buffer(
 	memset(&xo, 0, sizeof(xo));
 
 	diff_output_init(
-		(git_diff_output *)&xo, opts, file_cb, hunk_cb, data_cb, payload);
+		&xo.output, opts, file_cb, hunk_cb, data_cb, payload);
 	git_xdiff_init(&xo, opts);
 
 	if (!old_path && buf_path)
@@ -556,13 +551,13 @@ int git_diff_blob_to_buffer(
 	error = diff_patch_from_blob_and_buffer(
 		&pd, &xo, old_blob, old_path, buf, buflen, buf_path, opts);
 
-	git_diff_patch_free((git_diff_patch *)&pd);
+	git_patch_free(&pd.patch);
 
 	return error;
 }
 
-int git_diff_patch_from_blob_and_buffer(
-	git_diff_patch **out,
+int git_patch_from_blob_and_buffer(
+	git_patch **out,
 	const git_blob *old_blob,
 	const char *old_path,
 	const char *buf,
@@ -582,35 +577,31 @@ int git_diff_patch_from_blob_and_buffer(
 
 	memset(&xo, 0, sizeof(xo));
 
-	diff_output_to_patch((git_diff_output *)&xo, &pd->patch);
+	diff_output_to_patch(&xo.output, &pd->patch);
 	git_xdiff_init(&xo, opts);
 
 	error = diff_patch_from_blob_and_buffer(
 		pd, &xo, old_blob, old_path, buf, buflen, buf_path, opts);
 
 	if (!error)
-		*out = (git_diff_patch *)pd;
+		*out = (git_patch *)pd;
 	else
-		git_diff_patch_free((git_diff_patch *)pd);
+		git_patch_free((git_patch *)pd);
 
 	return error;
 }
 
-int git_diff_get_patch(
-	git_diff_patch **patch_ptr,
-	const git_diff_delta **delta_ptr,
-	git_diff_list *diff,
-	size_t idx)
+int git_patch_from_diff(
+	git_patch **patch_ptr, git_diff *diff, size_t idx)
 {
 	int error = 0;
 	git_xdiff_output xo;
 	git_diff_delta *delta = NULL;
-	git_diff_patch *patch = NULL;
+	git_patch *patch = NULL;
 
 	if (patch_ptr) *patch_ptr = NULL;
-	if (delta_ptr) *delta_ptr = NULL;
 
-	if (diff_required(diff, "git_diff_get_patch") < 0)
+	if (diff_required(diff, "git_patch_from_diff") < 0)
 		return -1;
 
 	delta = git_vector_get(&diff->deltas, idx);
@@ -619,9 +610,6 @@ int git_diff_get_patch(
 		return GIT_ENOTFOUND;
 	}
 
-	if (delta_ptr)
-		*delta_ptr = delta;
-
 	if (git_diff_delta__should_skip(&diff->opts, delta))
 		return 0;
 
@@ -634,13 +622,13 @@ int git_diff_get_patch(
 	if ((error = diff_patch_alloc_from_diff(&patch, diff, idx)) < 0)
 		return error;
 
-	diff_output_to_patch((git_diff_output *)&xo, patch);
+	diff_output_to_patch(&xo.output, patch);
 	git_xdiff_init(&xo, &diff->opts);
 
-	error = diff_patch_file_callback(patch, (git_diff_output *)&xo);
+	error = diff_patch_file_callback(patch, &xo.output);
 
 	if (!error)
-		error = diff_patch_generate(patch, (git_diff_output *)&xo);
+		error = diff_patch_generate(patch, &xo.output);
 
 	if (!error) {
 		/* if cumulative diff size is < 0.5 total size, flatten the patch */
@@ -648,7 +636,7 @@ int git_diff_get_patch(
 	}
 
 	if (error || !patch_ptr)
-		git_diff_patch_free(patch);
+		git_patch_free(patch);
 	else
 		*patch_ptr = patch;
 
@@ -657,36 +645,36 @@ int git_diff_get_patch(
 	return error;
 }
 
-void git_diff_patch_free(git_diff_patch *patch)
+void git_patch_free(git_patch *patch)
 {
 	if (patch)
 		GIT_REFCOUNT_DEC(patch, diff_patch_free);
 }
 
-const git_diff_delta *git_diff_patch_delta(git_diff_patch *patch)
+const git_diff_delta *git_patch_get_delta(git_patch *patch)
 {
 	assert(patch);
 	return patch->delta;
 }
 
-size_t git_diff_patch_num_hunks(git_diff_patch *patch)
+size_t git_patch_num_hunks(git_patch *patch)
 {
 	assert(patch);
 	return git_array_size(patch->hunks);
 }
 
-int git_diff_patch_line_stats(
+int git_patch_line_stats(
 	size_t *total_ctxt,
 	size_t *total_adds,
 	size_t *total_dels,
-	const git_diff_patch *patch)
+	const git_patch *patch)
 {
 	size_t totals[3], idx;
 
 	memset(totals, 0, sizeof(totals));
 
 	for (idx = 0; idx < git_array_size(patch->lines); ++idx) {
-		diff_patch_line *line = git_array_get(patch->lines, idx);
+		git_diff_line *line = git_array_get(patch->lines, idx);
 		if (!line)
 			continue;
 
@@ -718,12 +706,10 @@ static int diff_error_outofrange(const char *thing)
 	return GIT_ENOTFOUND;
 }
 
-int git_diff_patch_get_hunk(
-	const git_diff_range **range,
-	const char **header,
-	size_t *header_len,
+int git_patch_get_hunk(
+	const git_diff_hunk **out,
 	size_t *lines_in_hunk,
-	git_diff_patch *patch,
+	git_patch *patch,
 	size_t hunk_idx)
 {
 	diff_patch_hunk *hunk;
@@ -732,21 +718,17 @@ int git_diff_patch_get_hunk(
 	hunk = git_array_get(patch->hunks, hunk_idx);
 
 	if (!hunk) {
-		if (range) *range = NULL;
-		if (header) *header = NULL;
-		if (header_len) *header_len = 0;
+		if (out) *out = NULL;
 		if (lines_in_hunk) *lines_in_hunk = 0;
 		return diff_error_outofrange("hunk");
 	}
 
-	if (range) *range = &hunk->range;
-	if (header) *header = hunk->header;
-	if (header_len) *header_len = hunk->header_len;
+	if (out) *out = &hunk->hunk;
 	if (lines_in_hunk) *lines_in_hunk = hunk->line_count;
 	return 0;
 }
 
-int git_diff_patch_num_lines_in_hunk(git_diff_patch *patch, size_t hunk_idx)
+int git_patch_num_lines_in_hunk(git_patch *patch, size_t hunk_idx)
 {
 	diff_patch_hunk *hunk;
 	assert(patch);
@@ -756,82 +738,96 @@ int git_diff_patch_num_lines_in_hunk(git_diff_patch *patch, size_t hunk_idx)
 	return (int)hunk->line_count;
 }
 
-int git_diff_patch_get_line_in_hunk(
-	char *line_origin,
-	const char **content,
-	size_t *content_len,
-	int *old_lineno,
-	int *new_lineno,
-	git_diff_patch *patch,
+int git_patch_get_line_in_hunk(
+	const git_diff_line **out,
+	git_patch *patch,
 	size_t hunk_idx,
 	size_t line_of_hunk)
 {
 	diff_patch_hunk *hunk;
-	diff_patch_line *line;
-	const char *thing;
+	git_diff_line *line;
 
 	assert(patch);
 
 	if (!(hunk = git_array_get(patch->hunks, hunk_idx))) {
-		thing = "hunk";
-		goto notfound;
+		if (out) *out = NULL;
+		return diff_error_outofrange("hunk");
 	}
 
 	if (line_of_hunk >= hunk->line_count ||
 		!(line = git_array_get(
 			patch->lines, hunk->line_start + line_of_hunk))) {
-		thing = "line";
-		goto notfound;
+		if (out) *out = NULL;
+		return diff_error_outofrange("line");
 	}
 
-	if (line_origin) *line_origin = line->origin;
-	if (content) *content = line->ptr;
-	if (content_len) *content_len = line->len;
-	if (old_lineno) *old_lineno = (int)line->oldno;
-	if (new_lineno) *new_lineno = (int)line->newno;
-
+	if (out) *out = line;
 	return 0;
+}
 
-notfound:
-	if (line_origin) *line_origin = GIT_DIFF_LINE_CONTEXT;
-	if (content) *content = NULL;
-	if (content_len) *content_len = 0;
-	if (old_lineno) *old_lineno = -1;
-	if (new_lineno) *new_lineno = -1;
+size_t git_patch_size(
+	git_patch *patch,
+	int include_context,
+	int include_hunk_headers,
+	int include_file_headers)
+{
+	size_t out;
 
-	return diff_error_outofrange(thing);
+	assert(patch);
+
+	out = patch->content_size;
+
+	if (!include_context)
+		out -= patch->context_size;
+
+	if (include_hunk_headers)
+		out += patch->header_size;
+
+	if (include_file_headers) {
+		git_buf file_header = GIT_BUF_INIT;
+
+		if (git_diff_delta__format_file_header(
+				&file_header, patch->delta, NULL, NULL, 0) < 0)
+			giterr_clear();
+		else
+			out += git_buf_len(&file_header);
+
+		git_buf_free(&file_header);
+	}
+
+	return out;
 }
 
-git_diff_list *git_diff_patch__diff(git_diff_patch *patch)
+git_diff *git_patch__diff(git_patch *patch)
 {
 	return patch->diff;
 }
 
-git_diff_driver *git_diff_patch__driver(git_diff_patch *patch)
+git_diff_driver *git_patch__driver(git_patch *patch)
 {
 	/* ofile driver is representative for whole patch */
 	return patch->ofile.driver;
 }
 
-void git_diff_patch__old_data(
-	char **ptr, size_t *len, git_diff_patch *patch)
+void git_patch__old_data(
+	char **ptr, size_t *len, git_patch *patch)
 {
 	*ptr = patch->ofile.map.data;
 	*len = patch->ofile.map.len;
 }
 
-void git_diff_patch__new_data(
-	char **ptr, size_t *len, git_diff_patch *patch)
+void git_patch__new_data(
+	char **ptr, size_t *len, git_patch *patch)
 {
 	*ptr = patch->nfile.map.data;
 	*len = patch->nfile.map.len;
 }
 
-int git_diff_patch__invoke_callbacks(
-	git_diff_patch *patch,
+int git_patch__invoke_callbacks(
+	git_patch *patch,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
-	git_diff_data_cb line_cb,
+	git_diff_line_cb line_cb,
 	void *payload)
 {
 	int error = 0;
@@ -846,18 +842,16 @@ int git_diff_patch__invoke_callbacks(
 	for (i = 0; !error && i < git_array_size(patch->hunks); ++i) {
 		diff_patch_hunk *h = git_array_get(patch->hunks, i);
 
-		error = hunk_cb(
-			patch->delta, &h->range, h->header, h->header_len, payload);
+		error = hunk_cb(patch->delta, &h->hunk, payload);
 
 		if (!line_cb)
 			continue;
 
 		for (j = 0; !error && j < h->line_count; ++j) {
-			diff_patch_line *l =
+			git_diff_line *l =
 				git_array_get(patch->lines, h->line_start + j);
 
-			error = line_cb(
-				patch->delta, &h->range, l->origin, l->ptr, l->len, payload);
+			error = line_cb(patch->delta, &h->hunk, l, payload);
 		}
 	}
 
@@ -876,12 +870,10 @@ static int diff_patch_file_cb(
 
 static int diff_patch_hunk_cb(
 	const git_diff_delta *delta,
-	const git_diff_range *range,
-	const char *header,
-	size_t header_len,
+	const git_diff_hunk *hunk_,
 	void *payload)
 {
-	git_diff_patch *patch = payload;
+	git_patch *patch = payload;
 	diff_patch_hunk *hunk;
 
 	GIT_UNUSED(delta);
@@ -889,36 +881,28 @@ static int diff_patch_hunk_cb(
 	hunk = git_array_alloc(patch->hunks);
 	GITERR_CHECK_ALLOC(hunk);
 
-	memcpy(&hunk->range, range, sizeof(hunk->range));
+	memcpy(&hunk->hunk, hunk_, sizeof(hunk->hunk));
 
-	assert(header_len + 1 < sizeof(hunk->header));
-	memcpy(&hunk->header, header, header_len);
-	hunk->header[header_len] = '\0';
-	hunk->header_len = header_len;
+	patch->header_size += hunk_->header_len;
 
 	hunk->line_start = git_array_size(patch->lines);
 	hunk->line_count = 0;
 
-	patch->oldno = range->old_start;
-	patch->newno = range->new_start;
-
 	return 0;
 }
 
 static int diff_patch_line_cb(
 	const git_diff_delta *delta,
-	const git_diff_range *range,
-	char line_origin,
-	const char *content,
-	size_t content_len,
+	const git_diff_hunk *hunk_,
+	const git_diff_line *line_,
 	void *payload)
 {
-	git_diff_patch *patch = payload;
+	git_patch *patch = payload;
 	diff_patch_hunk *hunk;
-	diff_patch_line *line;
+	git_diff_line   *line;
 
 	GIT_UNUSED(delta);
-	GIT_UNUSED(range);
+	GIT_UNUSED(hunk_);
 
 	hunk = git_array_last(patch->hunks);
 	GITERR_CHECK_ALLOC(hunk);
@@ -926,39 +910,20 @@ static int diff_patch_line_cb(
 	line = git_array_alloc(patch->lines);
 	GITERR_CHECK_ALLOC(line);
 
-	line->ptr = content;
-	line->len = content_len;
-	line->origin = line_origin;
-
-	patch->content_size += content_len;
+	memcpy(line, line_, sizeof(*line));
 
 	/* do some bookkeeping so we can provide old/new line numbers */
 
-	for (line->lines = 0; content_len > 0; --content_len) {
-		if (*content++ == '\n')
-			++line->lines;
-	}
+	patch->content_size += line->content_len;
 
-	switch (line_origin) {
-	case GIT_DIFF_LINE_ADDITION:
-	case GIT_DIFF_LINE_DEL_EOFNL:
-		line->oldno = -1;
-		line->newno = patch->newno;
-		patch->newno += line->lines;
-		break;
-	case GIT_DIFF_LINE_DELETION:
-	case GIT_DIFF_LINE_ADD_EOFNL:
-		line->oldno = patch->oldno;
-		line->newno = -1;
-		patch->oldno += line->lines;
-		break;
-	default:
-		line->oldno = patch->oldno;
-		line->newno = patch->newno;
-		patch->oldno += line->lines;
-		patch->newno += line->lines;
-		break;
-	}
+	if (line->origin == GIT_DIFF_LINE_ADDITION ||
+		line->origin == GIT_DIFF_LINE_DELETION)
+		patch->content_size += 1;
+	else if (line->origin == GIT_DIFF_LINE_CONTEXT) {
+		patch->content_size += 1;
+		patch->context_size += line->content_len + 1;
+	} else if (line->origin == GIT_DIFF_LINE_CONTEXT_EOFNL)
+		patch->context_size += line->content_len;
 
 	hunk->line_count++;
 
@@ -970,7 +935,7 @@ static void diff_output_init(
 	const git_diff_options *opts,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
-	git_diff_data_cb data_cb,
+	git_diff_line_cb data_cb,
 	void *payload)
 {
 	GIT_UNUSED(opts);
@@ -983,7 +948,7 @@ static void diff_output_init(
 	out->payload = payload;
 }
 
-static void diff_output_to_patch(git_diff_output *out, git_diff_patch *patch)
+static void diff_output_to_patch(git_diff_output *out, git_patch *patch)
 {
 	diff_output_init(
 		out, NULL,
diff --git a/src/diff_patch.h b/src/diff_patch.h
index 56af14600..df2ba4c31 100644
--- a/src/diff_patch.h
+++ b/src/diff_patch.h
@@ -11,19 +11,20 @@
 #include "diff.h"
 #include "diff_file.h"
 #include "array.h"
+#include "git2/patch.h"
 
-extern git_diff_list *git_diff_patch__diff(git_diff_patch *);
+extern git_diff *git_patch__diff(git_patch *);
 
-extern git_diff_driver *git_diff_patch__driver(git_diff_patch *);
+extern git_diff_driver *git_patch__driver(git_patch *);
 
-extern void git_diff_patch__old_data(char **, size_t *, git_diff_patch *);
-extern void git_diff_patch__new_data(char **, size_t *, git_diff_patch *);
+extern void git_patch__old_data(char **, size_t *, git_patch *);
+extern void git_patch__new_data(char **, size_t *, git_patch *);
 
-extern int git_diff_patch__invoke_callbacks(
-	git_diff_patch *patch,
+extern int git_patch__invoke_callbacks(
+	git_patch *patch,
 	git_diff_file_cb file_cb,
 	git_diff_hunk_cb hunk_cb,
-	git_diff_data_cb line_cb,
+	git_diff_line_cb line_cb,
 	void *payload);
 
 typedef struct git_diff_output git_diff_output;
@@ -31,7 +32,7 @@ struct git_diff_output {
 	/* these callbacks are issued with the diff data */
 	git_diff_file_cb file_cb;
 	git_diff_hunk_cb hunk_cb;
-	git_diff_data_cb data_cb;
+	git_diff_line_cb data_cb;
 	void *payload;
 
 	/* this records the actual error in cases where it may be obscured */
@@ -40,7 +41,7 @@ struct git_diff_output {
 	/* this callback is used to do the diff and drive the other callbacks.
 	 * see diff_xdiff.h for how to use this in practice for now.
 	 */
-	int (*diff_cb)(git_diff_output *output, git_diff_patch *patch);
+	int (*diff_cb)(git_diff_output *output, git_patch *patch);
 };
 
 #endif
diff --git a/src/diff_print.c b/src/diff_print.c
index 0de548813..b04b11515 100644
--- a/src/diff_print.c
+++ b/src/diff_print.c
@@ -7,26 +7,39 @@
 #include "common.h"
 #include "diff.h"
 #include "diff_patch.h"
-#include "buffer.h"
+#include "fileops.h"
 
 typedef struct {
-	git_diff_list *diff;
-	git_diff_data_cb print_cb;
+	git_diff *diff;
+	git_diff_format_t format;
+	git_diff_line_cb print_cb;
 	void *payload;
 	git_buf *buf;
+	uint32_t flags;
 	int oid_strlen;
+	git_diff_line line;
 } diff_print_info;
 
 static int diff_print_info_init(
 	diff_print_info *pi,
-	git_buf *out, git_diff_list *diff, git_diff_data_cb cb, void *payload)
+	git_buf *out,
+	git_diff *diff,
+	git_diff_format_t format,
+	git_diff_line_cb cb,
+	void *payload)
 {
 	pi->diff     = diff;
+	pi->format   = format;
 	pi->print_cb = cb;
 	pi->payload  = payload;
 	pi->buf      = out;
 
-	if (!diff || !diff->repo)
+	if (diff)
+		pi->flags = diff->opts.flags;
+
+	if (diff && diff->opts.oid_abbrev != 0)
+		pi->oid_strlen = diff->opts.oid_abbrev;
+	else if (!diff || !diff->repo)
 		pi->oid_strlen = GIT_ABBREV_DEFAULT;
 	else if (git_repository__cvar(
 		&pi->oid_strlen, diff->repo, GIT_CVAR_ABBREV) < 0)
@@ -39,6 +52,11 @@ static int diff_print_info_init(
 	else if (pi->oid_strlen > GIT_OID_HEXSZ + 1)
 		pi->oid_strlen = GIT_OID_HEXSZ + 1;
 
+	memset(&pi->line, 0, sizeof(pi->line));
+	pi->line.old_lineno = -1;
+	pi->line.new_lineno = -1;
+	pi->line.num_lines  = 1;
+
 	return 0;
 }
 
@@ -46,7 +64,7 @@ static char diff_pick_suffix(int mode)
 {
 	if (S_ISDIR(mode))
 		return '/';
-	else if (mode & 0100) /* -V536 */
+	else if (GIT_PERMS_IS_EXEC(mode)) /* -V536 */
 		/* in git, modes are very regular, so we must have 0100755 mode */
 		return '*';
 	else
@@ -77,7 +95,35 @@ static int callback_error(void)
 	return GIT_EUSER;
 }
 
-static int diff_print_one_compact(
+static int diff_print_one_name_only(
+	const git_diff_delta *delta, float progress, void *data)
+{
+	diff_print_info *pi = data;
+	git_buf *out = pi->buf;
+
+	GIT_UNUSED(progress);
+
+	if ((pi->flags & GIT_DIFF_SHOW_UNMODIFIED) == 0 &&
+		delta->status == GIT_DELTA_UNMODIFIED)
+		return 0;
+
+	git_buf_clear(out);
+
+	if (git_buf_puts(out, delta->new_file.path) < 0 ||
+		git_buf_putc(out, '\n'))
+		return -1;
+
+	pi->line.origin      = GIT_DIFF_LINE_FILE_HDR;
+	pi->line.content     = git_buf_cstr(out);
+	pi->line.content_len = git_buf_len(out);
+
+	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
+		return callback_error();
+
+	return 0;
+}
+
+static int diff_print_one_name_status(
 	const git_diff_delta *delta, float progress, void *data)
 {
 	diff_print_info *pi = data;
@@ -88,7 +134,7 @@ static int diff_print_one_compact(
 
 	GIT_UNUSED(progress);
 
-	if (code == ' ')
+	if ((pi->flags & GIT_DIFF_SHOW_UNMODIFIED) == 0 && code == ' ')
 		return 0;
 
 	old_suffix = diff_pick_suffix(delta->old_file.mode);
@@ -98,12 +144,12 @@ static int diff_print_one_compact(
 
 	if (delta->old_file.path != delta->new_file.path &&
 		strcomp(delta->old_file.path,delta->new_file.path) != 0)
-		git_buf_printf(out, "%c\t%s%c -> %s%c\n", code,
+		git_buf_printf(out, "%c\t%s%c %s%c\n", code,
 			delta->old_file.path, old_suffix, delta->new_file.path, new_suffix);
 	else if (delta->old_file.mode != delta->new_file.mode &&
 		delta->old_file.mode != 0 && delta->new_file.mode != 0)
-		git_buf_printf(out, "%c\t%s%c (%o -> %o)\n", code,
-			delta->old_file.path, new_suffix, delta->old_file.mode, delta->new_file.mode);
+		git_buf_printf(out, "%c\t%s%c %s%c\n", code,
+			delta->old_file.path, old_suffix, delta->new_file.path, new_suffix);
 	else if (old_suffix != ' ')
 		git_buf_printf(out, "%c\t%s%c\n", code, delta->old_file.path, old_suffix);
 	else
@@ -112,31 +158,16 @@ static int diff_print_one_compact(
 	if (git_buf_oom(out))
 		return -1;
 
-	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_FILE_HDR,
-			git_buf_cstr(out), git_buf_len(out), pi->payload))
+	pi->line.origin      = GIT_DIFF_LINE_FILE_HDR;
+	pi->line.content     = git_buf_cstr(out);
+	pi->line.content_len = git_buf_len(out);
+
+	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
 		return callback_error();
 
 	return 0;
 }
 
-/* print a git_diff_list to a print callback in compact format */
-int git_diff_print_compact(
-	git_diff_list *diff,
-	git_diff_data_cb print_cb,
-	void *payload)
-{
-	int error;
-	git_buf buf = GIT_BUF_INIT;
-	diff_print_info pi;
-
-	if (!(error = diff_print_info_init(&pi, &buf, diff, print_cb, payload)))
-		error = git_diff_foreach(diff, diff_print_one_compact, NULL, NULL, &pi);
-
-	git_buf_free(&buf);
-
-	return error;
-}
-
 static int diff_print_one_raw(
 	const git_diff_delta *delta, float progress, void *data)
 {
@@ -147,7 +178,7 @@ static int diff_print_one_raw(
 
 	GIT_UNUSED(progress);
 
-	if (code == ' ')
+	if ((pi->flags & GIT_DIFF_SHOW_UNMODIFIED) == 0 && code == ' ')
 		return 0;
 
 	git_buf_clear(out);
@@ -173,38 +204,23 @@ static int diff_print_one_raw(
 	if (git_buf_oom(out))
 		return -1;
 
-	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_FILE_HDR,
-			git_buf_cstr(out), git_buf_len(out), pi->payload))
+	pi->line.origin      = GIT_DIFF_LINE_FILE_HDR;
+	pi->line.content     = git_buf_cstr(out);
+	pi->line.content_len = git_buf_len(out);
+
+	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
 		return callback_error();
 
 	return 0;
 }
 
-/* print a git_diff_list to a print callback in raw output format */
-int git_diff_print_raw(
-	git_diff_list *diff,
-	git_diff_data_cb print_cb,
-	void *payload)
+static int diff_print_oid_range(
+	git_buf *out, const git_diff_delta *delta, int oid_strlen)
 {
-	int error;
-	git_buf buf = GIT_BUF_INIT;
-	diff_print_info pi;
-
-	if (!(error = diff_print_info_init(&pi, &buf, diff, print_cb, payload)))
-		error = git_diff_foreach(diff, diff_print_one_raw, NULL, NULL, &pi);
-
-	git_buf_free(&buf);
-
-	return error;
-}
-
-static int diff_print_oid_range(diff_print_info *pi, const git_diff_delta *delta)
-{
-	git_buf *out = pi->buf;
 	char start_oid[GIT_OID_HEXSZ+1], end_oid[GIT_OID_HEXSZ+1];
 
-	git_oid_tostr(start_oid, pi->oid_strlen, &delta->old_file.oid);
-	git_oid_tostr(end_oid, pi->oid_strlen, &delta->new_file.oid);
+	git_oid_tostr(start_oid, oid_strlen, &delta->old_file.oid);
+	git_oid_tostr(end_oid, oid_strlen, &delta->new_file.oid);
 
 	/* TODO: Match git diff more closely */
 	if (delta->old_file.mode == delta->new_file.mode) {
@@ -228,70 +244,102 @@ static int diff_print_oid_range(diff_print_info *pi, const git_diff_delta *delta
 	return 0;
 }
 
-static int diff_print_patch_file(
-	const git_diff_delta *delta, float progress, void *data)
+static int diff_delta_format_with_paths(
+	git_buf *out,
+	const git_diff_delta *delta,
+	const char *oldpfx,
+	const char *newpfx,
+	const char *template)
 {
-	diff_print_info *pi = data;
-	const char *oldpfx = pi->diff ? pi->diff->opts.old_prefix : NULL;
 	const char *oldpath = delta->old_file.path;
-	const char *newpfx = pi->diff ? pi->diff->opts.new_prefix : NULL;
 	const char *newpath = delta->new_file.path;
-	uint32_t opts_flags = pi->diff ? pi->diff->opts.flags : GIT_DIFF_NORMAL;
 
-	GIT_UNUSED(progress);
+	if (git_oid_iszero(&delta->old_file.oid)) {
+		oldpfx = "";
+		oldpath = "/dev/null";
+	}
+	if (git_oid_iszero(&delta->new_file.oid)) {
+		newpfx = "";
+		newpath = "/dev/null";
+	}
 
-	if (S_ISDIR(delta->new_file.mode) ||
-		delta->status == GIT_DELTA_UNMODIFIED ||
-		delta->status == GIT_DELTA_IGNORED ||
-		(delta->status == GIT_DELTA_UNTRACKED &&
-		 (opts_flags & GIT_DIFF_INCLUDE_UNTRACKED_CONTENT) == 0))
-		return 0;
+	return git_buf_printf(out, template, oldpfx, oldpath, newpfx, newpath);
+}
 
+int git_diff_delta__format_file_header(
+	git_buf *out,
+	const git_diff_delta *delta,
+	const char *oldpfx,
+	const char *newpfx,
+	int oid_strlen)
+{
 	if (!oldpfx)
 		oldpfx = DIFF_OLD_PREFIX_DEFAULT;
 	if (!newpfx)
 		newpfx = DIFF_NEW_PREFIX_DEFAULT;
+	if (!oid_strlen)
+		oid_strlen = GIT_ABBREV_DEFAULT + 1;
 
-	git_buf_clear(pi->buf);
-	git_buf_printf(pi->buf, "diff --git %s%s %s%s\n",
+	git_buf_clear(out);
+
+	git_buf_printf(out, "diff --git %s%s %s%s\n",
 		oldpfx, delta->old_file.path, newpfx, delta->new_file.path);
 
-	if (diff_print_oid_range(pi, delta) < 0)
+	if (diff_print_oid_range(out, delta, oid_strlen) < 0)
 		return -1;
 
-	if (git_oid_iszero(&delta->old_file.oid)) {
-		oldpfx = "";
-		oldpath = "/dev/null";
-	}
-	if (git_oid_iszero(&delta->new_file.oid)) {
-		newpfx = "";
-		newpath = "/dev/null";
-	}
+	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0)
+		diff_delta_format_with_paths(
+			out, delta, oldpfx, newpfx, "--- %s%s\n+++ %s%s\n");
 
-	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0) {
-		git_buf_printf(pi->buf, "--- %s%s\n", oldpfx, oldpath);
-		git_buf_printf(pi->buf, "+++ %s%s\n", newpfx, newpath);
-	}
+	return git_buf_oom(out) ? -1 : 0;
+}
+
+static int diff_print_patch_file(
+	const git_diff_delta *delta, float progress, void *data)
+{
+	diff_print_info *pi = data;
+	const char *oldpfx =
+		pi->diff ? pi->diff->opts.old_prefix : DIFF_OLD_PREFIX_DEFAULT;
+	const char *newpfx =
+		pi->diff ? pi->diff->opts.new_prefix : DIFF_NEW_PREFIX_DEFAULT;
 
-	if (git_buf_oom(pi->buf))
+	GIT_UNUSED(progress);
+
+	if (S_ISDIR(delta->new_file.mode) ||
+		delta->status == GIT_DELTA_UNMODIFIED ||
+		delta->status == GIT_DELTA_IGNORED ||
+		(delta->status == GIT_DELTA_UNTRACKED &&
+		 (pi->flags & GIT_DIFF_SHOW_UNTRACKED_CONTENT) == 0))
+		return 0;
+
+	if (git_diff_delta__format_file_header(
+			pi->buf, delta, oldpfx, newpfx, pi->oid_strlen) < 0)
 		return -1;
 
-	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_FILE_HDR,
-			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
+	pi->line.origin      = GIT_DIFF_LINE_FILE_HDR;
+	pi->line.content     = git_buf_cstr(pi->buf);
+	pi->line.content_len = git_buf_len(pi->buf);
+
+	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
 		return callback_error();
 
 	if ((delta->flags & GIT_DIFF_FLAG_BINARY) == 0)
 		return 0;
 
 	git_buf_clear(pi->buf);
-	git_buf_printf(
-		pi->buf, "Binary files %s%s and %s%s differ\n",
-		oldpfx, oldpath, newpfx, newpath);
-	if (git_buf_oom(pi->buf))
+
+	if (diff_delta_format_with_paths(
+			pi->buf, delta, oldpfx, newpfx,
+			"Binary files %s%s and %s%s differ\n") < 0)
 		return -1;
 
-	if (pi->print_cb(delta, NULL, GIT_DIFF_LINE_BINARY,
-			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
+	pi->line.origin      = GIT_DIFF_LINE_BINARY;
+	pi->line.content     = git_buf_cstr(pi->buf);
+	pi->line.content_len = git_buf_len(pi->buf);
+	pi->line.num_lines   = 1;
+
+	if (pi->print_cb(delta, NULL, &pi->line, pi->payload))
 		return callback_error();
 
 	return 0;
@@ -299,9 +347,7 @@ static int diff_print_patch_file(
 
 static int diff_print_patch_hunk(
 	const git_diff_delta *d,
-	const git_diff_range *r,
-	const char *header,
-	size_t header_len,
+	const git_diff_hunk *h,
 	void *data)
 {
 	diff_print_info *pi = data;
@@ -309,12 +355,11 @@ static int diff_print_patch_hunk(
 	if (S_ISDIR(d->new_file.mode))
 		return 0;
 
-	git_buf_clear(pi->buf);
-	if (git_buf_printf(pi->buf, "%.*s", (int)header_len, header) < 0)
-		return -1;
+	pi->line.origin      = GIT_DIFF_LINE_HUNK_HDR;
+	pi->line.content     = h->header;
+	pi->line.content_len = h->header_len;
 
-	if (pi->print_cb(d, r, GIT_DIFF_LINE_HUNK_HDR,
-			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
+	if (pi->print_cb(d, h, &pi->line, pi->payload))
 		return callback_error();
 
 	return 0;
@@ -322,10 +367,8 @@ static int diff_print_patch_hunk(
 
 static int diff_print_patch_line(
 	const git_diff_delta *delta,
-	const git_diff_range *range,
-	char line_origin, /* GIT_DIFF_LINE value from above */
-	const char *content,
-	size_t content_len,
+	const git_diff_hunk *hunk,
+	const git_diff_line *line,
 	void *data)
 {
 	diff_print_info *pi = data;
@@ -333,49 +376,63 @@ static int diff_print_patch_line(
 	if (S_ISDIR(delta->new_file.mode))
 		return 0;
 
-	git_buf_clear(pi->buf);
-
-	if (line_origin == GIT_DIFF_LINE_ADDITION ||
-		line_origin == GIT_DIFF_LINE_DELETION ||
-		line_origin == GIT_DIFF_LINE_CONTEXT)
-		git_buf_printf(pi->buf, "%c%.*s", line_origin, (int)content_len, content);
-	else if (content_len > 0)
-		git_buf_printf(pi->buf, "%.*s", (int)content_len, content);
-
-	if (git_buf_oom(pi->buf))
-		return -1;
-
-	if (pi->print_cb(delta, range, line_origin,
-			git_buf_cstr(pi->buf), git_buf_len(pi->buf), pi->payload))
+	if (pi->print_cb(delta, hunk, line, pi->payload))
 		return callback_error();
 
 	return 0;
 }
 
-/* print a git_diff_list to an output callback in patch format */
-int git_diff_print_patch(
-	git_diff_list *diff,
-	git_diff_data_cb print_cb,
+/* print a git_diff to an output callback */
+int git_diff_print(
+	git_diff *diff,
+	git_diff_format_t format,
+	git_diff_line_cb print_cb,
 	void *payload)
 {
 	int error;
 	git_buf buf = GIT_BUF_INIT;
 	diff_print_info pi;
+	git_diff_file_cb print_file = NULL;
+	git_diff_hunk_cb print_hunk = NULL;
+	git_diff_line_cb print_line = NULL;
+
+	switch (format) {
+	case GIT_DIFF_FORMAT_PATCH:
+		print_file = diff_print_patch_file;
+		print_hunk = diff_print_patch_hunk;
+		print_line = diff_print_patch_line;
+		break;
+	case GIT_DIFF_FORMAT_PATCH_HEADER:
+		print_file = diff_print_patch_file;
+		break;
+	case GIT_DIFF_FORMAT_RAW:
+		print_file = diff_print_one_raw;
+		break;
+	case GIT_DIFF_FORMAT_NAME_ONLY:
+		print_file = diff_print_one_name_only;
+		break;
+	case GIT_DIFF_FORMAT_NAME_STATUS:
+		print_file = diff_print_one_name_status;
+		break;
+	default:
+		giterr_set(GITERR_INVALID, "Unknown diff output format (%d)", format);
+		return -1;
+	}
 
-	if (!(error = diff_print_info_init(&pi, &buf, diff, print_cb, payload)))
+	if (!(error = diff_print_info_init(
+			&pi, &buf, diff, format, print_cb, payload)))
 		error = git_diff_foreach(
-			diff, diff_print_patch_file, diff_print_patch_hunk,
-			diff_print_patch_line, &pi);
+			diff, print_file, print_hunk, print_line, &pi);
 
 	git_buf_free(&buf);
 
 	return error;
 }
 
-/* print a git_diff_patch to an output callback */
-int git_diff_patch_print(
-	git_diff_patch *patch,
-	git_diff_data_cb print_cb,
+/* print a git_patch to an output callback */
+int git_patch_print(
+	git_patch *patch,
+	git_diff_line_cb print_cb,
 	void *payload)
 {
 	int error;
@@ -385,8 +442,9 @@ int git_diff_patch_print(
 	assert(patch && print_cb);
 
 	if (!(error = diff_print_info_init(
-			&pi, &temp, git_diff_patch__diff(patch), print_cb, payload)))
-		error = git_diff_patch__invoke_callbacks(
+			&pi, &temp, git_patch__diff(patch),
+			GIT_DIFF_FORMAT_PATCH, print_cb, payload)))
+		error = git_patch__invoke_callbacks(
 			patch, diff_print_patch_file, diff_print_patch_hunk,
 			diff_print_patch_line, &pi);
 
@@ -397,26 +455,30 @@ int git_diff_patch_print(
 
 static int diff_print_to_buffer_cb(
 	const git_diff_delta *delta,
-	const git_diff_range *range,
-	char line_origin,
-	const char *content,
-	size_t content_len,
+	const git_diff_hunk *hunk,
+	const git_diff_line *line,
 	void *payload)
 {
 	git_buf *output = payload;
-	GIT_UNUSED(delta); GIT_UNUSED(range); GIT_UNUSED(line_origin);
-	return git_buf_put(output, content, content_len);
+	GIT_UNUSED(delta); GIT_UNUSED(hunk);
+
+	if (line->origin == GIT_DIFF_LINE_ADDITION ||
+		line->origin == GIT_DIFF_LINE_DELETION ||
+		line->origin == GIT_DIFF_LINE_CONTEXT)
+		git_buf_putc(output, line->origin);
+
+	return git_buf_put(output, line->content, line->content_len);
 }
 
-/* print a git_diff_patch to a string buffer */
-int git_diff_patch_to_str(
+/* print a git_patch to a string buffer */
+int git_patch_to_str(
 	char **string,
-	git_diff_patch *patch)
+	git_patch *patch)
 {
 	int error;
 	git_buf output = GIT_BUF_INIT;
 
-	error = git_diff_patch_print(patch, diff_print_to_buffer_cb, &output);
+	error = git_patch_print(patch, diff_print_to_buffer_cb, &output);
 
 	/* GIT_EUSER means git_buf_put in print_to_buffer_cb returned -1,
 	 * meaning a memory allocation failure, so just map to -1...
diff --git a/src/diff_tform.c b/src/diff_tform.c
index 8c4e96ecf..28a9cc70d 100644
--- a/src/diff_tform.c
+++ b/src/diff_tform.c
@@ -46,7 +46,9 @@ fail:
 }
 
 static git_diff_delta *diff_delta__merge_like_cgit(
-	const git_diff_delta *a, const git_diff_delta *b, git_pool *pool)
+	const git_diff_delta *a,
+	const git_diff_delta *b,
+	git_pool *pool)
 {
 	git_diff_delta *dup;
 
@@ -96,15 +98,46 @@ static git_diff_delta *diff_delta__merge_like_cgit(
 	return dup;
 }
 
-int git_diff_merge(
-	git_diff_list *onto,
-	const git_diff_list *from)
+static git_diff_delta *diff_delta__merge_like_cgit_reversed(
+	const git_diff_delta *a,
+	const git_diff_delta *b,
+	git_pool *pool)
+{
+	git_diff_delta *dup;
+
+	/* reversed version of above logic */
+
+	if (a->status == GIT_DELTA_UNMODIFIED)
+		return diff_delta__dup(b, pool);
+
+	if ((dup = diff_delta__dup(a, pool)) == NULL)
+		return NULL;
+
+	if (b->status == GIT_DELTA_UNMODIFIED || b->status == GIT_DELTA_UNTRACKED)
+		return dup;
+
+	if (dup->status == GIT_DELTA_DELETED) {
+		if (b->status == GIT_DELTA_ADDED)
+			dup->status = GIT_DELTA_UNMODIFIED;
+	} else {
+		dup->status = b->status;
+	}
+
+	git_oid_cpy(&dup->old_file.oid, &b->old_file.oid);
+	dup->old_file.mode  = b->old_file.mode;
+	dup->old_file.size  = b->old_file.size;
+	dup->old_file.flags = b->old_file.flags;
+
+	return dup;
+}
+
+int git_diff_merge(git_diff *onto, const git_diff *from)
 {
 	int error = 0;
 	git_pool onto_pool;
 	git_vector onto_new;
 	git_diff_delta *delta;
-	bool ignore_case = false;
+	bool ignore_case, reversed;
 	unsigned int i, j;
 
 	assert(onto && from);
@@ -112,26 +145,26 @@ int git_diff_merge(
 	if (!from->deltas.length)
 		return 0;
 
+	ignore_case = ((onto->opts.flags & GIT_DIFF_IGNORE_CASE) != 0);
+	reversed    = ((onto->opts.flags & GIT_DIFF_REVERSE) != 0);
+
+	if (ignore_case != ((from->opts.flags & GIT_DIFF_IGNORE_CASE) != 0) ||
+		reversed    != ((from->opts.flags & GIT_DIFF_REVERSE) != 0)) {
+		giterr_set(GITERR_INVALID,
+			"Attempt to merge diffs created with conflicting options");
+		return -1;
+	}
+
 	if (git_vector_init(
 			&onto_new, onto->deltas.length, git_diff_delta__cmp) < 0 ||
 		git_pool_init(&onto_pool, 1, 0) < 0)
 		return -1;
 
-	if ((onto->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0 ||
-		(from->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0)
-	{
-		ignore_case = true;
-
-		/* This function currently only supports merging diff lists that
-		 * are sorted identically. */
-		assert((onto->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0 &&
-				(from->opts.flags & GIT_DIFF_DELTAS_ARE_ICASE) != 0);
-	}
-
 	for (i = 0, j = 0; i < onto->deltas.length || j < from->deltas.length; ) {
 		git_diff_delta *o = GIT_VECTOR_GET(&onto->deltas, i);
 		const git_diff_delta *f = GIT_VECTOR_GET(&from->deltas, j);
-		int cmp = !f ? -1 : !o ? 1 : STRCMP_CASESELECT(ignore_case, o->old_file.path, f->old_file.path);
+		int cmp = !f ? -1 : !o ? 1 :
+			STRCMP_CASESELECT(ignore_case, o->old_file.path, f->old_file.path);
 
 		if (cmp < 0) {
 			delta = diff_delta__dup(o, &onto_pool);
@@ -140,7 +173,9 @@ int git_diff_merge(
 			delta = diff_delta__dup(f, &onto_pool);
 			j++;
 		} else {
-			delta = diff_delta__merge_like_cgit(o, f, &onto_pool);
+			delta = reversed ?
+				diff_delta__merge_like_cgit_reversed(o, f, &onto_pool) :
+				diff_delta__merge_like_cgit(o, f, &onto_pool);
 			i++;
 			j++;
 		}
@@ -160,7 +195,11 @@ int git_diff_merge(
 	if (!error) {
 		git_vector_swap(&onto->deltas, &onto_new);
 		git_pool_swap(&onto->pool, &onto_pool);
-		onto->new_src = from->new_src;
+
+		if ((onto->opts.flags & GIT_DIFF_REVERSE) != 0)
+			onto->old_src = from->old_src;
+		else
+			onto->new_src = from->new_src;
 
 		/* prefix strings also come from old pool, so recreate those.*/
 		onto->opts.old_prefix =
@@ -230,9 +269,9 @@ int git_diff_find_similar__calc_similarity(
 #define DEFAULT_RENAME_LIMIT 200
 
 static int normalize_find_opts(
-	git_diff_list *diff,
+	git_diff *diff,
 	git_diff_find_options *opts,
-	git_diff_find_options *given)
+	const git_diff_find_options *given)
 {
 	git_config *cfg = NULL;
 
@@ -328,7 +367,7 @@ static int normalize_find_opts(
 }
 
 static int apply_splits_and_deletes(
-	git_diff_list *diff, size_t expected_size, bool actually_split)
+	git_diff *diff, size_t expected_size, bool actually_split)
 {
 	git_vector onto = GIT_VECTOR_INIT;
 	size_t i;
@@ -350,6 +389,7 @@ static int apply_splits_and_deletes(
 				goto on_error;
 
 			deleted->status = GIT_DELTA_DELETED;
+			deleted->nfiles = 1;
 			memset(&deleted->new_file, 0, sizeof(deleted->new_file));
 			deleted->new_file.path = deleted->old_file.path;
 			deleted->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
@@ -361,6 +401,7 @@ static int apply_splits_and_deletes(
 				delta->status = GIT_DELTA_UNTRACKED;
 			else
 				delta->status = GIT_DELTA_ADDED;
+			delta->nfiles = 1;
 			memset(&delta->old_file, 0, sizeof(delta->old_file));
 			delta->old_file.path = delta->new_file.path;
 			delta->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
@@ -402,63 +443,105 @@ on_error:
 	return -1;
 }
 
-GIT_INLINE(git_diff_file *) similarity_get_file(git_diff_list *diff, size_t idx)
+GIT_INLINE(git_diff_file *) similarity_get_file(git_diff *diff, size_t idx)
 {
 	git_diff_delta *delta = git_vector_get(&diff->deltas, idx / 2);
 	return (idx & 1) ? &delta->new_file : &delta->old_file;
 }
 
-static int similarity_calc(
-	git_diff_list *diff,
+typedef struct {
+	size_t idx;
+	git_iterator_type_t src;
+	git_repository *repo;
+	git_diff_file *file;
+	git_buf data;
+	git_odb_object *odb_obj;
+	git_blob *blob;
+} similarity_info;
+
+static int similarity_init(
+	similarity_info *info, git_diff *diff, size_t file_idx)
+{
+	info->idx  = file_idx;
+	info->src  = (file_idx & 1) ? diff->new_src : diff->old_src;
+	info->repo = diff->repo;
+	info->file = similarity_get_file(diff, file_idx);
+	info->odb_obj = NULL;
+	info->blob = NULL;
+	git_buf_init(&info->data, 0);
+
+	if (info->file->size > 0)
+		return 0;
+
+	return git_diff_file__resolve_zero_size(
+		info->file, &info->odb_obj, info->repo);
+}
+
+static int similarity_sig(
+	similarity_info *info,
 	const git_diff_find_options *opts,
-	size_t file_idx,
 	void **cache)
 {
 	int error = 0;
-	git_diff_file *file = similarity_get_file(diff, file_idx);
-	git_iterator_type_t src = (file_idx & 1) ? diff->new_src : diff->old_src;
-
-	if (src == GIT_ITERATOR_TYPE_WORKDIR) { /* compute hashsig from file */
-		git_buf path = GIT_BUF_INIT;
-
-		/* TODO: apply wd-to-odb filters to file data if necessary */
+	git_diff_file *file = info->file;
 
+	if (info->src == GIT_ITERATOR_TYPE_WORKDIR) {
 		if ((error = git_buf_joinpath(
-				 &path, git_repository_workdir(diff->repo), file->path)) < 0)
+			&info->data, git_repository_workdir(info->repo), file->path)) < 0)
 			return error;
 
 		/* if path is not a regular file, just skip this item */
-		if (git_path_isfile(path.ptr))
-			error = opts->metric->file_signature(
-				&cache[file_idx], file, path.ptr, opts->metric->payload);
+		if (!git_path_isfile(info->data.ptr))
+			return 0;
 
-		git_buf_free(&path);
-	} else { /* compute hashsig from blob buffer */
-		git_blob *blob = NULL;
-		git_off_t blobsize;
+		/* TODO: apply wd-to-odb filters to file data if necessary */
 
-		/* TODO: add max size threshold a la diff? */
+		error = opts->metric->file_signature(
+			&cache[info->idx], info->file,
+			info->data.ptr, opts->metric->payload);
+	} else {
+		/* if we didn't initially know the size, we might have an odb_obj
+		 * around from earlier, so convert that, otherwise load the blob now
+		 */
+		if (info->odb_obj != NULL)
+			error = git_object__from_odb_object(
+				(git_object **)&info->blob, info->repo,
+				info->odb_obj, GIT_OBJ_BLOB);
+		else
+			error = git_blob_lookup(&info->blob, info->repo, &file->oid);
 
-		if (git_blob_lookup(&blob, diff->repo, &file->oid) < 0) {
+		if (error < 0) {
 			/* if lookup fails, just skip this item in similarity calc */
 			giterr_clear();
-			return 0;
-		}
+		} else {
+			size_t sz;
 
-		blobsize = git_blob_rawsize(blob);
-		if (!git__is_sizet(blobsize)) /* ? what to do ? */
-			blobsize = (size_t)-1;
+			/* index size may not be actual blob size if filtered */
+			if (file->size != git_blob_rawsize(info->blob))
+				file->size = git_blob_rawsize(info->blob);
 
-		error = opts->metric->buffer_signature(
-			&cache[file_idx], file, git_blob_rawcontent(blob),
-			(size_t)blobsize, opts->metric->payload);
+			sz = (size_t)(git__is_sizet(file->size) ? file->size : -1);
 
-		git_blob_free(blob);
+			error = opts->metric->buffer_signature(
+				&cache[info->idx], info->file,
+				git_blob_rawcontent(info->blob), sz, opts->metric->payload);
+		}
 	}
 
 	return error;
 }
 
+static void similarity_unload(similarity_info *info)
+{
+	if (info->odb_obj)
+		git_odb_object_free(info->odb_obj);
+
+	if (info->blob)
+		git_blob_free(info->blob);
+	else
+		git_buf_free(&info->data);
+}
+
 #define FLAG_SET(opts,flag_name) (((opts)->flags & flag_name) != 0)
 
 /* - score < 0 means files cannot be compared
@@ -467,7 +550,7 @@ static int similarity_calc(
  */
 static int similarity_measure(
 	int *score,
-	git_diff_list *diff,
+	git_diff *diff,
 	const git_diff_find_options *opts,
 	void **cache,
 	size_t a_idx,
@@ -476,6 +559,8 @@ static int similarity_measure(
 	git_diff_file *a_file = similarity_get_file(diff, a_idx);
 	git_diff_file *b_file = similarity_get_file(diff, b_idx);
 	bool exact_match = FLAG_SET(opts, GIT_DIFF_FIND_EXACT_MATCH_ONLY);
+	int error = 0;
+	similarity_info a_info, b_info;
 
 	*score = -1;
 
@@ -510,23 +595,48 @@ static int similarity_measure(
 		return 0;
 	}
 
+	memset(&a_info, 0, sizeof(a_info));
+	memset(&b_info, 0, sizeof(b_info));
+
+	/* set up similarity data (will try to update missing file sizes) */
+	if (!cache[a_idx] && (error = similarity_init(&a_info, diff, a_idx)) < 0)
+		return error;
+	if (!cache[b_idx] && (error = similarity_init(&b_info, diff, b_idx)) < 0)
+		goto cleanup;
+
+	/* check if file sizes are nowhere near each other */
+	if (a_file->size > 127 &&
+		b_file->size > 127 &&
+		(a_file->size > (b_file->size << 3) ||
+		 b_file->size > (a_file->size << 3)))
+		goto cleanup;
+
 	/* update signature cache if needed */
-	if (!cache[a_idx] && similarity_calc(diff, opts, a_idx, cache) < 0)
-		return -1;
-	if (!cache[b_idx] && similarity_calc(diff, opts, b_idx, cache) < 0)
-		return -1;
+	if (!cache[a_idx]) {
+		if ((error = similarity_sig(&a_info, opts, cache)) < 0)
+			goto cleanup;
+	}
+	if (!cache[b_idx]) {
+		if ((error = similarity_sig(&b_info, opts, cache)) < 0)
+			goto cleanup;
+	}
 
-	/* some metrics may not wish to process this file (too big / too small) */
-	if (!cache[a_idx] || !cache[b_idx])
-		return 0;
+	/* calculate similarity provided that the metric choose to process
+	 * both the a and b files (some may not if file is too big, etc).
+	 */
+	if (cache[a_idx] && cache[b_idx])
+		error = opts->metric->similarity(
+			score, cache[a_idx], cache[b_idx], opts->metric->payload);
+
+cleanup:
+	similarity_unload(&a_info);
+	similarity_unload(&b_info);
 
-	/* compare signatures */
-	return opts->metric->similarity(
-		score, cache[a_idx], cache[b_idx], opts->metric->payload);
+	return error;
 }
 
 static int calc_self_similarity(
-	git_diff_list *diff,
+	git_diff *diff,
 	const git_diff_find_options *opts,
 	size_t delta_idx,
 	void **cache)
@@ -543,7 +653,7 @@ static int calc_self_similarity(
 		return error;
 
 	if (similarity >= 0) {
-		delta->similarity = (uint32_t)similarity;
+		delta->similarity = (uint16_t)similarity;
 		delta->flags |= GIT_DIFF_FLAG__HAS_SELF_SIMILARITY;
 	}
 
@@ -551,7 +661,7 @@ static int calc_self_similarity(
 }
 
 static bool is_rename_target(
-	git_diff_list *diff,
+	git_diff *diff,
 	const git_diff_find_options *opts,
 	size_t delta_idx,
 	void **cache)
@@ -590,11 +700,13 @@ static bool is_rename_target(
 		return false;
 
 	case GIT_DELTA_UNTRACKED:
-	case GIT_DELTA_IGNORED:
 		if (!FLAG_SET(opts, GIT_DIFF_FIND_FOR_UNTRACKED))
 			return false;
 		break;
 
+	case GIT_DELTA_IGNORED:
+		return false;
+
 	default: /* all other status values should be checked */
 		break;
 	}
@@ -604,7 +716,7 @@ static bool is_rename_target(
 }
 
 static bool is_rename_source(
-	git_diff_list *diff,
+	git_diff *diff,
 	const git_diff_find_options *opts,
 	size_t delta_idx,
 	void **cache)
@@ -674,42 +786,49 @@ GIT_INLINE(bool) delta_is_new_only(git_diff_delta *delta)
 }
 
 GIT_INLINE(void) delta_make_rename(
-	git_diff_delta *to, const git_diff_delta *from, uint32_t similarity)
+	git_diff_delta *to, const git_diff_delta *from, uint16_t similarity)
 {
 	to->status     = GIT_DELTA_RENAMED;
 	to->similarity = similarity;
+	to->nfiles     = 2;
 	memcpy(&to->old_file, &from->old_file, sizeof(to->old_file));
 	to->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
 }
 
 typedef struct {
-	uint32_t idx;
-	uint32_t similarity;
+	size_t   idx;
+	uint16_t similarity;
 } diff_find_match;
 
 int git_diff_find_similar(
-	git_diff_list *diff,
-	git_diff_find_options *given_opts)
+	git_diff *diff,
+	const git_diff_find_options *given_opts)
 {
-	size_t i, j, sigcache_size;
-	int error = 0, similarity;
-	git_diff_delta *from, *to;
+	size_t s, t;
+	int error = 0, result;
+	uint16_t similarity;
+	git_diff_delta *src, *tgt;
 	git_diff_find_options opts;
-	size_t num_srcs = 0, num_tgts = 0, tried_srcs = 0, tried_tgts = 0;
+	size_t num_deltas, num_srcs = 0, num_tgts = 0;
+	size_t tried_srcs = 0, tried_tgts = 0;
 	size_t num_rewrites = 0, num_updates = 0, num_bumped = 0;
 	void **sigcache; /* cache of similarity metric file signatures */
-	diff_find_match *match_srcs = NULL, *match_tgts = NULL, *best_match;
+	diff_find_match *tgt2src = NULL;
+	diff_find_match *src2tgt = NULL;
+	diff_find_match *tgt2src_copy = NULL;
+	diff_find_match *best_match;
 	git_diff_file swap;
 
 	if ((error = normalize_find_opts(diff, &opts, given_opts)) < 0)
 		return error;
 
+	num_deltas = diff->deltas.length;
+
 	/* TODO: maybe abort if deltas.length > rename_limit ??? */
-	if (!git__is_uint32(diff->deltas.length))
+	if (!git__is_uint32(num_deltas))
 		return 0;
 
-	sigcache_size = diff->deltas.length * 2; /* keep size b/c diff may change */
-	sigcache = git__calloc(sigcache_size, sizeof(void *));
+	sigcache = git__calloc(num_deltas * 2, sizeof(void *));
 	GITERR_CHECK_ALLOC(sigcache);
 
 	/* Label rename sources and targets
@@ -717,22 +836,30 @@ int git_diff_find_similar(
 	 * This will also set self-similarity scores for MODIFIED files and
 	 * mark them for splitting if break-rewrites is enabled
 	 */
-	git_vector_foreach(&diff->deltas, i, to) {
-		if (is_rename_source(diff, &opts, i, sigcache))
+	git_vector_foreach(&diff->deltas, t, tgt) {
+		if (is_rename_source(diff, &opts, t, sigcache))
 			++num_srcs;
 
-		if (is_rename_target(diff, &opts, i, sigcache))
+		if (is_rename_target(diff, &opts, t, sigcache))
 			++num_tgts;
+
+		if ((tgt->flags & GIT_DIFF_FLAG__TO_SPLIT) != 0)
+			num_rewrites++;
 	}
 
 	/* if there are no candidate srcs or tgts, we're done */
 	if (!num_srcs || !num_tgts)
 		goto cleanup;
 
-	match_tgts = git__calloc(diff->deltas.length, sizeof(diff_find_match));
-	GITERR_CHECK_ALLOC(match_tgts);
-	match_srcs = git__calloc(diff->deltas.length, sizeof(diff_find_match));
-	GITERR_CHECK_ALLOC(match_srcs);
+	src2tgt = git__calloc(num_deltas, sizeof(diff_find_match));
+	GITERR_CHECK_ALLOC(src2tgt);
+	tgt2src = git__calloc(num_deltas, sizeof(diff_find_match));
+	GITERR_CHECK_ALLOC(tgt2src);
+
+	if (FLAG_SET(&opts, GIT_DIFF_FIND_COPIES)) {
+		tgt2src_copy = git__calloc(num_deltas, sizeof(diff_find_match));
+		GITERR_CHECK_ALLOC(tgt2src_copy);
+	}
 
 	/*
 	 * Find best-fit matches for rename / copy candidates
@@ -741,47 +868,62 @@ int git_diff_find_similar(
 find_best_matches:
 	tried_tgts = num_bumped = 0;
 
-	git_vector_foreach(&diff->deltas, i, to) {
+	git_vector_foreach(&diff->deltas, t, tgt) {
 		/* skip things that are not rename targets */
-		if ((to->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0)
+		if ((tgt->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0)
 			continue;
 
 		tried_srcs = 0;
 
-		git_vector_foreach(&diff->deltas, j, from) {
+		git_vector_foreach(&diff->deltas, s, src) {
 			/* skip things that are not rename sources */
-			if ((from->flags & GIT_DIFF_FLAG__IS_RENAME_SOURCE) == 0)
+			if ((src->flags & GIT_DIFF_FLAG__IS_RENAME_SOURCE) == 0)
 				continue;
 
 			/* calculate similarity for this pair and find best match */
-			if (i == j)
-				similarity = -1; /* don't measure self-similarity here */
+			if (s == t)
+				result = -1; /* don't measure self-similarity here */
 			else if ((error = similarity_measure(
-				&similarity, diff, &opts, sigcache, 2 * j, 2 * i + 1)) < 0)
+				&result, diff, &opts, sigcache, 2 * s, 2 * t + 1)) < 0)
 				goto cleanup;
 
-			/* if this pairing is better for the src and the tgt, keep it */
-			if (similarity > 0 &&
-				match_tgts[i].similarity < (uint32_t)similarity &&
-				match_srcs[j].similarity < (uint32_t)similarity)
+			if (result < 0)
+				continue;
+			similarity = (uint16_t)result;
+
+			/* is this a better rename? */
+			if (tgt2src[t].similarity < similarity &&
+				src2tgt[s].similarity < similarity)
 			{
-				if (match_tgts[i].similarity > 0) {
-					match_tgts[match_srcs[j].idx].similarity = 0;
-					match_srcs[match_tgts[i].idx].similarity = 0;
-					++num_bumped;
+				/* eject old mapping */
+				if (src2tgt[s].similarity > 0) {
+					tgt2src[src2tgt[s].idx].similarity = 0;
+					num_bumped++;
+				}
+				if (tgt2src[t].similarity > 0) {
+					src2tgt[tgt2src[t].idx].similarity = 0;
+					num_bumped++;
 				}
 
-				match_tgts[i].similarity = (uint32_t)similarity;
-				match_tgts[i].idx = (uint32_t)j;
+				/* write new mapping */
+				tgt2src[t].idx = s;
+				tgt2src[t].similarity = similarity;
+				src2tgt[s].idx = t;
+				src2tgt[s].similarity = similarity;
+			}
 
-				match_srcs[j].similarity = (uint32_t)similarity;
-				match_srcs[j].idx = (uint32_t)i;
+			/* keep best absolute match for copies */
+			if (tgt2src_copy != NULL &&
+				tgt2src_copy[t].similarity < similarity)
+			{
+				tgt2src_copy[t].idx = s;
+				tgt2src_copy[t].similarity = similarity;
 			}
 
 			if (++tried_srcs >= num_srcs)
 				break;
 
-			/* cap on maximum targets we'll examine (per "to" file) */
+			/* cap on maximum targets we'll examine (per "tgt" file) */
 			if (tried_srcs > opts.rename_limit)
 				break;
 		}
@@ -799,18 +941,21 @@ find_best_matches:
 
 	tried_tgts = 0;
 
-	git_vector_foreach(&diff->deltas, i, to) {
+	git_vector_foreach(&diff->deltas, t, tgt) {
 		/* skip things that are not rename targets */
-		if ((to->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0)
+		if ((tgt->flags & GIT_DIFF_FLAG__IS_RENAME_TARGET) == 0)
 			continue;
 
 		/* check if this delta was the target of a similarity */
-		best_match = &match_tgts[i];
-		if (!best_match->similarity)
+		if (tgt2src[t].similarity)
+			best_match = &tgt2src[t];
+		else if (tgt2src_copy && tgt2src_copy[t].similarity)
+			best_match = &tgt2src_copy[t];
+		else
 			continue;
 
-		j = best_match->idx;
-		from = GIT_VECTOR_GET(&diff->deltas, j);
+		s = best_match->idx;
+		src = GIT_VECTOR_GET(&diff->deltas, s);
 
 		/* possible scenarios:
 		 * 1. from DELETE to ADD/UNTRACK/IGNORE = RENAME
@@ -820,101 +965,114 @@ find_best_matches:
 		 * 5. from OTHER to ADD/UNTRACK/IGNORE = OTHER + COPY
 		 */
 
-		if (from->status == GIT_DELTA_DELETED) {
+		if (src->status == GIT_DELTA_DELETED) {
 
-			if (delta_is_new_only(to)) {
+			if (delta_is_new_only(tgt)) {
 
 				if (best_match->similarity < opts.rename_threshold)
 					continue;
 
-				delta_make_rename(to, from, best_match->similarity);
+				delta_make_rename(tgt, src, best_match->similarity);
 
-				from->flags |= GIT_DIFF_FLAG__TO_DELETE;
+				src->flags |= GIT_DIFF_FLAG__TO_DELETE;
 				num_rewrites++;
 			} else {
-				assert(delta_is_split(to));
+				assert(delta_is_split(tgt));
 
 				if (best_match->similarity < opts.rename_from_rewrite_threshold)
 					continue;
 
-				memcpy(&swap, &to->old_file, sizeof(swap));
+				memcpy(&swap, &tgt->old_file, sizeof(swap));
 
-				delta_make_rename(to, from, best_match->similarity);
+				delta_make_rename(tgt, src, best_match->similarity);
 				num_rewrites--;
 
-				from->status = GIT_DELTA_DELETED;
-				memcpy(&from->old_file, &swap, sizeof(from->old_file));
-				memset(&from->new_file, 0, sizeof(from->new_file));
-				from->new_file.path = from->old_file.path;
-				from->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+				assert(src->status == GIT_DELTA_DELETED);
+				memcpy(&src->old_file, &swap, sizeof(src->old_file));
+				memset(&src->new_file, 0, sizeof(src->new_file));
+				src->new_file.path = src->old_file.path;
+				src->new_file.flags |= GIT_DIFF_FLAG_VALID_OID;
 
 				num_updates++;
+
+				if (src2tgt[t].similarity > 0 && src2tgt[t].idx > t) {
+					/* what used to be at src t is now at src s */
+					tgt2src[src2tgt[t].idx].idx = s;
+				}
 			}
 		}
 
-		else if (delta_is_split(from)) {
+		else if (delta_is_split(src)) {
 
-			if (delta_is_new_only(to)) {
+			if (delta_is_new_only(tgt)) {
 
 				if (best_match->similarity < opts.rename_threshold)
 					continue;
 
-				delta_make_rename(to, from, best_match->similarity);
+				delta_make_rename(tgt, src, best_match->similarity);
 
-				from->status = (diff->new_src == GIT_ITERATOR_TYPE_WORKDIR) ?
+				src->status = (diff->new_src == GIT_ITERATOR_TYPE_WORKDIR) ?
 					GIT_DELTA_UNTRACKED : GIT_DELTA_ADDED;
-				memset(&from->old_file, 0, sizeof(from->old_file));
-				from->old_file.path = from->new_file.path;
-				from->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
+				src->nfiles = 1;
+				memset(&src->old_file, 0, sizeof(src->old_file));
+				src->old_file.path = src->new_file.path;
+				src->old_file.flags |= GIT_DIFF_FLAG_VALID_OID;
 
-				from->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
+				src->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
 				num_rewrites--;
 
 				num_updates++;
 			} else {
-				assert(delta_is_split(from));
+				assert(delta_is_split(src));
 
 				if (best_match->similarity < opts.rename_from_rewrite_threshold)
 					continue;
 
-				memcpy(&swap, &to->old_file, sizeof(swap));
+				memcpy(&swap, &tgt->old_file, sizeof(swap));
 
-				delta_make_rename(to, from, best_match->similarity);
+				delta_make_rename(tgt, src, best_match->similarity);
 				num_rewrites--;
 				num_updates++;
 
-				memcpy(&from->old_file, &swap, sizeof(from->old_file));
+				memcpy(&src->old_file, &swap, sizeof(src->old_file));
 
 				/* if we've just swapped the new element into the correct
 				 * place, clear the SPLIT flag
 				 */
-				if (match_tgts[j].idx == i &&
-					match_tgts[j].similarity >
+				if (tgt2src[s].idx == t &&
+					tgt2src[s].similarity >
 					opts.rename_from_rewrite_threshold) {
-
-					from->status     = GIT_DELTA_RENAMED;
-					from->similarity = match_tgts[j].similarity;
-					match_tgts[j].similarity = 0;
-					from->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
+					src->status     = GIT_DELTA_RENAMED;
+					src->similarity = tgt2src[s].similarity;
+					tgt2src[s].similarity = 0;
+					src->flags &= ~GIT_DIFF_FLAG__TO_SPLIT;
 					num_rewrites--;
 				}
 				/* otherwise, if we just overwrote a source, update mapping */
-				else if (j > i && match_srcs[i].similarity > 0) {
-					match_tgts[match_srcs[i].idx].idx = j;
+				else if (src2tgt[t].similarity > 0 && src2tgt[t].idx > t) {
+					/* what used to be at src t is now at src s */
+					tgt2src[src2tgt[t].idx].idx = s;
 				}
 
 				num_updates++;
 			}
 		}
 
-		else if (delta_is_new_only(to)) {
-			if (!FLAG_SET(&opts, GIT_DIFF_FIND_COPIES) ||
-				best_match->similarity < opts.copy_threshold)
+		else if (delta_is_new_only(tgt)) {
+			if (!FLAG_SET(&opts, GIT_DIFF_FIND_COPIES))
 				continue;
 
-			to->status     = GIT_DELTA_COPIED;
-			to->similarity = best_match->similarity;
-			memcpy(&to->old_file, &from->old_file, sizeof(to->old_file));
+			if (tgt2src_copy[t].similarity < opts.copy_threshold)
+				continue;
+
+			/* always use best possible source for copy */
+			best_match = &tgt2src_copy[t];
+			src = GIT_VECTOR_GET(&diff->deltas, best_match->idx);
+
+			tgt->status     = GIT_DELTA_COPIED;
+			tgt->similarity = best_match->similarity;
+			tgt->nfiles     = 2;
+			memcpy(&tgt->old_file, &src->old_file, sizeof(tgt->old_file));
 
 			num_updates++;
 		}
@@ -927,15 +1085,17 @@ find_best_matches:
 	if (num_rewrites > 0 || num_updates > 0)
 		error = apply_splits_and_deletes(
 			diff, diff->deltas.length - num_rewrites,
-			FLAG_SET(&opts, GIT_DIFF_BREAK_REWRITES));
+			FLAG_SET(&opts, GIT_DIFF_BREAK_REWRITES) &&
+			!FLAG_SET(&opts, GIT_DIFF_BREAK_REWRITES_FOR_RENAMES_ONLY));
 
 cleanup:
-	git__free(match_srcs);
-	git__free(match_tgts);
+	git__free(tgt2src);
+	git__free(src2tgt);
+	git__free(tgt2src_copy);
 
-	for (i = 0; i < sigcache_size; ++i) {
-		if (sigcache[i] != NULL)
-			opts.metric->free_signature(sigcache[i], opts.metric->payload);
+	for (t = 0; t < num_deltas * 2; ++t) {
+		if (sigcache[t] != NULL)
+			opts.metric->free_signature(sigcache[t], opts.metric->payload);
 	}
 	git__free(sigcache);
 
diff --git a/src/diff_xdiff.c b/src/diff_xdiff.c
index 7694fb996..e0bc11f7f 100644
--- a/src/diff_xdiff.c
+++ b/src/diff_xdiff.c
@@ -24,26 +24,26 @@ static int git_xdiff_scan_int(const char **str, int *value)
 	return (digits > 0) ? 0 : -1;
 }
 
-static int git_xdiff_parse_hunk(git_diff_range *range, const char *header)
+static int git_xdiff_parse_hunk(git_diff_hunk *hunk, const char *header)
 {
 	/* expect something of the form "@@ -%d[,%d] +%d[,%d] @@" */
 	if (*header != '@')
 		return -1;
-	if (git_xdiff_scan_int(&header, &range->old_start) < 0)
+	if (git_xdiff_scan_int(&header, &hunk->old_start) < 0)
 		return -1;
 	if (*header == ',') {
-		if (git_xdiff_scan_int(&header, &range->old_lines) < 0)
+		if (git_xdiff_scan_int(&header, &hunk->old_lines) < 0)
 			return -1;
 	} else
-		range->old_lines = 1;
-	if (git_xdiff_scan_int(&header, &range->new_start) < 0)
+		hunk->old_lines = 1;
+	if (git_xdiff_scan_int(&header, &hunk->new_start) < 0)
 		return -1;
 	if (*header == ',') {
-		if (git_xdiff_scan_int(&header, &range->new_lines) < 0)
+		if (git_xdiff_scan_int(&header, &hunk->new_lines) < 0)
 			return -1;
 	} else
-		range->new_lines = 1;
-	if (range->old_start < 0 || range->new_start < 0)
+		hunk->new_lines = 1;
+	if (hunk->old_start < 0 || hunk->new_start < 0)
 		return -1;
 
 	return 0;
@@ -51,38 +51,104 @@ static int git_xdiff_parse_hunk(git_diff_range *range, const char *header)
 
 typedef struct {
 	git_xdiff_output *xo;
-	git_diff_patch *patch;
-	git_diff_range range;
+	git_patch *patch;
+	git_diff_hunk hunk;
+	int old_lineno, new_lineno;
+	mmfile_t xd_old_data, xd_new_data;
 } git_xdiff_info;
 
+static int diff_update_lines(
+	git_xdiff_info *info,
+	git_diff_line *line,
+	const char *content,
+	size_t content_len)
+{
+	const char *scan = content, *scan_end = content + content_len;
+
+	for (line->num_lines = 0; scan < scan_end; ++scan)
+		if (*scan == '\n')
+			++line->num_lines;
+
+	line->content     = content;
+	line->content_len = content_len;
+
+	/* expect " "/"-"/"+", then data */
+	switch (line->origin) {
+	case GIT_DIFF_LINE_ADDITION:
+	case GIT_DIFF_LINE_DEL_EOFNL:
+		line->old_lineno = -1;
+		line->new_lineno = info->new_lineno;
+		info->new_lineno += (int)line->num_lines;
+		break;
+	case GIT_DIFF_LINE_DELETION:
+	case GIT_DIFF_LINE_ADD_EOFNL:
+		line->old_lineno = info->old_lineno;
+		line->new_lineno = -1;
+		info->old_lineno += (int)line->num_lines;
+		break;
+	case GIT_DIFF_LINE_CONTEXT:
+	case GIT_DIFF_LINE_CONTEXT_EOFNL:
+		line->old_lineno = info->old_lineno;
+		line->new_lineno = info->new_lineno;
+		info->old_lineno += (int)line->num_lines;
+		info->new_lineno += (int)line->num_lines;
+		break;
+	default:
+		giterr_set(GITERR_INVALID, "Unknown diff line origin %02x",
+			(unsigned int)line->origin);
+		return -1;
+	}
+
+	return 0;
+}
+
 static int git_xdiff_cb(void *priv, mmbuffer_t *bufs, int len)
 {
 	git_xdiff_info *info = priv;
-	git_diff_patch *patch = info->patch;
-	const git_diff_delta *delta = git_diff_patch_delta(patch);
+	git_patch *patch = info->patch;
+	const git_diff_delta *delta = git_patch_get_delta(patch);
 	git_diff_output *output = &info->xo->output;
+	git_diff_line line;
 
 	if (len == 1) {
-		output->error = git_xdiff_parse_hunk(&info->range, bufs[0].ptr);
+		output->error = git_xdiff_parse_hunk(&info->hunk, bufs[0].ptr);
 		if (output->error < 0)
 			return output->error;
 
+		info->hunk.header_len = bufs[0].size;
+		if (info->hunk.header_len >= sizeof(info->hunk.header))
+			info->hunk.header_len = sizeof(info->hunk.header) - 1;
+		memcpy(info->hunk.header, bufs[0].ptr, info->hunk.header_len);
+		info->hunk.header[info->hunk.header_len] = '\0';
+
 		if (output->hunk_cb != NULL &&
-			output->hunk_cb(delta, &info->range,
-				bufs[0].ptr, bufs[0].size, output->payload))
+			output->hunk_cb(delta, &info->hunk, output->payload))
 			output->error = GIT_EUSER;
+
+		info->old_lineno = info->hunk.old_start;
+		info->new_lineno = info->hunk.new_start;
 	}
 
 	if (len == 2 || len == 3) {
 		/* expect " "/"-"/"+", then data */
-		char origin =
+		line.origin =
 			(*bufs[0].ptr == '+') ? GIT_DIFF_LINE_ADDITION :
 			(*bufs[0].ptr == '-') ? GIT_DIFF_LINE_DELETION :
 			GIT_DIFF_LINE_CONTEXT;
 
-		if (output->data_cb != NULL &&
-			output->data_cb(delta, &info->range,
-				origin, bufs[1].ptr, bufs[1].size, output->payload))
+		if (line.origin == GIT_DIFF_LINE_ADDITION)
+			line.content_offset = bufs[1].ptr - info->xd_new_data.ptr;
+		else if (line.origin == GIT_DIFF_LINE_DELETION)
+			line.content_offset = bufs[1].ptr - info->xd_old_data.ptr;
+		else
+			line.content_offset = -1;
+
+		output->error = diff_update_lines(
+			info, &line, bufs[1].ptr, bufs[1].size);
+
+		if (!output->error &&
+			output->data_cb != NULL &&
+			output->data_cb(delta, &info->hunk, &line, output->payload))
 			output->error = GIT_EUSER;
 	}
 
@@ -92,26 +158,30 @@ static int git_xdiff_cb(void *priv, mmbuffer_t *bufs, int len)
 		 * If we have a '-' and a third buf, then we have removed a line
 		 * with out a newline but added a blank line, so ADD_EOFNL.
 		 */
-		char origin =
+		line.origin =
 			(*bufs[0].ptr == '+') ? GIT_DIFF_LINE_DEL_EOFNL :
 			(*bufs[0].ptr == '-') ? GIT_DIFF_LINE_ADD_EOFNL :
 			GIT_DIFF_LINE_CONTEXT_EOFNL;
 
-		if (output->data_cb != NULL &&
-			output->data_cb(delta, &info->range,
-				origin, bufs[2].ptr, bufs[2].size, output->payload))
+		line.content_offset = -1;
+
+		output->error = diff_update_lines(
+			info, &line, bufs[2].ptr, bufs[2].size);
+
+		if (!output->error &&
+			output->data_cb != NULL &&
+			output->data_cb(delta, &info->hunk, &line, output->payload))
 			output->error = GIT_EUSER;
 	}
 
 	return output->error;
 }
 
-static int git_xdiff(git_diff_output *output, git_diff_patch *patch)
+static int git_xdiff(git_diff_output *output, git_patch *patch)
 {
 	git_xdiff_output *xo = (git_xdiff_output *)output;
 	git_xdiff_info info;
 	git_diff_find_context_payload findctxt;
-	mmfile_t xd_old_data, xd_new_data;
 
 	memset(&info, 0, sizeof(info));
 	info.patch = patch;
@@ -120,7 +190,7 @@ static int git_xdiff(git_diff_output *output, git_diff_patch *patch)
 	xo->callback.priv = &info;
 
 	git_diff_find_context_init(
-		&xo->config.find_func, &findctxt, git_diff_patch__driver(patch));
+		&xo->config.find_func, &findctxt, git_patch__driver(patch));
 	xo->config.find_func_priv = &findctxt;
 
 	if (xo->config.find_func != NULL)
@@ -132,10 +202,10 @@ static int git_xdiff(git_diff_output *output, git_diff_patch *patch)
 	 * updates are needed to xo->params.flags
 	 */
 
-	git_diff_patch__old_data(&xd_old_data.ptr, &xd_old_data.size, patch);
-	git_diff_patch__new_data(&xd_new_data.ptr, &xd_new_data.size, patch);
+	git_patch__old_data(&info.xd_old_data.ptr, &info.xd_old_data.size, patch);
+	git_patch__new_data(&info.xd_new_data.ptr, &info.xd_new_data.size, patch);
 
-	xdl_diff(&xd_old_data, &xd_new_data,
+	xdl_diff(&info.xd_old_data, &info.xd_new_data,
 		&xo->params, &xo->config, &xo->callback);
 
 	git_diff_find_context_clear(&findctxt);
@@ -145,7 +215,7 @@ static int git_xdiff(git_diff_output *output, git_diff_patch *patch)
 
 void git_xdiff_init(git_xdiff_output *xo, const git_diff_options *opts)
 {
-	uint32_t flags = opts ? opts->flags : GIT_DIFF_NORMAL;
+	uint32_t flags = opts ? opts->flags : 0;
 
 	xo->output.diff_cb = git_xdiff;
 
@@ -161,6 +231,11 @@ void git_xdiff_init(git_xdiff_output *xo, const git_diff_options *opts)
 	if (flags & GIT_DIFF_IGNORE_WHITESPACE_EOL)
 		xo->params.flags |= XDF_IGNORE_WHITESPACE_AT_EOL;
 
+	if (flags & GIT_DIFF_PATIENCE)
+		xo->params.flags |= XDF_PATIENCE_DIFF;
+	if (flags & GIT_DIFF_MINIMAL)
+		xo->params.flags |= XDF_NEED_MINIMAL;
+
 	memset(&xo->callback, 0, sizeof(xo->callback));
 	xo->callback.outf = git_xdiff_cb;
 }
diff --git a/src/errors.c b/src/errors.c
index e2629f69e..d04da4ca9 100644
--- a/src/errors.c
+++ b/src/errors.c
@@ -112,8 +112,25 @@ void giterr_clear(void)
 #endif
 }
 
+int giterr_detach(git_error *cpy)
+{
+	git_error *error = GIT_GLOBAL->last_error;
+
+	assert(cpy);
+
+	if (!error)
+		return -1;
+
+	cpy->message = error->message;
+	cpy->klass = error->klass;
+
+	error->message = NULL;
+	giterr_clear();
+
+	return 0;
+}
+
 const git_error *giterr_last(void)
 {
 	return GIT_GLOBAL->last_error;
 }
-
diff --git a/src/fetch.c b/src/fetch.c
index 03fad5fec..276591821 100644
--- a/src/fetch.c
+++ b/src/fetch.c
@@ -19,55 +19,47 @@
 #include "repository.h"
 #include "refs.h"
 
-struct filter_payload {
-	git_remote *remote;
-	const git_refspec *spec, *tagspec;
-	git_odb *odb;
-	int found_head;
-};
-
-static int filter_ref__cb(git_remote_head *head, void *payload)
+static int maybe_want(git_remote *remote, git_remote_head *head, git_odb *odb, git_refspec *tagspec)
 {
-	struct filter_payload *p = payload;
 	int match = 0;
 
 	if (!git_reference_is_valid_name(head->name))
 		return 0;
 
-	if (!p->found_head && strcmp(head->name, GIT_HEAD_FILE) == 0)
-		p->found_head = 1;
-	else if (p->remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL) {
+	if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL) {
 		/*
 		 * If tagopt is --tags, then we only use the default
 		 * tags refspec and ignore the remote's
 		 */
-		if (git_refspec_src_matches(p->tagspec, head->name))
+		if (git_refspec_src_matches(tagspec, head->name))
 			match = 1;
 		else
 			return 0;
-	} else if (git_remote__matching_refspec(p->remote, head->name))
+	} else if (git_remote__matching_refspec(remote, head->name))
 			match = 1;
 
 	if (!match)
 		return 0;
 
 	/* If we have the object, mark it so we don't ask for it */
-	if (git_odb_exists(p->odb, &head->oid))
+	if (git_odb_exists(odb, &head->oid))
 		head->local = 1;
 	else
-		p->remote->need_pack = 1;
+		remote->need_pack = 1;
 
-	return git_vector_insert(&p->remote->refs, head);
+	return git_vector_insert(&remote->refs, head);
 }
 
 static int filter_wants(git_remote *remote)
 {
-	struct filter_payload p;
-	git_refspec tagspec;
-	int error = -1;
+	git_remote_head **heads;
+	git_refspec tagspec, head;
+	int error = 0;
+	git_odb *odb;
+	size_t i, heads_len;
 
 	git_vector_clear(&remote->refs);
-	if (git_refspec__parse(&tagspec, GIT_REFSPEC_TAGS, true) < 0)
+	if ((error = git_refspec__parse(&tagspec, GIT_REFSPEC_TAGS, true)) < 0)
 		return error;
 
 	/*
@@ -76,14 +68,27 @@ static int filter_wants(git_remote *remote)
 	 * not interested in any particular branch but just the remote's
 	 * HEAD, which will be stored in FETCH_HEAD after the fetch.
 	 */
-	p.tagspec = &tagspec;
-	p.found_head = 0;
-	p.remote = remote;
+	if (remote->active_refspecs.length == 0) {
+		if ((error = git_refspec__parse(&head, "HEAD", true)) < 0)
+			goto cleanup;
 
-	if (git_repository_odb__weakptr(&p.odb, remote->repo) < 0)
+		error = git_refspec__dwim_one(&remote->active_refspecs, &head, &remote->refs);
+		git_refspec__free(&head);
+
+		if (error < 0)
+			goto cleanup;
+	}
+
+	if (git_repository_odb__weakptr(&odb, remote->repo) < 0)
+		goto cleanup;
+
+	if (git_remote_ls((const git_remote_head ***)&heads, &heads_len, remote) < 0)
 		goto cleanup;
 
-	error = git_remote_ls(remote, filter_ref__cb, &p);
+	for (i = 0; i < heads_len; i++) {
+		if ((error = maybe_want(remote, heads[i], odb, &tagspec)) < 0)
+			break;
+	}
 
 cleanup:
 	git_refspec__free(&tagspec);
@@ -106,7 +111,7 @@ int git_fetch_negotiate(git_remote *remote)
 	}
 
 	/* Don't try to negotiate when we don't want anything */
-	if (remote->refs.length == 0 || !remote->need_pack)
+	if (!remote->need_pack)
 		return 0;
 
 	/*
@@ -119,15 +124,13 @@ int git_fetch_negotiate(git_remote *remote)
 		remote->refs.length);
 }
 
-int git_fetch_download_pack(
-	git_remote *remote,
-	git_transfer_progress_callback progress_cb,
-	void *progress_payload)
+int git_fetch_download_pack(git_remote *remote)
 {
 	git_transport *t = remote->transport;
 
 	if(!remote->need_pack)
 		return 0;
 
-	return t->download_pack(t, remote->repo, &remote->stats, progress_cb, progress_payload);
+	return t->download_pack(t, remote->repo, &remote->stats,
+				remote->callbacks.transfer_progress, remote->callbacks.payload);
 }
diff --git a/src/fetch.h b/src/fetch.h
index 059251d04..9605da1b5 100644
--- a/src/fetch.h
+++ b/src/fetch.h
@@ -11,10 +11,7 @@
 
 int git_fetch_negotiate(git_remote *remote);
 
-int git_fetch_download_pack(
-		git_remote *remote,
-		git_transfer_progress_callback progress_cb,
-		void *progress_payload);
+int git_fetch_download_pack(git_remote *remote);
 
 int git_fetch__download_pack(
 		git_transport *t,
diff --git a/src/fetchhead.c b/src/fetchhead.c
index 4dcebb857..67089d13d 100644
--- a/src/fetchhead.c
+++ b/src/fetchhead.c
@@ -74,6 +74,7 @@ static int fetchhead_ref_write(
 {
 	char oid[GIT_OID_HEXSZ + 1];
 	const char *type, *name;
+	int head = 0;
 
 	assert(file && fetchhead_ref);
 
@@ -87,11 +88,16 @@ static int fetchhead_ref_write(
 		GIT_REFS_TAGS_DIR) == 0) {
 		type = "tag ";
 		name = fetchhead_ref->ref_name + strlen(GIT_REFS_TAGS_DIR);
+	} else if (!git__strcmp(fetchhead_ref->ref_name, GIT_HEAD_FILE)) {
+		head = 1;
 	} else {
 		type = "";
 		name = fetchhead_ref->ref_name;
 	}
 
+	if (head)
+		return git_filebuf_printf(file, "%s\t\t%s\n", oid, fetchhead_ref->remote_url);
+
 	return git_filebuf_printf(file, "%s\t%s\t%s'%s' of %s\n",
 		oid,
 		(fetchhead_ref->is_merge) ? "" : "not-for-merge",
@@ -112,7 +118,7 @@ int git_fetchhead_write(git_repository *repo, git_vector *fetchhead_refs)
 	if (git_buf_joinpath(&path, repo->path_repository, GIT_FETCH_HEAD_FILE) < 0)
 		return -1;
 
-	if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_FORCE) < 0) {
+	if (git_filebuf_open(&file, path.ptr, GIT_FILEBUF_FORCE, GIT_REFS_FILE_MODE) < 0) {
 		git_buf_free(&path);
 		return -1;
 	}
@@ -124,7 +130,7 @@ int git_fetchhead_write(git_repository *repo, git_vector *fetchhead_refs)
 	git_vector_foreach(fetchhead_refs, i, fetchhead_ref)
 		fetchhead_ref_write(&file, fetchhead_ref);
 
-	return git_filebuf_commit(&file, GIT_REFS_FILE_MODE);
+	return git_filebuf_commit(&file);
 }
 
 static int fetchhead_ref_parse(
diff --git a/src/filebuf.c b/src/filebuf.c
index 246ae34e7..9c3dae811 100644
--- a/src/filebuf.c
+++ b/src/filebuf.c
@@ -10,8 +10,6 @@
 #include "filebuf.h"
 #include "fileops.h"
 
-#define GIT_LOCK_FILE_MODE 0644
-
 static const size_t WRITE_BUFFER_SIZE = (4096 * 2);
 
 enum buferr_t {
@@ -44,7 +42,7 @@ static int verify_last_error(git_filebuf *file)
 	}
 }
 
-static int lock_file(git_filebuf *file, int flags)
+static int lock_file(git_filebuf *file, int flags, mode_t mode)
 {
 	if (git_path_exists(file->path_lock) == true) {
 		if (flags & GIT_FILEBUF_FORCE)
@@ -53,20 +51,20 @@ static int lock_file(git_filebuf *file, int flags)
 			giterr_clear(); /* actual OS error code just confuses */
 			giterr_set(GITERR_OS,
 				"Failed to lock file '%s' for writing", file->path_lock);
-			return -1;
+			return GIT_ELOCKED;
 		}
 	}
 
 	/* create path to the file buffer is required */
 	if (flags & GIT_FILEBUF_FORCE) {
 		/* XXX: Should dirmode here be configurable? Or is 0777 always fine? */
-		file->fd = git_futils_creat_locked_withpath(file->path_lock, 0777, GIT_LOCK_FILE_MODE);
+		file->fd = git_futils_creat_locked_withpath(file->path_lock, 0777, mode);
 	} else {
-		file->fd = git_futils_creat_locked(file->path_lock, GIT_LOCK_FILE_MODE);
+		file->fd = git_futils_creat_locked(file->path_lock, mode);
 	}
 
 	if (file->fd < 0)
-		return -1;
+		return file->fd;
 
 	file->fd_is_open = true;
 
@@ -195,9 +193,9 @@ static int write_deflate(git_filebuf *file, void *source, size_t len)
 	return 0;
 }
 
-int git_filebuf_open(git_filebuf *file, const char *path, int flags)
+int git_filebuf_open(git_filebuf *file, const char *path, int flags, mode_t mode)
 {
-	int compression;
+	int compression, error = -1;
 	size_t path_len;
 
 	/* opening an already open buffer is a programming error;
@@ -255,7 +253,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 		git_buf tmp_path = GIT_BUF_INIT;
 
 		/* Open the file as temporary for locking */
-		file->fd = git_futils_mktmp(&tmp_path, path);
+		file->fd = git_futils_mktmp(&tmp_path, path, mode);
 
 		if (file->fd < 0) {
 			git_buf_free(&tmp_path);
@@ -282,7 +280,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 		memcpy(file->path_lock + path_len, GIT_FILELOCK_EXTENSION, GIT_FILELOCK_EXTLENGTH);
 
 		/* open the file for locking */
-		if (lock_file(file, flags) < 0)
+		if ((error = lock_file(file, flags, mode)) < 0)
 			goto cleanup;
 	}
 
@@ -290,7 +288,7 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 
 cleanup:
 	git_filebuf_cleanup(file);
-	return -1;
+	return error;
 }
 
 int git_filebuf_hash(git_oid *oid, git_filebuf *file)
@@ -309,16 +307,16 @@ int git_filebuf_hash(git_oid *oid, git_filebuf *file)
 	return 0;
 }
 
-int git_filebuf_commit_at(git_filebuf *file, const char *path, mode_t mode)
+int git_filebuf_commit_at(git_filebuf *file, const char *path)
 {
 	git__free(file->path_original);
 	file->path_original = git__strdup(path);
 	GITERR_CHECK_ALLOC(file->path_original);
 
-	return git_filebuf_commit(file, mode);
+	return git_filebuf_commit(file);
 }
 
-int git_filebuf_commit(git_filebuf *file, mode_t mode)
+int git_filebuf_commit(git_filebuf *file)
 {
 	/* temporary files cannot be committed */
 	assert(file && file->path_original);
@@ -338,11 +336,6 @@ int git_filebuf_commit(git_filebuf *file, mode_t mode)
 
 	file->fd = -1;
 
-	if (p_chmod(file->path_lock, mode)) {
-		giterr_set(GITERR_OS, "Failed to set attributes for file at '%s'", file->path_lock);
-		goto on_error;
-	}
-
 	p_unlink(file->path_original);
 
 	if (p_rename(file->path_lock, file->path_original) < 0) {
diff --git a/src/filebuf.h b/src/filebuf.h
index 823af81bf..044af5405 100644
--- a/src/filebuf.h
+++ b/src/filebuf.h
@@ -77,9 +77,9 @@ int git_filebuf_write(git_filebuf *lock, const void *buff, size_t len);
 int git_filebuf_reserve(git_filebuf *file, void **buff, size_t len);
 int git_filebuf_printf(git_filebuf *file, const char *format, ...) GIT_FORMAT_PRINTF(2, 3);
 
-int git_filebuf_open(git_filebuf *lock, const char *path, int flags);
-int git_filebuf_commit(git_filebuf *lock, mode_t mode);
-int git_filebuf_commit_at(git_filebuf *lock, const char *path, mode_t mode);
+int git_filebuf_open(git_filebuf *lock, const char *path, int flags, mode_t mode);
+int git_filebuf_commit(git_filebuf *lock);
+int git_filebuf_commit_at(git_filebuf *lock, const char *path);
 void git_filebuf_cleanup(git_filebuf *lock);
 int git_filebuf_hash(git_oid *oid, git_filebuf *file);
 int git_filebuf_flush(git_filebuf *file);
diff --git a/src/fileops.c b/src/fileops.c
index d5f6acfad..5763b370b 100644
--- a/src/fileops.c
+++ b/src/fileops.c
@@ -6,6 +6,7 @@
  */
 #include "common.h"
 #include "fileops.h"
+#include "global.h"
 #include <ctype.h>
 #if GIT_WIN32
 #include "win32/findfile.h"
@@ -18,9 +19,12 @@ int git_futils_mkpath2file(const char *file_path, const mode_t mode)
 		GIT_MKDIR_PATH | GIT_MKDIR_SKIP_LAST | GIT_MKDIR_VERIFY_DIR);
 }
 
-int git_futils_mktmp(git_buf *path_out, const char *filename)
+int git_futils_mktmp(git_buf *path_out, const char *filename, mode_t mode)
 {
 	int fd;
+	mode_t mask;
+
+	p_umask(mask = p_umask(0));
 
 	git_buf_sets(path_out, filename);
 	git_buf_puts(path_out, "_git2_XXXXXX");
@@ -34,6 +38,12 @@ int git_futils_mktmp(git_buf *path_out, const char *filename)
 		return -1;
 	}
 
+	if (p_chmod(path_out->ptr, (mode & ~mask))) {
+		giterr_set(GITERR_OS,
+			"Failed to set permissions on file '%s'", path_out->ptr);
+		return -1;
+	}
+
 	return fd;
 }
 
@@ -55,22 +65,12 @@ int git_futils_creat_withpath(const char *path, const mode_t dirmode, const mode
 
 int git_futils_creat_locked(const char *path, const mode_t mode)
 {
-	int fd;
-
-#ifdef GIT_WIN32
-	wchar_t buf[GIT_WIN_PATH];
-
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
-	fd = _wopen(buf, O_WRONLY | O_CREAT | O_TRUNC |
-		O_EXCL | O_BINARY | O_CLOEXEC, mode);
-#else
-	fd = open(path, O_WRONLY | O_CREAT | O_TRUNC |
+	int fd = p_open(path, O_WRONLY | O_CREAT | O_TRUNC |
 		O_EXCL | O_BINARY | O_CLOEXEC, mode);
-#endif
 
 	if (fd < 0) {
 		giterr_set(GITERR_OS, "Failed to create locked file '%s'", path);
-		return -1;
+		return errno == EEXIST ? GIT_ELOCKED : -1;
 	}
 
 	return fd;
@@ -87,11 +87,8 @@ int git_futils_creat_locked_withpath(const char *path, const mode_t dirmode, con
 int git_futils_open_ro(const char *path)
 {
 	int fd = p_open(path, O_RDONLY);
-	if (fd < 0) {
-		if (errno == ENOENT || errno == ENOTDIR)
-			fd = GIT_ENOTFOUND;
-		giterr_set(GITERR_OS, "Failed to open '%s'", path);
-	}
+	if (fd < 0)
+		return git_path_set_error(errno, path, "open");
 	return fd;
 }
 
@@ -110,7 +107,7 @@ git_off_t git_futils_filesize(git_file fd)
 mode_t git_futils_canonical_mode(mode_t raw_mode)
 {
 	if (S_ISREG(raw_mode))
-		return S_IFREG | GIT_CANONICAL_PERMS(raw_mode);
+		return S_IFREG | GIT_PERMS_CANONICAL(raw_mode);
 	else if (S_ISLNK(raw_mode))
 		return S_IFLNK;
 	else if (S_ISGITLINK(raw_mode))
@@ -156,11 +153,16 @@ int git_futils_readbuffer_updated(
 	if (updated != NULL)
 		*updated = 0;
 
-	if ((fd = git_futils_open_ro(path)) < 0)
-		return fd;
+	if (p_stat(path, &st) < 0)
+		return git_path_set_error(errno, path, "stat");
 
-	if (p_fstat(fd, &st) < 0 || S_ISDIR(st.st_mode) || !git__is_sizet(st.st_size+1)) {
-		p_close(fd);
+
+	if (S_ISDIR(st.st_mode)) {
+		giterr_set(GITERR_INVALID, "requested file is a directory");
+		return GIT_ENOTFOUND;
+	}
+
+	if (!git__is_sizet(st.st_size+1)) {
 		giterr_set(GITERR_OS, "Invalid regular file stat for '%s'", path);
 		return -1;
 	}
@@ -177,7 +179,6 @@ int git_futils_readbuffer_updated(
 		changed = true;
 
 	if (!changed) {
-		p_close(fd);
 		return 0;
 	}
 
@@ -186,6 +187,9 @@ int git_futils_readbuffer_updated(
 	if (size != NULL)
 		*size = (size_t)st.st_size;
 
+	if ((fd = git_futils_open_ro(path)) < 0)
+		return fd;
+
 	if (git_futils_readbuffer_fd(buf, fd, (size_t)st.st_size) < 0) {
 		p_close(fd);
 		return -1;
@@ -222,6 +226,7 @@ int git_futils_writebuffer(
 	if ((error = p_write(fd, git_buf_cstr(buf), git_buf_len(buf))) < 0) {
 		giterr_set(GITERR_OS, "Could not write to '%s'", path);
 		(void)p_close(fd);
+		return error;
 	}
 
 	if ((error = p_close(fd)) < 0)
@@ -347,12 +352,11 @@ int git_futils_mkdir(
 
 		/* make directory */
 		if (p_mkdir(make_path.ptr, mode) < 0) {
-			int tmp_errno = errno;
+			int tmp_errno = giterr_system_last();
 
 			/* ignore error if directory already exists */
-			if (p_stat(make_path.ptr, &st) < 0 ||
-				!(S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) {
-				errno = tmp_errno;
+			if (p_stat(make_path.ptr, &st) < 0 || !S_ISDIR(st.st_mode)) {
+				giterr_system_set(tmp_errno);
 				giterr_set(GITERR_OS, "Failed to make directory '%s'", make_path.ptr);
 				goto done;
 			}
@@ -400,8 +404,11 @@ typedef struct {
 	size_t baselen;
 	uint32_t flags;
 	int error;
+	int depth;
 } futils__rmdir_data;
 
+#define FUTILS_MAX_DEPTH 100
+
 static int futils__error_cannot_rmdir(const char *path, const char *filemsg)
 {
 	if (filemsg)
@@ -440,51 +447,60 @@ static int futils__rm_first_parent(git_buf *path, const char *ceiling)
 
 static int futils__rmdir_recurs_foreach(void *opaque, git_buf *path)
 {
-	struct stat st;
 	futils__rmdir_data *data = opaque;
+	int error = data->error;
+	struct stat st;
+
+	if (data->depth > FUTILS_MAX_DEPTH)
+		error = futils__error_cannot_rmdir(
+			path->ptr, "directory nesting too deep");
 
-	if ((data->error = p_lstat_posixly(path->ptr, &st)) < 0) {
+	else if ((error = p_lstat_posixly(path->ptr, &st)) < 0) {
 		if (errno == ENOENT)
-			data->error = 0;
+			error = 0;
 		else if (errno == ENOTDIR) {
 			/* asked to remove a/b/c/d/e and a/b is a normal file */
 			if ((data->flags & GIT_RMDIR_REMOVE_BLOCKERS) != 0)
-				data->error = futils__rm_first_parent(path, data->base);
+				error = futils__rm_first_parent(path, data->base);
 			else
 				futils__error_cannot_rmdir(
 					path->ptr, "parent is not directory");
 		}
 		else
-			futils__error_cannot_rmdir(path->ptr, "cannot access");
+			error = git_path_set_error(errno, path->ptr, "rmdir");
 	}
 
 	else if (S_ISDIR(st.st_mode)) {
-		int error = git_path_direach(path, futils__rmdir_recurs_foreach, data);
+		data->depth++;
+
+		error = git_path_direach(path, 0, futils__rmdir_recurs_foreach, data);
 		if (error < 0)
 			return (error == GIT_EUSER) ? data->error : error;
 
-		data->error = p_rmdir(path->ptr);
+		data->depth--;
+
+		if (data->depth == 0 && (data->flags & GIT_RMDIR_SKIP_ROOT) != 0)
+			return data->error;
 
-		if (data->error < 0) {
+		if ((error = p_rmdir(path->ptr)) < 0) {
 			if ((data->flags & GIT_RMDIR_SKIP_NONEMPTY) != 0 &&
 				(errno == ENOTEMPTY || errno == EEXIST || errno == EBUSY))
-				data->error = 0;
+				error = 0;
 			else
-				futils__error_cannot_rmdir(path->ptr, NULL);
+				error = git_path_set_error(errno, path->ptr, "rmdir");
 		}
 	}
 
 	else if ((data->flags & GIT_RMDIR_REMOVE_FILES) != 0) {
-		data->error = p_unlink(path->ptr);
-
-		if (data->error < 0)
-			futils__error_cannot_rmdir(path->ptr, "cannot be removed");
+		if (p_unlink(path->ptr) < 0)
+			error = git_path_set_error(errno, path->ptr, "remove");
 	}
 
 	else if ((data->flags & GIT_RMDIR_SKIP_NONEMPTY) == 0)
-		data->error = futils__error_cannot_rmdir(path->ptr, "still present");
+		error = futils__error_cannot_rmdir(path->ptr, "still present");
 
-	return data->error;
+	data->error = error;
+	return error;
 }
 
 static int futils__rmdir_empty_parent(void *opaque, git_buf *path)
@@ -507,7 +523,7 @@ static int futils__rmdir_empty_parent(void *opaque, git_buf *path)
 			giterr_clear();
 			error = GIT_ITEROVER;
 		} else {
-			futils__error_cannot_rmdir(git_buf_cstr(path), NULL);
+			error = git_path_set_error(errno, git_buf_cstr(path), "rmdir");
 		}
 	}
 
@@ -519,7 +535,7 @@ int git_futils_rmdir_r(
 {
 	int error;
 	git_buf fullpath = GIT_BUF_INIT;
-	futils__rmdir_data data;
+	futils__rmdir_data data = { 0 };
 
 	/* build path and find "root" where we should start calling mkdir */
 	if (git_path_join_unrooted(&fullpath, path, base, NULL) < 0)
@@ -528,7 +544,6 @@ int git_futils_rmdir_r(
 	data.base    = base ? base : "";
 	data.baselen = base ? strlen(base) : 0;
 	data.flags   = flags;
-	data.error   = 0;
 
 	error = futils__rmdir_recurs_foreach(&data, &fullpath);
 
@@ -546,46 +561,11 @@ int git_futils_rmdir_r(
 	return error;
 }
 
-int git_futils_cleanupdir_r(const char *path)
-{
-	int error;
-	git_buf fullpath = GIT_BUF_INIT;
-	futils__rmdir_data data;
-
-	if ((error = git_buf_put(&fullpath, path, strlen(path))) < 0)
-		goto clean_up;
-
-	data.base    = "";
-	data.baselen = 0;
-	data.flags   = GIT_RMDIR_REMOVE_FILES;
-	data.error   = 0;
-
-	if (!git_path_exists(path)) {
-		giterr_set(GITERR_OS, "Path does not exist: %s" , path);
-		error = GIT_ERROR;
-		goto clean_up;
-	}
-
-	if (!git_path_isdir(path)) {
-		giterr_set(GITERR_OS, "Path is not a directory: %s" , path);
-		error = GIT_ERROR;
-		goto clean_up;
-	}
-
-	error = git_path_direach(&fullpath, futils__rmdir_recurs_foreach, &data);
-	if (error == GIT_EUSER)
-		error = data.error;
-
-clean_up:
-	git_buf_free(&fullpath);
-	return error;
-}
-
 
 static int git_futils_guess_system_dirs(git_buf *out)
 {
 #ifdef GIT_WIN32
-	return git_win32__find_system_dirs(out);
+	return git_win32__find_system_dirs(out, L"etc\\");
 #else
 	return git_buf_sets(out, "/etc");
 #endif
@@ -617,17 +597,48 @@ static int git_futils_guess_xdg_dirs(git_buf *out)
 #endif
 }
 
+static int git_futils_guess_template_dirs(git_buf *out)
+{
+#ifdef GIT_WIN32
+	return git_win32__find_system_dirs(out, L"share\\git-core\\templates");
+#else
+	return git_buf_sets(out, "/usr/share/git-core/templates");
+#endif
+}
+
 typedef int (*git_futils_dirs_guess_cb)(git_buf *out);
 
 static git_buf git_futils__dirs[GIT_FUTILS_DIR__MAX] =
-	{ GIT_BUF_INIT, GIT_BUF_INIT, GIT_BUF_INIT };
+	{ GIT_BUF_INIT, GIT_BUF_INIT, GIT_BUF_INIT, GIT_BUF_INIT };
 
 static git_futils_dirs_guess_cb git_futils__dir_guess[GIT_FUTILS_DIR__MAX] = {
 	git_futils_guess_system_dirs,
 	git_futils_guess_global_dirs,
 	git_futils_guess_xdg_dirs,
+	git_futils_guess_template_dirs,
 };
 
+void git_futils_dirs_global_shutdown(void)
+{
+	int i;
+	for (i = 0; i < GIT_FUTILS_DIR__MAX; ++i)
+		git_buf_free(&git_futils__dirs[i]);
+}
+
+int git_futils_dirs_global_init(void)
+{
+	git_futils_dir_t i;
+	const git_buf *path;
+	int error = 0;
+
+	for (i = 0; !error && i < GIT_FUTILS_DIR__MAX; i++)
+		error = git_futils_dirs_get(&path, i);
+
+	git__on_shutdown(git_futils_dirs_global_shutdown);
+
+	return error;
+}
+
 static int git_futils_check_selector(git_futils_dir_t which)
 {
 	if (which < GIT_FUTILS_DIR__MAX)
@@ -707,13 +718,6 @@ int git_futils_dirs_set(git_futils_dir_t which, const char *search_path)
 	return git_buf_oom(&git_futils__dirs[which]) ? -1 : 0;
 }
 
-void git_futils_dirs_free(void)
-{
-	int i;
-	for (i = 0; i < GIT_FUTILS_DIR__MAX; ++i)
-		git_buf_free(&git_futils__dirs[i]);
-}
-
 static int git_futils_find_in_dirlist(
 	git_buf *path, const char *name, git_futils_dir_t which, const char *label)
 {
@@ -734,7 +738,8 @@ static int git_futils_find_in_dirlist(
 			continue;
 
 		GITERR_CHECK_ERROR(git_buf_set(path, scan, len));
-		GITERR_CHECK_ERROR(git_buf_joinpath(path, path->ptr, name));
+		if (name)
+			GITERR_CHECK_ERROR(git_buf_joinpath(path, path->ptr, name));
 
 		if (git_path_exists(path->ptr))
 			return 0;
@@ -763,6 +768,12 @@ int git_futils_find_xdg_file(git_buf *path, const char *filename)
 		path, filename, GIT_FUTILS_DIR_XDG, "global/xdg");
 }
 
+int git_futils_find_template_dir(git_buf *path)
+{
+	return git_futils_find_in_dirlist(
+		path, NULL, GIT_FUTILS_DIR_TEMPLATE, "template");
+}
+
 int git_futils_fake_symlink(const char *old, const char *new)
 {
 	int retcode = GIT_ERROR;
@@ -807,11 +818,8 @@ int git_futils_cp(const char *from, const char *to, mode_t filemode)
 		return ifd;
 
 	if ((ofd = p_open(to, O_WRONLY | O_CREAT | O_EXCL, filemode)) < 0) {
-		if (errno == ENOENT || errno == ENOTDIR)
-			ofd = GIT_ENOTFOUND;
-		giterr_set(GITERR_OS, "Failed to open '%s' for writing", to);
 		p_close(ifd);
-		return ofd;
+		return git_path_set_error(errno, to, "open for writing");
 	}
 
 	return cp_by_fd(ifd, ofd, true);
@@ -850,6 +858,7 @@ typedef struct {
 	uint32_t flags;
 	uint32_t mkdir_flags;
 	mode_t dirmode;
+	int error;
 } cp_r_info;
 
 #define GIT_CPDIR__MKDIR_DONE_FOR_TO_ROOT (1u << 10)
@@ -888,20 +897,22 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		return 0;
 
 	if (git_buf_joinpath(
-			&info->to, info->to_root, from->ptr + info->from_prefix) < 0)
-		return -1;
+			&info->to, info->to_root, from->ptr + info->from_prefix) < 0) {
+		error = -1;
+		goto exit;
+	}
 
-	if (p_lstat(info->to.ptr, &to_st) < 0) {
-		if (errno != ENOENT && errno != ENOTDIR) {
-			giterr_set(GITERR_OS,
-				"Could not access %s while copying files", info->to.ptr);
-			return -1;
-		}
-	} else
+	if (!(error = git_path_lstat(info->to.ptr, &to_st)))
 		exists = true;
+	else if (error != GIT_ENOTFOUND)
+		goto exit;
+	else {
+		giterr_clear();
+		error = 0;
+	}
 
 	if ((error = git_path_lstat(from->ptr, &from_st)) < 0)
-		return error;
+		goto exit;
 
 	if (S_ISDIR(from_st.st_mode)) {
 		mode_t oldmode = info->dirmode;
@@ -915,13 +926,17 @@ static int _cp_r_callback(void *ref, git_buf *from)
 			error = _cp_r_mkdir(info, from);
 
 		/* recurse onto target directory */
-		if (!error && (!exists || S_ISDIR(to_st.st_mode)))
-			error = git_path_direach(from, _cp_r_callback, info);
+		if (!error && (!exists || S_ISDIR(to_st.st_mode))) {
+			error = git_path_direach(from, 0, _cp_r_callback, info);
+
+			if (error == GIT_EUSER)
+				error = info->error;
+		}
 
 		if (oldmode != 0)
 			info->dirmode = oldmode;
 
-		return error;
+		goto exit;
 	}
 
 	if (exists) {
@@ -931,7 +946,8 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		if (p_unlink(info->to.ptr) < 0) {
 			giterr_set(GITERR_OS, "Cannot overwrite existing file '%s'",
 				info->to.ptr);
-			return -1;
+			error = -1;
+			goto exit;
 		}
 	}
 
@@ -944,7 +960,7 @@ static int _cp_r_callback(void *ref, git_buf *from)
 	/* Make container directory on demand if needed */
 	if ((info->flags & GIT_CPDIR_CREATE_EMPTY_DIRS) == 0 &&
 		(error = _cp_r_mkdir(info, from)) < 0)
-		return error;
+		goto exit;
 
 	/* make symlink or regular file */
 	if (S_ISLNK(from_st.st_mode))
@@ -953,11 +969,13 @@ static int _cp_r_callback(void *ref, git_buf *from)
 		mode_t usemode = from_st.st_mode;
 
 		if ((info->flags & GIT_CPDIR_SIMPLE_TO_MODE) != 0)
-			usemode = (usemode & 0111) ? 0777 : 0666;
+			usemode = GIT_PERMS_FOR_WRITE(usemode);
 
 		error = git_futils_cp(from->ptr, info->to.ptr, usemode);
 	}
 
+exit:
+	info->error = error;
 	return error;
 }
 
@@ -978,6 +996,7 @@ int git_futils_cp_r(
 	info.flags   = flags;
 	info.dirmode = dirmode;
 	info.from_prefix = path.size;
+	info.error = 0;
 	git_buf_init(&info.to, 0);
 
 	/* precalculate mkdir flags */
@@ -999,6 +1018,9 @@ int git_futils_cp_r(
 	git_buf_free(&path);
 	git_buf_free(&info.to);
 
+	if (error == GIT_EUSER)
+		error = info.error;
+
 	return error;
 }
 
diff --git a/src/fileops.h b/src/fileops.h
index f4e059c83..636c9b67d 100644
--- a/src/fileops.h
+++ b/src/fileops.h
@@ -115,12 +115,7 @@ extern int git_futils_mkpath2file(const char *path, const mode_t mode);
  * * GIT_RMDIR_EMPTY_PARENTS   - remove containing directories up to base
  *       if removing this item leaves them empty
  * * GIT_RMDIR_REMOVE_BLOCKERS - remove blocking file that causes ENOTDIR
- *
- * The old values translate into the new as follows:
- *
- * * GIT_DIRREMOVAL_EMPTY_HIERARCHY == GIT_RMDIR_EMPTY_HIERARCHY
- * * GIT_DIRREMOVAL_FILES_AND_DIRS  ~= GIT_RMDIR_REMOVE_FILES
- * * GIT_DIRREMOVAL_ONLY_EMPTY_DIRS == GIT_RMDIR_SKIP_NONEMPTY
+ * * GIT_RMDIR_SKIP_ROOT       - don't remove root directory itself
  */
 typedef enum {
 	GIT_RMDIR_EMPTY_HIERARCHY = 0,
@@ -128,6 +123,7 @@ typedef enum {
 	GIT_RMDIR_SKIP_NONEMPTY   = (1 << 1),
 	GIT_RMDIR_EMPTY_PARENTS   = (1 << 2),
 	GIT_RMDIR_REMOVE_BLOCKERS = (1 << 3),
+	GIT_RMDIR_SKIP_ROOT       = (1 << 4),
 } git_futils_rmdir_flags;
 
 /**
@@ -141,19 +137,11 @@ typedef enum {
 extern int git_futils_rmdir_r(const char *path, const char *base, uint32_t flags);
 
 /**
- * Remove all files and directories beneath the specified path.
- *
- * @param path Path to the top level directory to process.
- * @return 0 on success; -1 on error.
- */
-extern int git_futils_cleanupdir_r(const char *path);
-
-/**
  * Create and open a temporary file with a `_git2_` suffix.
  * Writes the filename into path_out.
  * @return On success, an open file descriptor, else an error code < 0.
  */
-extern int git_futils_mktmp(git_buf *path_out, const char *filename);
+extern int git_futils_mktmp(git_buf *path_out, const char *filename, mode_t mode);
 
 /**
  * Move a file on the filesystem, create the
@@ -223,9 +211,13 @@ extern int git_futils_open_ro(const char *path);
  */
 extern git_off_t git_futils_filesize(git_file fd);
 
+#define GIT_PERMS_IS_EXEC(MODE)		(((MODE) & 0111) != 0)
+#define GIT_PERMS_CANONICAL(MODE)	(GIT_PERMS_IS_EXEC(MODE) ? 0755 : 0644)
+#define GIT_PERMS_FOR_WRITE(MODE)   (GIT_PERMS_IS_EXEC(MODE) ? 0777 : 0666)
+
 #define GIT_MODE_PERMS_MASK			0777
-#define GIT_CANONICAL_PERMS(MODE)	(((MODE) & 0100) ? 0755 : 0644)
-#define GIT_MODE_TYPE(MODE)			((MODE) & ~GIT_MODE_PERMS_MASK)
+#define GIT_MODE_TYPE_MASK			0170000
+#define GIT_MODE_TYPE(MODE)			((MODE) & GIT_MODE_TYPE_MASK)
 #define GIT_MODE_ISBLOB(MODE)		(GIT_MODE_TYPE(MODE) == GIT_MODE_TYPE(GIT_FILEMODE_BLOB))
 
 /**
@@ -244,7 +236,7 @@ extern mode_t git_futils_canonical_mode(mode_t raw_mode);
  * @param out buffer to populate with the mapping information.
  * @param fd open descriptor to configure the mapping from.
  * @param begin first byte to map, this should be page aligned.
- * @param end number of bytes to map.
+ * @param len number of bytes to map.
  * @return
  * - 0 on success;
  * - -1 on error.
@@ -278,7 +270,7 @@ extern void git_futils_mmap_free(git_map *map);
 /**
  * Find a "global" file (i.e. one in a user's home directory).
  *
- * @param pathbuf buffer to write the full path into
+ * @param path buffer to write the full path into
  * @param filename name of file to find in the home directory
  * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
  */
@@ -287,7 +279,7 @@ extern int git_futils_find_global_file(git_buf *path, const char *filename);
 /**
  * Find an "XDG" file (i.e. one in user's XDG config path).
  *
- * @param pathbuf buffer to write the full path into
+ * @param path buffer to write the full path into
  * @param filename name of file to find in the home directory
  * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
  */
@@ -296,20 +288,36 @@ extern int git_futils_find_xdg_file(git_buf *path, const char *filename);
 /**
  * Find a "system" file (i.e. one shared for all users of the system).
  *
- * @param pathbuf buffer to write the full path into
+ * @param path buffer to write the full path into
  * @param filename name of file to find in the home directory
  * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
  */
 extern int git_futils_find_system_file(git_buf *path, const char *filename);
 
+/**
+ * Find template directory.
+ *
+ * @param path buffer to write the full path into
+ * @return 0 if found, GIT_ENOTFOUND if not found, or -1 on other OS error
+ */
+extern int git_futils_find_template_dir(git_buf *path);
+
 typedef enum {
 	GIT_FUTILS_DIR_SYSTEM = 0,
 	GIT_FUTILS_DIR_GLOBAL = 1,
 	GIT_FUTILS_DIR_XDG    = 2,
-	GIT_FUTILS_DIR__MAX   = 3,
+	GIT_FUTILS_DIR_TEMPLATE = 3,
+	GIT_FUTILS_DIR__MAX   = 4,
 } git_futils_dir_t;
 
 /**
+ * Configures global data for configuration file search paths.
+ *
+ * @return 0 on success, <0 on failure
+ */
+extern int git_futils_dirs_global_init(void);
+
+/**
  * Get the search path for global/system/xdg files
  *
  * @param out pointer to git_buf containing search path
@@ -343,11 +351,6 @@ extern int git_futils_dirs_get_str(
 extern int git_futils_dirs_set(git_futils_dir_t which, const char *paths);
 
 /**
- * Release / reset all search paths
- */
-extern void git_futils_dirs_free(void);
-
-/**
  * Create a "fake" symlink (text file containing the target path).
  *
  * @param new symlink file to be created
@@ -396,4 +399,9 @@ extern int git_futils_filestamp_check(
 extern void git_futils_filestamp_set(
 	git_futils_filestamp *tgt, const git_futils_filestamp *src);
 
+/**
+ * Free the configuration file search paths.
+ */
+extern void git_futils_dirs_global_shutdown(void);
+
 #endif /* INCLUDE_fileops_h__ */
diff --git a/src/filter.c b/src/filter.c
index 9f749dcbd..9f866fe88 100644
--- a/src/filter.c
+++ b/src/filter.c
@@ -10,68 +10,594 @@
 #include "hash.h"
 #include "filter.h"
 #include "repository.h"
+#include "global.h"
+#include "git2/sys/filter.h"
 #include "git2/config.h"
 #include "blob.h"
+#include "attr_file.h"
+#include "array.h"
 
-int git_filters_load(git_vector *filters, git_repository *repo, const char *path, int mode)
+struct git_filter_source {
+	git_repository *repo;
+	const char     *path;
+	git_oid         oid;  /* zero if unknown (which is likely) */
+	uint16_t        filemode; /* zero if unknown */
+	git_filter_mode_t mode;
+};
+
+typedef struct {
+	git_filter *filter;
+	void *payload;
+} git_filter_entry;
+
+struct git_filter_list {
+	git_array_t(git_filter_entry) filters;
+	git_filter_source source;
+	char path[GIT_FLEX_ARRAY];
+};
+
+typedef struct {
+	const char *filter_name;
+	git_filter *filter;
+	int priority;
+	int initialized;
+	size_t nattrs, nmatches;
+	char *attrdata;
+	const char *attrs[GIT_FLEX_ARRAY];
+} git_filter_def;
+
+static int filter_def_priority_cmp(const void *a, const void *b)
 {
-	int error;
+	int pa = ((const git_filter_def *)a)->priority;
+	int pb = ((const git_filter_def *)b)->priority;
+	return (pa < pb) ? -1 : (pa > pb) ? 1 : 0;
+}
 
-	if (mode == GIT_FILTER_TO_ODB) {
-		/* Load the CRLF cleanup filter when writing to the ODB */
-		error = git_filter_add__crlf_to_odb(filters, repo, path);
-		if (error < 0)
-			return error;
-	} else {
-		error = git_filter_add__crlf_to_workdir(filters, repo, path);
-		if (error < 0)
-			return error;
+struct filter_registry {
+	git_vector filters;
+};
+
+static struct filter_registry *git__filter_registry = NULL;
+
+static void filter_registry_shutdown(void)
+{
+	struct filter_registry *reg = NULL;
+	size_t pos;
+	git_filter_def *fdef;
+
+	if ((reg = git__swap(git__filter_registry, NULL)) == NULL)
+		return;
+
+	git_vector_foreach(&reg->filters, pos, fdef) {
+		if (fdef->initialized && fdef->filter && fdef->filter->shutdown) {
+			fdef->filter->shutdown(fdef->filter);
+			fdef->initialized = false;
+		}
+
+		git__free(fdef->attrdata);
+		git__free(fdef);
 	}
 
-	return (int)filters->length;
+	git_vector_free(&reg->filters);
+	git__free(reg);
 }
 
-void git_filters_free(git_vector *filters)
+static int filter_registry_initialize(void)
 {
-	size_t i;
-	git_filter *filter;
+	int error = 0;
+	struct filter_registry *reg;
+
+	if (git__filter_registry)
+		return 0;
+
+	reg = git__calloc(1, sizeof(struct filter_registry));
+	GITERR_CHECK_ALLOC(reg);
+
+	if ((error = git_vector_init(
+			&reg->filters, 2, filter_def_priority_cmp)) < 0)
+		goto cleanup;
+
+	reg = git__compare_and_swap(&git__filter_registry, NULL, reg);
+	if (reg != NULL)
+		goto cleanup;
+
+	git__on_shutdown(filter_registry_shutdown);
+
+	/* try to register both default filters */
+	{
+		git_filter *crlf = git_crlf_filter_new();
+		git_filter *ident = git_ident_filter_new();
+
+		if (crlf && git_filter_register(
+				GIT_FILTER_CRLF, crlf, GIT_FILTER_CRLF_PRIORITY) < 0)
+			crlf = NULL;
+		if (ident && git_filter_register(
+				GIT_FILTER_IDENT, ident, GIT_FILTER_IDENT_PRIORITY) < 0)
+			ident = NULL;
+
+		if (!crlf || !ident)
+			return -1;
+	}
+
+	return 0;
+
+cleanup:
+	git_vector_free(&reg->filters);
+	git__free(reg);
+	return error;
+}
+
+static int filter_def_scan_attrs(
+	git_buf *attrs, size_t *nattr, size_t *nmatch, const char *attr_str)
+{
+	const char *start, *scan = attr_str;
+	int has_eq;
+
+	*nattr = *nmatch = 0;
+
+	if (!scan)
+		return 0;
+
+	while (*scan) {
+		while (git__isspace(*scan)) scan++;
+
+		for (start = scan, has_eq = 0; *scan && !git__isspace(*scan); ++scan) {
+			if (*scan == '=')
+				has_eq = 1;
+		}
+
+		if (scan > start) {
+			(*nattr)++;
+			if (has_eq || *start == '-' || *start == '+' || *start == '!')
+				(*nmatch)++;
 
-	git_vector_foreach(filters, i, filter) {
-		if (filter->do_free != NULL)
-			filter->do_free(filter);
-		else
-			git__free(filter);
+			if (has_eq)
+				git_buf_putc(attrs, '=');
+			git_buf_put(attrs, start, scan - start);
+			git_buf_putc(attrs, '\0');
+		}
 	}
 
-	git_vector_free(filters);
+	return 0;
 }
 
-int git_filters_apply(git_buf *dest, git_buf *source, git_vector *filters)
+static void filter_def_set_attrs(git_filter_def *fdef)
 {
+	char *scan = fdef->attrdata;
 	size_t i;
-	unsigned int src;
-	git_buf *dbuffer[2];
 
-	dbuffer[0] = source;
-	dbuffer[1] = dest;
+	for (i = 0; i < fdef->nattrs; ++i) {
+		const char *name, *value;
+
+		switch (*scan) {
+		case '=':
+			name = scan + 1;
+			for (scan++; *scan != '='; scan++) /* find '=' */;
+			*scan++ = '\0';
+			value = scan;
+			break;
+		case '-':
+			name = scan + 1; value = git_attr__false; break;
+		case '+':
+			name = scan + 1; value = git_attr__true;  break;
+		case '!':
+			name = scan + 1; value = git_attr__unset; break;
+		default:
+			name = scan;     value = NULL; break;
+		}
+
+		fdef->attrs[i] = name;
+		fdef->attrs[i + fdef->nattrs] = value;
+
+		scan += strlen(scan) + 1;
+	}
+}
+
+static int filter_def_name_key_check(const void *key, const void *fdef)
+{
+	const char *name =
+		fdef ? ((const git_filter_def *)fdef)->filter_name : NULL;
+	return name ? git__strcmp(key, name) : -1;
+}
 
-	src = 0;
+static int filter_def_filter_key_check(const void *key, const void *fdef)
+{
+	const void *filter = fdef ? ((const git_filter_def *)fdef)->filter : NULL;
+	return (key == filter) ? 0 : -1;
+}
+
+static int filter_registry_find(size_t *pos, const char *name)
+{
+	return git_vector_search2(
+		pos, &git__filter_registry->filters, filter_def_name_key_check, name);
+}
 
-	if (git_buf_len(source) == 0) {
-		git_buf_clear(dest);
+static git_filter_def *filter_registry_lookup(size_t *pos, const char *name)
+{
+	git_filter_def *fdef = NULL;
+
+	if (!filter_registry_find(pos, name))
+		fdef = git_vector_get(&git__filter_registry->filters, *pos);
+
+	return fdef;
+}
+
+int git_filter_register(
+	const char *name, git_filter *filter, int priority)
+{
+	git_filter_def *fdef;
+	size_t nattr = 0, nmatch = 0;
+	git_buf attrs = GIT_BUF_INIT;
+
+	if (filter_registry_initialize() < 0)
+		return -1;
+
+	if (!filter_registry_find(NULL, name)) {
+		giterr_set(
+			GITERR_FILTER, "Attempt to reregister existing filter '%s'", name);
+		return GIT_EEXISTS;
+	}
+
+	if (filter_def_scan_attrs(&attrs, &nattr, &nmatch, filter->attributes) < 0)
+		return -1;
+
+	fdef = git__calloc(
+		sizeof(git_filter_def) + 2 * nattr * sizeof(char *), 1);
+	GITERR_CHECK_ALLOC(fdef);
+
+	fdef->filter_name = name;
+	fdef->filter      = filter;
+	fdef->priority    = priority;
+	fdef->nattrs      = nattr;
+	fdef->nmatches    = nmatch;
+	fdef->attrdata    = git_buf_detach(&attrs);
+
+	filter_def_set_attrs(fdef);
+
+	if (git_vector_insert(&git__filter_registry->filters, fdef) < 0) {
+		git__free(fdef->attrdata);
+		git__free(fdef);
+		return -1;
+	}
+
+	git_vector_sort(&git__filter_registry->filters);
+	return 0;
+}
+
+int git_filter_unregister(const char *name)
+{
+	size_t pos;
+	git_filter_def *fdef;
+
+	/* cannot unregister default filters */
+	if (!strcmp(GIT_FILTER_CRLF, name) || !strcmp(GIT_FILTER_IDENT, name)) {
+		giterr_set(GITERR_FILTER, "Cannot unregister filter '%s'", name);
+		return -1;
+	}
+
+	if ((fdef = filter_registry_lookup(&pos, name)) == NULL) {
+		giterr_set(GITERR_FILTER, "Cannot find filter '%s' to unregister", name);
+		return GIT_ENOTFOUND;
+	}
+
+	(void)git_vector_remove(&git__filter_registry->filters, pos);
+
+	if (fdef->initialized && fdef->filter && fdef->filter->shutdown) {
+		fdef->filter->shutdown(fdef->filter);
+		fdef->initialized = false;
+	}
+
+	git__free(fdef->attrdata);
+	git__free(fdef);
+
+	return 0;
+}
+
+static int filter_initialize(git_filter_def *fdef)
+{
+	int error = 0;
+
+	if (!fdef->initialized &&
+		fdef->filter &&
+		fdef->filter->initialize &&
+		(error = fdef->filter->initialize(fdef->filter)) < 0)
+	{
+		/* auto-unregister if initialize fails */
+		git_filter_unregister(fdef->filter_name);
+		return error;
+	}
+
+	fdef->initialized = true;
+	return 0;
+}
+
+git_filter *git_filter_lookup(const char *name)
+{
+	size_t pos;
+	git_filter_def *fdef;
+
+	if (filter_registry_initialize() < 0)
+		return NULL;
+
+	if ((fdef = filter_registry_lookup(&pos, name)) == NULL)
+		return NULL;
+
+	if (!fdef->initialized && filter_initialize(fdef) < 0)
+		return NULL;
+
+	return fdef->filter;
+}
+
+void git_filter_free(git_filter *filter)
+{
+	git__free(filter);
+}
+
+git_repository *git_filter_source_repo(const git_filter_source *src)
+{
+	return src->repo;
+}
+
+const char *git_filter_source_path(const git_filter_source *src)
+{
+	return src->path;
+}
+
+uint16_t git_filter_source_filemode(const git_filter_source *src)
+{
+	return src->filemode;
+}
+
+const git_oid *git_filter_source_id(const git_filter_source *src)
+{
+	return git_oid_iszero(&src->oid) ? NULL : &src->oid;
+}
+
+git_filter_mode_t git_filter_source_mode(const git_filter_source *src)
+{
+	return src->mode;
+}
+
+static int filter_list_new(
+	git_filter_list **out, const git_filter_source *src)
+{
+	git_filter_list *fl = NULL;
+	size_t pathlen = src->path ? strlen(src->path) : 0;
+
+	fl = git__calloc(1, sizeof(git_filter_list) + pathlen + 1);
+	GITERR_CHECK_ALLOC(fl);
+
+	if (src->path)
+		memcpy(fl->path, src->path, pathlen);
+	fl->source.repo = src->repo;
+	fl->source.path = fl->path;
+	fl->source.mode = src->mode;
+
+	*out = fl;
+	return 0;
+}
+
+static int filter_list_check_attributes(
+	const char ***out, git_filter_def *fdef, const git_filter_source *src)
+{
+	int error;
+	size_t i;
+	const char **strs = git__calloc(fdef->nattrs, sizeof(const char *));
+	GITERR_CHECK_ALLOC(strs);
+
+	error = git_attr_get_many(
+		strs, src->repo, 0, src->path, fdef->nattrs, fdef->attrs);
+
+	/* if no values were found but no matches are needed, it's okay! */
+	if (error == GIT_ENOTFOUND && !fdef->nmatches) {
+		giterr_clear();
+		git__free((void *)strs);
 		return 0;
 	}
 
-	/* Pre-grow the destination buffer to more or less the size
-	 * we expect it to have */
-	if (git_buf_grow(dest, git_buf_len(source)) < 0)
+	for (i = 0; !error && i < fdef->nattrs; ++i) {
+		const char *want = fdef->attrs[fdef->nattrs + i];
+		git_attr_t want_type, found_type;
+
+		if (!want)
+			continue;
+
+		want_type  = git_attr_value(want);
+		found_type = git_attr_value(strs[i]);
+
+		if (want_type != found_type ||
+			(want_type == GIT_ATTR_VALUE_T && strcmp(want, strs[i])))
+			error = GIT_ENOTFOUND;
+	}
+
+	if (error)
+		git__free((void *)strs);
+	else
+		*out = strs;
+
+	return error;
+}
+
+int git_filter_list_new(
+	git_filter_list **out, git_repository *repo, git_filter_mode_t mode)
+{
+	git_filter_source src = { 0 };
+	src.repo = repo;
+	src.path = NULL;
+	src.mode = mode;
+	return filter_list_new(out, &src);
+}
+
+int git_filter_list_load(
+	git_filter_list **filters,
+	git_repository *repo,
+	git_blob *blob, /* can be NULL */
+	const char *path,
+	git_filter_mode_t mode)
+{
+	int error = 0;
+	git_filter_list *fl = NULL;
+	git_filter_source src = { 0 };
+	git_filter_entry *fe;
+	size_t idx;
+	git_filter_def *fdef;
+
+	if (filter_registry_initialize() < 0)
+		return -1;
+
+	src.repo = repo;
+	src.path = path;
+	src.mode = mode;
+	if (blob)
+		git_oid_cpy(&src.oid, git_blob_id(blob));
+
+	git_vector_foreach(&git__filter_registry->filters, idx, fdef) {
+		const char **values = NULL;
+		void *payload = NULL;
+
+		if (!fdef || !fdef->filter)
+			continue;
+
+		if (fdef->nattrs > 0) {
+			error = filter_list_check_attributes(&values, fdef, &src);
+			if (error == GIT_ENOTFOUND) {
+				error = 0;
+				continue;
+			} else if (error < 0)
+				break;
+		}
+
+		if (!fdef->initialized && (error = filter_initialize(fdef)) < 0)
+			break;
+
+		if (fdef->filter->check)
+			error = fdef->filter->check(
+				fdef->filter, &payload, &src, values);
+
+		git__free((void *)values);
+
+		if (error == GIT_PASSTHROUGH)
+			error = 0;
+		else if (error < 0)
+			break;
+		else {
+			if (!fl && (error = filter_list_new(&fl, &src)) < 0)
+				return error;
+
+			fe = git_array_alloc(fl->filters);
+			GITERR_CHECK_ALLOC(fe);
+			fe->filter  = fdef->filter;
+			fe->payload = payload;
+		}
+	}
+
+	if (error && fl != NULL) {
+		git_array_clear(fl->filters);
+		git__free(fl);
+		fl = NULL;
+	}
+
+	*filters = fl;
+	return error;
+}
+
+void git_filter_list_free(git_filter_list *fl)
+{
+	uint32_t i;
+
+	if (!fl)
+		return;
+
+	for (i = 0; i < git_array_size(fl->filters); ++i) {
+		git_filter_entry *fe = git_array_get(fl->filters, i);
+		if (fe->filter->cleanup)
+			fe->filter->cleanup(fe->filter, fe->payload);
+	}
+
+	git_array_clear(fl->filters);
+	git__free(fl);
+}
+
+int git_filter_list_push(
+	git_filter_list *fl, git_filter *filter, void *payload)
+{
+	int error = 0;
+	size_t pos;
+	git_filter_def *fdef;
+	git_filter_entry *fe;
+
+	assert(fl && filter);
+
+	if (git_vector_search2(
+			&pos, &git__filter_registry->filters,
+			filter_def_filter_key_check, filter) < 0) {
+		giterr_set(GITERR_FILTER, "Cannot use an unregistered filter");
 		return -1;
+	}
+
+	fdef = git_vector_get(&git__filter_registry->filters, pos);
+
+	if (!fdef->initialized && (error = filter_initialize(fdef)) < 0)
+		return error;
+
+	fe = git_array_alloc(fl->filters);
+	GITERR_CHECK_ALLOC(fe);
+	fe->filter  = filter;
+	fe->payload = payload;
+
+	return 0;
+}
+
+size_t git_filter_list_length(const git_filter_list *fl)
+{
+	return fl ? git_array_size(fl->filters) : 0;
+}
+
+static int filter_list_out_buffer_from_raw(
+	git_buf *out, const void *ptr, size_t size)
+{
+	if (git_buf_is_allocated(out))
+		git_buf_free(out);
 
-	for (i = 0; i < filters->length; ++i) {
-		git_filter *filter = git_vector_get(filters, i);
-		unsigned int dst = 1 - src;
+	if (!size) {
+		git_buf_init(out, 0);
+	} else {
+		out->ptr   = (char *)ptr;
+		out->asize = 0;
+		out->size  = size;
+	}
+
+	return 0;
+}
+
+int git_filter_list_apply_to_data(
+	git_buf *tgt, git_filter_list *fl, git_buf *src)
+{
+	int error = 0;
+	uint32_t i;
+	git_buf *dbuffer[2], local = GIT_BUF_INIT;
+	unsigned int si = 0;
 
-		git_buf_clear(dbuffer[dst]);
+	if (!fl)
+		return filter_list_out_buffer_from_raw(tgt, src->ptr, src->size);
+
+	dbuffer[0] = src;
+	dbuffer[1] = tgt;
+
+	/* if `src` buffer is reallocable, then use it, otherwise copy it */
+	if (!git_buf_is_allocated(src)) {
+		if (git_buf_set(&local, src->ptr, src->size) < 0)
+			return -1;
+		dbuffer[0] = &local;
+	}
+
+	for (i = 0; i < git_array_size(fl->filters); ++i) {
+		unsigned int di = 1 - si;
+		uint32_t fidx = (fl->source.mode == GIT_FILTER_TO_WORKTREE) ?
+			i : git_array_size(fl->filters) - 1 - i;
+		git_filter_entry *fe = git_array_get(fl->filters, fidx);
+
+		dbuffer[di]->size = 0;
 
 		/* Apply the filter from dbuffer[src] to the other buffer;
 		 * if the filtering is canceled by the user mid-filter,
@@ -79,16 +605,72 @@ int git_filters_apply(git_buf *dest, git_buf *source, git_vector *filters)
 		 * of the double buffering (so that the text goes through
 		 * cleanly).
 		 */
-		if (filter->apply(filter, dbuffer[dst], dbuffer[src]) == 0)
-			src = dst;
 
-		if (git_buf_oom(dbuffer[dst]))
-			return -1;
+		error = fe->filter->apply(
+			fe->filter, &fe->payload, dbuffer[di], dbuffer[si], &fl->source);
+
+		if (error == GIT_PASSTHROUGH) {
+			/* PASSTHROUGH means filter decided not to process the buffer */
+			error = 0;
+		} else if (!error) {
+			git_buf_shorten(dbuffer[di], 0); /* force NUL termination */
+			si = di; /* swap buffers */
+		} else {
+			tgt->size = 0;
+			return error;
+		}
 	}
 
 	/* Ensure that the output ends up in dbuffer[1] (i.e. the dest) */
-	if (src != 1)
-		git_buf_swap(dest, source);
+	if (si != 1)
+		git_buf_swap(dbuffer[0], dbuffer[1]);
+
+	git_buf_free(&local); /* don't leak if we allocated locally */
 
 	return 0;
 }
+
+int git_filter_list_apply_to_file(
+	git_buf *out,
+	git_filter_list *filters,
+	git_repository *repo,
+	const char *path)
+{
+	int error;
+	const char *base = repo ? git_repository_workdir(repo) : NULL;
+	git_buf abspath = GIT_BUF_INIT, raw = GIT_BUF_INIT;
+
+	if (!(error = git_path_join_unrooted(&abspath, path, base, NULL)) &&
+		!(error = git_futils_readbuffer(&raw, abspath.ptr)))
+	{
+		error = git_filter_list_apply_to_data(out, filters, &raw);
+
+		git_buf_free(&raw);
+	}
+
+	git_buf_free(&abspath);
+	return error;
+}
+
+int git_filter_list_apply_to_blob(
+	git_buf *out,
+	git_filter_list *filters,
+	git_blob *blob)
+{
+	git_buf in = GIT_BUF_INIT;
+	git_off_t rawsize = git_blob_rawsize(blob);
+
+	if (!git__is_sizet(rawsize)) {
+		giterr_set(GITERR_OS, "Blob is too large to filter");
+		return -1;
+	}
+
+	in.ptr   = (char *)git_blob_rawcontent(blob);
+	in.asize = 0;
+	in.size  = (size_t)rawsize;
+
+	if (filters)
+		git_oid_cpy(&filters->source.oid, git_blob_id(blob));
+
+	return git_filter_list_apply_to_data(out, filters, &in);
+}
diff --git a/src/filter.h b/src/filter.h
index 42a44ebdb..d0ace0f9a 100644
--- a/src/filter.h
+++ b/src/filter.h
@@ -8,19 +8,7 @@
 #define INCLUDE_filter_h__
 
 #include "common.h"
-#include "buffer.h"
-#include "git2/odb.h"
-#include "git2/repository.h"
-
-typedef struct git_filter {
-	int (*apply)(struct git_filter *self, git_buf *dest, const git_buf *source);
-	void (*do_free)(struct git_filter *self);
-} git_filter;
-
-typedef enum {
-	GIT_FILTER_TO_WORKTREE,
-	GIT_FILTER_TO_ODB
-} git_filter_mode;
+#include "git2/filter.h"
 
 typedef enum {
 	GIT_CRLF_GUESS = -1,
@@ -31,64 +19,13 @@ typedef enum {
 	GIT_CRLF_AUTO,
 } git_crlf_t;
 
-/*
- * FILTER API
- */
-
-/*
- * For any given path in the working directory, fill the `filters`
- * array with the relevant filters that need to be applied.
- *
- * Mode is either `GIT_FILTER_TO_WORKTREE` if you need to load the
- * filters that will be used when checking out a file to the working
- * directory, or `GIT_FILTER_TO_ODB` for the filters used when writing
- * a file to the ODB.
- *
- * @param filters Vector where to store all the loaded filters
- * @param repo Repository object that contains `path`
- * @param path Relative path of the file to be filtered
- * @param mode Filtering direction (WT->ODB or ODB->WT)
- * @return the number of filters loaded for the file (0 if the file
- *	doesn't need filtering), or a negative error code
- */
-extern int git_filters_load(git_vector *filters, git_repository *repo, const char *path, int mode);
-
-/*
- * Apply one or more filters to a file.
- *
- * The file must have been loaded as a `git_buf` object. Both the `source`
- * and `dest` buffers are owned by the caller and must be freed once
- * they are no longer needed.
- *
- * NOTE: Because of the double-buffering schema, the `source` buffer that contains
- * the original file may be tampered once the filtering is complete. Regardless, 
- * the `dest` buffer will always contain the final result of the filtering
- *
- * @param dest Buffer to store the result of the filtering
- * @param source Buffer containing the document to filter
- * @param filters A non-empty vector of filters as supplied by `git_filters_load`
- * @return 0 on success, an error code otherwise
- */
-extern int git_filters_apply(git_buf *dest, git_buf *source, git_vector *filters);
-
-/*
- * Free the `filters` array generated by `git_filters_load`.
- *
- * Note that this frees both the array and its contents. The array will
- * be clean/reusable after this call.
- *
- * @param filters A filters array as supplied by `git_filters_load`
- */
-extern void git_filters_free(git_vector *filters);
+extern void git_filter_free(git_filter *filter);
 
 /*
  * Available filters
  */
 
-/* Strip CRLF, from Worktree to ODB */
-extern int git_filter_add__crlf_to_odb(git_vector *filters, git_repository *repo, const char *path);
-
-/* Add CRLF, from ODB to worktree */
-extern int git_filter_add__crlf_to_workdir(git_vector *filters, git_repository *repo, const char *path);
+extern git_filter *git_crlf_filter_new(void);
+extern git_filter *git_ident_filter_new(void);
 
 #endif
diff --git a/src/global.c b/src/global.c
index 2d40ca2fc..7d39c6fa8 100644
--- a/src/global.c
+++ b/src/global.c
@@ -14,6 +14,28 @@
 
 git_mutex git__mwindow_mutex;
 
+#define MAX_SHUTDOWN_CB 8
+
+git_global_shutdown_fn git__shutdown_callbacks[MAX_SHUTDOWN_CB];
+git_atomic git__n_shutdown_callbacks;
+
+void git__on_shutdown(git_global_shutdown_fn callback)
+{
+	int count = git_atomic_inc(&git__n_shutdown_callbacks);
+	assert(count <= MAX_SHUTDOWN_CB);
+	git__shutdown_callbacks[count - 1] = callback;
+}
+
+static void git__shutdown(void)
+{
+	int pos;
+
+	while ((pos = git_atomic_dec(&git__n_shutdown_callbacks)) >= 0) {
+		if (git__shutdown_callbacks[pos])
+			git__shutdown_callbacks[pos]();
+	}
+}
+
 /**
  * Handle the global state with TLS
  *
@@ -51,47 +73,69 @@ git_mutex git__mwindow_mutex;
 #if defined(GIT_THREADS) && defined(GIT_WIN32)
 
 static DWORD _tls_index;
-static int _tls_init = 0;
+static DWORD _mutex = 0;
+static DWORD _n_inits = 0;
 
-int git_threads_init(void)
+static int synchronized_threads_init()
 {
 	int error;
 
-	if (_tls_init)
-		return 0;
-
 	_tls_index = TlsAlloc();
 	if (git_mutex_init(&git__mwindow_mutex))
 		return -1;
 
 	/* Initialize any other subsystems that have global state */
 	if ((error = git_hash_global_init()) >= 0)
-		_tls_init = 1;
+		error = git_futils_dirs_global_init();
 
-	if (error == 0)
-		_tls_init = 1;
+	win32_pthread_initialize();
 
-	GIT_MEMORY_BARRIER;
+	return error;
+}
+
+int git_threads_init(void)
+{
+	int error = 0;
+
+	/* Enter the lock */
+	while (InterlockedCompareExchange(&_mutex, 1, 0)) { Sleep(0); }
+
+	/* Only do work on a 0 -> 1 transition of the refcount */
+	if (1 == ++_n_inits)
+		error = synchronized_threads_init();
+
+	/* Exit the lock */
+	InterlockedExchange(&_mutex, 0);
 
 	return error;
 }
 
-void git_threads_shutdown(void)
+static void synchronized_threads_shutdown()
 {
+	/* Shut down any subsystems that have global state */
+	git__shutdown();
 	TlsFree(_tls_index);
-	_tls_init = 0;
 	git_mutex_free(&git__mwindow_mutex);
+}
 
-	/* Shut down any subsystems that have global state */
-	git_hash_global_shutdown();
-	git_futils_dirs_free();
+void git_threads_shutdown(void)
+{
+	/* Enter the lock */
+	while (InterlockedCompareExchange(&_mutex, 1, 0)) { Sleep(0); }
+
+	/* Only do work on a 1 -> 0 transition of the refcount */
+	if (0 == --_n_inits)
+		synchronized_threads_shutdown();
+
+	/* Exit the lock */
+	InterlockedExchange(&_mutex, 0);
 }
 
 git_global_st *git__global_state(void)
 {
 	void *ptr;
 
-	assert(_tls_init);
+	assert(_n_inits);
 
 	if ((ptr = TlsGetValue(_tls_index)) != NULL)
 		return ptr;
@@ -108,55 +152,58 @@ git_global_st *git__global_state(void)
 #elif defined(GIT_THREADS) && defined(_POSIX_THREADS)
 
 static pthread_key_t _tls_key;
-static int _tls_init = 0;
+static pthread_once_t _once_init = PTHREAD_ONCE_INIT;
+static git_atomic git__n_inits;
+int init_error = 0;
 
 static void cb__free_status(void *st)
 {
 	git__free(st);
 }
 
-int git_threads_init(void)
+static void init_once(void)
 {
-	int error = 0;
-
-	if (_tls_init)
-		return 0;
-
-	if (git_mutex_init(&git__mwindow_mutex))
-		return -1;
+	if ((init_error = git_mutex_init(&git__mwindow_mutex)) != 0)
+		return;
 	pthread_key_create(&_tls_key, &cb__free_status);
 
 	/* Initialize any other subsystems that have global state */
-	if ((error = git_hash_global_init()) >= 0)
-		_tls_init = 1;
+	if ((init_error = git_hash_global_init()) >= 0)
+		init_error = git_futils_dirs_global_init();
 
 	GIT_MEMORY_BARRIER;
+}
 
-	return error;
+int git_threads_init(void)
+{
+	pthread_once(&_once_init, init_once);
+	git_atomic_inc(&git__n_inits);
+	return init_error;
 }
 
 void git_threads_shutdown(void)
 {
-	if (_tls_init) {
-		void *ptr = pthread_getspecific(_tls_key);
-		pthread_setspecific(_tls_key, NULL);
-		git__free(ptr);
-	}
+	pthread_once_t new_once = PTHREAD_ONCE_INIT;
 
-	pthread_key_delete(_tls_key);
-	_tls_init = 0;
-	git_mutex_free(&git__mwindow_mutex);
+	if (git_atomic_dec(&git__n_inits) > 0) return;
 
 	/* Shut down any subsystems that have global state */
-	git_hash_global_shutdown();
-	git_futils_dirs_free();
+	git__shutdown();
+
+	void *ptr = pthread_getspecific(_tls_key);
+	pthread_setspecific(_tls_key, NULL);
+	git__free(ptr);
+
+	pthread_key_delete(_tls_key);
+	git_mutex_free(&git__mwindow_mutex);
+	_once_init = new_once;
 }
 
 git_global_st *git__global_state(void)
 {
 	void *ptr;
 
-	assert(_tls_init);
+	assert(git__n_inits.val);
 
 	if ((ptr = pthread_getspecific(_tls_key)) != NULL)
 		return ptr;
@@ -176,15 +223,14 @@ static git_global_st __state;
 
 int git_threads_init(void)
 {
-	/* noop */ 
+	/* noop */
 	return 0;
 }
 
 void git_threads_shutdown(void)
 {
 	/* Shut down any subsystems that have global state */
-	git_hash_global_shutdown();
-	git_futils_dirs_free();
+	git__shutdown();
 }
 
 git_global_st *git__global_state(void)
diff --git a/src/global.h b/src/global.h
index badbc0883..778250376 100644
--- a/src/global.h
+++ b/src/global.h
@@ -21,4 +21,8 @@ extern git_mutex git__mwindow_mutex;
 
 #define GIT_GLOBAL (git__global_state())
 
+typedef void (*git_global_shutdown_fn)(void);
+
+extern void git__on_shutdown(git_global_shutdown_fn callback);
+
 #endif
diff --git a/src/hash.h b/src/hash.h
index 5b848981f..c47f33549 100644
--- a/src/hash.h
+++ b/src/hash.h
@@ -13,8 +13,6 @@ typedef struct git_hash_prov git_hash_prov;
 typedef struct git_hash_ctx git_hash_ctx;
 
 int git_hash_global_init(void);
-void git_hash_global_shutdown(void);
-
 int git_hash_ctx_init(git_hash_ctx *ctx);
 void git_hash_ctx_cleanup(git_hash_ctx *ctx);
 
diff --git a/src/hash/hash_generic.h b/src/hash/hash_generic.h
index 6b60c98c4..daeb1cda8 100644
--- a/src/hash/hash_generic.h
+++ b/src/hash/hash_generic.h
@@ -17,7 +17,6 @@ struct git_hash_ctx {
 };
 
 #define git_hash_global_init() 0
-#define git_hash_global_shutdown() /* noop */
 #define git_hash_ctx_init(ctx) git_hash_init(ctx)
 #define git_hash_ctx_cleanup(ctx)
 
diff --git a/src/hash/hash_openssl.h b/src/hash/hash_openssl.h
index f83279a5a..9a55d472d 100644
--- a/src/hash/hash_openssl.h
+++ b/src/hash/hash_openssl.h
@@ -17,7 +17,6 @@ struct git_hash_ctx {
 };
 
 #define git_hash_global_init() 0
-#define git_hash_global_shutdown() /* noop */
 #define git_hash_ctx_init(ctx) git_hash_init(ctx)
 #define git_hash_ctx_cleanup(ctx)
 
diff --git a/src/hash/hash_win32.c b/src/hash/hash_win32.c
index 43d54ca6d..bb2231364 100644
--- a/src/hash/hash_win32.c
+++ b/src/hash/hash_win32.c
@@ -20,33 +20,16 @@ static struct git_hash_prov hash_prov = {0};
 /* Initialize CNG, if available */
 GIT_INLINE(int) hash_cng_prov_init(void)
 {
-	OSVERSIONINFOEX version_test = {0};
-	DWORD version_test_mask;
-	DWORDLONG version_condition_mask = 0;
 	char dll_path[MAX_PATH];
 	DWORD dll_path_len, size_len;
 
-	return -1;
-
 	/* Only use CNG on Windows 2008 / Vista SP1  or better (Windows 6.0 SP1) */
-	version_test.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEX);
-	version_test.dwMajorVersion = 6;
-	version_test.dwMinorVersion = 0;
-	version_test.wServicePackMajor = 1;
-	version_test.wServicePackMinor = 0;
-
-	version_test_mask = (VER_MAJORVERSION | VER_MINORVERSION | VER_SERVICEPACKMAJOR | VER_SERVICEPACKMINOR);
-
-	VER_SET_CONDITION(version_condition_mask, VER_MAJORVERSION, VER_GREATER_EQUAL);
-	VER_SET_CONDITION(version_condition_mask, VER_MINORVERSION, VER_GREATER_EQUAL);
-	VER_SET_CONDITION(version_condition_mask, VER_SERVICEPACKMAJOR, VER_GREATER_EQUAL);
-	VER_SET_CONDITION(version_condition_mask, VER_SERVICEPACKMINOR, VER_GREATER_EQUAL);
-
-	if (!VerifyVersionInfo(&version_test, version_test_mask, version_condition_mask))
+	if (!git_has_win32_version(6, 0, 1))
 		return -1;
 
 	/* Load bcrypt.dll explicitly from the system directory */
-	if ((dll_path_len = GetSystemDirectory(dll_path, MAX_PATH)) == 0 || dll_path_len > MAX_PATH ||
+	if ((dll_path_len = GetSystemDirectory(dll_path, MAX_PATH)) == 0 ||
+		dll_path_len > MAX_PATH ||
 		StringCchCat(dll_path, MAX_PATH, "\\") < 0 ||
 		StringCchCat(dll_path, MAX_PATH, GIT_HASH_CNG_DLL_NAME) < 0 ||
 		(hash_prov.prov.cng.dll = LoadLibrary(dll_path)) == NULL)
@@ -106,7 +89,15 @@ GIT_INLINE(void) hash_cryptoapi_prov_shutdown(void)
 	hash_prov.type = INVALID;
 }
 
-int git_hash_global_init()
+static void git_hash_global_shutdown(void)
+{
+	if (hash_prov.type == CNG)
+		hash_cng_prov_shutdown();
+	else if(hash_prov.type == CRYPTOAPI)
+		hash_cryptoapi_prov_shutdown();
+}
+
+int git_hash_global_init(void)
 {
 	int error = 0;
 
@@ -116,15 +107,9 @@ int git_hash_global_init()
 	if ((error = hash_cng_prov_init()) < 0)
 		error = hash_cryptoapi_prov_init();
 
-	return error;	
-}
+	git__on_shutdown(git_hash_global_shutdown);
 
-void git_hash_global_shutdown()
-{
-	if (hash_prov.type == CNG)
-		hash_cng_prov_shutdown();
-	else if(hash_prov.type == CRYPTOAPI)
-		hash_cryptoapi_prov_shutdown();
+	return error;
 }
 
 /* CryptoAPI: available in Windows XP and newer */
diff --git a/src/hashsig.c b/src/hashsig.c
index ab8d8b3f0..109f966ba 100644
--- a/src/hashsig.c
+++ b/src/hashsig.c
@@ -13,12 +13,15 @@ typedef uint64_t hashsig_state;
 
 #define HASHSIG_SCALE 100
 
-#define HASHSIG_HASH_WINDOW 32
-#define HASHSIG_HASH_START	0
+#define HASHSIG_MAX_RUN 80
+#define HASHSIG_HASH_START	0x012345678ABCDEF0LL
 #define HASHSIG_HASH_SHIFT  5
-#define HASHSIG_HASH_MASK   0x7FFFFFFF
+
+#define HASHSIG_HASH_MIX(S,CH) \
+	(S) = ((S) << HASHSIG_HASH_SHIFT) - (S) + (hashsig_state)(CH)
 
 #define HASHSIG_HEAP_SIZE ((1 << 7) - 1)
+#define HASHSIG_HEAP_MIN_SIZE 4
 
 typedef int (*hashsig_cmp)(const void *a, const void *b, void *);
 
@@ -28,14 +31,6 @@ typedef struct {
 	hashsig_t values[HASHSIG_HEAP_SIZE];
 } hashsig_heap;
 
-typedef struct {
-	hashsig_state state, shift_n;
-	char window[HASHSIG_HASH_WINDOW];
-	int win_len, win_pos, saw_lf;
-} hashsig_in_progress;
-
-#define HASHSIG_IN_PROGRESS_INIT { HASHSIG_HASH_START, 1, {0}, 0, 0, 1 }
-
 struct git_hashsig {
 	hashsig_heap mins;
 	hashsig_heap maxs;
@@ -43,8 +38,8 @@ struct git_hashsig {
 	int considered;
 };
 
-#define HEAP_LCHILD_OF(I) (((I)*2)+1)
-#define HEAP_RCHILD_OF(I) (((I)*2)+2)
+#define HEAP_LCHILD_OF(I) (((I)<<1)+1)
+#define HEAP_RCHILD_OF(I) (((I)<<1)+2)
 #define HEAP_PARENT_OF(I) (((I)-1)>>1)
 
 static void hashsig_heap_init(hashsig_heap *h, hashsig_cmp cmp)
@@ -115,134 +110,109 @@ static void hashsig_heap_sort(hashsig_heap *h)
 
 static void hashsig_heap_insert(hashsig_heap *h, hashsig_t val)
 {
-	/* if heap is full, pop top if new element should replace it */
-	if (h->size == h->asize && h->cmp(&val, &h->values[0], NULL) > 0) {
-		h->size--;
-		h->values[0] = h->values[h->size];
-		hashsig_heap_down(h, 0);
-	}
-
 	/* if heap is not full, insert new element */
 	if (h->size < h->asize) {
 		h->values[h->size++] = val;
 		hashsig_heap_up(h, h->size - 1);
 	}
-}
-
-GIT_INLINE(bool) hashsig_include_char(
-	char ch, git_hashsig_option_t opt, int *saw_lf)
-{
-	if ((opt & GIT_HASHSIG_IGNORE_WHITESPACE) && git__isspace(ch))
-		return false;
-
-	if (opt & GIT_HASHSIG_SMART_WHITESPACE) {
-		if (ch == '\r' || (*saw_lf && git__isspace(ch)))
-			return false;
 
-		*saw_lf = (ch == '\n');
+	/* if heap is full, pop top if new element should replace it */
+	else if (h->cmp(&val, &h->values[0], NULL) > 0) {
+		h->size--;
+		h->values[0] = h->values[h->size];
+		hashsig_heap_down(h, 0);
 	}
 
-	return true;
 }
 
-static void hashsig_initial_window(
-	git_hashsig *sig,
-	const char **data,
-	size_t size,
-	hashsig_in_progress *prog)
-{
-	hashsig_state state, shift_n;
-	int win_len;
-	const char *scan, *end;
-
-	/* init until we have processed at least HASHSIG_HASH_WINDOW data */
-
-	if (prog->win_len >= HASHSIG_HASH_WINDOW)
-		return;
-
-	state   = prog->state;
-	win_len = prog->win_len;
-	shift_n = prog->shift_n;
-
-	scan = *data;
-	end  = scan + size;
-
-	while (scan < end && win_len < HASHSIG_HASH_WINDOW) {
-		char ch = *scan++;
-
-		if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf))
-			continue;
-
-		state = (state * HASHSIG_HASH_SHIFT + ch) & HASHSIG_HASH_MASK;
-
-		if (!win_len)
-			shift_n = 1;
-		else
-			shift_n = (shift_n * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK;
-
-		prog->window[win_len++] = ch;
-	}
-
-	/* insert initial hash if we just finished */
+typedef struct {
+	int use_ignores;
+	uint8_t ignore_ch[256];
+} hashsig_in_progress;
 
-	if (win_len == HASHSIG_HASH_WINDOW) {
-		hashsig_heap_insert(&sig->mins, (hashsig_t)state);
-		hashsig_heap_insert(&sig->maxs, (hashsig_t)state);
-		sig->considered = 1;
+static void hashsig_in_progress_init(
+	hashsig_in_progress *prog, git_hashsig *sig)
+{
+	int i;
+
+	switch (sig->opt) {
+	case GIT_HASHSIG_IGNORE_WHITESPACE:
+		for (i = 0; i < 256; ++i)
+			prog->ignore_ch[i] = git__isspace_nonlf(i);
+		prog->use_ignores = 1;
+		break;
+	case GIT_HASHSIG_SMART_WHITESPACE:
+		for (i = 0; i < 256; ++i)
+			prog->ignore_ch[i] = git__isspace(i);
+		prog->use_ignores = 1;
+		break;
+	default:
+		memset(prog, 0, sizeof(*prog));
+		break;
 	}
-
-	prog->state   = state;
-	prog->win_len = win_len;
-	prog->shift_n = shift_n;
-
-	*data = scan;
 }
 
+#define HASHSIG_IN_PROGRESS_INIT { 1 }
+
 static int hashsig_add_hashes(
 	git_hashsig *sig,
-	const char *data,
+	const uint8_t *data,
 	size_t size,
 	hashsig_in_progress *prog)
 {
-	const char *scan = data, *end = data + size;
-	hashsig_state state, shift_n, rmv;
-
-	if (prog->win_len < HASHSIG_HASH_WINDOW)
-		hashsig_initial_window(sig, &scan, size, prog);
-
-	state   = prog->state;
-	shift_n = prog->shift_n;
-
-	/* advance window, adding new chars and removing old */
-
-	for (; scan < end; ++scan) {
-		char ch = *scan;
-
-		if (!hashsig_include_char(ch, sig->opt, &prog->saw_lf))
-			continue;
-
-		rmv = shift_n * prog->window[prog->win_pos];
+	const uint8_t *scan = data, *end = data + size;
+	hashsig_state state = HASHSIG_HASH_START;
+	int use_ignores = prog->use_ignores, len;
+	uint8_t ch;
+
+	while (scan < end) {
+		state = HASHSIG_HASH_START;
+
+		for (len = 0; scan < end && len < HASHSIG_MAX_RUN; ) {
+			ch = *scan;
+
+			if (use_ignores)
+				for (; scan < end && git__isspace_nonlf(ch); ch = *scan)
+					++scan;
+			else if (sig->opt != GIT_HASHSIG_NORMAL)
+				for (; scan < end && ch == '\r'; ch = *scan)
+					++scan;
+
+			/* peek at next character to decide what to do next */
+			if (sig->opt == GIT_HASHSIG_SMART_WHITESPACE)
+				use_ignores = (ch == '\n');
+
+			if (scan >= end)
+				break;
+			++scan;
+
+			/* check run terminator */
+			if (ch == '\n' || ch == '\0')
+				break;
+
+			++len;
+			HASHSIG_HASH_MIX(state, ch);
+		}
 
-		state = (state - rmv) & HASHSIG_HASH_MASK;
-		state = (state * HASHSIG_HASH_SHIFT) & HASHSIG_HASH_MASK;
-		state = (state + ch) & HASHSIG_HASH_MASK;
+		if (len > 0) {
+			hashsig_heap_insert(&sig->mins, (hashsig_t)state);
+			hashsig_heap_insert(&sig->maxs, (hashsig_t)state);
 
-		hashsig_heap_insert(&sig->mins, (hashsig_t)state);
-		hashsig_heap_insert(&sig->maxs, (hashsig_t)state);
-		sig->considered++;
+			sig->considered++;
 
-		prog->window[prog->win_pos] = ch;
-		prog->win_pos = (prog->win_pos + 1) % HASHSIG_HASH_WINDOW;
+			while (scan < end && (*scan == '\n' || !*scan))
+				++scan;
+		}
 	}
 
-	prog->state = state;
+	prog->use_ignores = use_ignores;
 
 	return 0;
 }
 
 static int hashsig_finalize_hashes(git_hashsig *sig)
 {
-	if (sig->mins.size < HASHSIG_HEAP_SIZE) {
+	if (sig->mins.size < HASHSIG_HEAP_MIN_SIZE) {
 		giterr_set(GITERR_INVALID,
 			"File too small for similarity signature calculation");
 		return GIT_EBUFS;
@@ -274,11 +244,13 @@ int git_hashsig_create(
 	git_hashsig_option_t opts)
 {
 	int error;
-	hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT;
+	hashsig_in_progress prog;
 	git_hashsig *sig = hashsig_alloc(opts);
 	GITERR_CHECK_ALLOC(sig);
 
-	error = hashsig_add_hashes(sig, buf, buflen, &prog);
+	hashsig_in_progress_init(&prog, sig);
+
+	error = hashsig_add_hashes(sig, (const uint8_t *)buf, buflen, &prog);
 
 	if (!error)
 		error = hashsig_finalize_hashes(sig);
@@ -296,10 +268,10 @@ int git_hashsig_create_fromfile(
 	const char *path,
 	git_hashsig_option_t opts)
 {
-	char buf[4096];
+	uint8_t buf[0x1000];
 	ssize_t buflen = 0;
 	int error = 0, fd;
-	hashsig_in_progress prog = HASHSIG_IN_PROGRESS_INIT;
+	hashsig_in_progress prog;
 	git_hashsig *sig = hashsig_alloc(opts);
 	GITERR_CHECK_ALLOC(sig);
 
@@ -308,6 +280,8 @@ int git_hashsig_create_fromfile(
 		return fd;
 	}
 
+	hashsig_in_progress_init(&prog, sig);
+
 	while (!error) {
 		if ((buflen = p_read(fd, buf, sizeof(buf))) <= 0) {
 			if ((error = (int)buflen) < 0)
@@ -362,6 +336,12 @@ static int hashsig_heap_compare(const hashsig_heap *a, const hashsig_heap *b)
 
 int git_hashsig_compare(const git_hashsig *a, const git_hashsig *b)
 {
-	return (hashsig_heap_compare(&a->mins, &b->mins) +
-			hashsig_heap_compare(&a->maxs, &b->maxs)) / 2;
+	/* if we have fewer than the maximum number of elements, then just use
+	 * one array since the two arrays will be the same
+	 */
+	if (a->mins.size < HASHSIG_HEAP_SIZE)
+		return hashsig_heap_compare(&a->mins, &b->mins);
+	else
+		return (hashsig_heap_compare(&a->mins, &b->mins) +
+				hashsig_heap_compare(&a->maxs, &b->maxs)) / 2;
 }
diff --git a/src/ident.c b/src/ident.c
new file mode 100644
index 000000000..51630879d
--- /dev/null
+++ b/src/ident.c
@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "git2/sys/filter.h"
+#include "filter.h"
+#include "buffer.h"
+#include "buf_text.h"
+
+static int ident_find_id(
+	const char **id_start, const char **id_end, const char *start, size_t len)
+{
+	const char *end = start + len, *found = NULL;
+
+	while (len > 3 && (found = memchr(start, '$', len)) != NULL) {
+		size_t remaining = (size_t)(end - found) - 1;
+		if (remaining < 3)
+			return GIT_ENOTFOUND;
+
+		start = found + 1;
+		len   = remaining;
+
+		if (start[0] == 'I' && start[1] == 'd')
+			break;
+	}
+
+	if (len < 3 || !found)
+		return GIT_ENOTFOUND;
+	*id_start = found;
+
+	if ((found = memchr(start + 2, '$', len - 2)) == NULL)
+		return GIT_ENOTFOUND;
+
+	*id_end = found + 1;
+	return 0;
+}
+
+static int ident_insert_id(
+	git_buf *to, const git_buf *from, const git_filter_source *src)
+{
+	char oid[GIT_OID_HEXSZ+1];
+	const char *id_start, *id_end, *from_end = from->ptr + from->size;
+	size_t need_size;
+
+	/* replace $Id$ with blob id */
+
+	if (!git_filter_source_id(src))
+		return GIT_PASSTHROUGH;
+
+	git_oid_tostr(oid, sizeof(oid), git_filter_source_id(src));
+
+	if (ident_find_id(&id_start, &id_end, from->ptr, from->size) < 0)
+		return GIT_PASSTHROUGH;
+
+	need_size = (size_t)(id_start - from->ptr) +
+		5 /* "$Id: " */ + GIT_OID_HEXSZ + 1 /* "$" */ +
+		(size_t)(from_end - id_end);
+
+	if (git_buf_grow(to, need_size) < 0)
+		return -1;
+
+	git_buf_set(to, from->ptr, (size_t)(id_start - from->ptr));
+	git_buf_put(to, "$Id: ", 5);
+	git_buf_put(to, oid, GIT_OID_HEXSZ);
+	git_buf_putc(to, '$');
+	git_buf_put(to, id_end, (size_t)(from_end - id_end));
+
+	return git_buf_oom(to) ? -1 : 0;
+}
+
+static int ident_remove_id(
+	git_buf *to, const git_buf *from)
+{
+	const char *id_start, *id_end, *from_end = from->ptr + from->size;
+	size_t need_size;
+
+	if (ident_find_id(&id_start, &id_end, from->ptr, from->size) < 0)
+		return GIT_PASSTHROUGH;
+
+	need_size = (size_t)(id_start - from->ptr) +
+		4 /* "$Id$" */ + (size_t)(from_end - id_end);
+
+	if (git_buf_grow(to, need_size) < 0)
+		return -1;
+
+	git_buf_set(to, from->ptr, (size_t)(id_start - from->ptr));
+	git_buf_put(to, "$Id$", 4);
+	git_buf_put(to, id_end, (size_t)(from_end - id_end));
+
+	return git_buf_oom(to) ? -1 : 0;
+}
+
+static int ident_apply(
+	git_filter     *self,
+	void          **payload,
+	git_buf        *to,
+	const git_buf  *from,
+	const git_filter_source *src)
+{
+	GIT_UNUSED(self); GIT_UNUSED(payload);
+
+	/* Don't filter binary files */
+	if (git_buf_text_is_binary(from))
+		return GIT_PASSTHROUGH;
+
+	if (git_filter_source_mode(src) == GIT_FILTER_SMUDGE)
+		return ident_insert_id(to, from, src);
+	else
+		return ident_remove_id(to, from);
+}
+
+git_filter *git_ident_filter_new(void)
+{
+	git_filter *f = git__calloc(1, sizeof(git_filter));
+
+	f->version = GIT_FILTER_VERSION;
+	f->attributes = "+ident"; /* apply to files with ident attribute set */
+	f->shutdown = git_filter_free;
+	f->apply    = ident_apply;
+
+	return f;
+}
diff --git a/src/ignore.c b/src/ignore.c
index cc90b0c61..27d7c7ec4 100644
--- a/src/ignore.c
+++ b/src/ignore.c
@@ -1,4 +1,5 @@
 #include "git2/ignore.h"
+#include "common.h"
 #include "ignore.h"
 #include "attr.h"
 #include "path.h"
@@ -37,7 +38,7 @@ static int parse_ignore_file(
 			GITERR_CHECK_ALLOC(match);
 		}
 
-		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE;
+		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE | GIT_ATTR_FNMATCH_ALLOWNEG;
 
 		if (!(error = git_attr_fnmatch__parse(
 			match, ignores->pool, context, &scan)))
@@ -159,17 +160,36 @@ int git_ignore__push_dir(git_ignores *ign, const char *dir)
 {
 	if (git_buf_joinpath(&ign->dir, ign->dir.ptr, dir) < 0)
 		return -1;
-	else
-		return push_ignore_file(
-			ign->repo, ign, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE);
+
+	return push_ignore_file(
+		ign->repo, ign, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE);
 }
 
 int git_ignore__pop_dir(git_ignores *ign)
 {
 	if (ign->ign_path.length > 0) {
 		git_attr_file *file = git_vector_last(&ign->ign_path);
-		if (git__suffixcmp(ign->dir.ptr, file->key + 2) == 0)
+		const char *start, *end, *scan;
+		size_t keylen;
+
+		/* - ign->dir looks something like "a/b" (or "a/b/c/d")
+		 * - file->key looks something like "0#a/b/.gitignore
+		 *
+		 * We are popping the last directory off ign->dir.  We also want to
+		 * remove the file from the vector if the directory part of the key
+		 * matches the ign->dir path.  We need to test if the "a/b" part of
+		 * the file key matches the path we are about to pop.
+		 */
+
+		for (start = end = scan = &file->key[2]; *scan; ++scan)
+			if (*scan == '/')
+				end = scan; /* point 'end' to last '/' in key */
+		keylen = (end - start) + 1;
+
+		if (ign->dir.size >= keylen &&
+			!memcmp(ign->dir.ptr + ign->dir.size - keylen, start, keylen))
 			git_vector_pop(&ign->ign_path);
+
 		git_buf_rtruncate_at_char(&ign->dir, '/');
 	}
 	return 0;
@@ -298,12 +318,9 @@ int git_ignore_path_is_ignored(
 		path.full.size = (tail - path.full.ptr);
 		path.is_dir = (tail == end) ? full_is_dir : true;
 
-		/* update ignores for new path fragment */
-		if (path.basename == path.path)
-			error = git_ignore__for_path(repo, path.path, &ignores);
-		else
-			error = git_ignore__push_dir(&ignores, path.basename);
-		if (error < 0)
+		/* initialize ignores the first time through */
+		if (path.basename == path.path &&
+			(error = git_ignore__for_path(repo, path.path, &ignores)) < 0)
 			break;
 
 		/* first process builtins - success means path was found */
@@ -327,6 +344,10 @@ int git_ignore_path_is_ignored(
 		if (tail == end)
 			break;
 
+		/* now add this directory to list of ignores */
+		if ((error = git_ignore__push_dir(&ignores, path.path)) < 0)
+			break;
+
 		/* reinstate divider in path */
 		*tail = '/';
 		while (*tail == '/') tail++;
diff --git a/src/ignore.h b/src/ignore.h
index cc114b001..851c824bf 100644
--- a/src/ignore.h
+++ b/src/ignore.h
@@ -24,14 +24,15 @@
  */
 typedef struct {
 	git_repository *repo;
-	git_buf dir;
+	git_buf dir; /* current directory reflected in ign_path */
 	git_attr_file *ign_internal;
 	git_vector ign_path;
 	git_vector ign_global;
 	int ignore_case;
 } git_ignores;
 
-extern int git_ignore__for_path(git_repository *repo, const char *path, git_ignores *ign);
+extern int git_ignore__for_path(
+	git_repository *repo, const char *path, git_ignores *ign);
 
 extern int git_ignore__push_dir(git_ignores *ign, const char *dir);
 
diff --git a/src/index.c b/src/index.c
index 1d46779bf..09e7b2346 100644
--- a/src/index.c
+++ b/src/index.c
@@ -16,6 +16,7 @@
 #include "iterator.h"
 #include "pathspec.h"
 #include "ignore.h"
+#include "blob.h"
 
 #include "git2/odb.h"
 #include "git2/oid.h"
@@ -101,8 +102,6 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 static bool is_index_extended(git_index *index);
 static int write_index(git_index *index, git_filebuf *file);
 
-static int index_find(size_t *at_pos, git_index *index, const char *path, int stage);
-
 static void index_entry_free(git_index_entry *entry);
 static void index_entry_reuc_free(git_index_reuc_entry *reuc);
 
@@ -114,7 +113,7 @@ static int index_srch(const void *key, const void *array_member)
 
 	ret = strcmp(srch_key->path, entry->path);
 
-	if (ret == 0)
+	if (ret == 0 && srch_key->stage != GIT_INDEX_STAGE_ANY)
 		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
 
 	return ret;
@@ -128,7 +127,7 @@ static int index_isrch(const void *key, const void *array_member)
 
 	ret = strcasecmp(srch_key->path, entry->path);
 
-	if (ret == 0)
+	if (ret == 0 && srch_key->stage != GIT_INDEX_STAGE_ANY)
 		ret = srch_key->stage - GIT_IDXENTRY_STAGE(entry);
 
 	return ret;
@@ -261,6 +260,22 @@ static int reuc_icmp(const void *a, const void *b)
 	return strcasecmp(info_a->path, info_b->path);
 }
 
+static void index_entry_reuc_free(git_index_reuc_entry *reuc)
+{
+	if (!reuc)
+		return;
+	git__free(reuc->path);
+	git__free(reuc);
+}
+
+static void index_entry_free(git_index_entry *entry)
+{
+	if (!entry)
+		return;
+	git__free(entry->path);
+	git__free(entry);
+}
+
 static unsigned int index_create_mode(unsigned int mode)
 {
 	if (S_ISLNK(mode))
@@ -269,7 +284,7 @@ static unsigned int index_create_mode(unsigned int mode)
 	if (S_ISDIR(mode) || (mode & S_IFMT) == (S_IFLNK | S_IFDIR))
 		return (S_IFLNK | S_IFDIR);
 
-	return S_IFREG | ((mode & 0100) ? 0755 : 0644);
+	return S_IFREG | GIT_PERMS_CANONICAL(mode);
 }
 
 static unsigned int index_merge_mode(
@@ -306,6 +321,7 @@ void git_index__set_ignore_case(git_index *index, bool ignore_case)
 int git_index_open(git_index **index_out, const char *index_path)
 {
 	git_index *index;
+	int error;
 
 	assert(index_out);
 
@@ -331,10 +347,15 @@ int git_index_open(git_index **index_out, const char *index_path)
 	index->entries_search_path = index_srch_path;
 	index->reuc_search = reuc_srch;
 
+	if ((index_path != NULL) && ((error = git_index_read(index, true)) < 0)) {
+		git_index_free(index);
+		return error;
+	}
+
 	*index_out = index;
 	GIT_REFCOUNT_INC(index);
 
-	return (index_path != NULL) ? git_index_read(index) : 0;
+	return 0;
 }
 
 int git_index_new(git_index **out)
@@ -367,11 +388,8 @@ static void index_entries_free(git_vector *entries)
 {
 	size_t i;
 
-	for (i = 0; i < entries->length; ++i) {
-		git_index_entry *e = git_vector_get(entries, i);
-		git__free(e->path);
-		git__free(e);
-	}
+	for (i = 0; i < entries->length; ++i)
+		index_entry_free(git__swap(entries->contents[i], NULL));
 
 	git_vector_clear(entries);
 }
@@ -398,7 +416,7 @@ static int create_index_error(int error, const char *msg)
 
 int git_index_set_caps(git_index *index, unsigned int caps)
 {
-	int old_ignore_case;
+	unsigned int old_ignore_case;
 
 	assert(index);
 
@@ -426,7 +444,7 @@ int git_index_set_caps(git_index *index, unsigned int caps)
 	}
 
 	if (old_ignore_case != index->ignore_case) {
-		git_index__set_ignore_case(index, index->ignore_case);
+		git_index__set_ignore_case(index, (bool)index->ignore_case);
 	}
 
 	return 0;
@@ -439,24 +457,26 @@ unsigned int git_index_caps(const git_index *index)
 			(index->no_symlinks ? GIT_INDEXCAP_NO_SYMLINKS : 0));
 }
 
-int git_index_read(git_index *index)
+int git_index_read(git_index *index, int force)
 {
 	int error = 0, updated;
 	git_buf buffer = GIT_BUF_INIT;
-	git_futils_filestamp stamp = {0};
+	git_futils_filestamp stamp = index->stamp;
 
 	if (!index->index_file_path)
 		return create_index_error(-1,
 			"Failed to read index: The index is in-memory only");
 
-	if (!index->on_disk || git_path_exists(index->index_file_path) == false) {
-		git_index_clear(index);
-		index->on_disk = 0;
+	index->on_disk = git_path_exists(index->index_file_path);
+
+	if (!index->on_disk) {
+		if (force)
+			git_index_clear(index);
 		return 0;
 	}
 
 	updated = git_futils_filestamp_check(&stamp, index->index_file_path);
-	if (updated <= 0)
+	if (updated < 0 || (!updated && !force))
 		return updated;
 
 	error = git_futils_readbuffer(&buffer, index->index_file_path);
@@ -486,15 +506,19 @@ int git_index_write(git_index *index)
 	git_vector_sort(&index->reuc);
 
 	if ((error = git_filebuf_open(
-			 &file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS)) < 0)
+		&file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS, GIT_INDEX_FILE_MODE)) < 0) {
+		if (error == GIT_ELOCKED)
+			giterr_set(GITERR_INDEX, "The index is locked. This might be due to a concurrrent or crashed process");
+
 		return error;
+	}
 
 	if ((error = write_index(index, &file)) < 0) {
 		git_filebuf_cleanup(&file);
 		return error;
 	}
 
-	if ((error = git_filebuf_commit(&file, GIT_INDEX_FILE_MODE)) < 0)
+	if ((error = git_filebuf_commit(&file)) < 0)
 		return error;
 
 	error = git_futils_filestamp_check(&index->stamp, index->index_file_path);
@@ -505,6 +529,12 @@ int git_index_write(git_index *index)
 	return 0;
 }
 
+const char * git_index_path(git_index *index)
+{
+	assert(index);
+	return index->index_file_path;
+}
+
 int git_index_write_tree(git_oid *oid, git_index *index)
 {
 	git_repository *repo;
@@ -549,7 +579,7 @@ const git_index_entry *git_index_get_bypath(
 
 	git_vector_sort(&index->entries);
 
-	if (index_find(&pos, index, path, stage) < 0) {
+	if (git_index__find(&pos, index, path, stage) < 0) {
 		giterr_set(GITERR_INDEX, "Index does not contain %s", path);
 		return NULL;
 	}
@@ -557,7 +587,8 @@ const git_index_entry *git_index_get_bypath(
 	return git_index_get_byindex(index, pos);
 }
 
-void git_index_entry__init_from_stat(git_index_entry *entry, struct stat *st)
+void git_index_entry__init_from_stat(
+	git_index_entry *entry, struct stat *st, bool trust_mode)
 {
 	entry->ctime.seconds = (git_time_t)st->st_ctime;
 	entry->mtime.seconds = (git_time_t)st->st_mtime;
@@ -565,7 +596,8 @@ void git_index_entry__init_from_stat(git_index_entry *entry, struct stat *st)
 	/* entry->ctime.nanoseconds = st->st_ctimensec; */
 	entry->dev  = st->st_rdev;
 	entry->ino  = st->st_ino;
-	entry->mode = index_create_mode(st->st_mode);
+	entry->mode = (!trust_mode && S_ISREG(st->st_mode)) ?
+		index_create_mode(0666) : index_create_mode(st->st_mode);
 	entry->uid  = st->st_uid;
 	entry->gid  = st->st_gid;
 	entry->file_size = st->st_size;
@@ -587,48 +619,29 @@ int git_index_entry__cmp_icase(const void *a, const void *b)
 	return strcasecmp(entry_a->path, entry_b->path);
 }
 
-static int index_entry_init(git_index_entry **entry_out, git_index *index, const char *rel_path)
+static int index_entry_init(
+	git_index_entry **entry_out, git_index *index, const char *rel_path)
 {
+	int error = 0;
 	git_index_entry *entry = NULL;
 	struct stat st;
 	git_oid oid;
-	const char *workdir;
-	git_buf full_path = GIT_BUF_INIT;
-	int error;
 
 	if (INDEX_OWNER(index) == NULL)
 		return create_index_error(-1,
 			"Could not initialize index entry. "
 			"Index is not backed up by an existing repository.");
 
-	workdir = git_repository_workdir(INDEX_OWNER(index));
-
-	if (!workdir)
-		return create_index_error(GIT_EBAREREPO,
-			"Could not initialize index entry. Repository is bare");
-
-	if ((error = git_buf_joinpath(&full_path, workdir, rel_path)) < 0)
-		return error;
-
-	if ((error = git_path_lstat(full_path.ptr, &st)) < 0) {
-		git_buf_free(&full_path);
-		return error;
-	}
-
-	git_buf_free(&full_path); /* done with full path */
-
-	/* There is no need to validate the rel_path here, since it will be
-	 * immediately validated by the call to git_blob_create_fromfile.
-	 */
-
-	/* write the blob to disk and get the oid */
-	if ((error = git_blob_create_fromworkdir(&oid, INDEX_OWNER(index), rel_path)) < 0)
+	/* write the blob to disk and get the oid and stat info */
+	error = git_blob__create_from_paths(
+		&oid, &st, INDEX_OWNER(index), NULL, rel_path, 0, true);
+	if (error < 0)
 		return error;
 
 	entry = git__calloc(1, sizeof(git_index_entry));
 	GITERR_CHECK_ALLOC(entry);
 
-	git_index_entry__init_from_stat(entry, &st);
+	git_index_entry__init_from_stat(entry, &st, !index->distrust_filemode);
 
 	entry->oid = oid;
 	entry->path = git__strdup(rel_path);
@@ -670,15 +683,6 @@ static int index_entry_reuc_init(git_index_reuc_entry **reuc_out,
 	return 0;
 }
 
-static void index_entry_reuc_free(git_index_reuc_entry *reuc)
-{
-	if (!reuc)
-		return;
-
-	git__free(reuc->path);
-	git__free(reuc);
-}
-
 static git_index_entry *index_entry_dup(const git_index_entry *source_entry)
 {
 	git_index_entry *entry;
@@ -697,14 +701,6 @@ static git_index_entry *index_entry_dup(const git_index_entry *source_entry)
 	return entry;
 }
 
-static void index_entry_free(git_index_entry *entry)
-{
-	if (!entry)
-		return;
-	git__free(entry->path);
-	git__free(entry);
-}
-
 static int index_insert(git_index *index, git_index_entry *entry, int replace)
 {
 	size_t path_length, position;
@@ -723,7 +719,8 @@ static int index_insert(git_index *index, git_index_entry *entry, int replace)
 		entry->flags |= GIT_IDXENTRY_NAMEMASK;
 
 	/* look if an entry with this path already exists */
-	if (!index_find(&position, index, entry->path, GIT_IDXENTRY_STAGE(entry))) {
+	if (!git_index__find(
+			&position, index, entry->path, GIT_IDXENTRY_STAGE(entry))) {
 		existing = (git_index_entry **)&index->entries.contents[position];
 
 		/* update filemode to existing values if stat is not trusted */
@@ -835,7 +832,7 @@ int git_index_remove(git_index *index, const char *path, int stage)
 
 	git_vector_sort(&index->entries);
 
-	if (index_find(&position, index, path, stage) < 0) {
+	if (git_index__find(&position, index, path, stage) < 0) {
 		giterr_set(GITERR_INDEX, "Index does not contain %s at stage %d",
 			path, stage);
 		return GIT_ENOTFOUND;
@@ -891,7 +888,8 @@ int git_index_remove_directory(git_index *index, const char *dir, int stage)
 	return error;
 }
 
-static int index_find(size_t *at_pos, git_index *index, const char *path, int stage)
+int git_index__find(
+	size_t *at_pos, git_index *index, const char *path, int stage)
 {
 	struct entry_srch_key srch_key;
 
@@ -900,7 +898,8 @@ static int index_find(size_t *at_pos, git_index *index, const char *path, int st
 	srch_key.path = path;
 	srch_key.stage = stage;
 
-	return git_vector_bsearch2(at_pos, &index->entries, index->entries_search, &srch_key);
+	return git_vector_bsearch2(
+		at_pos, &index->entries, index->entries_search, &srch_key);
 }
 
 int git_index_find(size_t *at_pos, git_index *index, const char *path)
@@ -1357,14 +1356,11 @@ int git_index_reuc_remove(git_index *index, size_t position)
 void git_index_reuc_clear(git_index *index)
 {
 	size_t i;
-	git_index_reuc_entry *reuc;
 
 	assert(index);
 
-	git_vector_foreach(&index->reuc, i, reuc) {
-		git__free(reuc->path);
-		git__free(reuc);
-	}
+	for (i = 0; i < index->reuc.length; ++i)
+		index_entry_reuc_free(git__swap(index->reuc.contents[i], NULL));
 
 	git_vector_clear(&index->reuc);
 }
@@ -1389,7 +1385,7 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 	while (size) {
 		git_index_reuc_entry *lost;
 
-		len = strlen(buffer) + 1;
+		len = p_strnlen(buffer, size) + 1;
 		if (size <= len)
 			return index_error_invalid("reading reuc entries");
 
@@ -1409,14 +1405,18 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 
 			if (git__strtol32(&tmp, buffer, &endptr, 8) < 0 ||
 				!endptr || endptr == buffer || *endptr ||
-				(unsigned)tmp > UINT_MAX)
+				(unsigned)tmp > UINT_MAX) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry stage");
+			}
 
 			lost->mode[i] = tmp;
 
 			len = (endptr + 1) - buffer;
-			if (size <= len)
+			if (size <= len) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry stage");
+			}
 
 			size -= len;
 			buffer += len;
@@ -1426,8 +1426,10 @@ static int read_reuc(git_index *index, const char *buffer, size_t size)
 		for (i = 0; i < 3; i++) {
 			if (!lost->mode[i])
 				continue;
-			if (size < 20)
+			if (size < 20) {
+				index_entry_reuc_free(lost);
 				return index_error_invalid("reading reuc entry oid");
+			}
 
 			git_oid_fromraw(&lost->oid[i], (const unsigned char *) buffer);
 			size -= 20;
@@ -1456,7 +1458,7 @@ static int read_conflict_names(git_index *index, const char *buffer, size_t size
 		return -1;
 
 #define read_conflict_name(ptr) \
-	len = strlen(buffer) + 1; \
+	len = p_strnlen(buffer, size) + 1; \
 	if (size < len) \
 		return index_error_invalid("reading conflict name entries"); \
 	\
@@ -1583,7 +1585,8 @@ static size_t read_extension(git_index *index, const char *buffer, size_t buffer
 
 	total_size = dest.extension_size + sizeof(struct index_extension);
 
-	if (buffer_size < total_size ||
+	if (dest.extension_size > total_size ||
+		buffer_size < total_size ||
 		buffer_size - total_size < INDEX_FOOTER_SIZE)
 		return 0;
 
@@ -1958,8 +1961,9 @@ int git_index_entry_stage(const git_index_entry *entry)
 }
 
 typedef struct read_tree_data {
-	git_index *index;
 	git_vector *old_entries;
+	git_vector *new_entries;
+	git_vector_cmp entries_search;
 } read_tree_data;
 
 static int read_tree_cb(
@@ -1990,7 +1994,7 @@ static int read_tree_cb(
 		skey.stage = 0;
 
 		if (!git_vector_bsearch2(
-				&pos, data->old_entries, data->index->entries_search, &skey) &&
+				&pos, data->old_entries, data->entries_search, &skey) &&
 			(old_entry = git_vector_get(data->old_entries, pos)) != NULL &&
 			entry->mode == old_entry->mode &&
 			git_oid_equal(&entry->oid, &old_entry->oid))
@@ -2008,7 +2012,7 @@ static int read_tree_cb(
 	entry->path = git_buf_detach(&path);
 	git_buf_free(&path);
 
-	if (git_vector_insert(&data->index->entries, entry) < 0) {
+	if (git_vector_insert(data->new_entries, entry) < 0) {
 		index_entry_free(entry);
 		return -1;
 	}
@@ -2022,22 +2026,22 @@ int git_index_read_tree(git_index *index, const git_tree *tree)
 	git_vector entries = GIT_VECTOR_INIT;
 	read_tree_data data;
 
-	git_vector_sort(&index->entries);
+	git_vector_set_cmp(&entries, index->entries._cmp); /* match sort */
 
-	git_vector_set_cmp(&entries, index->entries._cmp);
-	git_vector_swap(&entries, &index->entries);
+	data.old_entries = &index->entries;
+	data.new_entries = &entries;
+	data.entries_search = index->entries_search;
 
-	git_index_clear(index);
-
-	data.index = index;
-	data.old_entries = &entries;
+	git_vector_sort(&index->entries);
 
 	error = git_tree_walk(tree, GIT_TREEWALK_POST, read_tree_cb, &data);
 
-	index_entries_free(&entries);
-	git_vector_free(&entries);
+	git_vector_sort(&entries);
 
-	git_vector_sort(&index->entries);
+	git_index_clear(index);
+
+	git_vector_swap(&entries, &index->entries);
+	git_vector_free(&entries);
 
 	return error;
 }
@@ -2059,7 +2063,7 @@ int git_index_add_all(
 	git_iterator *wditer = NULL;
 	const git_index_entry *wd = NULL;
 	git_index_entry *entry;
-	git_pathspec_context ps;
+	git_pathspec ps;
 	const char *match;
 	size_t existing;
 	bool no_fnmatch = (flags & GIT_INDEX_ADD_DISABLE_PATHSPEC_MATCH) != 0;
@@ -2080,7 +2084,7 @@ int git_index_add_all(
 	if (git_repository__cvar(&ignorecase, repo, GIT_CVAR_IGNORECASE) < 0)
 		return -1;
 
-	if ((error = git_pathspec_context_init(&ps, paths)) < 0)
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
 		return error;
 
 	/* optionally check that pathspec doesn't mention any ignored files */
@@ -2097,14 +2101,14 @@ int git_index_add_all(
 	while (!(error = git_iterator_advance(&wd, wditer))) {
 
 		/* check if path actually matches */
-		if (!git_pathspec_match_path(
-				&ps.pathspec, wd->path, no_fnmatch, ignorecase, &match))
+		if (!git_pathspec__match(
+				&ps.pathspec, wd->path, no_fnmatch, (bool)ignorecase, &match, NULL))
 			continue;
 
 		/* skip ignored items that are not already in the index */
 		if ((flags & GIT_INDEX_ADD_FORCE) == 0 &&
 			git_iterator_current_is_ignored(wditer) &&
-			index_find(&existing, index, wd->path, 0) < 0)
+			git_index__find(&existing, index, wd->path, 0) < 0)
 			continue;
 
 		/* issue notification callback if requested */
@@ -2154,7 +2158,7 @@ int git_index_add_all(
 
 cleanup:
 	git_iterator_free(wditer);
-	git_pathspec_context_free(&ps);
+	git_pathspec__clear(&ps);
 
 	return error;
 }
@@ -2174,13 +2178,13 @@ static int index_apply_to_all(
 {
 	int error = 0;
 	size_t i;
-	git_pathspec_context ps;
+	git_pathspec ps;
 	const char *match;
 	git_buf path = GIT_BUF_INIT;
 
 	assert(index);
 
-	if ((error = git_pathspec_context_init(&ps, paths)) < 0)
+	if ((error = git_pathspec__init(&ps, paths)) < 0)
 		return error;
 
 	git_vector_sort(&index->entries);
@@ -2189,8 +2193,9 @@ static int index_apply_to_all(
 		git_index_entry *entry = git_vector_get(&index->entries, i);
 
 		/* check if path actually matches */
-		if (!git_pathspec_match_path(
-				&ps.pathspec, entry->path, false, index->ignore_case, &match))
+		if (!git_pathspec__match(
+				&ps.pathspec, entry->path, false, (bool)index->ignore_case,
+				&match, NULL))
 			continue;
 
 		/* issue notification callback if requested */
@@ -2237,7 +2242,7 @@ static int index_apply_to_all(
 	}
 
 	git_buf_free(&path);
-	git_pathspec_context_free(&ps);
+	git_pathspec__clear(&ps);
 
 	return error;
 }
diff --git a/src/index.h b/src/index.h
index a59107a7b..4c448fabf 100644
--- a/src/index.h
+++ b/src/index.h
@@ -47,13 +47,17 @@ struct git_index_conflict_iterator {
 	size_t cur;
 };
 
-extern void git_index_entry__init_from_stat(git_index_entry *entry, struct stat *st);
+extern void git_index_entry__init_from_stat(
+	git_index_entry *entry, struct stat *st, bool trust_mode);
 
 extern size_t git_index__prefix_position(git_index *index, const char *path);
 
 extern int git_index_entry__cmp(const void *a, const void *b);
 extern int git_index_entry__cmp_icase(const void *a, const void *b);
 
+extern int git_index__find(
+	size_t *at_pos, git_index *index, const char *path, int stage);
+
 extern void git_index__set_ignore_case(git_index *index, bool ignore_case);
 
 #endif
diff --git a/src/indexer.c b/src/indexer.c
index 1b5339f23..df1ce7cfb 100644
--- a/src/indexer.c
+++ b/src/indexer.c
@@ -18,6 +18,7 @@
 #include "filebuf.h"
 #include "oid.h"
 #include "oidmap.h"
+#include "compress.h"
 
 #define UINT31_MAX (0x7FFFFFFF)
 
@@ -28,13 +29,15 @@ struct entry {
 	uint64_t offset_long;
 };
 
-struct git_indexer_stream {
+struct git_indexer {
 	unsigned int parsed_header :1,
 		opened_pack :1,
 		have_stream :1,
 		have_delta :1;
+	struct git_pack_header hdr;
 	struct git_pack_file *pack;
 	git_filebuf pack_file;
+	unsigned int mode;
 	git_off_t off;
 	git_off_t entry_start;
 	git_packfile_stream stream;
@@ -47,13 +50,21 @@ struct git_indexer_stream {
 	git_transfer_progress_callback progress_cb;
 	void *progress_payload;
 	char objbuf[8*1024];
+
+	/* Needed to look up objects which we want to inject to fix a thin pack */
+	git_odb *odb;
+
+	/* Fields for calculating the packfile trailer (hash of everything before it) */
+	char inbuf[GIT_OID_RAWSZ];
+	size_t inbuf_len;
+	git_hash_ctx trailer;
 };
 
 struct delta_info {
 	git_off_t delta_off;
 };
 
-const git_oid *git_indexer_stream_hash(const git_indexer_stream *idx)
+const git_oid *git_indexer_hash(const git_indexer *idx)
 {
 	return &idx->hash;
 }
@@ -106,28 +117,34 @@ static int objects_cmp(const void *a, const void *b)
 	return git_oid__cmp(&entrya->oid, &entryb->oid);
 }
 
-int git_indexer_stream_new(
-		git_indexer_stream **out,
+int git_indexer_new(
+		git_indexer **out,
 		const char *prefix,
+		unsigned int mode,
+		git_odb *odb,
 		git_transfer_progress_callback progress_cb,
 		void *progress_payload)
 {
-	git_indexer_stream *idx;
+	git_indexer *idx;
 	git_buf path = GIT_BUF_INIT;
 	static const char suff[] = "/pack";
 	int error;
 
-	idx = git__calloc(1, sizeof(git_indexer_stream));
+	idx = git__calloc(1, sizeof(git_indexer));
 	GITERR_CHECK_ALLOC(idx);
+	idx->odb = odb;
 	idx->progress_cb = progress_cb;
 	idx->progress_payload = progress_payload;
+	idx->mode = mode ? mode : GIT_PACK_FILE_MODE;
+	git_hash_ctx_init(&idx->trailer);
 
 	error = git_buf_joinpath(&path, prefix, suff);
 	if (error < 0)
 		goto cleanup;
 
 	error = git_filebuf_open(&idx->pack_file, path.ptr,
-				 GIT_FILEBUF_TEMPORARY | GIT_FILEBUF_DO_NOT_BUFFER);
+		GIT_FILEBUF_TEMPORARY | GIT_FILEBUF_DO_NOT_BUFFER,
+		idx->mode);
 	git_buf_free(&path);
 	if (error < 0)
 		goto cleanup;
@@ -143,7 +160,7 @@ cleanup:
 }
 
 /* Try to store the delta so we can try to resolve it later */
-static int store_delta(git_indexer_stream *idx)
+static int store_delta(git_indexer *idx)
 {
 	struct delta_info *delta;
 
@@ -166,7 +183,7 @@ static void hash_header(git_hash_ctx *ctx, git_off_t len, git_otype type)
 	git_hash_update(ctx, buffer, hdrlen);
 }
 
-static int hash_object_stream(git_indexer_stream *idx, git_packfile_stream *stream)
+static int hash_object_stream(git_indexer*idx, git_packfile_stream *stream)
 {
 	ssize_t read;
 
@@ -186,7 +203,7 @@ static int hash_object_stream(git_indexer_stream *idx, git_packfile_stream *stre
 }
 
 /* In order to create the packfile stream, we need to skip over the delta base description */
-static int advance_delta_offset(git_indexer_stream *idx, git_otype type)
+static int advance_delta_offset(git_indexer *idx, git_otype type)
 {
 	git_mwindow *w = NULL;
 
@@ -205,7 +222,7 @@ static int advance_delta_offset(git_indexer_stream *idx, git_otype type)
 }
 
 /* Read from the stream and discard any output */
-static int read_object_stream(git_indexer_stream *idx, git_packfile_stream *stream)
+static int read_object_stream(git_indexer *idx, git_packfile_stream *stream)
 {
 	ssize_t read;
 
@@ -245,7 +262,7 @@ static int crc_object(uint32_t *crc_out, git_mwindow_file *mwf, git_off_t start,
 	return 0;
 }
 
-static int store_object(git_indexer_stream *idx)
+static int store_object(git_indexer *idx)
 {
 	int i, error;
 	khiter_t k;
@@ -303,17 +320,10 @@ on_error:
 	return -1;
 }
 
-static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t entry_start)
+static int save_entry(git_indexer *idx, struct entry *entry, struct git_pack_entry *pentry, git_off_t entry_start)
 {
 	int i, error;
 	khiter_t k;
-	git_oid oid;
-	size_t entry_size;
-	struct entry *entry;
-	struct git_pack_entry *pentry;
-
-	entry = git__calloc(1, sizeof(*entry));
-	GITERR_CHECK_ALLOC(entry);
 
 	if (entry_start > UINT31_MAX) {
 		entry->offset = UINT32_MAX;
@@ -322,25 +332,43 @@ static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t ent
 		entry->offset = (uint32_t)entry_start;
 	}
 
-	/* FIXME: Parse the object instead of hashing it */
-	if (git_odb__hashobj(&oid, obj) < 0) {
-		giterr_set(GITERR_INDEXER, "Failed to hash object");
+	pentry->offset = entry_start;
+	k = kh_put(oid, idx->pack->idx_cache, &pentry->sha1, &error);
+	if (!error)
+		return -1;
+
+	kh_value(idx->pack->idx_cache, k) = pentry;
+
+	/* Add the object to the list */
+	if (git_vector_insert(&idx->objects, entry) < 0)
 		return -1;
+
+	for (i = entry->oid.id[0]; i < 256; ++i) {
+		idx->fanout[i]++;
 	}
 
-	pentry = git__calloc(1, sizeof(struct git_pack_entry));
-	GITERR_CHECK_ALLOC(pentry);
+	return 0;
+}
 
-	git_oid_cpy(&pentry->sha1, &oid);
-	pentry->offset = entry_start;
-	k = kh_put(oid, idx->pack->idx_cache, &pentry->sha1, &error);
-	if (!error) {
-		git__free(pentry);
+static int hash_and_save(git_indexer *idx, git_rawobj *obj, git_off_t entry_start)
+{
+	git_oid oid;
+	size_t entry_size;
+	struct entry *entry;
+	struct git_pack_entry *pentry;
+
+	entry = git__calloc(1, sizeof(*entry));
+	GITERR_CHECK_ALLOC(entry);
+
+	if (git_odb__hashobj(&oid, obj) < 0) {
+		giterr_set(GITERR_INDEXER, "Failed to hash object");
 		goto on_error;
 	}
 
-	kh_value(idx->pack->idx_cache, k) = pentry;
+	pentry = git__calloc(1, sizeof(struct git_pack_entry));
+	GITERR_CHECK_ALLOC(pentry);
 
+	git_oid_cpy(&pentry->sha1, &oid);
 	git_oid_cpy(&entry->oid, &oid);
 	entry->crc = crc32(0L, Z_NULL, 0);
 
@@ -348,15 +376,7 @@ static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t ent
 	if (crc_object(&entry->crc, &idx->pack->mwf, entry_start, entry_size) < 0)
 		goto on_error;
 
-	/* Add the object to the list */
-	if (git_vector_insert(&idx->objects, entry) < 0)
-		goto on_error;
-
-	for (i = oid.id[0]; i < 256; ++i) {
-		idx->fanout[i]++;
-	}
-
-	return 0;
+	return save_entry(idx, entry, pentry, entry_start);
 
 on_error:
 	git__free(entry);
@@ -364,17 +384,54 @@ on_error:
 	return -1;
 }
 
-static int do_progress_callback(git_indexer_stream *idx, git_transfer_progress *stats)
+static int do_progress_callback(git_indexer *idx, git_transfer_progress *stats)
 {
 	if (!idx->progress_cb) return 0;
 	return idx->progress_cb(stats, idx->progress_payload);
 }
 
-int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_transfer_progress *stats)
+/* Hash everything but the last 20B of input */
+static void hash_partially(git_indexer *idx, const uint8_t *data, size_t size)
+{
+	size_t to_expell, to_keep;
+
+	if (size == 0)
+		return;
+
+	/* Easy case, dump the buffer and the data minus the last 20 bytes */
+	if (size >= GIT_OID_RAWSZ) {
+		git_hash_update(&idx->trailer, idx->inbuf, idx->inbuf_len);
+		git_hash_update(&idx->trailer, data, size - GIT_OID_RAWSZ);
+
+		data += size - GIT_OID_RAWSZ;
+		memcpy(idx->inbuf, data, GIT_OID_RAWSZ);
+		idx->inbuf_len = GIT_OID_RAWSZ;
+		return;
+	}
+
+	/* We can just append */
+	if (idx->inbuf_len + size <= GIT_OID_RAWSZ) {
+		memcpy(idx->inbuf + idx->inbuf_len, data, size);
+		idx->inbuf_len += size;
+		return;
+	}
+
+	/* We need to partially drain the buffer and then append */
+	to_keep   = GIT_OID_RAWSZ - size;
+	to_expell = idx->inbuf_len - to_keep;
+
+	git_hash_update(&idx->trailer, idx->inbuf, to_expell);
+
+	memmove(idx->inbuf, idx->inbuf + to_expell, to_keep);
+	memcpy(idx->inbuf + to_keep, data, size);
+	idx->inbuf_len += size - to_expell;
+}
+
+int git_indexer_append(git_indexer *idx, const void *data, size_t size, git_transfer_progress *stats)
 {
 	int error = -1;
-	struct git_pack_header hdr;
 	size_t processed;
+	struct git_pack_header *hdr = &idx->hdr;
 	git_mwindow_file *mwf = &idx->pack->mwf;
 
 	assert(idx && data && stats);
@@ -384,6 +441,8 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 	if (git_filebuf_write(&idx->pack_file, data, size) < 0)
 		return -1;
 
+	hash_partially(idx, data, (int)size);
+
 	/* Make sure we set the new size of the pack */
 	if (idx->opened_pack) {
 		idx->pack->mwf.size += size;
@@ -399,14 +458,14 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 	if (!idx->parsed_header) {
 		unsigned int total_objects;
 
-		if ((unsigned)idx->pack->mwf.size < sizeof(hdr))
+		if ((unsigned)idx->pack->mwf.size < sizeof(struct git_pack_header))
 			return 0;
 
-		if (parse_header(&hdr, idx->pack) < 0)
+		if (parse_header(&idx->hdr, idx->pack) < 0)
 			return -1;
 
 		idx->parsed_header = 1;
-		idx->nr_objects = ntohl(hdr.hdr_entries);
+		idx->nr_objects = ntohl(hdr->hdr_entries);
 		idx->off = sizeof(struct git_pack_header);
 
 		/* for now, limit to 2^32 objects */
@@ -427,6 +486,9 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 			return -1;
 
 		stats->received_objects = 0;
+		stats->local_objects = 0;
+		stats->total_deltas = 0;
+		stats->indexed_deltas = 0;
 		processed = stats->indexed_objects = 0;
 		stats->total_objects = total_objects;
 		do_progress_callback(idx, stats);
@@ -512,6 +574,7 @@ int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t siz
 		stats->received_objects++;
 
 		if (do_progress_callback(idx, stats) != 0) {
+			giterr_clear();
 			error = GIT_EUSER;
 			goto on_error;
 		}
@@ -524,7 +587,7 @@ on_error:
 	return error;
 }
 
-static int index_path_stream(git_buf *path, git_indexer_stream *idx, const char *suffix)
+static int index_path(git_buf *path, git_indexer *idx, const char *suffix)
 {
 	const char prefix[] = "pack-";
 	size_t slash = (size_t)path->size;
@@ -546,68 +609,306 @@ static int index_path_stream(git_buf *path, git_indexer_stream *idx, const char
 	return git_buf_oom(path) ? -1 : 0;
 }
 
-static int resolve_deltas(git_indexer_stream *idx, git_transfer_progress *stats)
+/**
+ * Rewind the packfile by the trailer, as we might need to fix the
+ * packfile by injecting objects at the tail and must overwrite it.
+ */
+static git_off_t seek_back_trailer(git_indexer *idx)
+{
+	git_off_t off;
+
+	if ((off = p_lseek(idx->pack_file.fd, -GIT_OID_RAWSZ, SEEK_CUR)) < 0)
+		return -1;
+
+	idx->pack->mwf.size -= GIT_OID_RAWSZ;
+	git_mwindow_free_all(&idx->pack->mwf);
+
+	return off;
+}
+
+static int inject_object(git_indexer *idx, git_oid *id)
 {
+	git_odb_object *obj;
+	struct entry *entry;
+	struct git_pack_entry *pentry;
+	git_oid foo = {{0}};
+	unsigned char hdr[64];
+	git_buf buf = GIT_BUF_INIT;
+	git_off_t entry_start;
+	const void *data;
+	size_t len, hdr_len;
+	int error;
+
+	entry = git__calloc(1, sizeof(*entry));
+	GITERR_CHECK_ALLOC(entry);
+
+	entry_start = seek_back_trailer(idx);
+
+	if (git_odb_read(&obj, idx->odb, id) < 0)
+		return -1;
+
+	data = git_odb_object_data(obj);
+	len = git_odb_object_size(obj);
+
+	entry->crc = crc32(0L, Z_NULL, 0);
+
+	/* Write out the object header */
+	hdr_len = git_packfile__object_header(hdr, len, git_odb_object_type(obj));
+	git_filebuf_write(&idx->pack_file, hdr, hdr_len);
+	idx->pack->mwf.size += hdr_len;
+	entry->crc = crc32(entry->crc, hdr, hdr_len);
+
+	if ((error = git__compress(&buf, data, len)) < 0)
+		goto cleanup;
+
+	/* And then the compressed object */
+	git_filebuf_write(&idx->pack_file, buf.ptr, buf.size);
+	idx->pack->mwf.size += buf.size;
+	entry->crc = htonl(crc32(entry->crc, (unsigned char *)buf.ptr, (uInt)buf.size));
+	git_buf_free(&buf);
+
+	/* Write a fake trailer so the pack functions play ball */
+	if ((error = git_filebuf_write(&idx->pack_file, &foo, GIT_OID_RAWSZ)) < 0)
+		goto cleanup;
+
+	idx->pack->mwf.size += GIT_OID_RAWSZ;
+
+	pentry = git__calloc(1, sizeof(struct git_pack_entry));
+	GITERR_CHECK_ALLOC(pentry);
+
+	git_oid_cpy(&pentry->sha1, id);
+	git_oid_cpy(&entry->oid, id);
+	idx->off = entry_start + hdr_len + len;
+
+	if ((error = save_entry(idx, entry, pentry, entry_start)) < 0)
+		git__free(pentry);
+
+cleanup:
+	git_odb_object_free(obj);
+	return error;
+}
+
+static int fix_thin_pack(git_indexer *idx, git_transfer_progress *stats)
+{
+	int error, found_ref_delta = 0;
 	unsigned int i;
 	struct delta_info *delta;
+	size_t size;
+	git_otype type;
+	git_mwindow *w = NULL;
+	git_off_t curpos;
+	unsigned char *base_info;
+	unsigned int left = 0;
+	git_oid base;
+
+	assert(git_vector_length(&idx->deltas) > 0);
+
+	if (idx->odb == NULL) {
+		giterr_set(GITERR_INDEXER, "cannot fix a thin pack without an ODB");
+		return -1;
+	}
 
+	/* Loop until we find the first REF delta */
 	git_vector_foreach(&idx->deltas, i, delta) {
-		git_rawobj obj;
+		curpos = delta->delta_off;
+		error = git_packfile_unpack_header(&size, &type, &idx->pack->mwf, &w, &curpos);
+		git_mwindow_close(&w);
+		if (error < 0)
+			return error;
+
+		if (type == GIT_OBJ_REF_DELTA) {
+			found_ref_delta = 1;
+			break;
+		}
+	}
+
+	if (!found_ref_delta) {
+		giterr_set(GITERR_INDEXER, "no REF_DELTA found, cannot inject object");
+		return -1;
+	}
+
+	/* curpos now points to the base information, which is an OID */
+	base_info = git_mwindow_open(&idx->pack->mwf, &w, curpos, GIT_OID_RAWSZ, &left);
+	if (base_info == NULL) {
+		giterr_set(GITERR_INDEXER, "failed to map delta information");
+		return -1;
+	}
+
+	git_oid_fromraw(&base, base_info);
+	git_mwindow_close(&w);
+
+	if (inject_object(idx, &base) < 0)
+		return -1;
+
+	stats->local_objects++;
+
+	return 0;
+}
+
+static int resolve_deltas(git_indexer *idx, git_transfer_progress *stats)
+{
+	unsigned int i;
+	struct delta_info *delta;
+	int progressed = 0;
+
+	while (idx->deltas.length > 0) {
+		progressed = 0;
+		git_vector_foreach(&idx->deltas, i, delta) {
+			git_rawobj obj;
+
+			idx->off = delta->delta_off;
+			if (git_packfile_unpack(&obj, idx->pack, &idx->off) < 0)
+				continue;
+
+			if (hash_and_save(idx, &obj, delta->delta_off) < 0)
+				continue;
+
+			git__free(obj.data);
+			stats->indexed_objects++;
+			stats->indexed_deltas++;
+			progressed = 1;
+			do_progress_callback(idx, stats);
+
+			/*
+			 * Remove this delta from the list and
+			 * decrease i so we don't skip over the next
+			 * delta.
+			 */
+			git_vector_remove(&idx->deltas, i);
+			git__free(delta);
+			i--;
+		}
 
-		idx->off = delta->delta_off;
-		if (git_packfile_unpack(&obj, idx->pack, &idx->off) < 0)
+		if (!progressed && (fix_thin_pack(idx, stats) < 0)) {
+			giterr_set(GITERR_INDEXER, "missing delta bases");
 			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static int update_header_and_rehash(git_indexer *idx, git_transfer_progress *stats)
+{
+	void *ptr;
+	size_t chunk = 1024*1024;
+	git_off_t hashed = 0;
+	git_mwindow *w = NULL;
+	git_mwindow_file *mwf;
+	unsigned int left;
+	git_hash_ctx *ctx;
+
+	mwf = &idx->pack->mwf;
+	ctx = &idx->trailer;
+
+	git_hash_ctx_init(ctx);
+	git_mwindow_free_all(mwf);
 
-		if (hash_and_save(idx, &obj, delta->delta_off) < 0)
+	/* Update the header to include the numer of local objects we injected */
+	idx->hdr.hdr_entries = htonl(stats->total_objects + stats->local_objects);
+	if (p_lseek(idx->pack_file.fd, 0, SEEK_SET) < 0) {
+		giterr_set(GITERR_OS, "failed to seek to the beginning of the pack");
+		return -1;
+	}
+
+	if (p_write(idx->pack_file.fd, &idx->hdr, sizeof(struct git_pack_header)) < 0) {
+		giterr_set(GITERR_OS, "failed to update the pack header");
+		return -1;
+	}
+
+	/*
+	 * We now use the same technique as before to determine the
+	 * hash. We keep reading up to the end and let
+	 * hash_partially() keep the existing trailer out of the
+	 * calculation.
+	 */
+	idx->inbuf_len = 0;
+	while (hashed < mwf->size) {
+		ptr = git_mwindow_open(mwf, &w, hashed, chunk, &left);
+		if (ptr == NULL)
 			return -1;
 
-		git__free(obj.data);
-		stats->indexed_objects++;
-		do_progress_callback(idx, stats);
+		hash_partially(idx, ptr, left);
+		hashed += left;
+
+		git_mwindow_close(&w);
 	}
 
 	return 0;
 }
 
-int git_indexer_stream_finalize(git_indexer_stream *idx, git_transfer_progress *stats)
+int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 {
 	git_mwindow *w = NULL;
 	unsigned int i, long_offsets = 0, left;
 	struct git_pack_idx_header hdr;
 	git_buf filename = GIT_BUF_INIT;
 	struct entry *entry;
-	void *packfile_hash;
-	git_oid file_hash;
+	git_oid trailer_hash, file_hash;
 	git_hash_ctx ctx;
 	git_filebuf index_file = {0};
+	void *packfile_trailer;
 
 	if (git_hash_ctx_init(&ctx) < 0)
 		return -1;
 
 	/* Test for this before resolve_deltas(), as it plays with idx->off */
-	if (idx->off < idx->pack->mwf.size - GIT_OID_RAWSZ) {
-		giterr_set(GITERR_INDEXER, "Indexing error: unexpected data at the end of the pack");
+	if (idx->off < idx->pack->mwf.size - 20) {
+		giterr_set(GITERR_INDEXER, "unexpected data at the end of the pack");
 		return -1;
 	}
 
-	if (idx->deltas.length > 0)
-		if (resolve_deltas(idx, stats) < 0)
-			return -1;
+	packfile_trailer = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
+	if (packfile_trailer == NULL) {
+		git_mwindow_close(&w);
+		goto on_error;
+	}
+
+	/* Compare the packfile trailer as it was sent to us and what we calculated */
+	git_oid_fromraw(&file_hash, packfile_trailer);
+	git_mwindow_close(&w);
+
+	git_hash_final(&trailer_hash, &idx->trailer);
+	if (git_oid_cmp(&file_hash, &trailer_hash)) {
+		giterr_set(GITERR_INDEXER, "packfile trailer mismatch");
+		return -1;
+	}
+
+	/* Freeze the number of deltas */
+	stats->total_deltas = stats->total_objects - stats->indexed_objects;
+
+	if (resolve_deltas(idx, stats) < 0)
+		return -1;
 
 	if (stats->indexed_objects != stats->total_objects) {
-		giterr_set(GITERR_INDEXER, "Indexing error: early EOF");
+		giterr_set(GITERR_INDEXER, "early EOF");
 		return -1;
 	}
 
+	if (stats->local_objects > 0) {
+		if (update_header_and_rehash(idx, stats) < 0)
+			return -1;
+
+		git_hash_final(&trailer_hash, &idx->trailer);
+		if (p_lseek(idx->pack_file.fd, -GIT_OID_RAWSZ, SEEK_END) < 0)
+			return -1;
+
+		if (p_write(idx->pack_file.fd, &trailer_hash, GIT_OID_RAWSZ) < 0) {
+			giterr_set(GITERR_OS, "failed to update pack trailer");
+			return -1;
+		}
+	}
+
 	git_vector_sort(&idx->objects);
 
 	git_buf_sets(&filename, idx->pack->pack_name);
-	git_buf_truncate(&filename, filename.size - strlen("pack"));
+	git_buf_shorten(&filename, strlen("pack"));
 	git_buf_puts(&filename, "idx");
 	if (git_buf_oom(&filename))
 		return -1;
 
-	if (git_filebuf_open(&index_file, filename.ptr, GIT_FILEBUF_HASH_CONTENTS) < 0)
+	if (git_filebuf_open(&index_file, filename.ptr,
+		GIT_FILEBUF_HASH_CONTENTS, idx->mode) < 0)
 		goto on_error;
 
 	/* Write out the header */
@@ -658,30 +959,22 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_transfer_progress *
 		git_filebuf_write(&index_file, &split, sizeof(uint32_t) * 2);
 	}
 
-	/* Write out the packfile trailer */
-	packfile_hash = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
-	if (packfile_hash == NULL) {
-		git_mwindow_close(&w);
+	/* Write out the packfile trailer to the index */
+	if (git_filebuf_write(&index_file, &trailer_hash, GIT_OID_RAWSZ) < 0)
 		goto on_error;
-	}
-
-	memcpy(&file_hash, packfile_hash, GIT_OID_RAWSZ);
-	git_mwindow_close(&w);
-
-	git_filebuf_write(&index_file, &file_hash, sizeof(git_oid));
 
-	/* Write out the packfile trailer to the idx file as well */
-	if (git_filebuf_hash(&file_hash, &index_file) < 0)
+	/* Write out the hash of the idx */
+	if (git_filebuf_hash(&trailer_hash, &index_file) < 0)
 		goto on_error;
 
-	git_filebuf_write(&index_file, &file_hash, sizeof(git_oid));
+	git_filebuf_write(&index_file, &trailer_hash, sizeof(git_oid));
 
 	/* Figure out what the final name should be */
-	if (index_path_stream(&filename, idx, ".idx") < 0)
+	if (index_path(&filename, idx, ".idx") < 0)
 		goto on_error;
 
 	/* Commit file */
-	if (git_filebuf_commit_at(&index_file, filename.ptr, GIT_PACK_FILE_MODE) < 0)
+	if (git_filebuf_commit_at(&index_file, filename.ptr) < 0)
 		goto on_error;
 
 	git_mwindow_free_all(&idx->pack->mwf);
@@ -689,10 +982,10 @@ int git_indexer_stream_finalize(git_indexer_stream *idx, git_transfer_progress *
 	p_close(idx->pack->mwf.fd);
 	idx->pack->mwf.fd = -1;
 
-	if (index_path_stream(&filename, idx, ".pack") < 0)
+	if (index_path(&filename, idx, ".pack") < 0)
 		goto on_error;
 	/* And don't forget to rename the packfile to its new place. */
-	if (git_filebuf_commit_at(&idx->pack_file, filename.ptr, GIT_PACK_FILE_MODE) < 0)
+	if (git_filebuf_commit_at(&idx->pack_file, filename.ptr) < 0)
 		return -1;
 
 	git_buf_free(&filename);
@@ -706,7 +999,7 @@ on_error:
 	return -1;
 }
 
-void git_indexer_stream_free(git_indexer_stream *idx)
+void git_indexer_free(git_indexer *idx)
 {
 	khiter_t k;
 	unsigned int i;
diff --git a/src/iterator.c b/src/iterator.c
index 5917f63fd..8646399ab 100644
--- a/src/iterator.c
+++ b/src/iterator.c
@@ -893,6 +893,7 @@ struct fs_iterator {
 	git_index_entry entry;
 	git_buf path;
 	size_t root_len;
+	uint32_t dirload_flags;
 	int depth;
 
 	int (*enter_dir_cb)(fs_iterator *self);
@@ -986,12 +987,25 @@ static int fs_iterator__expand_dir(fs_iterator *fi)
 	GITERR_CHECK_ALLOC(ff);
 
 	error = git_path_dirload_with_stat(
-		fi->path.ptr, fi->root_len, iterator__ignore_case(fi),
+		fi->path.ptr, fi->root_len, fi->dirload_flags,
 		fi->base.start, fi->base.end, &ff->entries);
 
 	if (error < 0) {
+		git_error last_error = {0};
+
+		giterr_detach(&last_error);
+
+		/* these callbacks may clear the error message */
 		fs_iterator__free_frame(ff);
 		fs_iterator__advance_over(NULL, (git_iterator *)fi);
+		/* next time return value we skipped to */
+		fi->base.flags &= ~GIT_ITERATOR_FIRST_ACCESS;
+
+		if (last_error.message) {
+			giterr_set_str(last_error.klass, last_error.message);
+			free(last_error.message);
+		}
+
 		return error;
 	}
 
@@ -1174,7 +1188,7 @@ static int fs_iterator__update_entry(fs_iterator *fi)
 		return GIT_ITEROVER;
 
 	fi->entry.path = ps->path;
-	git_index_entry__init_from_stat(&fi->entry, &ps->st);
+	git_index_entry__init_from_stat(&fi->entry, &ps->st, true);
 
 	/* need different mode here to keep directories during iteration */
 	fi->entry.mode = git_futils_canonical_mode(ps->st.st_mode);
@@ -1207,6 +1221,11 @@ static int fs_iterator__initialize(
 	}
 	fi->root_len = fi->path.size;
 
+	fi->dirload_flags =
+		(iterator__ignore_case(fi) ? GIT_PATH_DIR_IGNORE_CASE : 0) |
+		(iterator__flag(fi, PRECOMPOSE_UNICODE) ?
+			GIT_PATH_DIR_PRECOMPOSE_UNICODE : 0);
+
 	if ((error = fs_iterator__expand_dir(fi)) < 0) {
 		if (error == GIT_ENOTFOUND || error == GIT_ITEROVER) {
 			giterr_clear();
@@ -1329,7 +1348,7 @@ int git_iterator_for_workdir_ext(
 	const char *start,
 	const char *end)
 {
-	int error;
+	int error, precompose = 0;
 	workdir_iterator *wi;
 
 	if (!repo_workdir) {
@@ -1350,12 +1369,18 @@ int git_iterator_for_workdir_ext(
 	wi->fi.update_entry_cb = workdir_iterator__update_entry;
 
 	if ((error = iterator__update_ignore_case((git_iterator *)wi, flags)) < 0 ||
-		(error = git_ignore__for_path(repo, "", &wi->ignores)) < 0)
+		(error = git_ignore__for_path(repo, ".gitignore", &wi->ignores)) < 0)
 	{
 		git_iterator_free((git_iterator *)wi);
 		return error;
 	}
 
+	/* try to look up precompose and set flag if appropriate */
+	if (git_repository__cvar(&precompose, repo, GIT_CVAR_PRECOMPOSE) < 0)
+		giterr_clear();
+	else if (precompose)
+		wi->fi.base.flags |= GIT_ITERATOR_PRECOMPOSE_UNICODE;
+
 	return fs_iterator__initialize(out, &wi->fi, repo_workdir);
 }
 
diff --git a/src/iterator.h b/src/iterator.h
index ea88fa6a2..751e139d0 100644
--- a/src/iterator.h
+++ b/src/iterator.h
@@ -24,13 +24,15 @@ typedef enum {
 
 typedef enum {
 	/** ignore case for entry sort order */
-	GIT_ITERATOR_IGNORE_CASE = (1 << 0),
+	GIT_ITERATOR_IGNORE_CASE = (1u << 0),
 	/** force case sensitivity for entry sort order */
-	GIT_ITERATOR_DONT_IGNORE_CASE = (1 << 1),
+	GIT_ITERATOR_DONT_IGNORE_CASE = (1u << 1),
 	/** return tree items in addition to blob items */
-	GIT_ITERATOR_INCLUDE_TREES    = (1 << 2),
+	GIT_ITERATOR_INCLUDE_TREES    = (1u << 2),
 	/** don't flatten trees, requiring advance_into (implies INCLUDE_TREES) */
-	GIT_ITERATOR_DONT_AUTOEXPAND  = (1 << 3),
+	GIT_ITERATOR_DONT_AUTOEXPAND  = (1u << 3),
+	/** convert precomposed unicode to decomposed unicode */
+	GIT_ITERATOR_PRECOMPOSE_UNICODE = (1u << 4),
 } git_iterator_flag_t;
 
 typedef struct {
diff --git a/src/merge.c b/src/merge.c
index 82d2e6f37..115867971 100644
--- a/src/merge.c
+++ b/src/merge.c
@@ -58,7 +58,7 @@ struct merge_diff_df_data {
 
 /* Merge base computation */
 
-int git_merge_base_many(git_oid *out, git_repository *repo, const git_oid input_array[], size_t length)
+int git_merge_base_many(git_oid *out, git_repository *repo, size_t length, const git_oid input_array[])
 {
 	git_revwalk *walk;
 	git_vector list;
@@ -285,7 +285,7 @@ int git_repository_mergehead_foreach(git_repository *repo,
 		if ((error = git_oid_fromstr(&oid, line)) < 0)
 			goto cleanup;
 
-		if (cb(&oid, payload) < 0) {
+		if (cb(&oid, payload) != 0) {
 			error = GIT_EUSER;
 			goto cleanup;
 		}
@@ -1615,17 +1615,14 @@ static int write_orig_head(
 {
 	git_filebuf file = GIT_FILEBUF_INIT;
 	git_buf file_path = GIT_BUF_INIT;
-	char orig_oid_str[GIT_OID_HEXSZ + 1];
 	int error = 0;
 
 	assert(repo && our_head);
 
-	git_oid_tostr(orig_oid_str, GIT_OID_HEXSZ+1, &our_head->oid);
-
 	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_ORIG_HEAD_FILE)) == 0 &&
-		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE)) == 0 &&
-		(error = git_filebuf_printf(&file, "%s\n", orig_oid_str)) == 0)
-		error = git_filebuf_commit(&file, 0666);
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_MERGE_FILE_MODE)) == 0 &&
+		(error = git_filebuf_printf(&file, "%s\n", our_head->oid_str)) == 0)
+		error = git_filebuf_commit(&file);
 
 	if (error < 0)
 		git_filebuf_cleanup(&file);
@@ -1642,24 +1639,21 @@ static int write_merge_head(
 {
 	git_filebuf file = GIT_FILEBUF_INIT;
 	git_buf file_path = GIT_BUF_INIT;
-	char merge_oid_str[GIT_OID_HEXSZ + 1];
 	size_t i;
 	int error = 0;
 
 	assert(repo && heads);
 
 	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_MERGE_HEAD_FILE)) < 0 ||
-		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE)) < 0)
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_MERGE_FILE_MODE)) < 0)
 		goto cleanup;
 
 	for (i = 0; i < heads_len; i++) {
-		git_oid_tostr(merge_oid_str, GIT_OID_HEXSZ+1, &heads[i]->oid);
-
-		if ((error = git_filebuf_printf(&file, "%s\n", merge_oid_str)) < 0)
+		if ((error = git_filebuf_printf(&file, "%s\n", heads[i]->oid_str)) < 0)
 			goto cleanup;
 	}
 
-	error = git_filebuf_commit(&file, 0666);
+	error = git_filebuf_commit(&file);
 
 cleanup:
 	if (error < 0)
@@ -1682,10 +1676,21 @@ static int write_merge_mode(git_repository *repo, unsigned int flags)
 	assert(repo);
 
 	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_MERGE_MODE_FILE)) < 0 ||
-		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE)) < 0)
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_MERGE_FILE_MODE)) < 0)
 		goto cleanup;
 
-	error = git_filebuf_commit(&file, 0666);
+	/*
+	 * no-ff is the only thing allowed here at present.  One would
+	 * presume they would be space-delimited when there are more, but
+	 * this needs to be revisited.
+	 */
+
+	if (flags & GIT_MERGE_NO_FASTFORWARD) {
+		if ((error = git_filebuf_write(&file, "no-ff", 5)) < 0)
+			goto cleanup;
+	}
+
+	error = git_filebuf_commit(&file);
 
 cleanup:
 	if (error < 0)
@@ -1890,7 +1895,6 @@ static int write_merge_msg(
 {
 	git_filebuf file = GIT_FILEBUF_INIT;
 	git_buf file_path = GIT_BUF_INIT;
-	char oid_str[GIT_OID_HEXSZ + 1];
 	struct merge_msg_entry *entries;
 	git_vector matching = GIT_VECTOR_INIT;
 	size_t i;
@@ -1902,14 +1906,16 @@ static int write_merge_msg(
 	entries = git__calloc(heads_len, sizeof(struct merge_msg_entry));
 	GITERR_CHECK_ALLOC(entries); 
 
-	if (git_vector_init(&matching, heads_len, NULL) < 0)
+	if (git_vector_init(&matching, heads_len, NULL) < 0) {
+		git__free(entries);
 		return -1;
+	}
 
 	for (i = 0; i < heads_len; i++)
 		entries[i].merge_head = heads[i];
 
 	if ((error = git_buf_joinpath(&file_path, repo->path_repository, GIT_MERGE_MSG_FILE)) < 0 ||
-		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE)) < 0 ||
+		(error = git_filebuf_open(&file, file_path.ptr, GIT_FILEBUF_FORCE, GIT_MERGE_FILE_MODE)) < 0 ||
 		(error = git_filebuf_write(&file, "Merge ", 6)) < 0)
 		goto cleanup;
 
@@ -1928,10 +1934,9 @@ static int write_merge_msg(
 		if (!msg_entry_is_oid(&entries[i]))
 			break;
 
-		git_oid_fmt(oid_str, &entries[i].merge_head->oid);
-		oid_str[GIT_OID_HEXSZ] = '\0';
-
-		if ((error = git_filebuf_printf(&file, "%scommit '%s'", (i > 0) ? "; " : "", oid_str)) < 0)
+		if ((error = git_filebuf_printf(&file,
+			"%scommit '%s'", (i > 0) ? "; " : "",
+			entries[i].merge_head->oid_str)) < 0)
 			goto cleanup;
 
 		entries[i].written = 1;
@@ -1978,15 +1983,13 @@ static int write_merge_msg(
 		if (merge_msg_entry_written(&entries[i]))
 			continue;
 
-		git_oid_fmt(oid_str, &entries[i].merge_head->oid);
-		oid_str[GIT_OID_HEXSZ] = '\0';
-
-		if ((error = git_filebuf_printf(&file, "; commit '%s'", oid_str)) < 0)
+		if ((error = git_filebuf_printf(&file, "; commit '%s'",
+			entries[i].merge_head->oid_str)) < 0)
 			goto cleanup;
 	}
 
 	if ((error = git_filebuf_printf(&file, "\n")) < 0 ||
-		(error = git_filebuf_commit(&file, 0666)) < 0)
+		(error = git_filebuf_commit(&file)) < 0)
 		goto cleanup;
 
 cleanup:
@@ -2001,6 +2004,477 @@ cleanup:
 	return error;
 }
 
+/* Merge branches */
+
+static int merge_ancestor_head(
+	git_merge_head **ancestor_head,
+	git_repository *repo,
+	const git_merge_head *our_head,
+	const git_merge_head **their_heads,
+	size_t their_heads_len)
+{
+	git_oid *oids, ancestor_oid;
+	size_t i;
+	int error = 0;
+
+	assert(repo && our_head && their_heads);
+
+	oids = git__calloc(their_heads_len + 1, sizeof(git_oid));
+	GITERR_CHECK_ALLOC(oids);
+
+	git_oid_cpy(&oids[0], git_commit_id(our_head->commit));
+
+	for (i = 0; i < their_heads_len; i++)
+		git_oid_cpy(&oids[i + 1], &their_heads[i]->oid);
+
+	if ((error = git_merge_base_many(&ancestor_oid, repo, their_heads_len + 1, oids)) < 0)
+		goto on_error;
+
+	error = git_merge_head_from_oid(ancestor_head, repo, &ancestor_oid);
+
+on_error:
+	git__free(oids);
+	return error;
+}
+
+GIT_INLINE(bool) merge_check_uptodate(
+	git_merge_result *result,
+	const git_merge_head *ancestor_head,
+	const git_merge_head *their_head)
+{
+	if (git_oid_cmp(&ancestor_head->oid, &their_head->oid) == 0) {
+		result->is_uptodate = 1;
+		return true;
+	}
+
+	return false;
+}
+
+GIT_INLINE(bool) merge_check_fastforward(
+	git_merge_result *result,
+	const git_merge_head *ancestor_head,
+	const git_merge_head *our_head,
+	const git_merge_head *their_head,
+	unsigned int flags)
+{
+	if ((flags & GIT_MERGE_NO_FASTFORWARD) == 0 &&
+		git_oid_cmp(&ancestor_head->oid, &our_head->oid) == 0) {
+		result->is_fastforward = 1;
+		git_oid_cpy(&result->fastforward_oid, &their_head->oid);
+
+		return true;
+	}
+
+	return false;
+}
+
+const char *merge_their_label(const char *branchname)
+{
+	const char *slash;
+
+	if ((slash = strrchr(branchname, '/')) == NULL)
+		return branchname;
+
+	if (*(slash+1) == '\0')
+		return "theirs";
+
+	return slash+1;
+}
+
+static int merge_normalize_opts(
+	git_repository *repo,
+	git_merge_opts *opts,
+	const git_merge_opts *given,
+	size_t their_heads_len,
+	const git_merge_head **their_heads)
+{
+	int error = 0;
+	unsigned int default_checkout_strategy = GIT_CHECKOUT_SAFE_CREATE |
+		GIT_CHECKOUT_ALLOW_CONFLICTS;
+
+	GIT_UNUSED(repo);
+
+	if (given != NULL)
+		memcpy(opts, given, sizeof(git_merge_opts));
+	else {
+		git_merge_opts default_opts = GIT_MERGE_OPTS_INIT;
+		memcpy(opts, &default_opts, sizeof(git_merge_opts));
+	}
+
+	if (!opts->checkout_opts.checkout_strategy)
+		opts->checkout_opts.checkout_strategy = default_checkout_strategy;
+
+	if (!opts->checkout_opts.our_label)
+		opts->checkout_opts.our_label = "HEAD";
+
+	if (!opts->checkout_opts.their_label) {
+		if (their_heads_len == 1 && their_heads[0]->ref_name)
+			opts->checkout_opts.their_label = merge_their_label(their_heads[0]->ref_name);
+		else if (their_heads_len == 1)
+			opts->checkout_opts.their_label = their_heads[0]->oid_str;
+		else
+			opts->checkout_opts.their_label = "theirs";
+	}
+
+	return error;
+}
+
+static int merge_affected_paths(git_vector *paths, git_repository *repo, git_index *index_new)
+{
+	git_tree *head_tree = NULL;
+	git_iterator *iter_head = NULL, *iter_new = NULL;
+	git_diff *merged_list = NULL;
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	git_diff_delta *delta;
+	size_t i;
+	const git_index_entry *e;
+	char *path;
+	int error = 0;
+
+	if ((error = git_repository_head_tree(&head_tree, repo)) < 0 ||
+		(error = git_iterator_for_tree(&iter_head, head_tree, GIT_ITERATOR_DONT_IGNORE_CASE, NULL, NULL)) < 0 ||
+		(error = git_iterator_for_index(&iter_new, index_new, GIT_ITERATOR_DONT_IGNORE_CASE, NULL, NULL)) < 0 ||
+		(error = git_diff__from_iterators(&merged_list, repo, iter_head, iter_new, &opts)) < 0)
+		goto done;
+
+	git_vector_foreach(&merged_list->deltas, i, delta) {
+		path = git__strdup(delta->new_file.path);
+		GITERR_CHECK_ALLOC(path);
+
+		if ((error = git_vector_insert(paths, path)) < 0)
+			goto on_error;
+	}
+
+	for (i = 0; i < git_index_entrycount(index_new); i++) {
+		e = git_index_get_byindex(index_new, i);
+
+		if (git_index_entry_stage(e) != 0 &&
+			(git_vector_last(paths) == NULL ||
+			strcmp(git_vector_last(paths), e->path) != 0)) {
+
+			path = git__strdup(e->path);
+			GITERR_CHECK_ALLOC(path);
+
+			if ((error = git_vector_insert(paths, path)) < 0)
+				goto on_error;
+		}
+	}
+
+	goto done;
+
+on_error:
+	git_vector_foreach(paths, i, path)
+		git__free(path);
+
+	git_vector_clear(paths);
+
+done:
+	git_tree_free(head_tree);
+	git_iterator_free(iter_head);
+	git_iterator_free(iter_new);
+	git_diff_free(merged_list);
+
+	return error;
+}
+
+static int merge_check_index(size_t *conflicts, git_repository *repo, git_index *index_new, git_vector *merged_paths)
+{
+	git_tree *head_tree = NULL;
+	git_index *index_repo = NULL;
+	git_iterator *iter_repo = NULL, *iter_new = NULL;
+	git_diff *staged_diff_list = NULL, *index_diff_list = NULL;
+	git_diff_delta *delta;
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	git_vector staged_paths = GIT_VECTOR_INIT;
+	size_t i;
+	int error = 0;
+
+	GIT_UNUSED(merged_paths);
+
+	*conflicts = 0;
+
+	/* No staged changes may exist unless the change staged is identical to
+	 * the result of the merge.  This allows one to apply to merge manually,
+	 * then run merge.  Any other staged change would be overwritten by
+	 * a reset merge.
+	 */
+	if ((error = git_repository_head_tree(&head_tree, repo)) < 0 ||
+		(error = git_repository_index(&index_repo, repo)) < 0 ||
+		(error = git_diff_tree_to_index(&staged_diff_list, repo, head_tree, index_repo, &opts)) < 0)
+		goto done;
+
+	if (staged_diff_list->deltas.length == 0)
+		goto done;
+
+	git_vector_foreach(&staged_diff_list->deltas, i, delta) {
+		if ((error = git_vector_insert(&staged_paths, (char *)delta->new_file.path)) < 0)
+			goto done;
+	}
+
+	opts.pathspec.count = staged_paths.length;
+	opts.pathspec.strings = (char **)staged_paths.contents;
+
+	if ((error = git_iterator_for_index(&iter_repo, index_repo, GIT_ITERATOR_DONT_IGNORE_CASE, NULL, NULL)) < 0 ||
+		(error = git_iterator_for_index(&iter_new, index_new, GIT_ITERATOR_DONT_IGNORE_CASE, NULL, NULL)) < 0 ||
+		(error = git_diff__from_iterators(&index_diff_list, repo, iter_repo, iter_new, &opts)) < 0)
+		goto done;
+
+	*conflicts = index_diff_list->deltas.length;
+
+done:
+	git_tree_free(head_tree);
+	git_index_free(index_repo);
+	git_iterator_free(iter_repo);
+	git_iterator_free(iter_new);
+	git_diff_free(staged_diff_list);
+	git_diff_free(index_diff_list);
+	git_vector_free(&staged_paths);
+
+	return error;
+}
+
+static int merge_check_workdir(size_t *conflicts, git_repository *repo, git_index *index_new, git_vector *merged_paths)
+{
+	git_tree *head_tree = NULL;
+	git_diff *wd_diff_list = NULL;
+	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
+	int error = 0;
+
+	GIT_UNUSED(index_new);
+
+	*conflicts = 0;
+
+	opts.flags |= GIT_DIFF_INCLUDE_UNTRACKED;
+
+	if ((error = git_repository_head_tree(&head_tree, repo)) < 0)
+		goto done;
+
+	/* Workdir changes may exist iff they do not conflict with changes that
+	 * will be applied by the merge (including conflicts).  Ensure that there
+	 * are no changes in the workdir to these paths.
+	 */
+	opts.pathspec.count = merged_paths->length;
+	opts.pathspec.strings = (char **)merged_paths->contents;
+
+	if ((error = git_diff_tree_to_workdir(&wd_diff_list, repo, head_tree, &opts)) < 0)
+		goto done;
+
+	*conflicts = wd_diff_list->deltas.length;
+
+done:
+	git_tree_free(head_tree);
+	git_diff_free(wd_diff_list);
+
+	return error;
+}
+
+static int merge_indexes(git_repository *repo, git_index *index_new)
+{
+	git_index *index_repo;
+	unsigned int index_repo_caps;
+	git_vector paths = GIT_VECTOR_INIT;
+	size_t index_conflicts = 0, wd_conflicts = 0, conflicts, i;
+	char *path;
+	const git_index_entry *e;
+	const git_index_name_entry *name;
+	const git_index_reuc_entry *reuc;
+	int error = 0;
+
+	if ((error = git_repository_index(&index_repo, repo)) < 0)
+		goto done;
+
+	/* Set the index to case sensitive to handle the merge */
+	index_repo_caps = git_index_caps(index_repo);
+
+	if ((error = git_index_set_caps(index_repo, (index_repo_caps & ~GIT_INDEXCAP_IGNORE_CASE))) < 0)
+		goto done;
+
+	/* Make sure the index and workdir state do not prevent merging */
+	if ((error = merge_affected_paths(&paths, repo, index_new)) < 0 ||
+		(error = merge_check_index(&index_conflicts, repo, index_new, &paths)) < 0 ||
+		(error = merge_check_workdir(&wd_conflicts, repo, index_new, &paths)) < 0)
+		goto done;
+
+	if ((conflicts = index_conflicts + wd_conflicts) > 0) {
+		giterr_set(GITERR_MERGE, "%d uncommitted change%s would be overwritten by merge",
+			conflicts, (conflicts != 1) ? "s" : "");
+		error = GIT_EMERGECONFLICT;
+
+		goto done;
+	}
+
+	/* Update the new index */
+	git_vector_foreach(&paths, i, path) {
+		if ((e = git_index_get_bypath(index_new, path, 0)) != NULL)
+			error = git_index_add(index_repo, e);
+		else
+			error = git_index_remove(index_repo, path, 0);
+	}
+
+	/* Add conflicts */
+	git_index_conflict_cleanup(index_repo);
+
+	for (i = 0; i < git_index_entrycount(index_new); i++) {
+		e = git_index_get_byindex(index_new, i);
+
+		if (git_index_entry_stage(e) != 0 &&
+			(error = git_index_add(index_repo, e)) < 0)
+			goto done;
+	}
+
+	/* Add name entries */
+	git_index_name_clear(index_repo);
+
+	for (i = 0; i < git_index_name_entrycount(index_new); i++) {
+		name = git_index_name_get_byindex(index_new, i);
+
+		if ((error = git_index_name_add(index_repo,
+			name->ancestor, name->ours, name->theirs)) < 0)
+			goto done;
+	}
+
+	/* Add the reuc */
+	git_index_reuc_clear(index_repo);
+
+	for (i = 0; i < git_index_reuc_entrycount(index_new); i++) {
+		reuc = (git_index_reuc_entry *)git_index_reuc_get_byindex(index_new, i);
+
+		if ((error = git_index_reuc_add(index_repo, reuc->path,
+			reuc->mode[0], &reuc->oid[0],
+			reuc->mode[1], &reuc->oid[1],
+			reuc->mode[2], &reuc->oid[2])) < 0)
+			goto done;
+	}
+
+done:
+	if (index_repo != NULL)
+        git_index_set_caps(index_repo, index_repo_caps);
+
+	git_index_free(index_repo);
+
+	git_vector_foreach(&paths, i, path)
+		git__free(path);
+
+	git_vector_free(&paths);
+
+	return error;
+}
+
+int git_merge(
+	git_merge_result **out,
+	git_repository *repo,
+	const git_merge_head **their_heads,
+	size_t their_heads_len,
+	const git_merge_opts *given_opts)
+{
+	git_merge_result *result;
+	git_merge_opts opts;
+	git_reference *our_ref = NULL;
+	git_merge_head *ancestor_head = NULL, *our_head = NULL;
+	git_tree *ancestor_tree = NULL, *our_tree = NULL, **their_trees = NULL;
+	git_index *index_new = NULL, *index_repo = NULL;
+	size_t i;
+	int error = 0;
+
+	assert(out && repo && their_heads);
+
+	*out = NULL;
+
+	if (their_heads_len != 1) {
+		giterr_set(GITERR_MERGE, "Can only merge a single branch");
+		return -1;
+	}
+
+	result = git__calloc(1, sizeof(git_merge_result));
+	GITERR_CHECK_ALLOC(result);
+
+	their_trees = git__calloc(their_heads_len, sizeof(git_tree *));
+	GITERR_CHECK_ALLOC(their_trees);
+
+	if ((error = merge_normalize_opts(repo, &opts, given_opts, their_heads_len, their_heads)) < 0)
+		goto on_error;
+
+	if ((error = git_repository__ensure_not_bare(repo, "merge")) < 0)
+		goto on_error;
+
+	if ((error = git_reference_lookup(&our_ref, repo, GIT_HEAD_FILE)) < 0 ||
+		(error = git_merge_head_from_ref(&our_head, repo, our_ref)) < 0)
+		goto on_error;
+
+	if ((error = merge_ancestor_head(&ancestor_head, repo, our_head, their_heads, their_heads_len)) < 0 &&
+		error != GIT_ENOTFOUND)
+		goto on_error;
+
+	if (their_heads_len == 1 &&
+		ancestor_head != NULL &&
+		(merge_check_uptodate(result, ancestor_head, their_heads[0]) ||
+		merge_check_fastforward(result, ancestor_head, our_head, their_heads[0], opts.merge_flags))) {
+		*out = result;
+		goto done;
+	}
+
+	/* If FASTFORWARD_ONLY is specified, fail. */
+	if ((opts.merge_flags & GIT_MERGE_FASTFORWARD_ONLY) ==
+		GIT_MERGE_FASTFORWARD_ONLY) {
+		giterr_set(GITERR_MERGE, "Not a fast-forward.");
+		error = GIT_ENONFASTFORWARD;
+		goto on_error;
+	}
+
+	/* Write the merge files to the repository. */
+	if ((error = git_merge__setup(repo, our_head, their_heads, their_heads_len, opts.merge_flags)) < 0)
+		goto on_error;
+
+	if (ancestor_head != NULL &&
+		(error = git_commit_tree(&ancestor_tree, ancestor_head->commit)) < 0)
+			goto on_error;
+
+	if ((error = git_commit_tree(&our_tree, our_head->commit)) < 0)
+		goto on_error;
+
+	for (i = 0; i < their_heads_len; i++) {
+		if ((error = git_commit_tree(&their_trees[i], their_heads[i]->commit)) < 0)
+			goto on_error;
+	}
+
+	/* TODO: recursive, octopus, etc... */
+
+	if ((error = git_merge_trees(&index_new, repo, ancestor_tree, our_tree, their_trees[0], &opts.merge_tree_opts)) < 0 ||
+		(error = merge_indexes(repo, index_new)) < 0 ||
+		(error = git_repository_index(&index_repo, repo)) < 0 ||
+		(error = git_checkout_index(repo, index_repo, &opts.checkout_opts)) < 0)
+		goto on_error;
+
+	result->index = index_new;
+
+	*out = result;
+	goto done;
+
+on_error:
+	git_repository_merge_cleanup(repo);
+
+	git_index_free(index_new);
+	git__free(result);
+
+done:
+	git_index_free(index_repo);
+
+	git_tree_free(ancestor_tree);
+	git_tree_free(our_tree);
+
+	for (i = 0; i < their_heads_len; i++)
+		git_tree_free(their_trees[i]);
+
+	git__free(their_trees);
+
+	git_merge_head_free(our_head);
+	git_merge_head_free(ancestor_head);
+
+	git_reference_free(our_ref);
+
+	return error;
+}
+
 int git_merge__setup(
 	git_repository *repo,
 	const git_merge_head *our_head,
@@ -2011,7 +2485,7 @@ int git_merge__setup(
 	int error = 0;
 
 	assert (repo && our_head && heads);
-	
+
 	if ((error = write_orig_head(repo, our_head)) == 0 &&
 		(error = write_merge_head(repo, heads, heads_len)) == 0 &&
 		(error = write_merge_mode(repo, flags)) == 0) {
@@ -2054,6 +2528,41 @@ cleanup:
 	return error;
 }
 
+/* Merge result data */
+
+int git_merge_result_is_uptodate(git_merge_result *merge_result)
+{
+	assert(merge_result);
+
+	return merge_result->is_uptodate;
+}
+
+int git_merge_result_is_fastforward(git_merge_result *merge_result)
+{
+	assert(merge_result);
+
+	return merge_result->is_fastforward;
+}
+
+int git_merge_result_fastforward_oid(git_oid *out, git_merge_result *merge_result)
+{
+	assert(out && merge_result);
+
+	git_oid_cpy(out, &merge_result->fastforward_oid);
+	return 0;
+}
+
+void git_merge_result_free(git_merge_result *merge_result)
+{
+	if (merge_result == NULL)
+		return;
+
+	git_index_free(merge_result->index);
+	merge_result->index = NULL;
+
+	git__free(merge_result);
+}
+
 /* Merge heads are the input to merge */
 
 static int merge_head_init(
@@ -2085,6 +2594,9 @@ static int merge_head_init(
 
 	git_oid_cpy(&head->oid, oid);
 
+	git_oid_fmt(head->oid_str, oid);
+	head->oid_str[GIT_OID_HEXSZ] = '\0';
+
 	if ((error = git_commit_lookup(&head->commit, repo, &head->oid)) < 0) {
 		git_merge_head_free(head);
 		return error;
diff --git a/src/merge.h b/src/merge.h
index ba6725de9..d7d1c67b7 100644
--- a/src/merge.h
+++ b/src/merge.h
@@ -16,6 +16,7 @@
 
 #define GIT_MERGE_MSG_FILE		"MERGE_MSG"
 #define GIT_MERGE_MODE_FILE		"MERGE_MODE"
+#define GIT_MERGE_FILE_MODE		0666
 
 #define GIT_MERGE_TREE_RENAME_THRESHOLD	50
 #define GIT_MERGE_TREE_TARGET_LIMIT		1000
@@ -113,9 +114,20 @@ struct git_merge_head {
 	char *remote_url;
 
 	git_oid oid;
+	char oid_str[GIT_OID_HEXSZ+1];
 	git_commit *commit;
 };
 
+/** Internal structure for merge results */
+struct git_merge_result {
+	bool is_uptodate;
+
+	bool is_fastforward;
+	git_oid fastforward_oid;
+
+	git_index *index;
+};
+
 int git_merge__bases_many(
 	git_commit_list **out,
 	git_revwalk *walk,
diff --git a/src/merge_file.c b/src/merge_file.c
index c3477ccb9..48fc46e57 100644
--- a/src/merge_file.c
+++ b/src/merge_file.c
@@ -47,7 +47,7 @@ GIT_INLINE(int) merge_file_best_mode(
 	 * assume executable.  Otherwise, if any mode changed from the ancestor,
 	 * use that one.
 	 */
-	if (GIT_MERGE_FILE_SIDE_EXISTS(ancestor)) {
+	if (!GIT_MERGE_FILE_SIDE_EXISTS(ancestor)) {
 		if (ours->mode == GIT_FILEMODE_BLOB_EXECUTABLE ||
 			theirs->mode == GIT_FILEMODE_BLOB_EXECUTABLE)
 			return GIT_FILEMODE_BLOB_EXECUTABLE;
diff --git a/src/netops.c b/src/netops.c
index 69179dd1c..ad27d84cf 100644
--- a/src/netops.c
+++ b/src/netops.c
@@ -19,10 +19,6 @@
 #	endif
 #endif
 
-#ifdef __FreeBSD__
-#	include <netinet/in.h>
-#endif
-
 #ifdef GIT_SSL
 # include <openssl/ssl.h>
 # include <openssl/err.h>
@@ -36,6 +32,7 @@
 #include "netops.h"
 #include "posix.h"
 #include "buffer.h"
+#include "http_parser.h"
 
 #ifdef GIT_WIN32
 static void net_set_error(const char *str)
@@ -577,55 +574,161 @@ int gitno_select_in(gitno_buffer *buf, long int sec, long int usec)
 	return select((int)buf->socket->socket + 1, &fds, NULL, NULL, &tv);
 }
 
+static const char *prefix_http = "http://";
+static const char *prefix_https = "https://";
+
+int gitno_connection_data_from_url(
+		gitno_connection_data *data,
+		const char *url,
+		const char *service_suffix)
+{
+	int error = -1;
+	const char *default_port = NULL, *path_search_start = NULL;
+	char *original_host = NULL;
+
+	/* service_suffix is optional */
+	assert(data && url);
+
+	/* Save these for comparison later */
+	original_host = data->host;
+	data->host = NULL;
+	gitno_connection_data_free_ptrs(data);
+
+	if (!git__prefixcmp(url, prefix_http)) {
+		path_search_start = url + strlen(prefix_http);
+		default_port = "80";
+
+		if (data->use_ssl) {
+			giterr_set(GITERR_NET, "Redirect from HTTPS to HTTP is not allowed");
+			goto cleanup;
+		}
+	} else if (!git__prefixcmp(url, prefix_https)) {
+		path_search_start = url + strlen(prefix_https);
+		default_port = "443";
+		data->use_ssl = true;
+	} else if (url[0] == '/')
+		default_port = data->use_ssl ? "443" : "80";
+
+	if (!default_port) {
+		giterr_set(GITERR_NET, "Unrecognized URL prefix");
+		goto cleanup;
+	}
+
+	error = gitno_extract_url_parts(
+		&data->host, &data->port, &data->path, &data->user, &data->pass,
+		url, default_port);
+
+	if (url[0] == '/') {
+		/* Relative redirect; reuse original host name and port */
+		path_search_start = url;
+		git__free(data->host);
+		data->host = original_host;
+		original_host = NULL;
+	}
+
+	if (!error) {
+		const char *path = strchr(path_search_start, '/');
+		size_t pathlen = strlen(path);
+		size_t suffixlen = service_suffix ? strlen(service_suffix) : 0;
+
+		if (suffixlen &&
+		    !memcmp(path + pathlen - suffixlen, service_suffix, suffixlen)) {
+			git__free(data->path);
+			data->path = git__strndup(path, pathlen - suffixlen);
+		} else {
+			git__free(data->path);
+			data->path = git__strdup(path);
+		}
+
+		/* Check for errors in the resulting data */
+		if (original_host && url[0] != '/' && strcmp(original_host, data->host)) {
+			giterr_set(GITERR_NET, "Cross host redirect not allowed");
+			error = -1;
+		}
+	}
+
+cleanup:
+	if (original_host) git__free(original_host);
+	return error;
+}
+
+void gitno_connection_data_free_ptrs(gitno_connection_data *d)
+{
+	git__free(d->host); d->host = NULL;
+	git__free(d->port); d->port = NULL;
+	git__free(d->path); d->path = NULL;
+	git__free(d->user); d->user = NULL;
+	git__free(d->pass); d->pass = NULL;
+}
+
+#define hex2c(c) ((c | 32) % 39 - 9)
+static char* unescape(char *str)
+{
+	int x, y;
+	int len = (int)strlen(str);
+
+	for (x=y=0; str[y]; ++x, ++y) {
+		if ((str[x] = str[y]) == '%') {
+			if (y < len-2 && isxdigit(str[y+1]) && isxdigit(str[y+2])) {
+				str[x] = (hex2c(str[y+1]) << 4) + hex2c(str[y+2]);
+				y += 2;
+			}
+		}
+	}
+	str[x] = '\0';
+	return str;
+}
+
 int gitno_extract_url_parts(
 		char **host,
 		char **port,
+		char **path,
 		char **username,
 		char **password,
 		const char *url,
 		const char *default_port)
 {
-	char *colon, *slash, *at, *end;
-	const char *start;
-
-	/*
-	 *
-	 * ==> [user[:pass]@]hostname.tld[:port]/resource
-	 */
-
-	colon = strchr(url, ':');
-	slash = strchr(url, '/');
-	at = strchr(url, '@');
+	struct http_parser_url u = {0};
+	const char *_host, *_port, *_path, *_userinfo;
 
-	if (slash == NULL) {
-		giterr_set(GITERR_NET, "Malformed URL: missing /");
-		return -1;
+	if (http_parser_parse_url(url, strlen(url), false, &u)) {
+		giterr_set(GITERR_NET, "Malformed URL '%s'", url);
+		return GIT_EINVALIDSPEC;
 	}
 
-	start = url;
-	if (at && at < slash) {
-		start = at+1;
-		*username = git__substrdup(url, at - url);
-	}
+	_host = url+u.field_data[UF_HOST].off;
+	_port = url+u.field_data[UF_PORT].off;
+	_path = url+u.field_data[UF_PATH].off;
+	_userinfo = url+u.field_data[UF_USERINFO].off;
 
-	if (colon && colon < at) {
-		git__free(*username);
-		*username = git__substrdup(url, colon-url);
-		*password = git__substrdup(colon+1, at-colon-1);
-		colon = strchr(at, ':');
+	if (u.field_set & (1 << UF_HOST)) {
+		*host = git__substrdup(_host, u.field_data[UF_HOST].len);
+		GITERR_CHECK_ALLOC(*host);
 	}
 
-	if (colon == NULL) {
+	if (u.field_set & (1 << UF_PORT))
+		*port = git__substrdup(_port, u.field_data[UF_PORT].len);
+	else
 		*port = git__strdup(default_port);
-	} else {
-		*port = git__substrdup(colon + 1, slash - colon - 1);
-	}
 	GITERR_CHECK_ALLOC(*port);
 
-	end = colon == NULL ? slash : colon;
+	if (u.field_set & (1 << UF_PATH)) {
+		*path = git__substrdup(_path, u.field_data[UF_PATH].len);
+		GITERR_CHECK_ALLOC(*path);
+	}
 
-	*host = git__substrdup(start, end - start);
-	GITERR_CHECK_ALLOC(*host);
+	if (u.field_set & (1 << UF_USERINFO)) {
+		const char *colon = memchr(_userinfo, ':', u.field_data[UF_USERINFO].len);
+		if (colon) {
+			*username = unescape(git__substrdup(_userinfo, colon - _userinfo));
+			*password = unescape(git__substrdup(colon+1, u.field_data[UF_USERINFO].len - (colon+1-_userinfo)));
+			GITERR_CHECK_ALLOC(*password);
+		} else {
+			*username = git__substrdup(_userinfo, u.field_data[UF_USERINFO].len);
+		}
+		GITERR_CHECK_ALLOC(*username);
+
+	}
 
 	return 0;
 }
diff --git a/src/netops.h b/src/netops.h
index d352bf3b6..666d66b12 100644
--- a/src/netops.h
+++ b/src/netops.h
@@ -66,9 +66,33 @@ int gitno_send(gitno_socket *socket, const char *msg, size_t len, int flags);
 int gitno_close(gitno_socket *s);
 int gitno_select_in(gitno_buffer *buf, long int sec, long int usec);
 
+typedef struct gitno_connection_data {
+	char *host;
+	char *port;
+	char *path;
+	char *user;
+	char *pass;
+	bool use_ssl;
+} gitno_connection_data;
+
+/*
+ * This replaces all the pointers in `data` with freshly-allocated strings,
+ * that the caller is responsible for freeing.
+ * `gitno_connection_data_free_ptrs` is good for this.
+ */
+
+int gitno_connection_data_from_url(
+		gitno_connection_data *data,
+		const char *url,
+		const char *service_suffix);
+
+/* This frees all the pointers IN the struct, but not the struct itself. */
+void gitno_connection_data_free_ptrs(gitno_connection_data *data);
+
 int gitno_extract_url_parts(
 		char **host,
 		char **port,
+		char **path,
 		char **username,
 		char **password,
 		const char *url,
diff --git a/src/object.c b/src/object.c
index 9b8ccdd3e..3fc984b45 100644
--- a/src/object.c
+++ b/src/object.c
@@ -364,3 +364,38 @@ int git_object_dup(git_object **dest, git_object *source)
 	*dest = source;
 	return 0;
 }
+
+int git_object_lookup_bypath(
+		git_object **out,
+		const git_object *treeish,
+		const char *path,
+		git_otype type)
+{
+	int error = -1;
+	git_tree *tree = NULL;
+	git_tree_entry *entry = NULL;
+
+	assert(out && treeish && path);
+
+	if ((error = git_object_peel((git_object**)&tree, treeish, GIT_OBJ_TREE) < 0) ||
+		 (error = git_tree_entry_bypath(&entry, tree, path)) < 0)
+	{
+		goto cleanup;
+	}
+
+	if (type != GIT_OBJ_ANY && git_tree_entry_type(entry) != type)
+	{
+		giterr_set(GITERR_OBJECT,
+				"object at path '%s' is not of the asked-for type %d",
+				path, type);
+		error = GIT_EINVALIDSPEC;
+		goto cleanup;
+	}
+
+	error = git_tree_entry_to_object(out, git_object_owner(treeish), entry);
+
+cleanup:
+	git_tree_entry_free(entry);
+	git_tree_free(tree);
+	return error;
+}
diff --git a/src/odb.c b/src/odb.c
index 8e62efd00..b208b279e 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -124,6 +124,13 @@ git_otype git_odb_object_type(git_odb_object *object)
 	return object->cached.type;
 }
 
+int git_odb_object_dup(git_odb_object **dest, git_odb_object *source)
+{
+	git_cached_obj_incref(source);
+	*dest = source;
+	return 0;
+}
+
 void git_odb_object_free(git_odb_object *object)
 {
 	if (object == NULL)
@@ -168,7 +175,6 @@ int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
 		error = -1;
 
 		goto done;
-		return -1;
 	}
 
 	error = git_hash_final(out, &ctx);
@@ -179,28 +185,30 @@ done:
 }
 
 int git_odb__hashfd_filtered(
-	git_oid *out, git_file fd, size_t size, git_otype type, git_vector *filters)
+	git_oid *out, git_file fd, size_t size, git_otype type, git_filter_list *fl)
 {
 	int error;
 	git_buf raw = GIT_BUF_INIT;
-	git_buf filtered = GIT_BUF_INIT;
 
-	if (!filters || !filters->length)
+	if (!fl)
 		return git_odb__hashfd(out, fd, size, type);
 
 	/* size of data is used in header, so we have to read the whole file
 	 * into memory to apply filters before beginning to calculate the hash
 	 */
 
-	if (!(error = git_futils_readbuffer_fd(&raw, fd, size)))
-		error = git_filters_apply(&filtered, &raw, filters);
+	if (!(error = git_futils_readbuffer_fd(&raw, fd, size))) {
+		git_buf post = GIT_BUF_INIT;
+
+		error = git_filter_list_apply_to_data(&post, fl, &raw);
 
-	git_buf_free(&raw);
+		git_buf_free(&raw);
 
-	if (!error)
-		error = git_odb_hash(out, filtered.ptr, filtered.size, type);
+		if (!error)
+			error = git_odb_hash(out, post.ptr, post.size, type);
 
-	git_buf_free(&filtered);
+		git_buf_free(&post);
+	}
 
 	return error;
 }
@@ -232,6 +240,7 @@ int git_odb__hashlink(git_oid *out, const char *path)
 		link_data[size] = '\0';
 		if (read_len != (ssize_t)size) {
 			giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path);
+			git__free(link_data);
 			return -1;
 		}
 
@@ -290,10 +299,10 @@ typedef struct {
 	git_otype type;
 } fake_wstream;
 
-static int fake_wstream__fwrite(git_oid *oid, git_odb_stream *_stream)
+static int fake_wstream__fwrite(git_odb_stream *_stream, const git_oid *oid)
 {
 	fake_wstream *stream = (fake_wstream *)_stream;
-	return _stream->backend->write(oid, _stream->backend, stream->buffer, stream->size, stream->type);
+	return _stream->backend->write(_stream->backend, oid, stream->buffer, stream->size, stream->type);
 }
 
 static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t len)
@@ -444,7 +453,7 @@ int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
 		return 0;
 	}
 
-	giterr_set(GITERR_ODB, "No ODB backend loaded at index " PRIuZ, pos);
+	giterr_set(GITERR_ODB, "No ODB backend loaded at index %" PRIuZ, pos);
 	return GIT_ENOTFOUND;
 }
 
@@ -483,7 +492,7 @@ static int add_default_backends(
 #endif
 
 	/* add the loose object backend */
-	if (git_odb_backend_loose(&loose, objects_dir, -1, 0) < 0 ||
+	if (git_odb_backend_loose(&loose, objects_dir, -1, 0, 0, 0) < 0 ||
 		add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates, inode) < 0)
 		return -1;
 
@@ -607,7 +616,6 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 	git_odb_object *object;
 	size_t i;
 	bool found = false;
-	bool refreshed = false;
 
 	assert(db && id);
 
@@ -616,23 +624,12 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 		return (int)true;
 	}
 
-attempt_lookup:
 	for (i = 0; i < db->backends.length && !found; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
 
 		if (b->exists != NULL)
-			found = b->exists(b, id);
-	}
-
-	if (!found && !refreshed) {
-		if (git_odb_refresh(db) < 0) {
-			giterr_clear();
-			return (int)false;
-		}
-
-		refreshed = true;
-		goto attempt_lookup;
+			found = (bool)b->exists(b, id);
 	}
 
 	return (int)found;
@@ -699,7 +696,6 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 {
 	size_t i, reads = 0;
 	int error;
-	bool refreshed = false;
 	git_rawobj raw;
 	git_odb_object *object;
 
@@ -709,7 +705,6 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 	if (*out != NULL)
 		return 0;
 
-attempt_lookup:
 	error = GIT_ENOTFOUND;
 
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
@@ -722,14 +717,6 @@ attempt_lookup:
 		}
 	}
 
-	if (error == GIT_ENOTFOUND && !refreshed) {
-		if ((error = git_odb_refresh(db)) < 0)
-			return error;
-
-		refreshed = true;
-		goto attempt_lookup;
-	}
-
 	if (error && error != GIT_PASSTHROUGH) {
 		if (!reads)
 			return git_odb__error_notfound("no match for id", id);
@@ -751,7 +738,7 @@ int git_odb_read_prefix(
 	git_oid found_full_oid = {{0}};
 	git_rawobj raw;
 	void *data = NULL;
-	bool found = false, refreshed = false;
+	bool found = false;
 	git_odb_object *object;
 
 	assert(out && db);
@@ -768,7 +755,6 @@ int git_odb_read_prefix(
 			return 0;
 	}
 
-attempt_lookup:
 	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
@@ -785,22 +771,16 @@ attempt_lookup:
 			git__free(data);
 			data = raw.data;
 
-			if (found && git_oid__cmp(&full_oid, &found_full_oid))
+			if (found && git_oid__cmp(&full_oid, &found_full_oid)) {
+				git__free(raw.data);
 				return git_odb__error_ambiguous("multiple matches for prefix");
+			}
 
 			found_full_oid = full_oid;
 			found = true;
 		}
 	}
 
-	if (!found && !refreshed) {
-		if ((error = git_odb_refresh(db)) < 0)
-			return error;
-
-		refreshed = true;
-		goto attempt_lookup;
-	}
-
 	if (!found)
 		return git_odb__error_notfound("no match for prefix", short_id);
 
@@ -848,7 +828,7 @@ int git_odb_write(
 			continue;
 
 		if (b->write != NULL)
-			error = b->write(oid, b, data, len, type);
+			error = b->write(b, oid, data, len, type);
 	}
 
 	if (!error || error == GIT_PASSTHROUGH)
@@ -862,17 +842,27 @@ int git_odb_write(
 		return error;
 
 	stream->write(stream, data, len);
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
+	error = stream->finalize_write(stream, oid);
+	git_odb_stream_free(stream);
 
 	return error;
 }
 
+static void hash_header(git_hash_ctx *ctx, size_t size, git_otype type)
+{
+	char header[64];
+	int hdrlen;
+
+	hdrlen = git_odb__format_object_header(header, sizeof(header), size, type);
+	git_hash_update(ctx, header, hdrlen);
+}
+
 int git_odb_open_wstream(
 	git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
 {
 	size_t i, writes = 0;
 	int error = GIT_ERROR;
+	git_hash_ctx *ctx;
 
 	assert(stream && db);
 
@@ -898,9 +888,71 @@ int git_odb_open_wstream(
 	if (error < 0 && !writes)
 		error = git_odb__error_unsupported_in_backend("write object");
 
+	ctx = git__malloc(sizeof(git_hash_ctx));
+	GITERR_CHECK_ALLOC(ctx);
+
+
+	git_hash_ctx_init(ctx);
+	hash_header(ctx, size, type);
+	(*stream)->hash_ctx = ctx;
+
+	(*stream)->declared_size = size;
+	(*stream)->received_bytes = 0;
+
 	return error;
 }
 
+static int git_odb_stream__invalid_length(
+	const git_odb_stream *stream,
+	const char *action)
+{
+	giterr_set(GITERR_ODB,
+		"Cannot %s - "
+		"Invalid length. %"PRIuZ" was expected. The "
+		"total size of the received chunks amounts to %"PRIuZ".",
+		action, stream->declared_size, stream->received_bytes);		
+
+	return -1;
+}
+
+int git_odb_stream_write(git_odb_stream *stream, const char *buffer, size_t len)
+{
+	git_hash_update(stream->hash_ctx, buffer, len);
+
+	stream->received_bytes += len;
+
+	if (stream->received_bytes > stream->declared_size)
+		return git_odb_stream__invalid_length(stream,
+			"stream_write()");
+
+	return stream->write(stream, buffer, len);
+}
+
+int git_odb_stream_finalize_write(git_oid *out, git_odb_stream *stream)
+{
+	if (stream->received_bytes != stream->declared_size)
+		return git_odb_stream__invalid_length(stream,
+			"stream_finalize_write()");
+
+	git_hash_final(out, stream->hash_ctx);
+
+	if (git_odb_exists(stream->backend->odb, out))
+		return 0;
+
+	return stream->finalize_write(stream, out);
+}
+
+int git_odb_stream_read(git_odb_stream *stream, char *buffer, size_t len)
+{
+	return stream->read(stream, buffer, len);
+}
+
+void git_odb_stream_free(git_odb_stream *stream)
+{
+	git__free(stream->hash_ctx);
+	stream->free(stream);
+}
+
 int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
 {
 	size_t i, reads = 0;
@@ -943,7 +995,7 @@ int git_odb_write_pack(struct git_odb_writepack **out, git_odb *db, git_transfer
 
 		if (b->writepack != NULL) {
 			++writes;
-			error = b->writepack(out, b, progress_cb, progress_payload);
+			error = b->writepack(out, b, db, progress_cb, progress_payload);
 		}
 	}
 
diff --git a/src/odb.h b/src/odb.h
index 0d9f9e2ea..61dd9a7fd 100644
--- a/src/odb.h
+++ b/src/odb.h
@@ -14,6 +14,7 @@
 #include "vector.h"
 #include "cache.h"
 #include "posix.h"
+#include "filter.h"
 
 #define GIT_OBJECTS_DIR "objects/"
 #define GIT_OBJECT_DIR_MODE 0777
@@ -66,7 +67,7 @@ int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type);
  * Acts just like git_odb__hashfd with the addition of filters...
  */
 int git_odb__hashfd_filtered(
-	git_oid *out, git_file fd, size_t len, git_otype type, git_vector *filters);
+	git_oid *out, git_file fd, size_t len, git_otype type, git_filter_list *fl);
 
 /*
  * Hash a `path`, assuming it could be a POSIX symlink: if the path is a
diff --git a/src/odb_loose.c b/src/odb_loose.c
index 76ed8e232..ced272b33 100644
--- a/src/odb_loose.c
+++ b/src/odb_loose.c
@@ -33,6 +33,8 @@ typedef struct loose_backend {
 
 	int object_zlib_level; /** loose object zlib compression level. */
 	int fsync_object_files; /** loose object file fsync flag. */
+	mode_t object_file_mode;
+	mode_t object_dir_mode;
 
 	size_t objects_dirlen;
 	char objects_dir[GIT_FLEX_ARRAY];
@@ -79,7 +81,7 @@ static int object_file_name(
 static int object_mkdir(const git_buf *name, const loose_backend *be)
 {
 	return git_futils_mkdir(
-		name->ptr + be->objects_dirlen, be->objects_dir, GIT_OBJECT_DIR_MODE,
+		name->ptr + be->objects_dirlen, be->objects_dir, be->object_dir_mode,
 		GIT_MKDIR_PATH | GIT_MKDIR_SKIP_LAST | GIT_MKDIR_VERIFY_DIR);
 }
 
@@ -499,7 +501,7 @@ static int fn_locate_object_short_oid(void *state, git_buf *pathbuf) {
 	}
 
 	if (sstate->found > 1)
-		return git_odb__error_ambiguous("multiple matches in loose objects");
+		return GIT_EAMBIGUOUS;
 
 	return 0;
 }
@@ -544,13 +546,17 @@ static int locate_object_short_oid(
 
 	/* Explore directory to find a unique object matching short_oid */
 	error = git_path_direach(
-		object_location, fn_locate_object_short_oid, &state);
-	if (error)
+		object_location, 0, fn_locate_object_short_oid, &state);
+
+	if (error && error != GIT_EUSER)
 		return error;
 
 	if (!state.found)
 		return git_odb__error_notfound("no matching loose object for prefix", short_oid);
 
+	if (state.found > 1)
+		return git_odb__error_ambiguous("multiple matches in loose objects");
+
 	/* Convert obtained hex formatted oid to raw */
 	error = git_oid_fromstr(res_oid, (char *)state.res_oid);
 	if (error)
@@ -641,10 +647,12 @@ static int loose_backend__read_prefix(
 {
 	int error = 0;
 
+	assert(len <= GIT_OID_HEXSZ);
+
 	if (len < GIT_OID_MINPREFIXLEN)
 		error = git_odb__error_ambiguous("prefix length too short");
 
-	else if (len >= GIT_OID_HEXSZ) {
+	else if (len == GIT_OID_HEXSZ) {
 		/* We can fall back to regular read method */
 		error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
 		if (!error)
@@ -739,7 +747,7 @@ static int foreach_cb(void *_state, git_buf *path)
 {
 	struct foreach_state *state = (struct foreach_state *) _state;
 
-	return git_path_direach(path, foreach_object_dir_cb, state);
+	return git_path_direach(path, 0, foreach_object_dir_cb, state);
 }
 
 static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb cb, void *data)
@@ -762,34 +770,26 @@ static int loose_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb
 	state.data = data;
 	state.dir_len = git_buf_len(&buf);
 
-	error = git_path_direach(&buf, foreach_cb, &state);
+	error = git_path_direach(&buf, 0, foreach_cb, &state);
 
 	git_buf_free(&buf);
 
 	return state.cb_error ? state.cb_error : error;
 }
 
-static int loose_backend__stream_fwrite(git_oid *oid, git_odb_stream *_stream)
+static int loose_backend__stream_fwrite(git_odb_stream *_stream, const git_oid *oid)
 {
 	loose_writestream *stream = (loose_writestream *)_stream;
 	loose_backend *backend = (loose_backend *)_stream->backend;
 	git_buf final_path = GIT_BUF_INIT;
 	int error = 0;
 
-	if (git_filebuf_hash(oid, &stream->fbuf) < 0 ||
-		object_file_name(&final_path, backend, oid) < 0 ||
+	if (object_file_name(&final_path, backend, oid) < 0 ||
 		object_mkdir(&final_path, backend) < 0)
 		error = -1;
-	/*
-	 * Don't try to add an existing object to the repository. This
-	 * is what git does and allows us to sidestep the fact that
-	 * we're not allowed to overwrite a read-only file on Windows.
-	 */
-	else if (git_path_exists(final_path.ptr) == true)
-		git_filebuf_cleanup(&stream->fbuf);
 	else
 		error = git_filebuf_commit_at(
-			&stream->fbuf, final_path.ptr, GIT_OBJECT_FILE_MODE);
+			&stream->fbuf, final_path.ptr);
 
 	git_buf_free(&final_path);
 
@@ -810,17 +810,6 @@ static void loose_backend__stream_free(git_odb_stream *_stream)
 	git__free(stream);
 }
 
-static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
-{
-	const char *type_str = git_object_type2string(obj_type);
-	int len = snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
-
-	assert(len > 0);				/* otherwise snprintf() is broken */
-	assert(((size_t)len) < n);		/* otherwise the caller is broken! */
-
-	return len+1;
-}
-
 static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, size_t length, git_otype type)
 {
 	loose_backend *backend;
@@ -834,7 +823,7 @@ static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_
 	backend = (loose_backend *)_backend;
 	*stream_out = NULL;
 
-	hdrlen = format_object_header(hdr, sizeof(hdr), length, type);
+	hdrlen = git_odb__format_object_header(hdr, sizeof(hdr), length, type);
 
 	stream = git__calloc(1, sizeof(loose_writestream));
 	GITERR_CHECK_ALLOC(stream);
@@ -848,9 +837,9 @@ static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_
 
 	if (git_buf_joinpath(&tmp_path, backend->objects_dir, "tmp_object") < 0 ||
 		git_filebuf_open(&stream->fbuf, tmp_path.ptr,
-			GIT_FILEBUF_HASH_CONTENTS |
 			GIT_FILEBUF_TEMPORARY |
-			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT)) < 0 ||
+			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT),
+			backend->object_file_mode) < 0 ||
 		stream->stream.write((git_odb_stream *)stream, hdr, hdrlen) < 0)
 	{
 		git_filebuf_cleanup(&stream->fbuf);
@@ -863,7 +852,7 @@ static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_
 	return !stream ? -1 : 0;
 }
 
-static int loose_backend__write(git_oid *oid, git_odb_backend *_backend, const void *data, size_t len, git_otype type)
+static int loose_backend__write(git_odb_backend *_backend, const git_oid *oid, const void *data, size_t len, git_otype type)
 {
 	int error = 0, header_len;
 	git_buf final_path = GIT_BUF_INIT;
@@ -874,12 +863,13 @@ static int loose_backend__write(git_oid *oid, git_odb_backend *_backend, const v
 	backend = (loose_backend *)_backend;
 
 	/* prepare the header for the file */
-	header_len = format_object_header(header, sizeof(header), len, type);
+	header_len = git_odb__format_object_header(header, sizeof(header), len, type);
 
 	if (git_buf_joinpath(&final_path, backend->objects_dir, "tmp_object") < 0 ||
 		git_filebuf_open(&fbuf, final_path.ptr,
 			GIT_FILEBUF_TEMPORARY |
-			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT)) < 0)
+			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT),
+			backend->object_file_mode) < 0)
 	{
 		error = -1;
 		goto cleanup;
@@ -890,7 +880,7 @@ static int loose_backend__write(git_oid *oid, git_odb_backend *_backend, const v
 
 	if (object_file_name(&final_path, backend, oid) < 0 ||
 		object_mkdir(&final_path, backend) < 0 ||
-		git_filebuf_commit_at(&fbuf, final_path.ptr, GIT_OBJECT_FILE_MODE) < 0)
+		git_filebuf_commit_at(&fbuf, final_path.ptr) < 0)
 		error = -1;
 
 cleanup:
@@ -913,7 +903,9 @@ int git_odb_backend_loose(
 	git_odb_backend **backend_out,
 	const char *objects_dir,
 	int compression_level,
-	int do_fsync)
+	int do_fsync,
+	unsigned int dir_mode,
+	unsigned int file_mode)
 {
 	loose_backend *backend;
 	size_t objects_dirlen;
@@ -934,8 +926,16 @@ int git_odb_backend_loose(
 	if (compression_level < 0)
 		compression_level = Z_BEST_SPEED;
 
+	if (dir_mode == 0)
+		dir_mode = GIT_OBJECT_DIR_MODE;
+
+	if (file_mode == 0)
+		file_mode = GIT_OBJECT_FILE_MODE;
+
 	backend->object_zlib_level = compression_level;
 	backend->fsync_object_files = do_fsync;
+	backend->object_dir_mode = dir_mode;
+	backend->object_file_mode = file_mode;
 
 	backend->parent.read = &loose_backend__read;
 	backend->parent.write = &loose_backend__write;
diff --git a/src/odb_pack.c b/src/odb_pack.c
index eec79259b..fd2ca0fd8 100644
--- a/src/odb_pack.c
+++ b/src/odb_pack.c
@@ -29,7 +29,7 @@ struct pack_backend {
 
 struct pack_writepack {
 	struct git_odb_writepack parent;
-	git_indexer_stream *indexer_stream;
+	git_indexer *indexer;
 };
 
 /**
@@ -259,23 +259,26 @@ static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backen
 	return git_odb__error_notfound("failed to find pack entry", oid);
 }
 
-static unsigned pack_entry_find_prefix_inner(
-		struct git_pack_entry *e,
-		struct pack_backend *backend,
-		const git_oid *short_oid,
-		size_t len,
-		struct git_pack_file *last_found)
+static int pack_entry_find_prefix(
+	struct git_pack_entry *e,
+	struct pack_backend *backend,
+	const git_oid *short_oid,
+	size_t len)
 {
 	int error;
 	size_t i;
-	unsigned found = 0;
+	git_oid found_full_oid = {{0}};
+	bool found = false;
+	struct git_pack_file *last_found = backend->last_found;
 
 	if (last_found) {
 		error = git_pack_entry_find(e, last_found, short_oid, len);
 		if (error == GIT_EAMBIGUOUS)
 			return error;
-		if (!error)
-			found = 1;
+		if (!error) {
+			git_oid_cpy(&found_full_oid, &e->sha1);
+			found = true;
+		}
 	}
 
 	for (i = 0; i < backend->packs.length; ++i) {
@@ -289,28 +292,16 @@ static unsigned pack_entry_find_prefix_inner(
 		if (error == GIT_EAMBIGUOUS)
 			return error;
 		if (!error) {
-			if (++found > 1)
-				break;
+			if (found && git_oid_cmp(&e->sha1, &found_full_oid))
+				return git_odb__error_ambiguous("found multiple pack entries");
+			git_oid_cpy(&found_full_oid, &e->sha1);
+			found = true;
 			backend->last_found = p;
 		}
 	}
 
-	return found;
-}
-
-static int pack_entry_find_prefix(
-	struct git_pack_entry *e,
-	struct pack_backend *backend,
-	const git_oid *short_oid,
-	size_t len)
-{
-	struct git_pack_file *last_found = backend->last_found;
-	unsigned int found = pack_entry_find_prefix_inner(e, backend, short_oid, len, last_found);
-
 	if (!found)
 		return git_odb__error_notfound("no matching pack entry for prefix", short_oid);
-	else if (found > 1)
-		return git_odb__error_ambiguous("found multiple pack entries");
 	else
 		return 0;
 }
@@ -340,19 +331,20 @@ static int pack_backend__refresh(git_odb_backend *_backend)
 	git_buf_sets(&path, backend->pack_folder);
 
 	/* reload all packs */
-	error = git_path_direach(&path, packfile_load__cb, (void *)backend);
+	error = git_path_direach(&path, 0, packfile_load__cb, backend);
 
 	git_buf_free(&path);
 
 	if (error < 0)
-		return error;
+		return -1;
 
 	git_vector_sort(&backend->packs);
 	return 0;
 }
 
-
-static int pack_backend__read_header(size_t *len_p, git_otype *type_p, struct git_odb_backend *backend, const git_oid *oid)
+static int pack_backend__read_header_internal(
+	size_t *len_p, git_otype *type_p,
+	struct git_odb_backend *backend, const git_oid *oid)
 {
 	struct git_pack_entry e;
 	int error;
@@ -365,7 +357,26 @@ static int pack_backend__read_header(size_t *len_p, git_otype *type_p, struct gi
 	return git_packfile_resolve_header(len_p, type_p, e.p, e.offset);
 }
 
-static int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+static int pack_backend__read_header(
+	size_t *len_p, git_otype *type_p,
+	struct git_odb_backend *backend, const git_oid *oid)
+{
+	int error;
+
+	error = pack_backend__read_header_internal(len_p, type_p, backend, oid);
+
+	if (error != GIT_ENOTFOUND)
+		return error;
+
+	if ((error = pack_backend__refresh(backend)) < 0)
+		return error;
+
+	return pack_backend__read_header_internal(len_p, type_p, backend, oid);
+}
+
+static int pack_backend__read_internal(
+	void **buffer_p, size_t *len_p, git_otype *type_p,
+	git_odb_backend *backend, const git_oid *oid)
 {
 	struct git_pack_entry e;
 	git_rawobj raw;
@@ -382,7 +393,24 @@ static int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p,
 	return 0;
 }
 
-static int pack_backend__read_prefix(
+static int pack_backend__read(
+	void **buffer_p, size_t *len_p, git_otype *type_p,
+	git_odb_backend *backend, const git_oid *oid)
+{
+	int error;
+
+	error = pack_backend__read_internal(buffer_p, len_p, type_p, backend, oid);
+
+	if (error != GIT_ENOTFOUND)
+		return error;
+
+	if ((error = pack_backend__refresh(backend)) < 0)
+		return error;
+
+	return pack_backend__read_internal(buffer_p, len_p, type_p, backend, oid);
+}
+
+static int pack_backend__read_prefix_internal(
 	git_oid *out_oid,
 	void **buffer_p,
 	size_t *len_p,
@@ -419,9 +447,45 @@ static int pack_backend__read_prefix(
 	return error;
 }
 
+static int pack_backend__read_prefix(
+	git_oid *out_oid,
+	void **buffer_p,
+	size_t *len_p,
+	git_otype *type_p,
+	git_odb_backend *backend,
+	const git_oid *short_oid,
+	size_t len)
+{
+	int error;
+
+	error = pack_backend__read_prefix_internal(
+		out_oid, buffer_p, len_p, type_p, backend, short_oid, len);
+
+	if (error != GIT_ENOTFOUND)
+		return error;
+
+	if ((error = pack_backend__refresh(backend)) < 0)
+		return error;
+
+	return pack_backend__read_prefix_internal(
+		out_oid, buffer_p, len_p, type_p, backend, short_oid, len);
+}
+
 static int pack_backend__exists(git_odb_backend *backend, const git_oid *oid)
 {
 	struct git_pack_entry e;
+	int error;
+
+	error = pack_entry_find(&e, (struct pack_backend *)backend, oid);
+
+	if (error != GIT_ENOTFOUND)
+		return error == 0;
+
+	if ((error = pack_backend__refresh(backend)) < 0) {
+		giterr_clear();
+		return (int)false;
+	}
+
 	return pack_entry_find(&e, (struct pack_backend *)backend, oid) == 0;
 }
 
@@ -447,13 +511,13 @@ static int pack_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb c
 	return 0;
 }
 
-static int pack_backend__writepack_add(struct git_odb_writepack *_writepack, const void *data, size_t size, git_transfer_progress *stats)
+static int pack_backend__writepack_append(struct git_odb_writepack *_writepack, const void *data, size_t size, git_transfer_progress *stats)
 {
 	struct pack_writepack *writepack = (struct pack_writepack *)_writepack;
 
 	assert(writepack);
 
-	return git_indexer_stream_add(writepack->indexer_stream, data, size, stats);
+	return git_indexer_append(writepack->indexer, data, size, stats);
 }
 
 static int pack_backend__writepack_commit(struct git_odb_writepack *_writepack, git_transfer_progress *stats)
@@ -462,7 +526,7 @@ static int pack_backend__writepack_commit(struct git_odb_writepack *_writepack,
 
 	assert(writepack);
 
-	return git_indexer_stream_finalize(writepack->indexer_stream, stats);
+	return git_indexer_commit(writepack->indexer, stats);
 }
 
 static void pack_backend__writepack_free(struct git_odb_writepack *_writepack)
@@ -471,12 +535,13 @@ static void pack_backend__writepack_free(struct git_odb_writepack *_writepack)
 
 	assert(writepack);
 
-	git_indexer_stream_free(writepack->indexer_stream);
+	git_indexer_free(writepack->indexer);
 	git__free(writepack);
 }
 
 static int pack_backend__writepack(struct git_odb_writepack **out,
 	git_odb_backend *_backend,
+        git_odb *odb,
 	git_transfer_progress_callback progress_cb,
 	void *progress_payload)
 {
@@ -492,14 +557,14 @@ static int pack_backend__writepack(struct git_odb_writepack **out,
 	writepack = git__calloc(1, sizeof(struct pack_writepack));
 	GITERR_CHECK_ALLOC(writepack);
 
-	if (git_indexer_stream_new(&writepack->indexer_stream,
-		backend->pack_folder, progress_cb, progress_payload) < 0) {
+	if (git_indexer_new(&writepack->indexer,
+		backend->pack_folder, 0, odb, progress_cb, progress_payload) < 0) {
 		git__free(writepack);
 		return -1;
 	}
 
 	writepack->parent.backend = _backend;
-	writepack->parent.add = pack_backend__writepack_add;
+	writepack->parent.append = pack_backend__writepack_append;
 	writepack->parent.commit = pack_backend__writepack_commit;
 	writepack->parent.free = pack_backend__writepack_free;
 
diff --git a/src/oid.c b/src/oid.c
index 8300e46c1..d56b6af24 100644
--- a/src/oid.c
+++ b/src/oid.c
@@ -211,7 +211,7 @@ int git_oid_strcmp(const git_oid *oid_a, const char *str)
 	for (a = oid_a->id; *str && (a - oid_a->id) < GIT_OID_RAWSZ; ++a) {
 		if ((hexval = git__fromhex(*str++)) < 0)
 			return -1;
-		strval = hexval << 4;
+		strval = (unsigned char)(hexval << 4);
 		if (*str) {
 			if ((hexval = git__fromhex(*str++)) < 0)
 				return -1;
@@ -369,8 +369,10 @@ int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid)
 	bool is_leaf;
 	node_index idx;
 
-	if (os->full)
+	if (os->full) {
+		giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
 		return -1;
+	}
 
 	if (text_oid == NULL)
 		return os->min_length;
@@ -396,12 +398,19 @@ int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid)
 			node->tail = NULL;
 
 			node = push_leaf(os, idx, git__fromhex(tail[0]), &tail[1]);
-			GITERR_CHECK_ALLOC(node);
+			if (node == NULL) {
+				if (os->full)
+					giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
+				return -1;
+			}
 		}
 
 		if (node->children[c] == 0) {
-			if (push_leaf(os, idx, c, &text_oid[i + 1]) == NULL)
+			if (push_leaf(os, idx, c, &text_oid[i + 1]) == NULL) {
+				if (os->full)
+					giterr_set(GITERR_INVALID, "Unable to shorten OID - OID set full");
 				return -1;
+			}
 			break;
 		}
 
diff --git a/src/oid.h b/src/oid.h
index 077d0a4c8..cfe7ca1b2 100644
--- a/src/oid.h
+++ b/src/oid.h
@@ -9,17 +9,8 @@
 
 #include "git2/oid.h"
 
-/*
- * Compare two oid structures.
- *
- * @param a first oid structure.
- * @param b second oid structure.
- * @return <0, 0, >0 if a < b, a == b, a > b.
- */
-GIT_INLINE(int) git_oid__cmp(const git_oid *a, const git_oid *b)
+GIT_INLINE(int) git_oid__hashcmp(const unsigned char *sha1, const unsigned char *sha2)
 {
-	const unsigned char *sha1 = a->id;
-	const unsigned char *sha2 = b->id;
 	int i;
 
 	for (i = 0; i < GIT_OID_RAWSZ; i++, sha1++, sha2++) {
@@ -30,4 +21,16 @@ GIT_INLINE(int) git_oid__cmp(const git_oid *a, const git_oid *b)
 	return 0;
 }
 
+/*
+ * Compare two oid structures.
+ *
+ * @param a first oid structure.
+ * @param b second oid structure.
+ * @return <0, 0, >0 if a < b, a == b, a > b.
+ */
+GIT_INLINE(int) git_oid__cmp(const git_oid *a, const git_oid *b)
+{
+	return git_oid__hashcmp(a->id, b->id);
+}
+
 #endif
diff --git a/src/pack-objects.c b/src/pack-objects.c
index 500104c55..2d62507f2 100644
--- a/src/pack-objects.c
+++ b/src/pack-objects.c
@@ -14,6 +14,7 @@
 #include "pack.h"
 #include "thread-utils.h"
 #include "tree.h"
+#include "util.h"
 
 #include "git2/pack.h"
 #include "git2/commit.h"
@@ -25,7 +26,7 @@ struct unpacked {
 	git_pobject *object;
 	void *data;
 	struct git_delta_index *index;
-	unsigned int depth;
+	int depth;
 };
 
 struct tree_walk_context {
@@ -34,7 +35,7 @@ struct tree_walk_context {
 };
 
 struct pack_write_context {
-	git_indexer_stream *indexer;
+	git_indexer *indexer;
 	git_transfer_progress *stats;
 };
 
@@ -57,6 +58,9 @@ struct pack_write_context {
 #define git_packbuilder__progress_lock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, progress_mutex, lock)
 #define git_packbuilder__progress_unlock(pb) GIT_PACKBUILDER__MUTEX_OP(pb, progress_mutex, unlock)
 
+/* The minimal interval between progress updates (in seconds). */
+#define MIN_PROGRESS_UPDATE_INTERVAL 0.5
+
 static unsigned name_hash(const char *name)
 {
 	unsigned c, hash = 0;
@@ -213,41 +217,19 @@ int git_packbuilder_insert(git_packbuilder *pb, const git_oid *oid,
 	kh_value(pb->object_ix, pos) = po;
 
 	pb->done = false;
-	return 0;
-}
-
-/*
- * The per-object header is a pretty dense thing, which is
- *  - first byte: low four bits are "size",
- *    then three bits of "type",
- *    with the high bit being "size continues".
- *  - each byte afterwards: low seven bits are size continuation,
- *    with the high bit being "size continues"
- */
-static int gen_pack_object_header(
-		unsigned char *hdr,
-		unsigned long size,
-		git_otype type)
-{
-	unsigned char *hdr_base;
-	unsigned char c;
-
-	assert(type >= GIT_OBJ_COMMIT && type <= GIT_OBJ_REF_DELTA);
-
-	/* TODO: add support for chunked objects; see git.git 6c0d19b1 */
-
-	c = (unsigned char)((type << 4) | (size & 15));
-	size >>= 4;
-	hdr_base = hdr;
 
-	while (size) {
-		*hdr++ = c | 0x80;
-		c = size & 0x7f;
-		size >>= 7;
+	if (pb->progress_cb) {
+		double current_time = git__timer();
+		if ((current_time - pb->last_progress_report_time) >= MIN_PROGRESS_UPDATE_INTERVAL) {
+			pb->last_progress_report_time = current_time;
+			if (pb->progress_cb(GIT_PACKBUILDER_ADDING_OBJECTS, pb->nr_objects, 0, pb->progress_cb_payload)) {
+				giterr_clear();
+				return GIT_EUSER;
+			}
+		}
 	}
-	*hdr++ = c;
 
-	return (int)(hdr - hdr_base);
+	return 0;
 }
 
 static int get_delta(void **out, git_odb *odb, git_pobject *po)
@@ -290,7 +272,7 @@ static int write_object(git_buf *buf, git_packbuilder *pb, git_pobject *po)
 	git_buf zbuf = GIT_BUF_INIT;
 	git_otype type;
 	unsigned char hdr[10];
-	unsigned int hdr_len;
+	size_t hdr_len;
 	unsigned long size;
 	void *data;
 
@@ -311,7 +293,7 @@ static int write_object(git_buf *buf, git_packbuilder *pb, git_pobject *po)
 	}
 
 	/* Write header */
-	hdr_len = gen_pack_object_header(hdr, size, type);
+	hdr_len = git_packfile__object_header(hdr, size, type);
 
 	if (git_buf_put(buf, (char *)hdr, hdr_len) < 0)
 		goto on_error;
@@ -505,8 +487,10 @@ static git_pobject **compute_write_order(git_packbuilder *pb)
 	/*
 	 * Mark objects that are at the tip of tags.
 	 */
-	if (git_tag_foreach(pb->repo, &cb_tag_foreach, pb) < 0)
+	if (git_tag_foreach(pb->repo, &cb_tag_foreach, pb) < 0) {
+		git__free(wo);
 		return NULL;
+	}
 
 	/*
 	 * Give the objects in the original recency order until
@@ -576,50 +560,52 @@ static int write_pack(git_packbuilder *pb,
 	git_buf buf = GIT_BUF_INIT;
 	enum write_one_status status;
 	struct git_pack_header ph;
+	git_oid entry_oid;
 	unsigned int i = 0;
+	int error = 0;
 
 	write_order = compute_write_order(pb);
-	if (write_order == NULL)
-		goto on_error;
+	if (write_order == NULL) {
+		error = -1;
+		goto done;
+	}
 
 	/* Write pack header */
 	ph.hdr_signature = htonl(PACK_SIGNATURE);
 	ph.hdr_version = htonl(PACK_VERSION);
 	ph.hdr_entries = htonl(pb->nr_objects);
 
-	if (cb(&ph, sizeof(ph), data) < 0)
-		goto on_error;
+	if ((error = cb(&ph, sizeof(ph), data)) < 0)
+		goto done;
 
-	if (git_hash_update(&pb->ctx, &ph, sizeof(ph)) < 0)
-		goto on_error;
+	if ((error = git_hash_update(&pb->ctx, &ph, sizeof(ph))) < 0)
+		goto done;
 
 	pb->nr_remaining = pb->nr_objects;
 	do {
 		pb->nr_written = 0;
 		for ( ; i < pb->nr_objects; ++i) {
 			po = write_order[i];
-			if (write_one(&buf, pb, po, &status) < 0)
-				goto on_error;
-			if (cb(buf.ptr, buf.size, data) < 0)
-				goto on_error;
+			if ((error = write_one(&buf, pb, po, &status)) < 0)
+				goto done;
+			if ((error = cb(buf.ptr, buf.size, data)) < 0)
+				goto done;
 			git_buf_clear(&buf);
 		}
 
 		pb->nr_remaining -= pb->nr_written;
 	} while (pb->nr_remaining && i < pb->nr_objects);
 
-	git__free(write_order);
-	git_buf_free(&buf);
 
-	if (git_hash_final(&pb->pack_oid, &pb->ctx) < 0)
-		goto on_error;
+	if ((error = git_hash_final(&entry_oid, &pb->ctx)) < 0)
+		goto done;
 
-	return cb(pb->pack_oid.id, GIT_OID_RAWSZ, data);
+	error = cb(entry_oid.id, GIT_OID_RAWSZ, data);
 
-on_error:
+done:
 	git__free(write_order);
 	git_buf_free(&buf);
-	return -1;
+	return error;
 }
 
 static int write_pack_buf(void *buf, size_t size, void *data)
@@ -674,7 +660,7 @@ static int delta_cacheable(git_packbuilder *pb, unsigned long src_size,
 }
 
 static int try_delta(git_packbuilder *pb, struct unpacked *trg,
-		     struct unpacked *src, unsigned int max_depth,
+		     struct unpacked *src, int max_depth,
 		     unsigned long *mem_usage, int *ret)
 {
 	git_pobject *trg_object = trg->object;
@@ -839,7 +825,7 @@ static unsigned long free_unpacked(struct unpacked *n)
 
 static int find_deltas(git_packbuilder *pb, git_pobject **list,
 		       unsigned int *list_size, unsigned int window,
-		       unsigned int depth)
+		       int depth)
 {
 	git_pobject *po;
 	git_buf zbuf = GIT_BUF_INIT;
@@ -854,8 +840,7 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list,
 
 	for (;;) {
 		struct unpacked *n = array + idx;
-		unsigned int max_depth;
-		int j, best_base = -1;
+		int max_depth, j, best_base = -1;
 
 		git_packbuilder__progress_lock(pb);
 		if (!*list_size) {
@@ -1048,7 +1033,7 @@ static void *threaded_find_deltas(void *arg)
 
 static int ll_find_deltas(git_packbuilder *pb, git_pobject **list,
 			  unsigned int list_size, unsigned int window,
-			  unsigned int depth)
+			  int depth)
 {
 	struct thread_params *p;
 	int i, ret, active_threads = 0;
@@ -1205,6 +1190,13 @@ static int prepare_pack(git_packbuilder *pb)
 	if (pb->nr_objects == 0 || pb->done)
 		return 0; /* nothing to do */
 
+	/*
+	 * Although we do not report progress during deltafication, we
+	 * at least report that we are in the deltafication stage
+	 */
+	if (pb->progress_cb)
+			pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload);
+
 	delta_list = git__malloc(pb->nr_objects * sizeof(*delta_list));
 	GITERR_CHECK_ALLOC(delta_list);
 
@@ -1250,40 +1242,48 @@ int git_packbuilder_write_buf(git_buf *buf, git_packbuilder *pb)
 static int write_cb(void *buf, size_t len, void *payload)
 {
 	struct pack_write_context *ctx = payload;
-	return git_indexer_stream_add(ctx->indexer, buf, len, ctx->stats);
+	return git_indexer_append(ctx->indexer, buf, len, ctx->stats);
 }
 
 int git_packbuilder_write(
 	git_packbuilder *pb,
 	const char *path,
+	unsigned int mode,
 	git_transfer_progress_callback progress_cb,
 	void *progress_cb_payload)
 {
-	git_indexer_stream *indexer;
+	git_indexer *indexer;
 	git_transfer_progress stats;
 	struct pack_write_context ctx;
 
 	PREPARE_PACK;
 
-	if (git_indexer_stream_new(
-		&indexer, path, progress_cb, progress_cb_payload) < 0)
+	if (git_indexer_new(
+		&indexer, path, mode, pb->odb, progress_cb, progress_cb_payload) < 0)
 		return -1;
 
 	ctx.indexer = indexer;
 	ctx.stats = &stats;
 
 	if (git_packbuilder_foreach(pb, write_cb, &ctx) < 0 ||
-		git_indexer_stream_finalize(indexer, &stats) < 0) {
-		git_indexer_stream_free(indexer);
+		git_indexer_commit(indexer, &stats) < 0) {
+		git_indexer_free(indexer);
 		return -1;
 	}
 
-	git_indexer_stream_free(indexer);
+	git_oid_cpy(&pb->pack_oid, git_indexer_hash(indexer));
+
+	git_indexer_free(indexer);
 	return 0;
 }
 
 #undef PREPARE_PACK
 
+const git_oid *git_packbuilder_hash(git_packbuilder *pb)
+{
+	return &pb->pack_oid;
+}
+
 static int cb_tree_walk(const char *root, const git_tree_entry *entry, void *payload)
 {
 	struct tree_walk_context *ctx = payload;
@@ -1346,6 +1346,17 @@ uint32_t git_packbuilder_written(git_packbuilder *pb)
 	return pb->nr_written;
 }
 
+int git_packbuilder_set_callbacks(git_packbuilder *pb, git_packbuilder_progress progress_cb, void *progress_cb_payload)
+{
+	if (!pb)
+		return -1;
+
+	pb->progress_cb = progress_cb;
+	pb->progress_cb_payload = progress_cb_payload;
+
+	return 0;
+}
+
 void git_packbuilder_free(git_packbuilder *pb)
 {
 	if (pb == NULL)
diff --git a/src/pack-objects.h b/src/pack-objects.h
index 8e7ba7f78..0c94a5a7a 100644
--- a/src/pack-objects.h
+++ b/src/pack-objects.h
@@ -16,6 +16,7 @@
 #include "netops.h"
 
 #include "git2/oid.h"
+#include "git2/pack.h"
 
 #define GIT_PACK_WINDOW 10 /* number of objects to possibly delta against */
 #define GIT_PACK_DEPTH 50 /* max delta depth */
@@ -79,6 +80,10 @@ struct git_packbuilder {
 
 	int nr_threads; /* nr of threads to use */
 
+	git_packbuilder_progress progress_cb;
+	void *progress_cb_payload;
+	double last_progress_report_time; /* the time progress was last reported */
+
 	bool done;
 };
 
diff --git a/src/pack.c b/src/pack.c
index 7ce7099e0..644b2d465 100644
--- a/src/pack.c
+++ b/src/pack.c
@@ -329,8 +329,10 @@ static int pack_index_open(struct git_pack_file *p)
 	memcpy(idx_name, p->pack_name, base_len);
 	memcpy(idx_name + base_len, ".idx", sizeof(".idx"));
 
-	if ((error = git_mutex_lock(&p->lock)) < 0)
+	if ((error = git_mutex_lock(&p->lock)) < 0) {
+		git__free(idx_name);
 		return error;
+	}
 
 	if (p->index_version == -1)
 		error = pack_index_check(idx_name, p);
@@ -362,6 +364,38 @@ static unsigned char *pack_window_open(
 	return git_mwindow_open(&p->mwf, w_cursor, offset, 20, left);
  }
 
+/*
+ * The per-object header is a pretty dense thing, which is
+ *  - first byte: low four bits are "size",
+ *    then three bits of "type",
+ *    with the high bit being "size continues".
+ *  - each byte afterwards: low seven bits are size continuation,
+ *    with the high bit being "size continues"
+ */
+size_t git_packfile__object_header(unsigned char *hdr, size_t size, git_otype type)
+{
+	unsigned char *hdr_base;
+	unsigned char c;
+
+	assert(type >= GIT_OBJ_COMMIT && type <= GIT_OBJ_REF_DELTA);
+
+	/* TODO: add support for chunked objects; see git.git 6c0d19b1 */
+
+	c = (unsigned char)((type << 4) | (size & 15));
+	size >>= 4;
+	hdr_base = hdr;
+
+	while (size) {
+		*hdr++ = c | 0x80;
+		c = size & 0x7f;
+		size >>= 7;
+	}
+	*hdr++ = c;
+
+	return (hdr - hdr_base);
+}
+
+
 static int packfile_unpack_header1(
 		unsigned long *usedp,
 		size_t *sizep,
@@ -820,7 +854,7 @@ void git_packfile_free(struct git_pack_file *p)
 
 	git_mwindow_free_all(&p->mwf);
 
-	if (p->mwf.fd != -1)
+	if (p->mwf.fd >= 0)
 		p_close(p->mwf.fd);
 
 	pack_index_free(p);
@@ -903,7 +937,8 @@ static int packfile_open(struct git_pack_file *p)
 cleanup:
 	giterr_set(GITERR_OS, "Invalid packfile '%s'", p->pack_name);
 
-	p_close(p->mwf.fd);
+	if (p->mwf.fd >= 0)
+		p_close(p->mwf.fd);
 	p->mwf.fd = -1;
 
 	git_mutex_unlock(&p->lock);
@@ -1107,8 +1142,11 @@ static int pack_entry_find_offset(
 		short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects);
 #endif
 
-	/* Use git.git lookup code */
+#ifdef GIT_USE_LOOKUP
 	pos = sha1_entry_pos(index, stride, 0, lo, hi, p->num_objects, short_oid->id);
+#else
+	pos = sha1_position(index, stride, lo, hi, short_oid->id);
+#endif
 
 	if (pos >= 0) {
 		/* An object matching exactly the oid was found */
diff --git a/src/pack.h b/src/pack.h
index aeeac9ce1..28146ab30 100644
--- a/src/pack.h
+++ b/src/pack.h
@@ -112,6 +112,8 @@ typedef struct git_packfile_stream {
 	git_mwindow *mw;
 } git_packfile_stream;
 
+size_t git_packfile__object_header(unsigned char *hdr, size_t size, git_otype type);
+
 int git_packfile_unpack_header(
 		size_t *size_p,
 		git_otype *type_p,
diff --git a/src/path.c b/src/path.c
index 6437979d5..750dd3ef7 100644
--- a/src/path.c
+++ b/src/path.c
@@ -8,7 +8,6 @@
 #include "path.h"
 #include "posix.h"
 #ifdef GIT_WIN32
-#include "win32/dir.h"
 #include "win32/posix.h"
 #else
 #include <dirent.h>
@@ -243,8 +242,8 @@ int git_path_root(const char *path)
 
 #ifdef GIT_WIN32
 	/* Are we dealing with a windows network path? */
-	else if ((path[0] == '/' && path[1] == '/') ||
-		(path[0] == '\\' && path[1] == '\\'))
+	else if ((path[0] == '/' && path[1] == '/' && path[2] != '/') ||
+		(path[0] == '\\' && path[1] == '\\' && path[2] != '\\'))
 	{
 		offset += 2;
 
@@ -484,74 +483,90 @@ bool git_path_isfile(const char *path)
 
 bool git_path_is_empty_dir(const char *path)
 {
-	git_buf pathbuf = GIT_BUF_INIT;
 	HANDLE hFind = INVALID_HANDLE_VALUE;
-	wchar_t wbuf[GIT_WIN_PATH];
+	git_win32_path wbuf;
+	int wbufsz;
 	WIN32_FIND_DATAW ffd;
 	bool retval = true;
 
-	if (!git_path_isdir(path)) return false;
+	if (!git_path_isdir(path))
+		return false;
 
-	git_buf_printf(&pathbuf, "%s\\*", path);
-	git__utf8_to_16(wbuf, GIT_WIN_PATH, git_buf_cstr(&pathbuf));
+	wbufsz = git_win32_path_from_c(wbuf, path);
+	if (!wbufsz || wbufsz + 2 > GIT_WIN_PATH_UTF16)
+		return false;
+	memcpy(&wbuf[wbufsz - 1], L"\\*", 3 * sizeof(wchar_t));
 
 	hFind = FindFirstFileW(wbuf, &ffd);
-	if (INVALID_HANDLE_VALUE == hFind) {
-		giterr_set(GITERR_OS, "Couldn't open '%s'", path);
+	if (INVALID_HANDLE_VALUE == hFind)
 		return false;
-	}
 
 	do {
 		if (!git_path_is_dot_or_dotdotW(ffd.cFileName)) {
 			retval = false;
+			break;
 		}
 	} while (FindNextFileW(hFind, &ffd) != 0);
 
 	FindClose(hFind);
-	git_buf_free(&pathbuf);
 	return retval;
 }
 
 #else
 
-bool git_path_is_empty_dir(const char *path)
+static int path_found_entry(void *payload, git_buf *path)
 {
-	DIR *dir = NULL;
-	struct dirent *e;
-	bool retval = true;
+	GIT_UNUSED(payload);
+	return !git_path_is_dot_or_dotdot(path->ptr);
+}
 
-	if (!git_path_isdir(path)) return false;
+bool git_path_is_empty_dir(const char *path)
+{
+	int error;
+	git_buf dir = GIT_BUF_INIT;
 
-	dir = opendir(path);
-	if (!dir) {
-		giterr_set(GITERR_OS, "Couldn't open '%s'", path);
+	if (!git_path_isdir(path))
 		return false;
-	}
 
-	while ((e = readdir(dir)) != NULL) {
-		if (!git_path_is_dot_or_dotdot(e->d_name)) {
-			giterr_set(GITERR_INVALID,
-						  "'%s' exists and is not an empty directory", path);
-			retval = false;
-			break;
-		}
-	}
-	closedir(dir);
+	if (!(error = git_buf_sets(&dir, path)))
+		error = git_path_direach(&dir, 0, path_found_entry, NULL);
 
-	return retval;
+	git_buf_free(&dir);
+
+	return !error;
 }
+
 #endif
 
-int git_path_lstat(const char *path, struct stat *st)
+int git_path_set_error(int errno_value, const char *path, const char *action)
 {
-	int err = 0;
-
-	if (p_lstat(path, st) < 0) {
-		err = (errno == ENOENT) ? GIT_ENOTFOUND : -1;
-		giterr_set(GITERR_OS, "Failed to stat file '%s'", path);
+	switch (errno_value) {
+	case ENOENT:
+	case ENOTDIR:
+		giterr_set(GITERR_OS, "Could not find '%s' to %s", path, action);
+		return GIT_ENOTFOUND;
+
+	case EINVAL:
+	case ENAMETOOLONG:
+		giterr_set(GITERR_OS, "Invalid path for filesystem '%s'", path);
+		return GIT_EINVALIDSPEC;
+
+	case EEXIST:
+		giterr_set(GITERR_OS, "Failed %s - '%s' already exists", action, path);
+		return GIT_EEXISTS;
+
+	default:
+		giterr_set(GITERR_OS, "Could not %s '%s'", action, path);
+		return -1;
 	}
+}
 
-	return err;
+int git_path_lstat(const char *path, struct stat *st)
+{
+	if (p_lstat(path, st) == 0)
+		return 0;
+
+	return git_path_set_error(errno, path, "stat");
 }
 
 static bool _check_dir_contents(
@@ -564,7 +579,7 @@ static bool _check_dir_contents(
 	size_t sub_size = strlen(sub);
 
 	/* leave base valid even if we could not make space for subdir */
-	if (git_buf_try_grow(dir, dir_size + sub_size + 2, false) < 0)
+	if (git_buf_try_grow(dir, dir_size + sub_size + 2, false, false) < 0)
 		return false;
 
 	/* save excursion */
@@ -603,7 +618,7 @@ int git_path_find_dir(git_buf *dir, const char *path, const char *base)
 	}
 
 	/* call dirname if this is not a directory */
-	if (!error && git_path_isdir(dir->ptr) == false)
+	if (!error) /* && git_path_isdir(dir->ptr) == false) */
 		error = git_path_dirname_r(dir, dir->ptr);
 
 	if (!error)
@@ -645,12 +660,33 @@ int git_path_resolve_relative(git_buf *path, size_t ceiling)
 			/* do nothing with singleton dot */;
 
 		else if (len == 2 && from[0] == '.' && from[1] == '.') {
-			while (to > base && to[-1] == '/') to--;
-			while (to > base && to[-1] != '/') to--;
-		}
+			/* error out if trying to up one from a hard base */
+			if (to == base && ceiling != 0) {
+				giterr_set(GITERR_INVALID,
+					"Cannot strip root component off url");
+				return -1;
+			}
 
-		else {
-			if (*next == '/')
+			/* no more path segments to strip,
+			 * use '../' as a new base path */
+			if (to == base) {
+				if (*next == '/')
+					len++;
+
+				if (to != from)
+					memmove(to, from, len);
+
+				to += len;
+				/* this is now the base, can't back up from a
+				 * relative prefix */
+				base = to;
+			} else {
+				/* back up a path segment */
+				while (to > base && to[-1] == '/') to--;
+				while (to > base && to[-1] != '/') to--;
+			}
+		} else {
+			if (*next == '/' && *from != '/')
 				len++;
 
 			if (to != from)
@@ -702,14 +738,108 @@ int git_path_cmp(
 	return (c1 < c2) ? -1 : (c1 > c2) ? 1 : 0;
 }
 
+bool git_path_has_non_ascii(const char *path, size_t pathlen)
+{
+	const uint8_t *scan = (const uint8_t *)path, *end;
+
+	for (end = scan + pathlen; scan < end; ++scan)
+		if (*scan & 0x80)
+			return true;
+
+	return false;
+}
+
+#ifdef GIT_USE_ICONV
+
+int git_path_iconv_init_precompose(git_path_iconv_t *ic)
+{
+	git_buf_init(&ic->buf, 0);
+	ic->map = iconv_open(GIT_PATH_REPO_ENCODING, GIT_PATH_NATIVE_ENCODING);
+	return 0;
+}
+
+void git_path_iconv_clear(git_path_iconv_t *ic)
+{
+	if (ic) {
+		if (ic->map != (iconv_t)-1)
+			iconv_close(ic->map);
+		git_buf_free(&ic->buf);
+	}
+}
+
+int git_path_iconv(git_path_iconv_t *ic, char **in, size_t *inlen)
+{
+	char *nfd = *in, *nfc;
+	size_t nfdlen = *inlen, nfclen, wantlen = nfdlen, rv;
+	int retry = 1;
+
+	if (!ic || ic->map == (iconv_t)-1 ||
+		!git_path_has_non_ascii(*in, *inlen))
+		return 0;
+
+	while (1) {
+		if (git_buf_grow(&ic->buf, wantlen) < 0)
+			return -1;
+
+		nfc    = ic->buf.ptr   + ic->buf.size;
+		nfclen = ic->buf.asize - ic->buf.size;
+
+		rv = iconv(ic->map, &nfd, &nfdlen, &nfc, &nfclen);
+
+		ic->buf.size = (nfc - ic->buf.ptr);
+
+		if (rv != (size_t)-1)
+			break;
+
+		if (errno != E2BIG)
+			goto fail;
+
+		/* make space for 2x the remaining data to be converted
+		 * (with per retry overhead to avoid infinite loops)
+		 */
+		wantlen = ic->buf.size + max(nfclen, nfdlen) * 2 + (size_t)(retry * 4);
+
+		if (retry++ > 4)
+			goto fail;
+	}
+
+	ic->buf.ptr[ic->buf.size] = '\0';
+
+	*in    = ic->buf.ptr;
+	*inlen = ic->buf.size;
+
+	return 0;
+
+fail:
+	giterr_set(GITERR_OS, "Unable to convert unicode path data");
+	return -1;
+}
+
+#endif
+
+#if defined(__sun) || defined(__GNU__)
+typedef char path_dirent_data[sizeof(struct dirent) + FILENAME_MAX + 1];
+#else
+typedef struct dirent path_dirent_data;
+#endif
+
 int git_path_direach(
 	git_buf *path,
+	uint32_t flags,
 	int (*fn)(void *, git_buf *),
 	void *arg)
 {
+	int error = 0;
 	ssize_t wd_len;
 	DIR *dir;
-	struct dirent *de, *de_buf;
+	path_dirent_data de_data;
+	struct dirent *de, *de_buf = (struct dirent *)&de_data;
+
+	(void)flags;
+
+#ifdef GIT_USE_ICONV
+	git_path_iconv_t ic = GIT_PATH_ICONV_INIT;
+#endif
 
 	if (git_path_to_dir(path) < 0)
 		return -1;
@@ -721,64 +851,80 @@ int git_path_direach(
 		return -1;
 	}
 
-#if defined(__sun) || defined(__GNU__)
-	de_buf = git__malloc(sizeof(struct dirent) + FILENAME_MAX + 1);
-#else
-	de_buf = git__malloc(sizeof(struct dirent));
+#ifdef GIT_USE_ICONV
+	if ((flags & GIT_PATH_DIR_PRECOMPOSE_UNICODE) != 0)
+		(void)git_path_iconv_init_precompose(&ic);
 #endif
 
 	while (p_readdir_r(dir, de_buf, &de) == 0 && de != NULL) {
-		int result;
+		char *de_path = de->d_name;
+		size_t de_len = strlen(de_path);
 
-		if (git_path_is_dot_or_dotdot(de->d_name))
+		if (git_path_is_dot_or_dotdot(de_path))
 			continue;
 
-		if (git_buf_puts(path, de->d_name) < 0) {
-			closedir(dir);
-			git__free(de_buf);
-			return -1;
-		}
+#ifdef GIT_USE_ICONV
+		if ((error = git_path_iconv(&ic, &de_path, &de_len)) < 0)
+			break;
+#endif
+				
+		if ((error = git_buf_put(path, de_path, de_len)) < 0)
+			break;
 
-		result = fn(arg, path);
+		error = fn(arg, path);
 
 		git_buf_truncate(path, wd_len); /* restore path */
 
-		if (result < 0) {
-			closedir(dir);
-			git__free(de_buf);
-			return -1;
+		if (error) {
+			error = GIT_EUSER;
+			break;
 		}
 	}
 
 	closedir(dir);
-	git__free(de_buf);
-	return 0;
+
+#ifdef GIT_USE_ICONV
+	git_path_iconv_clear(&ic);
+#endif
+
+	return error;
 }
 
 int git_path_dirload(
 	const char *path,
 	size_t prefix_len,
 	size_t alloc_extra,
+	unsigned int flags,
 	git_vector *contents)
 {
 	int error, need_slash;
 	DIR *dir;
-	struct dirent *de, *de_buf;
 	size_t path_len;
+	path_dirent_data de_data;
+	struct dirent *de, *de_buf = (struct dirent *)&de_data;
+
+	(void)flags;
+
+#ifdef GIT_USE_ICONV
+	git_path_iconv_t ic = GIT_PATH_ICONV_INIT;
+#endif
+
+	assert(path && contents);
 
-	assert(path != NULL && contents != NULL);
 	path_len = strlen(path);
-	assert(path_len > 0 && path_len >= prefix_len);
 
+	if (!path_len || path_len < prefix_len) {
+		giterr_set(GITERR_INVALID, "Invalid directory path '%s'", path);
+		return -1;
+	}
 	if ((dir = opendir(path)) == NULL) {
 		giterr_set(GITERR_OS, "Failed to open directory '%s'", path);
 		return -1;
 	}
 
-#if defined(__sun) || defined(__GNU__)
-	de_buf = git__malloc(sizeof(struct dirent) + FILENAME_MAX + 1);
-#else
-	de_buf = git__malloc(sizeof(struct dirent));
+#ifdef GIT_USE_ICONV
+	if ((flags & GIT_PATH_DIR_PRECOMPOSE_UNICODE) != 0)
+		(void)git_path_iconv_init_precompose(&ic);
 #endif
 
 	path += prefix_len;
@@ -786,34 +932,38 @@ int git_path_dirload(
 	need_slash = (path_len > 0 && path[path_len-1] != '/') ? 1 : 0;
 
 	while ((error = p_readdir_r(dir, de_buf, &de)) == 0 && de != NULL) {
-		char *entry_path;
-		size_t entry_len;
+		char *entry_path, *de_path = de->d_name;
+		size_t alloc_size, de_len = strlen(de_path);
 
-		if (git_path_is_dot_or_dotdot(de->d_name))
+		if (git_path_is_dot_or_dotdot(de_path))
 			continue;
 
-		entry_len = strlen(de->d_name);
+#ifdef GIT_USE_ICONV
+		if ((error = git_path_iconv(&ic, &de_path, &de_len)) < 0)
+			break;
+#endif
 
-		entry_path = git__malloc(
-			path_len + need_slash + entry_len + 1 + alloc_extra);
-		GITERR_CHECK_ALLOC(entry_path);
+		alloc_size = path_len + need_slash + de_len + 1 + alloc_extra;
+		if ((entry_path = git__calloc(alloc_size, 1)) == NULL) {
+			error = -1;
+			break;
+		}
 
 		if (path_len)
 			memcpy(entry_path, path, path_len);
 		if (need_slash)
 			entry_path[path_len] = '/';
-		memcpy(&entry_path[path_len + need_slash], de->d_name, entry_len);
-		entry_path[path_len + need_slash + entry_len] = '\0';
+		memcpy(&entry_path[path_len + need_slash], de_path, de_len);
 
-		if (git_vector_insert(contents, entry_path) < 0) {
-			closedir(dir);
-			git__free(de_buf);
-			return -1;
-		}
+		if ((error = git_vector_insert(contents, entry_path)) < 0)
+			break;
 	}
 
 	closedir(dir);
-	git__free(de_buf);
+
+#ifdef GIT_USE_ICONV
+	git_path_iconv_clear(&ic);
+#endif
 
 	if (error != 0)
 		giterr_set(GITERR_OS, "Failed to process directory entry in '%s'", path);
@@ -836,7 +986,7 @@ int git_path_with_stat_cmp_icase(const void *a, const void *b)
 int git_path_dirload_with_stat(
 	const char *path,
 	size_t prefix_len,
-	bool ignore_case,
+	unsigned int flags,
 	const char *start_stat,
 	const char *end_stat,
 	git_vector *contents)
@@ -853,13 +1003,14 @@ int git_path_dirload_with_stat(
 		return -1;
 
 	error = git_path_dirload(
-		path, prefix_len, sizeof(git_path_with_stat) + 1, contents);
+		path, prefix_len, sizeof(git_path_with_stat) + 1, flags, contents);
 	if (error < 0) {
 		git_buf_free(&full);
 		return error;
 	}
 
-	strncomp = ignore_case ? git__strncasecmp : git__strncmp;
+	strncomp = (flags & GIT_PATH_DIR_IGNORE_CASE) != 0 ?
+		git__strncasecmp : git__strncmp;
 
 	/* stat struct at start of git_path_with_stat, so shift path text */
 	git_vector_foreach(contents, i, ps) {
@@ -880,8 +1031,16 @@ int git_path_dirload_with_stat(
 		git_buf_truncate(&full, prefix_len);
 
 		if ((error = git_buf_joinpath(&full, full.ptr, ps->path)) < 0 ||
-			(error = git_path_lstat(full.ptr, &ps->st)) < 0)
+			(error = git_path_lstat(full.ptr, &ps->st)) < 0) {
+			if (error == GIT_ENOTFOUND) {
+				giterr_clear();
+				error = 0;
+				git_vector_remove(contents, i--);
+				continue;
+			}
+
 			break;
+		}
 
 		if (S_ISDIR(ps->st.st_mode)) {
 			if ((error = git_buf_joinpath(&full, full.ptr, ".git")) < 0)
diff --git a/src/path.h b/src/path.h
index ead4fa338..3daafd265 100644
--- a/src/path.h
+++ b/src/path.h
@@ -175,7 +175,6 @@ extern bool git_path_contains(git_buf *dir, const char *item);
  *
  * @param parent Directory path that might contain subdir
  * @param subdir Subdirectory name to look for in parent
- * @param append_if_exists If true, then subdir will be appended to the parent path if it does exist
  * @return true if subdirectory exists, false otherwise.
  */
 extern bool git_path_contains_dir(git_buf *parent, const char *subdir);
@@ -185,7 +184,6 @@ extern bool git_path_contains_dir(git_buf *parent, const char *subdir);
  *
  * @param dir Directory path that might contain file
  * @param file File name to look for in parent
- * @param append_if_exists If true, then file will be appended to the path if it does exist
  * @return true if file exists, false otherwise.
  */
 extern bool git_path_contains_file(git_buf *dir, const char *file);
@@ -244,21 +242,28 @@ extern int git_path_resolve_relative(git_buf *path, size_t ceiling);
  */
 extern int git_path_apply_relative(git_buf *target, const char *relpath);
 
+enum {
+	GIT_PATH_DIR_IGNORE_CASE = (1u << 0),
+	GIT_PATH_DIR_PRECOMPOSE_UNICODE = (1u << 1),
+};
+
 /**
  * Walk each directory entry, except '.' and '..', calling fn(state).
  *
- * @param pathbuf buffer the function reads the initial directory
+ * @param pathbuf Buffer the function reads the initial directory
  * 		path from, and updates with each successive entry's name.
- * @param fn function to invoke with each entry. The first arg is
- *		the input state and the second arg is pathbuf. The function
- *		may modify the pathbuf, but only by appending new text.
- * @param state to pass to fn as the first arg.
+ * @param flags Combination of GIT_PATH_DIR flags.
+ * @param callback Callback for each entry. Passed the `payload` and each
+ *		successive path inside the directory as a full path.  This may
+ *		safely append text to the pathbuf if needed.
+ * @param payload Passed to callback as first argument.
  * @return 0 on success, GIT_EUSER on non-zero callback, or error code
  */
 extern int git_path_direach(
 	git_buf *pathbuf,
-	int (*fn)(void *, git_buf *),
-	void *state);
+	uint32_t flags,
+	int (*callback)(void *payload, git_buf *path),
+	void *payload);
 
 /**
  * Sort function to order two paths
@@ -278,19 +283,19 @@ extern int git_path_cmp(
  * @param pathbuf Buffer the function reads the directory from and
  *		and updates with each successive name.
  * @param ceiling Prefix of path at which to stop walking up.  If NULL,
- *      this will walk all the way up to the root.  If not a prefix of
- *      pathbuf, the callback will be invoked a single time on the
- *      original input path.
- * @param fn Function to invoke on each path.  The first arg is the
- *		input satte and the second arg is the pathbuf.  The function
- *		should not modify the pathbuf.
+ *		this will walk all the way up to the root.  If not a prefix of
+ *		pathbuf, the callback will be invoked a single time on the
+ *		original input path.
+ * @param callback Function to invoke on each path.  Passed the `payload`
+ *		and the buffer containing the current path.  The path should not
+ *		be modified in any way.
  * @param state Passed to fn as the first ath.
  */
 extern int git_path_walk_up(
 	git_buf *pathbuf,
 	const char *ceiling,
-	int (*fn)(void *state, git_buf *),
-	void *state);
+	int (*callback)(void *payload, git_buf *path),
+	void *payload);
 
 /**
  * Load all directory entries (except '.' and '..') into a vector.
@@ -306,12 +311,14 @@ extern int git_path_walk_up(
  * 		prefix_len 3, the entries will look like "b/e1", "b/e2", etc.
  * @param alloc_extra Extra bytes to add to each string allocation in
  * 		case you want to append anything funny.
+ * @param flags Combination of GIT_PATH_DIR flags.
  * @param contents Vector to fill with directory entry names.
  */
 extern int git_path_dirload(
 	const char *path,
 	size_t prefix_len,
 	size_t alloc_extra,
+	uint32_t flags,
 	git_vector *contents);
 
 
@@ -338,7 +345,7 @@ extern int git_path_with_stat_cmp_icase(const void *a, const void *b);
  *
  * @param path The directory to read from
  * @param prefix_len The trailing part of path to prefix to entry paths
- * @param ignore_case How to sort and compare paths with start/end limits
+ * @param flags GIT_PATH_DIR flags from above
  * @param start_stat As optimization, only stat values after this prefix
  * @param end_stat As optimization, only stat values before this prefix
  * @param contents Vector to fill with git_path_with_stat structures
@@ -346,9 +353,78 @@ extern int git_path_with_stat_cmp_icase(const void *a, const void *b);
 extern int git_path_dirload_with_stat(
 	const char *path,
 	size_t prefix_len,
-	bool ignore_case,
+	uint32_t flags,
 	const char *start_stat,
 	const char *end_stat,
 	git_vector *contents);
 
+enum { GIT_PATH_NOTEQUAL = 0, GIT_PATH_EQUAL = 1, GIT_PATH_PREFIX = 2 };
+
+/*
+ * Determines if a path is equal to or potentially a child of another.
+ * @param parent The possible parent
+ * @param child The possible child
+ */
+GIT_INLINE(int) git_path_equal_or_prefixed(
+	const char *parent,
+	const char *child)
+{
+	const char *p = parent, *c = child;
+
+	while (*p && *c) {
+		if (*p++ != *c++)
+			return GIT_PATH_NOTEQUAL;
+	}
+
+	if (*p != '\0')
+		return GIT_PATH_NOTEQUAL;
+	if (*c == '\0')
+		return GIT_PATH_EQUAL;
+	if (*c == '/')
+		return GIT_PATH_PREFIX;
+
+	return GIT_PATH_NOTEQUAL;
+}
+
+/* translate errno to libgit2 error code and set error message */
+extern int git_path_set_error(
+	int errno_value, const char *path, const char *action);
+
+/* check if non-ascii characters are present in filename */
+extern bool git_path_has_non_ascii(const char *path, size_t pathlen);
+
+#define GIT_PATH_REPO_ENCODING "UTF-8"
+
+#ifdef __APPLE__
+#define GIT_PATH_NATIVE_ENCODING "UTF-8-MAC"
+#else
+#define GIT_PATH_NATIVE_ENCODING "UTF-8"
+#endif
+
+#ifdef GIT_USE_ICONV
+
+#include <iconv.h>
+
+typedef struct {
+	iconv_t map;
+	git_buf buf;
+} git_path_iconv_t;
+
+#define GIT_PATH_ICONV_INIT { (iconv_t)-1, GIT_BUF_INIT }
+
+/* Init iconv data for converting decomposed UTF-8 to precomposed */
+extern int git_path_iconv_init_precompose(git_path_iconv_t *ic);
+
+/* Clear allocated iconv data */
+extern void git_path_iconv_clear(git_path_iconv_t *ic);
+
+/*
+ * Rewrite `in` buffer using iconv map if necessary, replacing `in`
+ * pointer internal iconv buffer if rewrite happened.  The `in` pointer
+ * will be left unchanged if no rewrite was needed.
+ */
+extern int git_path_iconv(git_path_iconv_t *ic, char **in, size_t *inlen);
+
+#endif /* GIT_USE_ICONV */
+
 #endif
diff --git a/src/pathspec.c b/src/pathspec.c
index f029836d0..1e7e65e90 100644
--- a/src/pathspec.c
+++ b/src/pathspec.c
@@ -5,9 +5,16 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
+#include "git2/pathspec.h"
+#include "git2/diff.h"
 #include "pathspec.h"
 #include "buf_text.h"
 #include "attr_file.h"
+#include "iterator.h"
+#include "repository.h"
+#include "index.h"
+#include "bitvec.h"
+#include "diff.h"
 
 /* what is the common non-wildcard prefix for all items in the pathspec */
 char *git_pathspec_prefix(const git_strarray *pathspec)
@@ -56,7 +63,7 @@ bool git_pathspec_is_empty(const git_strarray *pathspec)
 }
 
 /* build a vector of fnmatch patterns to evaluate efficiently */
-int git_pathspec_init(
+int git_pathspec__vinit(
 	git_vector *vspec, const git_strarray *strspec, git_pool *strpool)
 {
 	size_t i;
@@ -76,7 +83,7 @@ int git_pathspec_init(
 		if (!match)
 			return -1;
 
-		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE;
+		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE | GIT_ATTR_FNMATCH_ALLOWNEG;
 
 		ret = git_attr_fnmatch__parse(match, strpool, NULL, &pattern);
 		if (ret == GIT_ENOTFOUND) {
@@ -93,7 +100,7 @@ int git_pathspec_init(
 }
 
 /* free data from the pathspec vector */
-void git_pathspec_free(git_vector *vspec)
+void git_pathspec__vfree(git_vector *vspec)
 {
 	git_attr_fnmatch *match;
 	unsigned int i;
@@ -106,88 +113,612 @@ void git_pathspec_free(git_vector *vspec)
 	git_vector_free(vspec);
 }
 
+struct pathspec_match_context {
+	int fnmatch_flags;
+	int (*strcomp)(const char *, const char *);
+	int (*strncomp)(const char *, const char *, size_t);
+};
+
+static void pathspec_match_context_init(
+	struct pathspec_match_context *ctxt,
+	bool disable_fnmatch,
+	bool casefold)
+{
+	if (disable_fnmatch)
+		ctxt->fnmatch_flags = -1;
+	else if (casefold)
+		ctxt->fnmatch_flags = FNM_CASEFOLD;
+	else
+		ctxt->fnmatch_flags = 0;
+
+	if (casefold) {
+		ctxt->strcomp  = git__strcasecmp;
+		ctxt->strncomp = git__strncasecmp;
+	} else {
+		ctxt->strcomp  = git__strcmp;
+		ctxt->strncomp = git__strncmp;
+	}
+}
+
+static int pathspec_match_one(
+	const git_attr_fnmatch *match,
+	struct pathspec_match_context *ctxt,
+	const char *path)
+{
+	int result = (match->flags & GIT_ATTR_FNMATCH_MATCH_ALL) ? 0 : FNM_NOMATCH;
+
+	if (result == FNM_NOMATCH)
+		result = ctxt->strcomp(match->pattern, path) ? FNM_NOMATCH : 0;
+
+	if (ctxt->fnmatch_flags >= 0 && result == FNM_NOMATCH)
+		result = p_fnmatch(match->pattern, path, ctxt->fnmatch_flags);
+
+	/* if we didn't match, look for exact dirname prefix match */
+	if (result == FNM_NOMATCH &&
+		(match->flags & GIT_ATTR_FNMATCH_HASWILD) == 0 &&
+		ctxt->strncomp(path, match->pattern, match->length) == 0 &&
+		path[match->length] == '/')
+		result = 0;
+
+	/* if we didn't match and this is a negative match, check for exact
+	 * match of filename with leading '!'
+	 */
+	if (result == FNM_NOMATCH &&
+		(match->flags & GIT_ATTR_FNMATCH_NEGATIVE) != 0 &&
+		*path == '!' &&
+		ctxt->strncomp(path + 1, match->pattern, match->length) == 0 &&
+		(!path[match->length + 1] || path[match->length + 1] == '/'))
+		return 1;
+
+	if (result == 0)
+		return (match->flags & GIT_ATTR_FNMATCH_NEGATIVE) ? 0 : 1;
+	return -1;
+}
+
+static int git_pathspec__match_at(
+	size_t *matched_at,
+	const git_vector *vspec,
+	struct pathspec_match_context *ctxt,
+	const char *path0,
+	const char *path1)
+{
+	int result = GIT_ENOTFOUND;
+	size_t i = 0;
+	const git_attr_fnmatch *match;
+
+	git_vector_foreach(vspec, i, match) {
+		if (path0 && (result = pathspec_match_one(match, ctxt, path0)) >= 0)
+			break;
+		if (path1 && (result = pathspec_match_one(match, ctxt, path1)) >= 0)
+			break;
+	}
+
+	*matched_at = i;
+	return result;
+}
+
 /* match a path against the vectorized pathspec */
-bool git_pathspec_match_path(
-	git_vector *vspec,
+bool git_pathspec__match(
+	const git_vector *vspec,
 	const char *path,
 	bool disable_fnmatch,
 	bool casefold,
-	const char **matched_pathspec)
+	const char **matched_pathspec,
+	size_t *matched_at)
 {
-	size_t i;
-	git_attr_fnmatch *match;
-	int fnmatch_flags = 0;
-	int (*use_strcmp)(const char *, const char *);
-	int (*use_strncmp)(const char *, const char *, size_t);
+	int result;
+	size_t pos;
+	struct pathspec_match_context ctxt;
 
 	if (matched_pathspec)
 		*matched_pathspec = NULL;
+	if (matched_at)
+		*matched_at = GIT_PATHSPEC_NOMATCH;
 
 	if (!vspec || !vspec->length)
 		return true;
 
-	if (disable_fnmatch)
-		fnmatch_flags = -1;
-	else if (casefold)
-		fnmatch_flags = FNM_CASEFOLD;
+	pathspec_match_context_init(&ctxt, disable_fnmatch, casefold);
 
-	if (casefold) {
-		use_strcmp  = git__strcasecmp;
-		use_strncmp = git__strncasecmp;
-	} else {
-		use_strcmp  = git__strcmp;
-		use_strncmp = git__strncmp;
+	result = git_pathspec__match_at(&pos, vspec, &ctxt, path, NULL);
+	if (result >= 0) {
+		if (matched_pathspec) {
+			const git_attr_fnmatch *match = git_vector_get(vspec, pos);
+			*matched_pathspec = match->pattern;
+		}
+
+		if (matched_at)
+			*matched_at = pos;
 	}
 
-	git_vector_foreach(vspec, i, match) {
-		int result = (match->flags & GIT_ATTR_FNMATCH_MATCH_ALL) ? 0 : FNM_NOMATCH;
+	return (result > 0);
+}
+
+
+int git_pathspec__init(git_pathspec *ps, const git_strarray *paths)
+{
+	int error = 0;
+
+	memset(ps, 0, sizeof(*ps));
+
+	ps->prefix = git_pathspec_prefix(paths);
+
+	if ((error = git_pool_init(&ps->pool, 1, 0)) < 0 ||
+		(error = git_pathspec__vinit(&ps->pathspec, paths, &ps->pool)) < 0)
+		git_pathspec__clear(ps);
+
+	return error;
+}
+
+void git_pathspec__clear(git_pathspec *ps)
+{
+	git__free(ps->prefix);
+	git_pathspec__vfree(&ps->pathspec);
+	git_pool_clear(&ps->pool);
+	memset(ps, 0, sizeof(*ps));
+}
+
+int git_pathspec_new(git_pathspec **out, const git_strarray *pathspec)
+{
+	int error = 0;
+	git_pathspec *ps = git__malloc(sizeof(git_pathspec));
+	GITERR_CHECK_ALLOC(ps);
+
+	if ((error = git_pathspec__init(ps, pathspec)) < 0) {
+		git__free(ps);
+		return error;
+	}
+
+	GIT_REFCOUNT_INC(ps);
+	*out = ps;
+	return 0;
+}
+
+static void pathspec_free(git_pathspec *ps)
+{
+	git_pathspec__clear(ps);
+	git__free(ps);
+}
+
+void git_pathspec_free(git_pathspec *ps)
+{
+	if (!ps)
+		return;
+	GIT_REFCOUNT_DEC(ps, pathspec_free);
+}
+
+int git_pathspec_matches_path(
+	const git_pathspec *ps, uint32_t flags, const char *path)
+{
+	bool no_fnmatch = (flags & GIT_PATHSPEC_NO_GLOB) != 0;
+	bool casefold =  (flags & GIT_PATHSPEC_IGNORE_CASE) != 0;
+
+	assert(ps && path);
+
+	return (0 != git_pathspec__match(
+		&ps->pathspec, path, no_fnmatch, casefold, NULL, NULL));
+}
+
+static void pathspec_match_free(git_pathspec_match_list *m)
+{
+	git_pathspec_free(m->pathspec);
+	m->pathspec = NULL;
+
+	git_array_clear(m->matches);
+	git_array_clear(m->failures);
+	git_pool_clear(&m->pool);
+	git__free(m);
+}
+
+static git_pathspec_match_list *pathspec_match_alloc(
+	git_pathspec *ps, int datatype)
+{
+	git_pathspec_match_list *m = git__calloc(1, sizeof(git_pathspec_match_list));
+
+	if (m != NULL && git_pool_init(&m->pool, 1, 0) < 0) {
+		pathspec_match_free(m);
+		m = NULL;
+	}
+
+	/* need to keep reference to pathspec and increment refcount because
+	 * failures array stores pointers to the pattern strings of the
+	 * pathspec that had no matches
+	 */
+	GIT_REFCOUNT_INC(ps);
+	m->pathspec = ps;
+	m->datatype = datatype;
+
+	return m;
+}
+
+GIT_INLINE(size_t) pathspec_mark_pattern(git_bitvec *used, size_t pos)
+{
+	if (!git_bitvec_get(used, pos)) {
+		git_bitvec_set(used, pos, true);
+		return 1;
+	}
+
+	return 0;
+}
+
+static size_t pathspec_mark_remaining(
+	git_bitvec *used,
+	git_vector *patterns,
+	struct pathspec_match_context *ctxt,
+	size_t start,
+	const char *path0,
+	const char *path1)
+{
+	size_t count = 0;
+
+	if (path1 == path0)
+		path1 = NULL;
+
+	for (; start < patterns->length; ++start) {
+		const git_attr_fnmatch *pat = git_vector_get(patterns, start);
+
+		if (git_bitvec_get(used, start))
+			continue;
+
+		if (path0 && pathspec_match_one(pat, ctxt, path0) > 0)
+			count += pathspec_mark_pattern(used, start);
+		else if (path1 && pathspec_match_one(pat, ctxt, path1) > 0)
+			count += pathspec_mark_pattern(used, start);
+	}
+
+	return count;
+}
+
+static int pathspec_build_failure_array(
+	git_pathspec_string_array_t *failures,
+	git_vector *patterns,
+	git_bitvec *used,
+	git_pool *pool)
+{
+	size_t pos;
+	char **failed;
+	const git_attr_fnmatch *pat;
+
+	for (pos = 0; pos < patterns->length; ++pos) {
+		if (git_bitvec_get(used, pos))
+			continue;
+
+		if ((failed = git_array_alloc(*failures)) == NULL)
+			return -1;
+
+		pat = git_vector_get(patterns, pos);
+
+		if ((*failed = git_pool_strdup(pool, pat->pattern)) == NULL)
+			return -1;
+	}
+
+	return 0;
+}
 
-		if (result == FNM_NOMATCH)
-			result = use_strcmp(match->pattern, path) ? FNM_NOMATCH : 0;
+static int pathspec_match_from_iterator(
+	git_pathspec_match_list **out,
+	git_iterator *iter,
+	uint32_t flags,
+	git_pathspec *ps)
+{
+	int error = 0;
+	git_pathspec_match_list *m = NULL;
+	const git_index_entry *entry = NULL;
+	struct pathspec_match_context ctxt;
+	git_vector *patterns = &ps->pathspec;
+	bool find_failures = out && (flags & GIT_PATHSPEC_FIND_FAILURES) != 0;
+	bool failures_only = !out || (flags & GIT_PATHSPEC_FAILURES_ONLY) != 0;
+	size_t pos, used_ct = 0, found_files = 0;
+	git_index *index = NULL;
+	git_bitvec used_patterns;
+	char **file;
+
+	if (git_bitvec_init(&used_patterns, patterns->length) < 0)
+		return -1;
+
+	if (out) {
+		*out = m = pathspec_match_alloc(ps, PATHSPEC_DATATYPE_STRINGS);
+		GITERR_CHECK_ALLOC(m);
+	}
+
+	if ((error = git_iterator_reset(iter, ps->prefix, ps->prefix)) < 0)
+		goto done;
+
+	if (git_iterator_type(iter) == GIT_ITERATOR_TYPE_WORKDIR &&
+		(error = git_repository_index__weakptr(
+			&index, git_iterator_owner(iter))) < 0)
+		goto done;
 
-		if (fnmatch_flags >= 0 && result == FNM_NOMATCH)
-			result = p_fnmatch(match->pattern, path, fnmatch_flags);
+	pathspec_match_context_init(
+		&ctxt, (flags & GIT_PATHSPEC_NO_GLOB) != 0,
+		git_iterator_ignore_case(iter));
 
-		/* if we didn't match, look for exact dirname prefix match */
-		if (result == FNM_NOMATCH &&
-			(match->flags & GIT_ATTR_FNMATCH_HASWILD) == 0 &&
-			use_strncmp(path, match->pattern, match->length) == 0 &&
-			path[match->length] == '/')
-			result = 0;
+	while (!(error = git_iterator_advance(&entry, iter))) {
+		/* search for match with entry->path */
+		int result = git_pathspec__match_at(
+			&pos, patterns, &ctxt, entry->path, NULL);
 
-		if (result == 0) {
-			if (matched_pathspec)
-				*matched_pathspec = match->pattern;
+		/* no matches for this path */
+		if (result < 0)
+			continue;
 
-			return (match->flags & GIT_ATTR_FNMATCH_NEGATIVE) ? false : true;
+		/* if result was a negative pattern match, then don't list file */
+		if (!result) {
+			used_ct += pathspec_mark_pattern(&used_patterns, pos);
+			continue;
+		}
+
+		/* check if path is ignored and untracked */
+		if (index != NULL &&
+			git_iterator_current_is_ignored(iter) &&
+			git_index__find(NULL, index, entry->path, GIT_INDEX_STAGE_ANY) < 0)
+			continue;
+
+		/* mark the matched pattern as used */
+		used_ct += pathspec_mark_pattern(&used_patterns, pos);
+		++found_files;
+
+		/* if find_failures is on, check if any later patterns also match */
+		if (find_failures && used_ct < patterns->length)
+			used_ct += pathspec_mark_remaining(
+				&used_patterns, patterns, &ctxt, pos + 1, entry->path, NULL);
+
+		/* if only looking at failures, exit early or just continue */
+		if (failures_only || !out) {
+			if (used_ct == patterns->length)
+				break;
+			continue;
+		}
+
+		/* insert matched path into matches array */
+		if ((file = (char **)git_array_alloc(m->matches)) == NULL ||
+			(*file = git_pool_strdup(&m->pool, entry->path)) == NULL) {
+			error = -1;
+			goto done;
 		}
 	}
 
-	return false;
+	if (error < 0 && error != GIT_ITEROVER)
+		goto done;
+	error = 0;
+
+	/* insert patterns that had no matches into failures array */
+	if (find_failures && used_ct < patterns->length &&
+		(error = pathspec_build_failure_array(
+			&m->failures, patterns, &used_patterns, &m->pool)) < 0)
+		goto done;
+
+	/* if every pattern failed to match, then we have failed */
+	if ((flags & GIT_PATHSPEC_NO_MATCH_ERROR) != 0 && !found_files) {
+		giterr_set(GITERR_INVALID, "No matching files were found");
+		error = GIT_ENOTFOUND;
+	}
+
+done:
+	git_bitvec_free(&used_patterns);
+
+	if (error < 0) {
+		pathspec_match_free(m);
+		if (out) *out = NULL;
+	}
+
+	return error;
 }
 
+static git_iterator_flag_t pathspec_match_iter_flags(uint32_t flags)
+{
+	git_iterator_flag_t f = 0;
 
-int git_pathspec_context_init(
-	git_pathspec_context *ctxt, const git_strarray *paths)
+	if ((flags & GIT_PATHSPEC_IGNORE_CASE) != 0)
+		f |= GIT_ITERATOR_IGNORE_CASE;
+	else if ((flags & GIT_PATHSPEC_USE_CASE) != 0)
+		f |= GIT_ITERATOR_DONT_IGNORE_CASE;
+
+	return f;
+}
+
+int git_pathspec_match_workdir(
+	git_pathspec_match_list **out,
+	git_repository *repo,
+	uint32_t flags,
+	git_pathspec *ps)
+{
+	int error = 0;
+	git_iterator *iter;
+
+	assert(repo);
+
+	if (!(error = git_iterator_for_workdir(
+			&iter, repo, pathspec_match_iter_flags(flags), NULL, NULL))) {
+
+		error = pathspec_match_from_iterator(out, iter, flags, ps);
+
+		git_iterator_free(iter);
+	}
+
+	return error;
+}
+
+int git_pathspec_match_index(
+	git_pathspec_match_list **out,
+	git_index *index,
+	uint32_t flags,
+	git_pathspec *ps)
 {
 	int error = 0;
+	git_iterator *iter;
+
+	assert(index);
 
-	memset(ctxt, 0, sizeof(*ctxt));
+	if (!(error = git_iterator_for_index(
+			&iter, index, pathspec_match_iter_flags(flags), NULL, NULL))) {
 
-	ctxt->prefix = git_pathspec_prefix(paths);
+		error = pathspec_match_from_iterator(out, iter, flags, ps);
 
-	if ((error = git_pool_init(&ctxt->pool, 1, 0)) < 0 ||
-		(error = git_pathspec_init(&ctxt->pathspec, paths, &ctxt->pool)) < 0)
-		git_pathspec_context_free(ctxt);
+		git_iterator_free(iter);
+	}
 
 	return error;
 }
 
-void git_pathspec_context_free(
-	git_pathspec_context *ctxt)
+int git_pathspec_match_tree(
+	git_pathspec_match_list **out,
+	git_tree *tree,
+	uint32_t flags,
+	git_pathspec *ps)
 {
-	git__free(ctxt->prefix);
-	git_pathspec_free(&ctxt->pathspec);
-	git_pool_clear(&ctxt->pool);
-	memset(ctxt, 0, sizeof(*ctxt));
+	int error = 0;
+	git_iterator *iter;
+
+	assert(tree);
+
+	if (!(error = git_iterator_for_tree(
+			&iter, tree, pathspec_match_iter_flags(flags), NULL, NULL))) {
+
+		error = pathspec_match_from_iterator(out, iter, flags, ps);
+
+		git_iterator_free(iter);
+	}
+
+	return error;
 }
+
+int git_pathspec_match_diff(
+	git_pathspec_match_list **out,
+	git_diff *diff,
+	uint32_t flags,
+	git_pathspec *ps)
+{
+	int error = 0;
+	git_pathspec_match_list *m = NULL;
+	struct pathspec_match_context ctxt;
+	git_vector *patterns = &ps->pathspec;
+	bool find_failures = out && (flags & GIT_PATHSPEC_FIND_FAILURES) != 0;
+	bool failures_only = !out || (flags & GIT_PATHSPEC_FAILURES_ONLY) != 0;
+	size_t i, pos, used_ct = 0, found_deltas = 0;
+	const git_diff_delta *delta, **match;
+	git_bitvec used_patterns;
+
+	assert(diff);
+
+	if (git_bitvec_init(&used_patterns, patterns->length) < 0)
+		return -1;
+
+	if (out) {
+		*out = m = pathspec_match_alloc(ps, PATHSPEC_DATATYPE_DIFF);
+		GITERR_CHECK_ALLOC(m);
+	}
+
+	pathspec_match_context_init(
+		&ctxt, (flags & GIT_PATHSPEC_NO_GLOB) != 0,
+		git_diff_is_sorted_icase(diff));
+
+	git_vector_foreach(&diff->deltas, i, delta) {
+		/* search for match with delta */
+		int result = git_pathspec__match_at(
+			&pos, patterns, &ctxt, delta->old_file.path, delta->new_file.path);
+
+		/* no matches for this path */
+		if (result < 0)
+			continue;
+
+		/* mark the matched pattern as used */
+		used_ct += pathspec_mark_pattern(&used_patterns, pos);
+
+		/* if result was a negative pattern match, then don't list file */
+		if (!result)
+			continue;
+
+		++found_deltas;
+
+		/* if find_failures is on, check if any later patterns also match */
+		if (find_failures && used_ct < patterns->length)
+			used_ct += pathspec_mark_remaining(
+				&used_patterns, patterns, &ctxt, pos + 1,
+				delta->old_file.path, delta->new_file.path);
+
+		/* if only looking at failures, exit early or just continue */
+		if (failures_only || !out) {
+			if (used_ct == patterns->length)
+				break;
+			continue;
+		}
+
+		/* insert matched delta into matches array */
+		if (!(match = (const git_diff_delta **)git_array_alloc(m->matches))) {
+			error = -1;
+			goto done;
+		} else {
+			*match = delta;
+		}
+	}
+
+	/* insert patterns that had no matches into failures array */
+	if (find_failures && used_ct < patterns->length &&
+		(error = pathspec_build_failure_array(
+			&m->failures, patterns, &used_patterns, &m->pool)) < 0)
+		goto done;
+
+	/* if every pattern failed to match, then we have failed */
+	if ((flags & GIT_PATHSPEC_NO_MATCH_ERROR) != 0 && !found_deltas) {
+		giterr_set(GITERR_INVALID, "No matching deltas were found");
+		error = GIT_ENOTFOUND;
+	}
+
+done:
+	git_bitvec_free(&used_patterns);
+
+	if (error < 0) {
+		pathspec_match_free(m);
+		if (out) *out = NULL;
+	}
+
+	return error;
+}
+
+void git_pathspec_match_list_free(git_pathspec_match_list *m)
+{
+	if (m)
+		pathspec_match_free(m);
+}
+
+size_t git_pathspec_match_list_entrycount(
+	const git_pathspec_match_list *m)
+{
+	return m ? git_array_size(m->matches) : 0;
+}
+
+const char *git_pathspec_match_list_entry(
+	const git_pathspec_match_list *m, size_t pos)
+{
+	if (!m || m->datatype != PATHSPEC_DATATYPE_STRINGS ||
+		!git_array_valid_index(m->matches, pos))
+		return NULL;
+
+	return *((const char **)git_array_get(m->matches, pos));
+}
+
+const git_diff_delta *git_pathspec_match_list_diff_entry(
+	const git_pathspec_match_list *m, size_t pos)
+{
+	if (!m || m->datatype != PATHSPEC_DATATYPE_DIFF ||
+		!git_array_valid_index(m->matches, pos))
+		return NULL;
+
+	return *((const git_diff_delta **)git_array_get(m->matches, pos));
+}
+
+size_t git_pathspec_match_list_failed_entrycount(
+	const git_pathspec_match_list *m)
+{
+	return m ? git_array_size(m->failures) : 0;
+}
+
+const char * git_pathspec_match_list_failed_entry(
+	const git_pathspec_match_list *m, size_t pos)
+{
+	char **entry = m ? git_array_get(m->failures, pos) : NULL;
+
+	return entry ? *entry : NULL;
+}
+
diff --git a/src/pathspec.h b/src/pathspec.h
index f6509df4c..40cd21c3f 100644
--- a/src/pathspec.h
+++ b/src/pathspec.h
@@ -8,9 +8,35 @@
 #define INCLUDE_pathspec_h__
 
 #include "common.h"
+#include <git2/pathspec.h>
 #include "buffer.h"
 #include "vector.h"
 #include "pool.h"
+#include "array.h"
+
+/* public compiled pathspec */
+struct git_pathspec {
+	git_refcount rc;
+	char *prefix;
+	git_vector pathspec;
+	git_pool pool;
+};
+
+enum {
+	PATHSPEC_DATATYPE_STRINGS = 0,
+	PATHSPEC_DATATYPE_DIFF = 1,
+};
+
+typedef git_array_t(char *) git_pathspec_string_array_t;
+
+/* public interface to pathspec matching */
+struct git_pathspec_match_list {
+	git_pathspec *pathspec;
+	git_array_t(void *) matches;
+	git_pathspec_string_array_t failures;
+	git_pool pool;
+	int datatype;
+};
 
 /* what is the common non-wildcard prefix for all items in the pathspec */
 extern char *git_pathspec_prefix(const git_strarray *pathspec);
@@ -19,36 +45,31 @@ extern char *git_pathspec_prefix(const git_strarray *pathspec);
 extern bool git_pathspec_is_empty(const git_strarray *pathspec);
 
 /* build a vector of fnmatch patterns to evaluate efficiently */
-extern int git_pathspec_init(
+extern int git_pathspec__vinit(
 	git_vector *vspec, const git_strarray *strspec, git_pool *strpool);
 
 /* free data from the pathspec vector */
-extern void git_pathspec_free(git_vector *vspec);
+extern void git_pathspec__vfree(git_vector *vspec);
+
+#define GIT_PATHSPEC_NOMATCH ((size_t)-1)
 
 /*
  * Match a path against the vectorized pathspec.
  * The matched pathspec is passed back into the `matched_pathspec` parameter,
  * unless it is passed as NULL by the caller.
  */
-extern bool git_pathspec_match_path(
-	git_vector *vspec,
+extern bool git_pathspec__match(
+	const git_vector *vspec,
 	const char *path,
 	bool disable_fnmatch,
 	bool casefold,
-	const char **matched_pathspec);
+	const char **matched_pathspec,
+	size_t *matched_at);
 
 /* easy pathspec setup */
 
-typedef struct {
-	char *prefix;
-	git_vector pathspec;
-	git_pool pool;
-} git_pathspec_context;
-
-extern int git_pathspec_context_init(
-	git_pathspec_context *ctxt, const git_strarray *paths);
+extern int git_pathspec__init(git_pathspec *ps, const git_strarray *paths);
 
-extern void git_pathspec_context_free(
-	git_pathspec_context *ctxt);
+extern void git_pathspec__clear(git_pathspec *ps);
 
 #endif
diff --git a/src/posix.h b/src/posix.h
index 40bcc1ab0..14c92b93d 100644
--- a/src/posix.h
+++ b/src/posix.h
@@ -39,7 +39,7 @@ typedef int git_file;
  *
  * Some of the methods are slightly wrapped to provide
  * saner defaults. Some of these methods are emulated
- * in Windows platforns.
+ * in Windows platforms.
  *
  * Use your manpages to check the docs on these.
  */
@@ -71,16 +71,12 @@ typedef int GIT_SOCKET;
 
 #define p_localtime_r localtime_r
 #define p_gmtime_r gmtime_r
-#define p_gettimeofday gettimeofday
 
 #else
 
 typedef SOCKET GIT_SOCKET;
-struct timezone;
 extern struct tm * p_localtime_r (const time_t *timer, struct tm *result);
 extern struct tm * p_gmtime_r (const time_t *timer, struct tm *result);
-extern int p_gettimeofday(struct timeval *tv, struct timezone *tz);
-
 
 #endif
 
@@ -93,6 +89,10 @@ extern int p_gettimeofday(struct timeval *tv, struct timezone *tz);
 #	include "unix/posix.h"
 #endif
 
+#ifndef __MINGW32__
+# define p_strnlen strnlen
+#endif
+
 #ifdef NO_READDIR_R
 #	include <dirent.h>
 GIT_INLINE(int) p_readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result)
diff --git a/src/pqueue.h b/src/pqueue.h
index ed7139285..9061f8279 100644
--- a/src/pqueue.h
+++ b/src/pqueue.h
@@ -48,7 +48,7 @@ typedef struct {
  *			should be preallocated
  * @param cmppri the callback function to compare two nodes of the queue
  *
- * @Return the handle or NULL for insufficent memory
+ * @return the handle or NULL for insufficent memory
  */
 int git_pqueue_init(git_pqueue *q, size_t n, git_pqueue_cmp cmppri);
 
@@ -83,8 +83,7 @@ int git_pqueue_insert(git_pqueue *q, void *d);
 
 /**
  * pop the highest-ranking item from the queue.
- * @param p the queue
- * @param d where to copy the entry to
+ * @param q the queue
  * @return NULL on error, otherwise the entry
  */
 void *git_pqueue_pop(git_pqueue *q);
@@ -93,7 +92,6 @@ void *git_pqueue_pop(git_pqueue *q);
 /**
  * access highest-ranking item without removing it.
  * @param q the queue
- * @param d the entry
  * @return NULL on error, otherwise the entry
  */
 void *git_pqueue_peek(git_pqueue *q);
diff --git a/src/push.c b/src/push.c
index 452d71789..3c9d5bb35 100644
--- a/src/push.c
+++ b/src/push.c
@@ -70,6 +70,25 @@ int git_push_set_options(git_push *push, const git_push_options *opts)
 	return 0;
 }
 
+int git_push_set_callbacks(
+	git_push *push,
+	git_packbuilder_progress pack_progress_cb,
+	void *pack_progress_cb_payload,
+	git_push_transfer_progress transfer_progress_cb,
+	void *transfer_progress_cb_payload)
+{
+	if (!push)
+		return -1;
+
+	push->pack_progress_cb = pack_progress_cb;
+	push->pack_progress_cb_payload = pack_progress_cb_payload;
+
+	push->transfer_progress_cb = transfer_progress_cb;
+	push->transfer_progress_cb_payload = transfer_progress_cb_payload;
+
+	return 0;
+}
+
 static void free_refspec(push_spec *spec)
 {
 	if (spec == NULL)
@@ -233,6 +252,37 @@ on_error:
 	return error;
 }
 
+/**
+ * Insert all tags until we find a non-tag object, which is returned
+ * in `out`.
+ */
+static int enqueue_tag(git_object **out, git_push *push, git_oid *id)
+{
+	git_object *obj = NULL, *target = NULL;
+	int error;
+
+	if ((error = git_object_lookup(&obj, push->repo, id, GIT_OBJ_TAG)) < 0)
+		return error;
+
+	while (git_object_type(obj) == GIT_OBJ_TAG) {
+		if ((error = git_packbuilder_insert(push->pb, git_object_id(obj), NULL)) < 0)
+			break;
+
+		if ((error = git_tag_target(&target, (git_tag *) obj)) < 0)
+			break;
+
+		git_object_free(obj);
+		obj = target;
+	}
+
+	if (error < 0)
+		git_object_free(obj);
+	else
+		*out = obj;
+
+	return error;
+}
+
 static int revwalk(git_vector *commits, git_push *push)
 {
 	git_remote_head *head;
@@ -265,21 +315,11 @@ static int revwalk(git_vector *commits, git_push *push)
 			goto on_error;
 
 		if (type == GIT_OBJ_TAG) {
-			git_tag *tag;
 			git_object *target;
 
-			if (git_packbuilder_insert(push->pb, &spec->loid, NULL) < 0)
-				goto on_error;
-
-			if (git_tag_lookup(&tag, push->repo, &spec->loid) < 0)
+			if ((error = enqueue_tag(&target, push, &spec->loid)) < 0)
 				goto on_error;
 
-			if (git_tag_peel(&target, tag) < 0) {
-				git_tag_free(tag);
-				goto on_error;
-			}
-			git_tag_free(tag);
-
 			if (git_object_type(target) == GIT_OBJ_COMMIT) {
 				if (git_revwalk_push(rw, git_object_id(target)) < 0) {
 					git_object_free(target);
@@ -542,7 +582,7 @@ static int calculate_work(git_push *push)
 
 static int do_push(git_push *push)
 {
-	int error;
+	int error = 0;
 	git_transport *transport = push->remote->transport;
 
 	if (!transport->push) {
@@ -562,28 +602,36 @@ static int do_push(git_push *push)
 
 	git_packbuilder_set_threads(push->pb, push->pb_parallelism);
 
+	if (push->pack_progress_cb)
+		if ((error = git_packbuilder_set_callbacks(push->pb, push->pack_progress_cb, push->pack_progress_cb_payload)) < 0)
+			goto on_error;
+
 	if ((error = calculate_work(push)) < 0 ||
 		(error = queue_objects(push)) < 0 ||
 		(error = transport->push(transport, push)) < 0)
 		goto on_error;
 
-	error = 0;
-
 on_error:
 	git_packbuilder_free(push->pb);
 	return error;
 }
 
-static int cb_filter_refs(git_remote_head *ref, void *data)
-{
-	git_remote *remote = (git_remote *) data;
-	return git_vector_insert(&remote->refs, ref);
-}
-
 static int filter_refs(git_remote *remote)
 {
+	const git_remote_head **heads;
+	size_t heads_len, i;
+
 	git_vector_clear(&remote->refs);
-	return git_remote_ls(remote, cb_filter_refs, remote);
+
+	if (git_remote_ls(&heads, &heads_len, remote) < 0)
+		return -1;
+
+	for (i = 0; i < heads_len; i++) {
+		if (git_vector_insert(&remote->refs, (void *)heads[i]) < 0)
+			return -1;
+	}
+
+	return 0;
 }
 
 int git_push_finish(git_push *push)
diff --git a/src/push.h b/src/push.h
index e982b8385..6c8bf7229 100644
--- a/src/push.h
+++ b/src/push.h
@@ -39,6 +39,11 @@ struct git_push {
 
 	/* options */
 	unsigned pb_parallelism;
+
+	git_packbuilder_progress pack_progress_cb;
+	void *pack_progress_cb_payload;
+	git_push_transfer_progress transfer_progress_cb;
+	void *transfer_progress_cb_payload;
 };
 
 /**
diff --git a/src/refdb.c b/src/refdb.c
index 4de7188b2..adb58806e 100644
--- a/src/refdb.c
+++ b/src/refdb.c
@@ -16,6 +16,7 @@
 #include "hash.h"
 #include "refdb.h"
 #include "refs.h"
+#include "reflog.h"
 
 int git_refdb_new(git_refdb **out, git_repository *repo)
 {
@@ -201,5 +202,20 @@ int git_refdb_rename(
 int git_refdb_delete(struct git_refdb *db, const char *ref_name)
 {
 	assert(db && db->backend);
-	return db->backend->delete(db->backend, ref_name);
+	return db->backend->del(db->backend, ref_name);
+}
+
+int git_refdb_reflog_read(git_reflog **out, git_refdb *db,  const char *name)
+{
+	int error;
+
+	assert(db && db->backend);
+
+	if ((error = db->backend->reflog_read(out, db->backend, name)) < 0)
+		return error;
+
+	GIT_REFCOUNT_INC(db);
+	(*out)->db = db;
+
+	return 0;
 }
diff --git a/src/refdb.h b/src/refdb.h
index 3aea37b62..0ee60d911 100644
--- a/src/refdb.h
+++ b/src/refdb.h
@@ -43,4 +43,8 @@ void git_refdb_iterator_free(git_reference_iterator *iter);
 int git_refdb_write(git_refdb *refdb, git_reference *ref, int force);
 int git_refdb_delete(git_refdb *refdb, const char *ref_name);
 
+int git_refdb_reflog_read(git_reflog **out, git_refdb *db,  const char *name);
+int git_refdb_reflog_write(git_reflog *reflog);
+
+
 #endif
diff --git a/src/refdb_fs.c b/src/refdb_fs.c
index b9e283ac5..62d5c1047 100644
--- a/src/refdb_fs.c
+++ b/src/refdb_fs.c
@@ -15,12 +15,15 @@
 #include "refdb.h"
 #include "refdb_fs.h"
 #include "iterator.h"
+#include "sortedcache.h"
+#include "signature.h"
 
 #include <git2/tag.h>
 #include <git2/object.h>
 #include <git2/refdb.h>
 #include <git2/sys/refdb_backend.h>
 #include <git2/sys/refs.h>
+#include <git2/sys/reflog.h>
 
 GIT__USE_STRMAP;
 
@@ -53,243 +56,151 @@ typedef struct refdb_fs_backend {
 	git_repository *repo;
 	char *path;
 
-	git_refcache refcache;
+	git_sortedcache *refcache;
 	int peeling_mode;
+	git_iterator_flag_t iterator_flags;
+	uint32_t direach_flags;
 } refdb_fs_backend;
 
-static int reference_read(
-	git_buf *file_content,
-	time_t *mtime,
-	const char *repo_path,
-	const char *ref_name,
-	int *updated)
+static int packref_cmp(const void *a_, const void *b_)
 {
-	git_buf path = GIT_BUF_INIT;
-	int result;
-
-	assert(file_content && repo_path && ref_name);
-
-	/* Determine the full path of the file */
-	if (git_buf_joinpath(&path, repo_path, ref_name) < 0)
-		return -1;
-
-	result = git_futils_readbuffer_updated(file_content, path.ptr, mtime, NULL, updated);
-	git_buf_free(&path);
-
-	return result;
-}
-
-static int packed_parse_oid(
-	struct packref **ref_out,
-	const char **buffer_out,
-	const char *buffer_end)
-{
-	struct packref *ref = NULL;
-
-	const char *buffer = *buffer_out;
-	const char *refname_begin, *refname_end;
-
-	size_t refname_len;
-	git_oid id;
-
-	refname_begin = (buffer + GIT_OID_HEXSZ + 1);
-	if (refname_begin >= buffer_end || refname_begin[-1] != ' ')
-		goto corrupt;
-
-	/* Is this a valid object id? */
-	if (git_oid_fromstr(&id, buffer) < 0)
-		goto corrupt;
-
-	refname_end = memchr(refname_begin, '\n', buffer_end - refname_begin);
-	if (refname_end == NULL)
-		refname_end = buffer_end;
-
-	if (refname_end[-1] == '\r')
-		refname_end--;
-
-	refname_len = refname_end - refname_begin;
-
-	ref = git__calloc(1, sizeof(struct packref) + refname_len + 1);
-	GITERR_CHECK_ALLOC(ref);
-
-	memcpy(ref->name, refname_begin, refname_len);
-	ref->name[refname_len] = 0;
-
-	git_oid_cpy(&ref->oid, &id);
-
-	*ref_out = ref;
-	*buffer_out = refname_end + 1;
-	return 0;
-
-corrupt:
-	git__free(ref);
-	giterr_set(GITERR_REFERENCE, "The packed references file is corrupted");
-	return -1;
+	const struct packref *a = a_, *b = b_;
+	return strcmp(a->name, b->name);
 }
 
-static int packed_parse_peel(
-	struct packref *tag_ref,
-	const char **buffer_out,
-	const char *buffer_end)
+static int packed_reload(refdb_fs_backend *backend)
 {
-	const char *buffer = *buffer_out + 1;
-
-	assert(buffer[-1] == '^');
-
-	/* Ensure it's not the first entry of the file */
-	if (tag_ref == NULL)
-		goto corrupt;
-
-	if (buffer + GIT_OID_HEXSZ > buffer_end)
-		goto corrupt;
-
-	/* Is this a valid object id? */
-	if (git_oid_fromstr(&tag_ref->peel, buffer) < 0)
-		goto corrupt;
-
-	buffer = buffer + GIT_OID_HEXSZ;
-	if (*buffer == '\r')
-		buffer++;
-
-	if (buffer != buffer_end) {
-		if (*buffer == '\n')
-			buffer++;
-		else
-			goto corrupt;
-	}
-
-	tag_ref->flags |= PACKREF_HAS_PEEL;
-	*buffer_out = buffer;
-	return 0;
-
-corrupt:
-	giterr_set(GITERR_REFERENCE, "The packed references file is corrupted");
-	return -1;
-}
-
-static int packed_load(refdb_fs_backend *backend)
-{
-	int result, updated;
-	git_buf packfile = GIT_BUF_INIT;
-	const char *buffer_start, *buffer_end;
-	git_refcache *ref_cache = &backend->refcache;
-
-	/* First we make sure we have allocated the hash table */
-	if (ref_cache->packfile == NULL) {
-		ref_cache->packfile = git_strmap_alloc();
-		GITERR_CHECK_ALLOC(ref_cache->packfile);
-	}
+	int error;
+	git_buf packedrefs = GIT_BUF_INIT;
+	char *scan, *eof, *eol;
 
-	if (backend->path == NULL)
+	if (!backend->path)
 		return 0;
 
-	result = reference_read(&packfile, &ref_cache->packfile_time,
-		backend->path, GIT_PACKEDREFS_FILE, &updated);
+	error = git_sortedcache_lockandload(backend->refcache, &packedrefs);
 
 	/*
-	 * If we couldn't find the file, we need to clear the table and
-	 * return. On any other error, we return that error. If everything
-	 * went fine and the file wasn't updated, then there's nothing new
-	 * for us here, so just return. Anything else means we need to
-	 * refresh the packed refs.
+	 * If we can't find the packed-refs, clear table and return.
+	 * Any other error just gets passed through.
+	 * If no error, and file wasn't changed, just return.
+	 * Anything else means we need to refresh the packed refs.
 	 */
-	if (result == GIT_ENOTFOUND) {
-		git_strmap_clear(ref_cache->packfile);
-		return 0;
+	if (error <= 0) {
+		if (error == GIT_ENOTFOUND) {
+			git_sortedcache_clear(backend->refcache, true);
+			giterr_clear();
+			error = 0;
+		}
+		return error;
 	}
 
-	if (result < 0)
-		return -1;
-
-	if (!updated)
-		return 0;
+	/* At this point, refresh the packed refs from the loaded buffer. */
 
-	/*
-	 * At this point, we want to refresh the packed refs. We already
-	 * have the contents in our buffer.
-	 */
-	git_strmap_clear(ref_cache->packfile);
+	git_sortedcache_clear(backend->refcache, false);
 
-	buffer_start = (const char *)packfile.ptr;
-	buffer_end = (const char *)(buffer_start) + packfile.size;
+	scan = (char *)packedrefs.ptr;
+	eof  = scan + packedrefs.size;
 
 	backend->peeling_mode = PEELING_NONE;
 
-	if (buffer_start[0] == '#') {
+	if (*scan == '#') {
 		static const char *traits_header = "# pack-refs with: ";
 
-		if (git__prefixcmp(buffer_start, traits_header) == 0) {
-			char *traits = (char *)buffer_start + strlen(traits_header);
-			char *traits_end = strchr(traits, '\n');
+		if (git__prefixcmp(scan, traits_header) == 0) {
+			scan += strlen(traits_header);
+			eol = strchr(scan, '\n');
 
-			if (traits_end == NULL)
+			if (!eol)
 				goto parse_failed;
+			*eol = '\0';
 
-			*traits_end = '\0';
-
-			if (strstr(traits, " fully-peeled ") != NULL) {
+			if (strstr(scan, " fully-peeled ") != NULL) {
 				backend->peeling_mode = PEELING_FULL;
-			} else if (strstr(traits, " peeled ") != NULL) {
+			} else if (strstr(scan, " peeled ") != NULL) {
 				backend->peeling_mode = PEELING_STANDARD;
 			}
 
-			buffer_start = traits_end + 1;
+			scan = eol + 1;
 		}
 	}
 
-	while (buffer_start < buffer_end && buffer_start[0] == '#') {
-		buffer_start = strchr(buffer_start, '\n');
-		if (buffer_start == NULL)
+	while (scan < eof && *scan == '#') {
+		if (!(eol = strchr(scan, '\n')))
 			goto parse_failed;
-
-		buffer_start++;
+		scan = eol + 1;
 	}
 
-	while (buffer_start < buffer_end) {
-		int err;
-		struct packref *ref = NULL;
+	while (scan < eof) {
+		struct packref *ref;
+		git_oid oid;
+
+		/* parse "<OID> <refname>\n" */
 
-		if (packed_parse_oid(&ref, &buffer_start, buffer_end) < 0)
+		if (git_oid_fromstr(&oid, scan) < 0)
 			goto parse_failed;
+		scan += GIT_OID_HEXSZ;
 
-		if (buffer_start[0] == '^') {
-			if (packed_parse_peel(ref, &buffer_start, buffer_end) < 0)
-				goto parse_failed;
-		} else if (backend->peeling_mode == PEELING_FULL ||
-							(backend->peeling_mode == PEELING_STANDARD &&
-							git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) == 0)) {
-			ref->flags |= PACKREF_CANNOT_PEEL;
-		}
+		if (*scan++ != ' ')
+			goto parse_failed;
+		if (!(eol = strchr(scan, '\n')))
+			goto parse_failed;
+		*eol = '\0';
+		if (eol[-1] == '\r')
+			eol[-1] = '\0';
 
-		git_strmap_insert(ref_cache->packfile, ref->name, ref, err);
-		if (err < 0)
+		if (git_sortedcache_upsert((void **)&ref, backend->refcache, scan) < 0)
 			goto parse_failed;
+		scan = eol + 1;
+
+		git_oid_cpy(&ref->oid, &oid);
+
+		/* look for optional "^<OID>\n" */
+
+		if (*scan == '^') {
+			if (git_oid_fromstr(&oid, scan + 1) < 0)
+				goto parse_failed;
+			scan += GIT_OID_HEXSZ + 1;
+
+			if (scan < eof) {
+				if (!(eol = strchr(scan, '\n')))
+					goto parse_failed;
+				scan = eol + 1;
+			}
+
+			git_oid_cpy(&ref->peel, &oid);
+			ref->flags |= PACKREF_HAS_PEEL;
+		}
+		else if (backend->peeling_mode == PEELING_FULL ||
+				(backend->peeling_mode == PEELING_STANDARD &&
+				 git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) == 0))
+			ref->flags |= PACKREF_CANNOT_PEEL;
 	}
 
-	git_buf_free(&packfile);
+	git_sortedcache_wunlock(backend->refcache);
+	git_buf_free(&packedrefs);
+
 	return 0;
 
 parse_failed:
-	git_strmap_free(ref_cache->packfile);
-	ref_cache->packfile = NULL;
-	git_buf_free(&packfile);
+	giterr_set(GITERR_REFERENCE, "Corrupted packed references file");
+
+	git_sortedcache_clear(backend->refcache, false);
+	git_sortedcache_wunlock(backend->refcache);
+	git_buf_free(&packedrefs);
+
 	return -1;
 }
 
-static int loose_parse_oid(git_oid *oid, const char *filename, git_buf *file_content)
+static int loose_parse_oid(
+	git_oid *oid, const char *filename, git_buf *file_content)
 {
-	size_t len;
-	const char *str;
+	const char *str = git_buf_cstr(file_content);
 
-	len = git_buf_len(file_content);
-	if (len < GIT_OID_HEXSZ)
+	if (git_buf_len(file_content) < GIT_OID_HEXSZ)
 		goto corrupted;
 
-	/* str is guranteed to be zero-terminated */
-	str = git_buf_cstr(file_content);
-
 	/* we need to get 40 OID characters from the file */
-	if (git_oid_fromstr(oid, git_buf_cstr(file_content)) < 0)
+	if (git_oid_fromstr(oid, str) < 0)
 		goto corrupted;
 
 	/* If the file is longer than 40 chars, the 41st must be a space */
@@ -302,68 +213,72 @@ corrupted:
 	return -1;
 }
 
-static int loose_lookup_to_packfile(
-	struct packref **ref_out,
-	refdb_fs_backend *backend,
-	const char *name)
+static int loose_readbuffer(git_buf *buf, const char *base, const char *path)
+{
+	int error;
+
+	/* build full path to file */
+	if ((error = git_buf_joinpath(buf, base, path)) < 0 ||
+		(error = git_futils_readbuffer(buf, buf->ptr)) < 0)
+		git_buf_free(buf);
+
+	return error;
+}
+
+static int loose_lookup_to_packfile(refdb_fs_backend *backend, const char *name)
 {
+	int error = 0;
 	git_buf ref_file = GIT_BUF_INIT;
 	struct packref *ref = NULL;
-	size_t name_len;
+	git_oid oid;
 
-	*ref_out = NULL;
+	/* if we fail to load the loose reference, assume someone changed
+	 * the filesystem under us and skip it...
+	 */
+	if (loose_readbuffer(&ref_file, backend->path, name) < 0) {
+		giterr_clear();
+		goto done;
+	}
 
-	if (reference_read(&ref_file, NULL, backend->path, name, NULL) < 0)
-		return -1;
+	/* skip symbolic refs */
+	if (!git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF))
+		goto done;
 
-	git_buf_rtrim(&ref_file);
+	/* parse OID from file */
+	if ((error = loose_parse_oid(&oid, name, &ref_file)) < 0)
+		goto done;
 
-	name_len = strlen(name);
-	ref = git__calloc(1, sizeof(struct packref) + name_len + 1);
-	GITERR_CHECK_ALLOC(ref);
+	git_sortedcache_wlock(backend->refcache);
 
-	memcpy(ref->name, name, name_len);
-	ref->name[name_len] = 0;
+	if (!(error = git_sortedcache_upsert(
+			(void **)&ref, backend->refcache, name))) {
 
-	if (loose_parse_oid(&ref->oid, name, &ref_file) < 0) {
-		git_buf_free(&ref_file);
-		git__free(ref);
-		return -1;
+		git_oid_cpy(&ref->oid, &oid);
+		ref->flags = PACKREF_WAS_LOOSE;
 	}
 
-	ref->flags = PACKREF_WAS_LOOSE;
+	git_sortedcache_wunlock(backend->refcache);
 
-	*ref_out = ref;
+done:
 	git_buf_free(&ref_file);
-	return 0;
+	return error;
 }
 
-
 static int _dirent_loose_load(void *data, git_buf *full_path)
 {
 	refdb_fs_backend *backend = (refdb_fs_backend *)data;
-	void *old_ref = NULL;
-	struct packref *ref;
 	const char *file_path;
-	int err;
-
-	if (git_path_isdir(full_path->ptr) == true)
-		return git_path_direach(full_path, _dirent_loose_load, backend);
 
-	file_path = full_path->ptr + strlen(backend->path);
+	if (git__suffixcmp(full_path->ptr, ".lock") == 0)
+		return 0;
 
-	if (loose_lookup_to_packfile(&ref, backend, file_path) < 0)
-		return -1;
+	if (git_path_isdir(full_path->ptr))
+		return git_path_direach(
+			full_path, backend->direach_flags, _dirent_loose_load, backend);
 
-	git_strmap_insert2(
-		backend->refcache.packfile, ref->name, ref, old_ref, err);
-	if (err < 0) {
-		git__free(ref);
-		return -1;
-	}
+	file_path = full_path->ptr + strlen(backend->path);
 
-	git__free(old_ref);
-	return 0;
+	return loose_lookup_to_packfile(backend, file_path);
 }
 
 /*
@@ -374,11 +289,8 @@ static int _dirent_loose_load(void *data, git_buf *full_path)
  */
 static int packed_loadloose(refdb_fs_backend *backend)
 {
+	int error;
 	git_buf refs_path = GIT_BUF_INIT;
-	int result;
-
-	/* the packfile must have been previously loaded! */
-	assert(backend->refcache.packfile);
 
 	if (git_buf_joinpath(&refs_path, backend->path, GIT_REFS_DIR) < 0)
 		return -1;
@@ -388,10 +300,12 @@ static int packed_loadloose(refdb_fs_backend *backend)
 	 * This will overwrite any old packed entries with their
 	 * updated loose versions
 	 */
-	result = git_path_direach(&refs_path, _dirent_loose_load, backend);
+	error = git_path_direach(
+		&refs_path, backend->direach_flags, _dirent_loose_load, backend);
+
 	git_buf_free(&refs_path);
 
-	return result;
+	return (error == GIT_EUSER) ? -1 : error;
 }
 
 static int refdb_fs_backend__exists(
@@ -399,23 +313,17 @@ static int refdb_fs_backend__exists(
 	git_refdb_backend *_backend,
 	const char *ref_name)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	git_buf ref_path = GIT_BUF_INIT;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
-
-	if (packed_load(backend) < 0)
-		return -1;
+	assert(backend);
 
-	if (git_buf_joinpath(&ref_path, backend->path, ref_name) < 0)
+	if (packed_reload(backend) < 0 ||
+		git_buf_joinpath(&ref_path, backend->path, ref_name) < 0)
 		return -1;
 
-	if (git_path_isfile(ref_path.ptr) == true ||
-		git_strmap_exists(backend->refcache.packfile, ref_path.ptr))
-		*exists = 1;
-	else
-		*exists = 0;
+	*exists = git_path_isfile(ref_path.ptr) ||
+		(git_sortedcache_lookup(backend->refcache, ref_name) != NULL);
 
 	git_buf_free(&ref_path);
 	return 0;
@@ -447,64 +355,39 @@ static int loose_lookup(
 	refdb_fs_backend *backend,
 	const char *ref_name)
 {
-	const char *target;
-	git_oid oid;
 	git_buf ref_file = GIT_BUF_INIT;
 	int error = 0;
 
-	error = reference_read(&ref_file, NULL, backend->path, ref_name, NULL);
+	if (out)
+		*out = NULL;
 
-	if (error < 0)
-		goto done;
+	if ((error = loose_readbuffer(&ref_file, backend->path, ref_name)) < 0)
+		/* cannot read loose ref file - gah */;
+	else if (git__prefixcmp(git_buf_cstr(&ref_file), GIT_SYMREF) == 0) {
+		const char *target;
 
-	if (git__prefixcmp((const char *)(ref_file.ptr), GIT_SYMREF) == 0) {
 		git_buf_rtrim(&ref_file);
 
-		if ((target = loose_parse_symbolic(&ref_file)) == NULL) {
+		if (!(target = loose_parse_symbolic(&ref_file)))
 			error = -1;
-			goto done;
-		}
-
-		*out = git_reference__alloc_symbolic(ref_name, target);
+		else if (out != NULL)
+			*out = git_reference__alloc_symbolic(ref_name, target);
 	} else {
-		if ((error = loose_parse_oid(&oid, ref_name, &ref_file)) < 0)
-			goto done;
+		git_oid oid;
 
-		*out = git_reference__alloc(ref_name, &oid, NULL);
+		if (!(error = loose_parse_oid(&oid, ref_name, &ref_file)) &&
+			out != NULL)
+			*out = git_reference__alloc(ref_name, &oid, NULL);
 	}
 
-	if (*out == NULL)
-		error = -1;
-
-done:
 	git_buf_free(&ref_file);
 	return error;
 }
 
-static int packed_map_entry(
-	struct packref **entry,
-	khiter_t *pos,
-	refdb_fs_backend *backend,
-	const char *ref_name)
+static int ref_error_notfound(const char *name)
 {
-	git_strmap *packfile_refs;
-
-	if (packed_load(backend) < 0)
-		return -1;
-
-	/* Look up on the packfile */
-	packfile_refs = backend->refcache.packfile;
-
-	*pos = git_strmap_lookup_index(packfile_refs, ref_name);
-
-	if (!git_strmap_valid_index(packfile_refs, *pos)) {
-		giterr_set(GITERR_REFERENCE, "Reference '%s' not found", ref_name);
-		return GIT_ENOTFOUND;
-	}
-
-	*entry = git_strmap_value_at(packfile_refs, *pos);
-
-	return 0;
+	giterr_set(GITERR_REFERENCE, "Reference '%s' not found", name);
+	return GIT_ENOTFOUND;
 }
 
 static int packed_lookup(
@@ -512,18 +395,27 @@ static int packed_lookup(
 	refdb_fs_backend *backend,
 	const char *ref_name)
 {
-	struct packref *entry;
-	khiter_t pos;
 	int error = 0;
+	struct packref *entry;
 
-	if ((error = packed_map_entry(&entry, &pos, backend, ref_name)) < 0)
-		return error;
+	if (packed_reload(backend) < 0)
+		return -1;
 
-	if ((*out = git_reference__alloc(ref_name,
-		&entry->oid, &entry->peel)) == NULL)
+	if (git_sortedcache_rlock(backend->refcache) < 0)
 		return -1;
 
-	return 0;
+	entry = git_sortedcache_lookup(backend->refcache, ref_name);
+	if (!entry) {
+		error = ref_error_notfound(ref_name);
+	} else {
+		*out = git_reference__alloc(ref_name, &entry->oid, &entry->peel);
+		if (!*out)
+			error = -1;
+	}
+
+	git_sortedcache_runlock(backend->refcache);
+
+	return error;
 }
 
 static int refdb_fs_backend__lookup(
@@ -531,73 +423,68 @@ static int refdb_fs_backend__lookup(
 	git_refdb_backend *_backend,
 	const char *ref_name)
 {
-	refdb_fs_backend *backend;
-	int result;
-
-	assert(_backend);
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
+	int error;
 
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	if ((result = loose_lookup(out, backend, ref_name)) == 0)
+	if (!(error = loose_lookup(out, backend, ref_name)))
 		return 0;
 
 	/* only try to lookup this reference on the packfile if it
 	 * wasn't found on the loose refs; not if there was a critical error */
-	if (result == GIT_ENOTFOUND) {
+	if (error == GIT_ENOTFOUND) {
 		giterr_clear();
-		result = packed_lookup(out, backend, ref_name);
+		error = packed_lookup(out, backend, ref_name);
 	}
 
-	return result;
+	return error;
 }
 
 typedef struct {
 	git_reference_iterator parent;
 
 	char *glob;
+
+	git_pool pool;
 	git_vector loose;
-	unsigned int loose_pos;
-	khiter_t packed_pos;
+
+	size_t loose_pos;
+	size_t packed_pos;
 } refdb_fs_iter;
 
 static void refdb_fs_backend__iterator_free(git_reference_iterator *_iter)
 {
 	refdb_fs_iter *iter = (refdb_fs_iter *) _iter;
-	char *loose_path;
-	size_t i;
-
-	git_vector_foreach(&iter->loose, i, loose_path) {
-		git__free(loose_path);
-	}
 
 	git_vector_free(&iter->loose);
-
-	git__free(iter->glob);
+	git_pool_clear(&iter->pool);
 	git__free(iter);
 }
 
 static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter)
 {
-	git_strmap *packfile = backend->refcache.packfile;
+	int error = 0;
 	git_buf path = GIT_BUF_INIT;
-	git_iterator *fsit;
+	git_iterator *fsit = NULL;
 	const git_index_entry *entry = NULL;
 
 	if (!backend->path) /* do nothing if no path for loose refs */
 		return 0;
 
-	if (git_buf_printf(&path, "%s/refs", backend->path) < 0)
-		return -1;
-
-	if (git_iterator_for_filesystem(&fsit, git_buf_cstr(&path), 0, NULL, NULL) < 0)
-		return -1;
+	if ((error = git_buf_printf(&path, "%s/refs", backend->path)) < 0 ||
+		(error = git_iterator_for_filesystem(
+			&fsit, path.ptr, backend->iterator_flags, NULL, NULL)) < 0) {
+		git_buf_free(&path);
+		return error;
+	}
 
-	git_vector_init(&iter->loose, 8, NULL);
-	git_buf_sets(&path, GIT_REFS_DIR);
+	error = git_buf_sets(&path, GIT_REFS_DIR);
 
-	while (!git_iterator_advance(&entry, fsit)) {
+	while (!error && !git_iterator_advance(&entry, fsit)) {
 		const char *ref_name;
-		khiter_t pos;
+		struct packref *ref;
+		char *ref_dup;
 
 		git_buf_truncate(&path, strlen(GIT_REFS_DIR));
 		git_buf_puts(&path, entry->path);
@@ -607,27 +494,32 @@ static int iter_load_loose_paths(refdb_fs_backend *backend, refdb_fs_iter *iter)
 			(iter->glob && p_fnmatch(iter->glob, ref_name, 0) != 0))
 			continue;
 
-		pos = git_strmap_lookup_index(packfile, ref_name);
-		if (git_strmap_valid_index(packfile, pos)) {
-			struct packref *ref = git_strmap_value_at(packfile, pos);
+		git_sortedcache_rlock(backend->refcache);
+		ref = git_sortedcache_lookup(backend->refcache, ref_name);
+		if (ref)
 			ref->flags |= PACKREF_SHADOWED;
-		}
+		git_sortedcache_runlock(backend->refcache);
 
-		git_vector_insert(&iter->loose, git__strdup(ref_name));
+		ref_dup = git_pool_strdup(&iter->pool, ref_name);
+		if (!ref_dup)
+			error = -1;
+		else
+			error = git_vector_insert(&iter->loose, ref_dup);
 	}
 
 	git_iterator_free(fsit);
 	git_buf_free(&path);
 
-	return 0;
+	return error;
 }
 
 static int refdb_fs_backend__iterator_next(
 	git_reference **out, git_reference_iterator *_iter)
 {
+	int error = GIT_ITEROVER;
 	refdb_fs_iter *iter = (refdb_fs_iter *)_iter;
 	refdb_fs_backend *backend = (refdb_fs_backend *)iter->parent.db->backend;
-	git_strmap *packfile = backend->refcache.packfile;
+	struct packref *ref;
 
 	while (iter->loose_pos < iter->loose.length) {
 		const char *path = git_vector_get(&iter->loose, iter->loose_pos++);
@@ -638,99 +530,102 @@ static int refdb_fs_backend__iterator_next(
 		giterr_clear();
 	}
 
-	while (iter->packed_pos < kh_end(packfile)) {
-		struct packref *ref = NULL;
-
-		while (!kh_exist(packfile, iter->packed_pos)) {
-			iter->packed_pos++;
-			if (iter->packed_pos == kh_end(packfile))
-				return GIT_ITEROVER;
-		}
+	git_sortedcache_rlock(backend->refcache);
 
-		ref = kh_val(packfile, iter->packed_pos);
-		iter->packed_pos++;
+	while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) {
+		ref = git_sortedcache_entry(backend->refcache, iter->packed_pos++);
+		if (!ref) /* stop now if another thread deleted refs and we past end */
+			break;
 
 		if (ref->flags & PACKREF_SHADOWED)
 			continue;
-
 		if (iter->glob && p_fnmatch(iter->glob, ref->name, 0) != 0)
 			continue;
 
 		*out = git_reference__alloc(ref->name, &ref->oid, &ref->peel);
-		if (*out == NULL)
-			return -1;
-
-		return 0;
+		error = (*out != NULL) ? 0 : -1;
+		break;
 	}
 
-	return GIT_ITEROVER;
+	git_sortedcache_runlock(backend->refcache);
+	return error;
 }
 
 static int refdb_fs_backend__iterator_next_name(
 	const char **out, git_reference_iterator *_iter)
 {
+	int error = GIT_ITEROVER;
 	refdb_fs_iter *iter = (refdb_fs_iter *)_iter;
 	refdb_fs_backend *backend = (refdb_fs_backend *)iter->parent.db->backend;
-	git_strmap *packfile = backend->refcache.packfile;
+	struct packref *ref;
 
 	while (iter->loose_pos < iter->loose.length) {
 		const char *path = git_vector_get(&iter->loose, iter->loose_pos++);
 
-		if (git_strmap_exists(packfile, path))
-			continue;
+		if (loose_lookup(NULL, backend, path) == 0) {
+			*out = path;
+			return 0;
+		}
 
-		*out = path;
-		return 0;
+		giterr_clear();
 	}
 
-	while (iter->packed_pos < kh_end(packfile)) {
-		while (!kh_exist(packfile, iter->packed_pos)) {
-			iter->packed_pos++;
-			if (iter->packed_pos == kh_end(packfile))
-				return GIT_ITEROVER;
-		}
+	git_sortedcache_rlock(backend->refcache);
 
-		*out = kh_key(packfile, iter->packed_pos);
-		iter->packed_pos++;
+	while (iter->packed_pos < git_sortedcache_entrycount(backend->refcache)) {
+		ref = git_sortedcache_entry(backend->refcache, iter->packed_pos++);
+		if (!ref) /* stop now if another thread deleted refs and we past end */
+			break;
 
-		if (iter->glob && p_fnmatch(iter->glob, *out, 0) != 0)
+		if (ref->flags & PACKREF_SHADOWED)
+			continue;
+		if (iter->glob && p_fnmatch(iter->glob, ref->name, 0) != 0)
 			continue;
 
-		return 0;
+		*out = ref->name;
+		error = 0;
+		break;
 	}
 
-	return GIT_ITEROVER;
+	git_sortedcache_runlock(backend->refcache);
+	return error;
 }
 
 static int refdb_fs_backend__iterator(
 	git_reference_iterator **out, git_refdb_backend *_backend, const char *glob)
 {
 	refdb_fs_iter *iter;
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	if (packed_load(backend) < 0)
+	if (packed_reload(backend) < 0)
 		return -1;
 
 	iter = git__calloc(1, sizeof(refdb_fs_iter));
 	GITERR_CHECK_ALLOC(iter);
 
-	if (glob != NULL)
-		iter->glob = git__strdup(glob);
+	if (git_pool_init(&iter->pool, 1, 0) < 0 ||
+		git_vector_init(&iter->loose, 8, NULL) < 0)
+		goto fail;
+
+	if (glob != NULL &&
+		(iter->glob = git_pool_strdup(&iter->pool, glob)) == NULL)
+		goto fail;
 
 	iter->parent.next = refdb_fs_backend__iterator_next;
 	iter->parent.next_name = refdb_fs_backend__iterator_next_name;
 	iter->parent.free = refdb_fs_backend__iterator_free;
 
-	if (iter_load_loose_paths(backend, iter) < 0) {
-		refdb_fs_backend__iterator_free((git_reference_iterator *)iter);
-		return -1;
-	}
+	if (iter_load_loose_paths(backend, iter) < 0)
+		goto fail;
 
 	*out = (git_reference_iterator *)iter;
 	return 0;
+
+fail:
+	refdb_fs_backend__iterator_free((git_reference_iterator *)iter);
+	return -1;
 }
 
 static bool ref_is_available(
@@ -756,33 +651,40 @@ static int reference_path_available(
 	const char* old_ref,
 	int force)
 {
-	struct packref *this_ref;
+	size_t i;
 
-	if (packed_load(backend) < 0)
+	if (packed_reload(backend) < 0)
 		return -1;
 
 	if (!force) {
 		int exists;
 
-		if (refdb_fs_backend__exists(&exists, (git_refdb_backend *)backend, new_ref) < 0)
+		if (refdb_fs_backend__exists(
+				&exists, (git_refdb_backend *)backend, new_ref) < 0)
 			return -1;
 
 		if (exists) {
 			giterr_set(GITERR_REFERENCE,
 				"Failed to write reference '%s': a reference with "
-				" that name already exists.", new_ref);
+				"that name already exists.", new_ref);
 			return GIT_EEXISTS;
 		}
 	}
 
-	git_strmap_foreach_value(backend->refcache.packfile, this_ref, {
-		if (!ref_is_available(old_ref, new_ref, this_ref->name)) {
+	git_sortedcache_rlock(backend->refcache);
+
+	for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) {
+		struct packref *ref = git_sortedcache_entry(backend->refcache, i);
+
+		if (ref && !ref_is_available(old_ref, new_ref, ref->name)) {
+			git_sortedcache_runlock(backend->refcache);
 			giterr_set(GITERR_REFERENCE,
-				"The path to reference '%s' collides with an existing one", new_ref);
+				"Path to reference '%s' collides with existing one", new_ref);
 			return -1;
 		}
-	});
-	
+	}
+
+	git_sortedcache_runlock(backend->refcache);
 	return 0;
 }
 
@@ -800,7 +702,7 @@ static int loose_write(refdb_fs_backend *backend, const git_reference *ref)
 	if (git_buf_joinpath(&ref_path, backend->path, ref->name) < 0)
 		return -1;
 
-	if (git_filebuf_open(&file, ref_path.ptr, GIT_FILEBUF_FORCE) < 0) {
+	if (git_filebuf_open(&file, ref_path.ptr, GIT_FILEBUF_FORCE, GIT_REFS_FILE_MODE) < 0) {
 		git_buf_free(&ref_path);
 		return -1;
 	}
@@ -809,27 +711,16 @@ static int loose_write(refdb_fs_backend *backend, const git_reference *ref)
 
 	if (ref->type == GIT_REF_OID) {
 		char oid[GIT_OID_HEXSZ + 1];
-
-		git_oid_fmt(oid, &ref->target.oid);
-		oid[GIT_OID_HEXSZ] = '\0';
+		git_oid_nfmt(oid, sizeof(oid), &ref->target.oid);
 
 		git_filebuf_printf(&file, "%s\n", oid);
-
 	} else if (ref->type == GIT_REF_SYMBOLIC) {
 		git_filebuf_printf(&file, GIT_SYMREF "%s\n", ref->target.symbolic);
 	} else {
 		assert(0); /* don't let this happen */
 	}
 
-	return git_filebuf_commit(&file, GIT_REFS_FILE_MODE);
-}
-
-static int packed_sort(const void *a, const void *b)
-{
-	const struct packref *ref_a = (const struct packref *)a;
-	const struct packref *ref_b = (const struct packref *)b;
-
-	return strcmp(ref_a->name, ref_b->name);
+	return git_filebuf_commit(&file);
 }
 
 /*
@@ -884,9 +775,7 @@ static int packed_find_peel(refdb_fs_backend *backend, struct packref *ref)
 static int packed_write_ref(struct packref *ref, git_filebuf *file)
 {
 	char oid[GIT_OID_HEXSZ + 1];
-
-	git_oid_fmt(oid, &ref->oid);
-	oid[GIT_OID_HEXSZ] = 0;
+	git_oid_nfmt(oid, sizeof(oid), &ref->oid);
 
 	/*
 	 * For references that peel to an object in the repo, we must
@@ -900,8 +789,7 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file)
 	 */
 	if (ref->flags & PACKREF_HAS_PEEL) {
 		char peel[GIT_OID_HEXSZ + 1];
-		git_oid_fmt(peel, &ref->peel);
-		peel[GIT_OID_HEXSZ] = 0;
+		git_oid_nfmt(peel, sizeof(peel), &ref->peel);
 
 		if (git_filebuf_printf(file, "%s %s\n^%s\n", oid, ref->name, peel) < 0)
 			return -1;
@@ -924,31 +812,30 @@ static int packed_write_ref(struct packref *ref, git_filebuf *file)
  * is well-written, because we are destructing references
  * here otherwise.
  */
-static int packed_remove_loose(
-	refdb_fs_backend *backend,
-	git_vector *packing_list)
+static int packed_remove_loose(refdb_fs_backend *backend)
 {
 	size_t i;
 	git_buf full_path = GIT_BUF_INIT;
 	int failed = 0;
 
-	for (i = 0; i < packing_list->length; ++i) {
-		struct packref *ref = git_vector_get(packing_list, i);
+	/* backend->refcache is already locked when this is called */
+
+	for (i = 0; i < git_sortedcache_entrycount(backend->refcache); ++i) {
+		struct packref *ref = git_sortedcache_entry(backend->refcache, i);
 
-		if ((ref->flags & PACKREF_WAS_LOOSE) == 0)
+		if (!ref || !(ref->flags & PACKREF_WAS_LOOSE))
 			continue;
 
 		if (git_buf_joinpath(&full_path, backend->path, ref->name) < 0)
 			return -1; /* critical; do not try to recover on oom */
 
-		if (git_path_exists(full_path.ptr) == true && p_unlink(full_path.ptr) < 0) {
+		if (git_path_exists(full_path.ptr) && p_unlink(full_path.ptr) < 0) {
 			if (failed)
 				continue;
 
 			giterr_set(GITERR_REFERENCE,
 				"Failed to remove loose reference '%s' after packing: %s",
 				full_path.ptr, strerror(errno));
-
 			failed = 1;
 		}
 
@@ -969,85 +856,53 @@ static int packed_remove_loose(
  */
 static int packed_write(refdb_fs_backend *backend)
 {
+	git_sortedcache *refcache = backend->refcache;
 	git_filebuf pack_file = GIT_FILEBUF_INIT;
 	size_t i;
-	git_buf pack_file_path = GIT_BUF_INIT;
-	git_vector packing_list;
-	unsigned int total_refs;
-
-	assert(backend && backend->refcache.packfile);
-
-	total_refs =
-		(unsigned int)git_strmap_num_entries(backend->refcache.packfile);
 
-	if (git_vector_init(&packing_list, total_refs, packed_sort) < 0)
+	/* lock the cache to updates while we do this */
+	if (git_sortedcache_wlock(refcache) < 0)
 		return -1;
 
-	/* Load all the packfile into a vector */
-	{
-		struct packref *reference;
-
-		/* cannot fail: vector already has the right size */
-		git_strmap_foreach_value(backend->refcache.packfile, reference, {
-			git_vector_insert(&packing_list, reference);
-		});
-	}
-
-	/* sort the vector so the entries appear sorted on the packfile */
-	git_vector_sort(&packing_list);
-
-	/* Now we can open the file! */
-	if (git_buf_joinpath(&pack_file_path,
-		backend->path, GIT_PACKEDREFS_FILE) < 0)
-		goto cleanup_memory;
-
-	if (git_filebuf_open(&pack_file, pack_file_path.ptr, 0) < 0)
-		goto cleanup_packfile;
+	/* Open the file! */
+	if (git_filebuf_open(&pack_file, git_sortedcache_path(refcache), 0, GIT_PACKEDREFS_FILE_MODE) < 0)
+		goto fail;
 
 	/* Packfiles have a header... apparently
 	 * This is in fact not required, but we might as well print it
 	 * just for kicks */
 	if (git_filebuf_printf(&pack_file, "%s\n", GIT_PACKEDREFS_HEADER) < 0)
-		goto cleanup_packfile;
+		goto fail;
 
-	for (i = 0; i < packing_list.length; ++i) {
-		struct packref *ref = (struct packref *)git_vector_get(&packing_list, i);
+	for (i = 0; i < git_sortedcache_entrycount(refcache); ++i) {
+		struct packref *ref = git_sortedcache_entry(refcache, i);
 
 		if (packed_find_peel(backend, ref) < 0)
-			goto cleanup_packfile;
+			goto fail;
 
 		if (packed_write_ref(ref, &pack_file) < 0)
-			goto cleanup_packfile;
+			goto fail;
 	}
 
 	/* if we've written all the references properly, we can commit
 	 * the packfile to make the changes effective */
-	if (git_filebuf_commit(&pack_file, GIT_PACKEDREFS_FILE_MODE) < 0)
-		goto cleanup_memory;
+	if (git_filebuf_commit(&pack_file) < 0)
+		goto fail;
 
 	/* when and only when the packfile has been properly written,
 	 * we can go ahead and remove the loose refs */
-	 if (packed_remove_loose(backend, &packing_list) < 0)
-		 goto cleanup_memory;
-
-	 {
-		struct stat st;
-		if (p_stat(pack_file_path.ptr, &st) == 0)
-			backend->refcache.packfile_time = st.st_mtime;
-	 }
+	if (packed_remove_loose(backend) < 0)
+		goto fail;
 
-	git_vector_free(&packing_list);
-	git_buf_free(&pack_file_path);
+	git_sortedcache_updated(refcache);
+	git_sortedcache_wunlock(refcache);
 
 	/* we're good now */
 	return 0;
 
-cleanup_packfile:
+fail:
 	git_filebuf_cleanup(&pack_file);
-
-cleanup_memory:
-	git_vector_free(&packing_list);
-	git_buf_free(&pack_file_path);
+	git_sortedcache_wunlock(refcache);
 
 	return -1;
 }
@@ -1057,11 +912,10 @@ static int refdb_fs_backend__write(
 	const git_reference *ref,
 	int force)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	int error;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
 	error = reference_path_available(backend, ref->name, NULL, force);
 	if (error < 0)
@@ -1074,17 +928,13 @@ static int refdb_fs_backend__delete(
 	git_refdb_backend *_backend,
 	const char *ref_name)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	git_buf loose_path = GIT_BUF_INIT;
-	struct packref *pack_ref;
-	khiter_t pack_ref_pos;
+	size_t pack_pos;
 	int error = 0;
 	bool loose_deleted = 0;
 
-	assert(_backend);
-	assert(ref_name);
-
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend && ref_name);
 
 	/* If a loose reference exists, remove it from the filesystem */
 	if (git_buf_joinpath(&loose_path, backend->path, ref_name) < 0)
@@ -1100,19 +950,23 @@ static int refdb_fs_backend__delete(
 	if (error != 0)
 		return error;
 
+	if (packed_reload(backend) < 0)
+		return -1;
+
 	/* If a packed reference exists, remove it from the packfile and repack */
-	error = packed_map_entry(&pack_ref, &pack_ref_pos, backend, ref_name);
+	if (git_sortedcache_wlock(backend->refcache) < 0)
+		return -1;
 
-	if (error == GIT_ENOTFOUND)
-		return loose_deleted ? 0 : GIT_ENOTFOUND;
+	if (!(error = git_sortedcache_lookup_index(
+			&pack_pos, backend->refcache, ref_name)))
+		error = git_sortedcache_remove(backend->refcache, pack_pos);
 
-	if (error == 0) {
-		git_strmap_delete_at(backend->refcache.packfile, pack_ref_pos);
-		git__free(pack_ref);
-		error = packed_write(backend);
-	}
+	git_sortedcache_wunlock(backend->refcache);
 
-	return error;
+	if (error == GIT_ENOTFOUND)
+		return loose_deleted ? 0 : ref_error_notfound(ref_name);
+
+	return packed_write(backend);
 }
 
 static int refdb_fs_backend__rename(
@@ -1122,53 +976,44 @@ static int refdb_fs_backend__rename(
 	const char *new_name,
 	int force)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 	git_reference *old, *new;
 	int error;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	error = reference_path_available(backend, new_name, old_name, force);
-	if (error < 0)
+	if ((error = reference_path_available(
+			backend, new_name, old_name, force)) < 0 ||
+		(error = refdb_fs_backend__lookup(&old, _backend, old_name)) < 0)
 		return error;
 
-	error = refdb_fs_backend__lookup(&old, _backend, old_name);
-	if (error < 0)
-		return error;
-
-	error = refdb_fs_backend__delete(_backend, old_name);
-	if (error < 0) {
+	if ((error = refdb_fs_backend__delete(_backend, old_name)) < 0) {
 		git_reference_free(old);
 		return error;
 	}
 
-	new = realloc(old, sizeof(git_reference) + strlen(new_name) + 1);
-	memcpy(new->name, new_name, strlen(new_name) + 1);
-
-	error = loose_write(backend, new);
-	if (error < 0) {
-		git_reference_free(new);
-		return error;
+	new = git_reference__set_name(old, new_name);
+	if (!new) {
+		git_reference_free(old);
+		return -1;
 	}
 
-	if (out) {
-		*out = new;
-	} else {
+	if ((error = loose_write(backend, new)) < 0 || out == NULL) {
 		git_reference_free(new);
+		return error;
 	}
 
+	*out = new;
 	return 0;
 }
 
 static int refdb_fs_backend__compress(git_refdb_backend *_backend)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	if (packed_load(backend) < 0 || /* load the existing packfile */
+	if (packed_reload(backend) < 0 || /* load the existing packfile */
 		packed_loadloose(backend) < 0 || /* add all the loose refs */
 		packed_write(backend) < 0) /* write back to disk */
 		return -1;
@@ -1176,29 +1021,13 @@ static int refdb_fs_backend__compress(git_refdb_backend *_backend)
 	return 0;
 }
 
-static void refcache_free(git_refcache *refs)
-{
-	assert(refs);
-
-	if (refs->packfile) {
-		struct packref *reference;
-
-		git_strmap_foreach_value(refs->packfile, reference, {
-			git__free(reference);
-		});
-
-		git_strmap_free(refs->packfile);
-	}
-}
-
 static void refdb_fs_backend__free(git_refdb_backend *_backend)
 {
-	refdb_fs_backend *backend;
+	refdb_fs_backend *backend = (refdb_fs_backend *)_backend;
 
-	assert(_backend);
-	backend = (refdb_fs_backend *)_backend;
+	assert(backend);
 
-	refcache_free(&backend->refcache);
+	git_sortedcache_free(backend->refcache);
 	git__free(backend->path);
 	git__free(backend);
 }
@@ -1222,7 +1051,7 @@ static int setup_namespace(git_buf *path, git_repository *repo)
 	if (parts == NULL)
 		return -1;
 
-	/**
+	/*
 	 * From `man gitnamespaces`:
 	 *  namespaces which include a / will expand to a hierarchy
 	 *  of namespaces; for example, GIT_NAMESPACE=foo/bar will store
@@ -1239,15 +1068,355 @@ static int setup_namespace(git_buf *path, git_repository *repo)
 	if (git_futils_mkdir_r(git_buf_cstr(path), repo->path_repository, 0777) < 0)
 		return -1;
 
-	/* Return the root of the namespaced path, i.e. without the trailing '/refs' */
+	/* Return root of the namespaced path, i.e. without the trailing '/refs' */
 	git_buf_rtruncate_at_char(path, '/');
 	return 0;
 }
 
+static int reflog_alloc(git_reflog **reflog, const char *name)
+{
+	git_reflog *log;
+
+	*reflog = NULL;
+
+	log = git__calloc(1, sizeof(git_reflog));
+	GITERR_CHECK_ALLOC(log);
+
+	log->ref_name = git__strdup(name);
+	GITERR_CHECK_ALLOC(log->ref_name);
+
+	if (git_vector_init(&log->entries, 0, NULL) < 0) {
+		git__free(log->ref_name);
+		git__free(log);
+		return -1;
+	}
+
+	*reflog = log;
+
+	return 0;
+}
+
+static int reflog_parse(git_reflog *log, const char *buf, size_t buf_size)
+{
+	const char *ptr;
+	git_reflog_entry *entry;
+
+#define seek_forward(_increase) do { \
+	if (_increase >= buf_size) { \
+		giterr_set(GITERR_INVALID, "Ran out of data while parsing reflog"); \
+		goto fail; \
+	} \
+	buf += _increase; \
+	buf_size -= _increase; \
+	} while (0)
+
+	while (buf_size > GIT_REFLOG_SIZE_MIN) {
+		entry = git__calloc(1, sizeof(git_reflog_entry));
+		GITERR_CHECK_ALLOC(entry);
+
+		entry->committer = git__malloc(sizeof(git_signature));
+		GITERR_CHECK_ALLOC(entry->committer);
+
+		if (git_oid_fromstrn(&entry->oid_old, buf, GIT_OID_HEXSZ) < 0)
+			goto fail;
+		seek_forward(GIT_OID_HEXSZ + 1);
+
+		if (git_oid_fromstrn(&entry->oid_cur, buf, GIT_OID_HEXSZ) < 0)
+			goto fail;
+		seek_forward(GIT_OID_HEXSZ + 1);
+
+		ptr = buf;
+
+		/* Seek forward to the end of the signature. */
+		while (*buf && *buf != '\t' && *buf != '\n')
+			seek_forward(1);
+
+		if (git_signature__parse(entry->committer, &ptr, buf + 1, NULL, *buf) < 0)
+			goto fail;
+
+		if (*buf == '\t') {
+			/* We got a message. Read everything till we reach LF. */
+			seek_forward(1);
+			ptr = buf;
+
+			while (*buf && *buf != '\n')
+				seek_forward(1);
+
+			entry->msg = git__strndup(ptr, buf - ptr);
+			GITERR_CHECK_ALLOC(entry->msg);
+		} else
+			entry->msg = NULL;
+
+		while (*buf && *buf == '\n' && buf_size > 1)
+			seek_forward(1);
+
+		if (git_vector_insert(&log->entries, entry) < 0)
+			goto fail;
+	}
+
+	return 0;
+
+#undef seek_forward
+
+fail:
+	if (entry)
+		git_reflog_entry__free(entry);
+
+	return -1;
+}
+
+static int create_new_reflog_file(const char *filepath)
+{
+	int fd, error;
+
+	if ((error = git_futils_mkpath2file(filepath, GIT_REFLOG_DIR_MODE)) < 0)
+		return error;
+
+	if ((fd = p_open(filepath,
+			O_WRONLY | O_CREAT | O_TRUNC,
+			GIT_REFLOG_FILE_MODE)) < 0)
+		return -1;
+
+	return p_close(fd);
+}
+
+GIT_INLINE(int) retrieve_reflog_path(git_buf *path, git_repository *repo, const char *name)
+{
+	return git_buf_join_n(path, '/', 3, repo->path_repository, GIT_REFLOG_DIR, name);
+}
+
+static int refdb_reflog_fs__read(git_reflog **out, git_refdb_backend *_backend, const char *name)
+{
+	int error = -1;
+	git_buf log_path = GIT_BUF_INIT;
+	git_buf log_file = GIT_BUF_INIT;
+	git_reflog *log = NULL;
+	git_repository *repo;
+	refdb_fs_backend *backend;
+
+	assert(out && _backend && name);
+
+	backend = (refdb_fs_backend *) _backend;
+	repo = backend->repo;
+
+	if (reflog_alloc(&log, name) < 0)
+		return -1;
+
+	if (retrieve_reflog_path(&log_path, repo, name) < 0)
+		goto cleanup;
+
+	error = git_futils_readbuffer(&log_file, git_buf_cstr(&log_path));
+	if (error < 0 && error != GIT_ENOTFOUND)
+		goto cleanup;
+
+	if ((error == GIT_ENOTFOUND) &&
+		((error = create_new_reflog_file(git_buf_cstr(&log_path))) < 0))
+		goto cleanup;
+ 
+	if ((error = reflog_parse(log,
+		git_buf_cstr(&log_file), git_buf_len(&log_file))) < 0)
+		goto cleanup;
+
+	*out = log;
+	goto success;
+
+cleanup:
+	git_reflog_free(log);
+
+success:
+	git_buf_free(&log_file);
+	git_buf_free(&log_path);
+
+	return error;
+}
+
+static int serialize_reflog_entry(
+	git_buf *buf,
+	const git_oid *oid_old,
+	const git_oid *oid_new,
+	const git_signature *committer,
+	const char *msg)
+{
+	char raw_old[GIT_OID_HEXSZ+1];
+	char raw_new[GIT_OID_HEXSZ+1];
+
+	git_oid_tostr(raw_old, GIT_OID_HEXSZ+1, oid_old);
+	git_oid_tostr(raw_new, GIT_OID_HEXSZ+1, oid_new);
+
+	git_buf_clear(buf);
+
+	git_buf_puts(buf, raw_old);
+	git_buf_putc(buf, ' ');
+	git_buf_puts(buf, raw_new);
+
+	git_signature__writebuf(buf, " ", committer);
+
+	/* drop trailing LF */
+	git_buf_rtrim(buf);
+
+	if (msg) {
+		git_buf_putc(buf, '\t');
+		git_buf_puts(buf, msg);
+	}
+
+	git_buf_putc(buf, '\n');
+
+	return git_buf_oom(buf);
+}
+
+static int refdb_reflog_fs__write(git_refdb_backend *_backend, git_reflog *reflog)
+{
+	int error = -1;
+	unsigned int i;
+	git_reflog_entry *entry;
+	git_repository *repo;
+	refdb_fs_backend *backend;
+	git_buf log_path = GIT_BUF_INIT;
+	git_buf log = GIT_BUF_INIT;
+	git_filebuf fbuf = GIT_FILEBUF_INIT;
+
+	assert(_backend && reflog);
+
+	backend = (refdb_fs_backend *) _backend;
+	repo = backend->repo;
+
+	if (retrieve_reflog_path(&log_path, repo, reflog->ref_name) < 0)
+		return -1;
+
+	if (!git_path_isfile(git_buf_cstr(&log_path))) {
+		giterr_set(GITERR_INVALID,
+			"Log file for reference '%s' doesn't exist.", reflog->ref_name);
+		goto cleanup;
+	}
+
+	if ((error = git_filebuf_open(&fbuf, git_buf_cstr(&log_path), 0, GIT_REFLOG_FILE_MODE)) < 0)
+		goto cleanup;
+
+	git_vector_foreach(&reflog->entries, i, entry) {
+		if (serialize_reflog_entry(&log, &(entry->oid_old), &(entry->oid_cur), entry->committer, entry->msg) < 0)
+			goto cleanup;
+
+		if ((error = git_filebuf_write(&fbuf, log.ptr, log.size)) < 0)
+			goto cleanup;
+	}
+
+	error = git_filebuf_commit(&fbuf);
+	goto success;
+
+cleanup:
+	git_filebuf_cleanup(&fbuf);
+
+success:
+	git_buf_free(&log);
+	git_buf_free(&log_path);
+	return error;
+}
+
+static int refdb_reflog_fs__rename(git_refdb_backend *_backend, const char *old_name, const char *new_name)
+{
+	int error = 0, fd;
+	git_buf old_path = GIT_BUF_INIT;
+	git_buf new_path = GIT_BUF_INIT;
+	git_buf temp_path = GIT_BUF_INIT;
+	git_buf normalized = GIT_BUF_INIT;
+	git_repository *repo;
+	refdb_fs_backend *backend;
+
+	assert(_backend && old_name && new_name);
+
+	backend = (refdb_fs_backend *) _backend;
+	repo = backend->repo;
+
+	if ((error = git_reference__normalize_name(
+		&normalized, new_name, GIT_REF_FORMAT_ALLOW_ONELEVEL)) < 0)
+			return error;
+
+	if (git_buf_joinpath(&temp_path, repo->path_repository, GIT_REFLOG_DIR) < 0)
+		return -1;
+
+	if (git_buf_joinpath(&old_path, git_buf_cstr(&temp_path), old_name) < 0)
+		return -1;
+
+	if (git_buf_joinpath(&new_path, git_buf_cstr(&temp_path), git_buf_cstr(&normalized)) < 0)
+		return -1;
+
+	/*
+	 * Move the reflog to a temporary place. This two-phase renaming is required
+	 * in order to cope with funny renaming use cases when one tries to move a reference
+	 * to a partially colliding namespace:
+	 *  - a/b -> a/b/c
+	 *  - a/b/c/d -> a/b/c
+	 */
+	if (git_buf_joinpath(&temp_path, git_buf_cstr(&temp_path), "temp_reflog") < 0)
+		return -1;
+
+	if ((fd = git_futils_mktmp(&temp_path, git_buf_cstr(&temp_path), GIT_REFLOG_FILE_MODE)) < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	p_close(fd);
+
+	if (p_rename(git_buf_cstr(&old_path), git_buf_cstr(&temp_path)) < 0) {
+		giterr_set(GITERR_OS, "Failed to rename reflog for %s", new_name);
+		error = -1;
+		goto cleanup;
+	}
+
+	if (git_path_isdir(git_buf_cstr(&new_path)) && 
+		(git_futils_rmdir_r(git_buf_cstr(&new_path), NULL, GIT_RMDIR_SKIP_NONEMPTY) < 0)) {
+		error = -1;
+		goto cleanup;
+	}
+
+	if (git_futils_mkpath2file(git_buf_cstr(&new_path), GIT_REFLOG_DIR_MODE) < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	if (p_rename(git_buf_cstr(&temp_path), git_buf_cstr(&new_path)) < 0) {
+		giterr_set(GITERR_OS, "Failed to rename reflog for %s", new_name);
+		error = -1;
+	}
+
+cleanup:
+	git_buf_free(&temp_path);
+	git_buf_free(&old_path);
+	git_buf_free(&new_path);
+	git_buf_free(&normalized);
+
+	return error;
+}
+
+static int refdb_reflog_fs__delete(git_refdb_backend *_backend, const char *name)
+{
+	int error;
+	git_buf path = GIT_BUF_INIT;
+
+	git_repository *repo;
+	refdb_fs_backend *backend;
+
+	assert(_backend && name);
+
+	backend = (refdb_fs_backend *) _backend;
+	repo = backend->repo;
+
+	error = retrieve_reflog_path(&path, repo, name);
+
+	if (!error && git_path_exists(path.ptr))
+		error = p_unlink(path.ptr);
+
+	git_buf_free(&path);
+
+	return error;
+
+}
+
 int git_refdb_backend_fs(
 	git_refdb_backend **backend_out,
 	git_repository *repository)
 {
+	int t = 0;
 	git_buf path = GIT_BUF_INIT;
 	refdb_fs_backend *backend;
 
@@ -1256,22 +1425,47 @@ int git_refdb_backend_fs(
 
 	backend->repo = repository;
 
-	if (setup_namespace(&path, repository) < 0) {
-		git__free(backend);
-		return -1;
-	}
+	if (setup_namespace(&path, repository) < 0)
+		goto fail;
 
 	backend->path = git_buf_detach(&path);
 
+	if (git_buf_joinpath(&path, backend->path, GIT_PACKEDREFS_FILE) < 0 ||
+		git_sortedcache_new(
+			&backend->refcache, offsetof(struct packref, name),
+			NULL, NULL, packref_cmp, git_buf_cstr(&path)) < 0)
+		goto fail;
+
+	git_buf_free(&path);
+
+	if (!git_repository__cvar(&t, backend->repo, GIT_CVAR_IGNORECASE) && t) {
+		backend->iterator_flags |= GIT_ITERATOR_IGNORE_CASE;
+		backend->direach_flags  |= GIT_PATH_DIR_IGNORE_CASE;
+	}
+	if (!git_repository__cvar(&t, backend->repo, GIT_CVAR_PRECOMPOSE) && t) {
+		backend->iterator_flags |= GIT_ITERATOR_PRECOMPOSE_UNICODE;
+		backend->direach_flags  |= GIT_PATH_DIR_PRECOMPOSE_UNICODE;
+	}
+
 	backend->parent.exists = &refdb_fs_backend__exists;
 	backend->parent.lookup = &refdb_fs_backend__lookup;
 	backend->parent.iterator = &refdb_fs_backend__iterator;
 	backend->parent.write = &refdb_fs_backend__write;
-	backend->parent.delete = &refdb_fs_backend__delete;
+	backend->parent.del = &refdb_fs_backend__delete;
 	backend->parent.rename = &refdb_fs_backend__rename;
 	backend->parent.compress = &refdb_fs_backend__compress;
 	backend->parent.free = &refdb_fs_backend__free;
+	backend->parent.reflog_read = &refdb_reflog_fs__read;
+	backend->parent.reflog_write = &refdb_reflog_fs__write;
+	backend->parent.reflog_rename = &refdb_reflog_fs__rename;
+	backend->parent.reflog_delete = &refdb_reflog_fs__delete;
 
 	*backend_out = (git_refdb_backend *)backend;
 	return 0;
+
+fail:
+	git_buf_free(&path);
+	git__free(backend->path);
+	git__free(backend);
+	return -1;
 }
diff --git a/src/reflog.c b/src/reflog.c
index 4cc20d2c7..cebb87d86 100644
--- a/src/reflog.c
+++ b/src/reflog.c
@@ -9,82 +9,16 @@
 #include "repository.h"
 #include "filebuf.h"
 #include "signature.h"
+#include "refdb.h"
 
-static int reflog_init(git_reflog **reflog, const git_reference *ref)
-{
-	git_reflog *log;
-
-	*reflog = NULL;
-
-	log = git__calloc(1, sizeof(git_reflog));
-	GITERR_CHECK_ALLOC(log);
-
-	log->ref_name = git__strdup(ref->name);
-	GITERR_CHECK_ALLOC(log->ref_name);
-
-	if (git_vector_init(&log->entries, 0, NULL) < 0) {
-		git__free(log->ref_name);
-		git__free(log);
-		return -1;
-	}
-
-	log->owner = git_reference_owner(ref);
-	*reflog = log;
+#include <git2/sys/refdb_backend.h>
 
-	return 0;
-}
-
-static int serialize_reflog_entry(
-	git_buf *buf,
-	const git_oid *oid_old,
-	const git_oid *oid_new,
-	const git_signature *committer,
-	const char *msg)
+git_reflog_entry *git_reflog_entry__alloc(void)
 {
-	char raw_old[GIT_OID_HEXSZ+1];
-	char raw_new[GIT_OID_HEXSZ+1];
-
-	git_oid_tostr(raw_old, GIT_OID_HEXSZ+1, oid_old);
-	git_oid_tostr(raw_new, GIT_OID_HEXSZ+1, oid_new);
-
-	git_buf_clear(buf);
-
-	git_buf_puts(buf, raw_old);
-	git_buf_putc(buf, ' ');
-	git_buf_puts(buf, raw_new);
-
-	git_signature__writebuf(buf, " ", committer);
-
-	/* drop trailing LF */
-	git_buf_rtrim(buf);
-
-	if (msg) {
-		git_buf_putc(buf, '\t');
-		git_buf_puts(buf, msg);
-	}
-
-	git_buf_putc(buf, '\n');
-
-	return git_buf_oom(buf);
-}
-
-static int reflog_entry_new(git_reflog_entry **entry)
-{
-	git_reflog_entry *e;
-
-	assert(entry);
-
-	e = git__malloc(sizeof(git_reflog_entry));
-	GITERR_CHECK_ALLOC(e);
-
-	memset(e, 0, sizeof(git_reflog_entry));
-
-	*entry = e;
-
-	return 0;
+	return git__calloc(1, sizeof(git_reflog_entry));
 }
 
-static void reflog_entry_free(git_reflog_entry *entry)
+void git_reflog_entry__free(git_reflog_entry *entry)
 {
 	git_signature_free(entry->committer);
 
@@ -92,75 +26,6 @@ static void reflog_entry_free(git_reflog_entry *entry)
 	git__free(entry);
 }
 
-static int reflog_parse(git_reflog *log, const char *buf, size_t buf_size)
-{
-	const char *ptr;
-	git_reflog_entry *entry;
-
-#define seek_forward(_increase) do { \
-	if (_increase >= buf_size) { \
-		giterr_set(GITERR_INVALID, "Ran out of data while parsing reflog"); \
-		goto fail; \
-	} \
-	buf += _increase; \
-	buf_size -= _increase; \
-	} while (0)
-
-	while (buf_size > GIT_REFLOG_SIZE_MIN) {
-		if (reflog_entry_new(&entry) < 0)
-			return -1;
-
-		entry->committer = git__malloc(sizeof(git_signature));
-		GITERR_CHECK_ALLOC(entry->committer);
-
-		if (git_oid_fromstrn(&entry->oid_old, buf, GIT_OID_HEXSZ) < 0)
-			goto fail;
-		seek_forward(GIT_OID_HEXSZ + 1);
-
-		if (git_oid_fromstrn(&entry->oid_cur, buf, GIT_OID_HEXSZ) < 0)
-			goto fail;
-		seek_forward(GIT_OID_HEXSZ + 1);
-
-		ptr = buf;
-
-		/* Seek forward to the end of the signature. */
-		while (*buf && *buf != '\t' && *buf != '\n')
-			seek_forward(1);
-
-		if (git_signature__parse(entry->committer, &ptr, buf + 1, NULL, *buf) < 0)
-			goto fail;
-
-		if (*buf == '\t') {
-			/* We got a message. Read everything till we reach LF. */
-			seek_forward(1);
-			ptr = buf;
-
-			while (*buf && *buf != '\n')
-				seek_forward(1);
-
-			entry->msg = git__strndup(ptr, buf - ptr);
-			GITERR_CHECK_ALLOC(entry->msg);
-		} else
-			entry->msg = NULL;
-
-		while (*buf && *buf == '\n' && buf_size > 1)
-			seek_forward(1);
-
-		if (git_vector_insert(&log->entries, entry) < 0)
-			goto fail;
-	}
-
-	return 0;
-
-#undef seek_forward
-
-fail:
-	if (entry)
-		reflog_entry_free(entry);
-
-	return -1;
-}
-
 void git_reflog_free(git_reflog *reflog)
 {
 	size_t i;
@@ -169,10 +34,13 @@ void git_reflog_free(git_reflog *reflog)
 	if (reflog == NULL)
 		return;
 
+	if (reflog->db)
+		GIT_REFCOUNT_DEC(reflog->db, git_refdb__free);
+
 	for (i=0; i < reflog->entries.length; i++) {
 		entry = git_vector_get(&reflog->entries, i);
 
-		reflog_entry_free(entry);
+		git_reflog_entry__free(entry);
 	}
 
 	git_vector_free(&reflog->entries);
@@ -180,115 +48,30 @@ void git_reflog_free(git_reflog *reflog)
 	git__free(reflog);
 }
 
-static int retrieve_reflog_path(git_buf *path, const git_reference *ref)
+int git_reflog_read(git_reflog **reflog, git_repository *repo,  const char *name)
 {
-	return git_buf_join_n(path, '/', 3,
-		git_reference_owner(ref)->path_repository, GIT_REFLOG_DIR, ref->name);
-}
+	git_refdb *refdb;
+	int error;
 
-static int create_new_reflog_file(const char *filepath)
-{
-	int fd, error;
+	assert(reflog && repo && name);
 
-	if ((error = git_futils_mkpath2file(filepath, GIT_REFLOG_DIR_MODE)) < 0)
+	if ((error = git_repository_refdb__weakptr(&refdb, repo)) < 0)
 		return error;
 
-	if ((fd = p_open(filepath,
-			O_WRONLY | O_CREAT | O_TRUNC,
-			GIT_REFLOG_FILE_MODE)) < 0)
-		return -1;
-
-	return p_close(fd);
-}
-
-int git_reflog_read(git_reflog **reflog, const git_reference *ref)
-{
-	int error = -1;
-	git_buf log_path = GIT_BUF_INIT;
-	git_buf log_file = GIT_BUF_INIT;
-	git_reflog *log = NULL;
-
-	assert(reflog && ref);
-
-	*reflog = NULL;
-
-	if (reflog_init(&log, ref) < 0)
-		return -1;
-
-	if (retrieve_reflog_path(&log_path, ref) < 0)
-		goto cleanup;
-
-	error = git_futils_readbuffer(&log_file, git_buf_cstr(&log_path));
-	if (error < 0 && error != GIT_ENOTFOUND)
-		goto cleanup;
-
-	if ((error == GIT_ENOTFOUND) &&
-		((error = create_new_reflog_file(git_buf_cstr(&log_path))) < 0))
-		goto cleanup;
-
-	if ((error = reflog_parse(log,
-		git_buf_cstr(&log_file), git_buf_len(&log_file))) < 0)
-		goto cleanup;
-
-	*reflog = log;
-	goto success;
-
-cleanup:
-	git_reflog_free(log);
-
-success:
-	git_buf_free(&log_file);
-	git_buf_free(&log_path);
-
-	return error;
+	return git_refdb_reflog_read(reflog, refdb, name);
 }
 
 int git_reflog_write(git_reflog *reflog)
 {
-	int error = -1;
-	unsigned int i;
-	git_reflog_entry *entry;
-	git_buf log_path = GIT_BUF_INIT;
-	git_buf log = GIT_BUF_INIT;
-	git_filebuf fbuf = GIT_FILEBUF_INIT;
+	git_refdb *db;
 
-	assert(reflog);
+	assert(reflog && reflog->db);
 
-	if (git_buf_join_n(&log_path, '/', 3,
-		git_repository_path(reflog->owner), GIT_REFLOG_DIR, reflog->ref_name) < 0)
-		return -1;
-
-	if (!git_path_isfile(git_buf_cstr(&log_path))) {
-		giterr_set(GITERR_INVALID,
-			"Log file for reference '%s' doesn't exist.", reflog->ref_name);
-		goto cleanup;
-	}
-
-	if ((error = git_filebuf_open(&fbuf, git_buf_cstr(&log_path), 0)) < 0)
-		goto cleanup;
-
-	git_vector_foreach(&reflog->entries, i, entry) {
-		if (serialize_reflog_entry(&log, &(entry->oid_old), &(entry->oid_cur), entry->committer, entry->msg) < 0)
-			goto cleanup;
-
-		if ((error = git_filebuf_write(&fbuf, log.ptr, log.size)) < 0)
-			goto cleanup;
-	}
-
-	error = git_filebuf_commit(&fbuf, GIT_REFLOG_FILE_MODE);
-	goto success;
-
-cleanup:
-	git_filebuf_cleanup(&fbuf);
-
-success:
-	git_buf_free(&log);
-	git_buf_free(&log_path);
-	return error;
+	db = reflog->db;
+	return db->backend->reflog_write(db->backend, reflog);
 }
 
-int git_reflog_append(git_reflog *reflog, const git_oid *new_oid,
-				const git_signature *committer, const char *msg)
+int git_reflog_append(git_reflog *reflog, const git_oid *new_oid, const git_signature *committer, const char *msg)
 {
 	git_reflog_entry *entry;
 	const git_reflog_entry *previous;
@@ -296,8 +79,8 @@ int git_reflog_append(git_reflog *reflog, const git_oid *new_oid,
 
 	assert(reflog && new_oid && committer);
 
-	if (reflog_entry_new(&entry) < 0)
-		return -1;
+	entry = git__calloc(1, sizeof(git_reflog_entry));
+	GITERR_CHECK_ALLOC(entry);
 
 	if ((entry->committer = git_signature_dup(committer)) == NULL)
 		goto cleanup;
@@ -333,94 +116,30 @@ int git_reflog_append(git_reflog *reflog, const git_oid *new_oid,
 	return 0;
 
 cleanup:
-	reflog_entry_free(entry);
+	git_reflog_entry__free(entry);
 	return -1;
 }
 
-int git_reflog_rename(git_reference *ref, const char *new_name)
+int git_reflog_rename(git_repository *repo, const char *old_name, const char *new_name)
 {
-	int error = 0, fd;
-	git_buf old_path = GIT_BUF_INIT;
-	git_buf new_path = GIT_BUF_INIT;
-	git_buf temp_path = GIT_BUF_INIT;
-	git_buf normalized = GIT_BUF_INIT;
-
-	assert(ref && new_name);
-
-	if ((error = git_reference__normalize_name(
-		&normalized, new_name, GIT_REF_FORMAT_ALLOW_ONELEVEL)) < 0)
-			return error;
-
-	if (git_buf_joinpath(&temp_path, git_reference_owner(ref)->path_repository, GIT_REFLOG_DIR) < 0)
-		return -1;
-
-	if (git_buf_joinpath(&old_path, git_buf_cstr(&temp_path), ref->name) < 0)
-		return -1;
-
-	if (git_buf_joinpath(&new_path, git_buf_cstr(&temp_path), git_buf_cstr(&normalized)) < 0)
-		return -1;
+	git_refdb *refdb;
+	int error;
 
-	/*
-	 * Move the reflog to a temporary place. This two-phase renaming is required
-	 * in order to cope with funny renaming use cases when one tries to move a reference
-	 * to a partially colliding namespace:
-	 *  - a/b -> a/b/c
-	 *  - a/b/c/d -> a/b/c
-	 */
-	if (git_buf_joinpath(&temp_path, git_buf_cstr(&temp_path), "temp_reflog") < 0)
+	if ((error = git_repository_refdb__weakptr(&refdb, repo)) < 0)
 		return -1;
 
-	if ((fd = git_futils_mktmp(&temp_path, git_buf_cstr(&temp_path))) < 0) {
-		error = -1;
-		goto cleanup;
-	}
-
-	p_close(fd);
-
-	if (p_rename(git_buf_cstr(&old_path), git_buf_cstr(&temp_path)) < 0) {
-		giterr_set(GITERR_OS, "Failed to rename reflog for %s", new_name);
-		error = -1;
-		goto cleanup;
-	}
-
-	if (git_path_isdir(git_buf_cstr(&new_path)) && 
-		(git_futils_rmdir_r(git_buf_cstr(&new_path), NULL, GIT_RMDIR_SKIP_NONEMPTY) < 0)) {
-		error = -1;
-		goto cleanup;
-	}
-
-	if (git_futils_mkpath2file(git_buf_cstr(&new_path), GIT_REFLOG_DIR_MODE) < 0) {
-		error = -1;
-		goto cleanup;
-	}
-
-	if (p_rename(git_buf_cstr(&temp_path), git_buf_cstr(&new_path)) < 0) {
-		giterr_set(GITERR_OS, "Failed to rename reflog for %s", new_name);
-		error = -1;
-	}
-
-cleanup:
-	git_buf_free(&temp_path);
-	git_buf_free(&old_path);
-	git_buf_free(&new_path);
-	git_buf_free(&normalized);
-
-	return error;
+	return refdb->backend->reflog_rename(refdb->backend, old_name, new_name);
 }
 
-int git_reflog_delete(git_reference *ref)
+int git_reflog_delete(git_repository *repo, const char *name)
 {
+	git_refdb *refdb;
 	int error;
-	git_buf path = GIT_BUF_INIT;
-
-	error = retrieve_reflog_path(&path, ref);
 
-	if (!error && git_path_exists(path.ptr))
-		error = p_unlink(path.ptr);
-
-	git_buf_free(&path);
+	if ((error = git_repository_refdb__weakptr(&refdb, repo)) < 0)
+		return -1;
 
-	return error;
+	return refdb->backend->reflog_delete(refdb->backend, name);
 }
 
 size_t git_reflog_entrycount(git_reflog *reflog)
@@ -429,11 +148,6 @@ size_t git_reflog_entrycount(git_reflog *reflog)
 	return reflog->entries.length;
 }
 
-GIT_INLINE(size_t) reflog_inverse_index(size_t idx, size_t total)
-{
-	return (total - 1) - idx;
-}
-
 const git_reflog_entry * git_reflog_entry_byindex(git_reflog *reflog, size_t idx)
 {
 	assert(reflog);
@@ -469,16 +183,11 @@ const char * git_reflog_entry_message(const git_reflog_entry *entry)
 	return entry->msg;
 }
 
-int git_reflog_drop(
-	git_reflog *reflog,
-	size_t idx,
-	int rewrite_previous_entry)
+int git_reflog_drop(git_reflog *reflog, size_t idx, int rewrite_previous_entry)
 {
 	size_t entrycount;
 	git_reflog_entry *entry, *previous;
 
-	assert(reflog);
-
 	entrycount = git_reflog_entrycount(reflog);
 
 	entry = (git_reflog_entry *)git_reflog_entry_byindex(reflog, idx);
@@ -488,7 +197,7 @@ int git_reflog_drop(
 		return GIT_ENOTFOUND;
 	}
 
-	reflog_entry_free(entry);
+	git_reflog_entry__free(entry);
 
 	if (git_vector_remove(
 			&reflog->entries, reflog_inverse_index(idx, entrycount)) < 0)
@@ -521,3 +230,22 @@ int git_reflog_drop(
 
 	return 0;
 }
+
+int git_reflog_append_to(git_repository *repo, const char *name, const git_oid *id,
+			 const git_signature *committer, const char *msg)
+{
+	int error;
+	git_reflog *reflog;
+
+	if ((error = git_reflog_read(&reflog, repo, name)) < 0)
+		return error;
+
+	if ((error = git_reflog_append(reflog, id, committer, msg)) < 0)
+		goto cleanup;
+
+	error = git_reflog_write(reflog);
+
+cleanup:
+	git_reflog_free(reflog);
+	return error;
+}
diff --git a/src/reflog.h b/src/reflog.h
index 9444ebd10..2d31ae47d 100644
--- a/src/reflog.h
+++ b/src/reflog.h
@@ -27,9 +27,14 @@ struct git_reflog_entry {
 };
 
 struct git_reflog {
+	git_refdb *db;
 	char *ref_name;
-	git_repository *owner;
 	git_vector entries;
 };
 
+GIT_INLINE(size_t) reflog_inverse_index(size_t idx, size_t total)
+{
+	return (total - 1) - idx;
+}
+
 #endif /* INCLUDE_reflog_h__ */
diff --git a/src/refs.c b/src/refs.c
index c0e460cc3..472a79890 100644
--- a/src/refs.c
+++ b/src/refs.c
@@ -88,6 +88,17 @@ git_reference *git_reference__alloc(
 	return ref;
 }
 
+git_reference *git_reference__set_name(
+	git_reference *ref, const char *name)
+{
+	size_t namelen = strlen(name);
+	git_reference *rewrite =
+		git__realloc(ref, sizeof(git_reference) + namelen + 1);
+	if (rewrite != NULL)
+		memcpy(rewrite->name, name, namelen + 1);
+	return rewrite;
+}
+
 void git_reference_free(git_reference *reference)
 {
 	if (reference == NULL)
@@ -127,6 +138,22 @@ int git_reference_name_to_id(
 	return 0;
 }
 
+static int reference_normalize_for_repo(
+	char *out,
+	size_t out_size,
+	git_repository *repo,
+	const char *name)
+{
+	int precompose;
+	unsigned int flags = GIT_REF_FORMAT_ALLOW_ONELEVEL;
+
+	if (!git_repository__cvar(&precompose, repo, GIT_CVAR_PRECOMPOSE) &&
+		precompose)
+		flags |= GIT_REF_FORMAT__PRECOMPOSE_UNICODE;
+
+	return git_reference_normalize_name(out, out_size, name, flags);
+}
+
 int git_reference_lookup_resolved(
 	git_reference **ref_out,
 	git_repository *repo,
@@ -148,13 +175,13 @@ int git_reference_lookup_resolved(
 	else if (max_nesting < 0)
 		max_nesting = DEFAULT_NESTING_LEVEL;
 
-	strncpy(scan_name, name, GIT_REFNAME_MAX);
 	scan_type = GIT_REF_SYMBOLIC;
 
-	if ((error = git_repository_refdb__weakptr(&refdb, repo)) < 0)
-		return -1;
+	if ((error = reference_normalize_for_repo(
+			scan_name, sizeof(scan_name), repo, name)) < 0)
+		return error;
 
-	if ((error = git_reference__normalize_name_lax(scan_name, GIT_REFNAME_MAX, name)) < 0)
+	if ((error = git_repository_refdb__weakptr(&refdb, repo)) < 0)
 		return error;
 
 	for (nesting = max_nesting;
@@ -162,7 +189,7 @@ int git_reference_lookup_resolved(
 		 nesting--)
 	{
 		if (nesting != max_nesting) {
-			strncpy(scan_name, ref->target.symbolic, GIT_REFNAME_MAX);
+			strncpy(scan_name, ref->target.symbolic, sizeof(scan_name));
 			git_reference_free(ref);
 		}
 
@@ -456,7 +483,7 @@ int git_reference_rename(
 	if (reference_has_log < 0)
 		return reference_has_log;
 
-	if (reference_has_log && (error = git_reflog_rename(ref, new_name)) < 0)
+	if (reference_has_log && (error = git_reflog_rename(git_reference_owner(ref), git_reference_name(ref), new_name)) < 0)
 		return error;
 
 	return 0;
@@ -700,17 +727,21 @@ static bool is_all_caps_and_underscore(const char *name, size_t len)
 	return true;
 }
 
+/* Inspired from https://github.com/git/git/blob/f06d47e7e0d9db709ee204ed13a8a7486149f494/refs.c#L36-100 */
 int git_reference__normalize_name(
 	git_buf *buf,
 	const char *name,
 	unsigned int flags)
 {
-	// Inspired from https://github.com/git/git/blob/f06d47e7e0d9db709ee204ed13a8a7486149f494/refs.c#L36-100
-
 	char *current;
 	int segment_len, segments_count = 0, error = GIT_EINVALIDSPEC;
 	unsigned int process_flags;
 	bool normalize = (buf != NULL);
+
+#ifdef GIT_USE_ICONV
+	git_path_iconv_t ic = GIT_PATH_ICONV_INIT;
+#endif
+
 	assert(name);
 
 	process_flags = flags;
@@ -722,6 +753,16 @@ int git_reference__normalize_name(
 	if (normalize)
 		git_buf_clear(buf);
 
+#ifdef GIT_USE_ICONV
+	if ((flags & GIT_REF_FORMAT__PRECOMPOSE_UNICODE) != 0) {
+		size_t namelen = strlen(current);
+		if ((error = git_path_iconv_init_precompose(&ic)) < 0 ||
+			(error = git_path_iconv(&ic, &current, &namelen)) < 0)
+			goto cleanup;
+		error = GIT_EINVALIDSPEC;
+	}
+#endif
+
 	while (true) {
 		segment_len = ensure_segment_validity(current);
 		if (segment_len < 0) {
@@ -798,6 +839,10 @@ cleanup:
 	if (error && normalize)
 		git_buf_free(buf);
 
+#ifdef GIT_USE_ICONV
+	git_path_iconv_clear(&ic);
+#endif
+
 	return error;
 }
 
@@ -952,6 +997,17 @@ int git_reference_is_remote(git_reference *ref)
 	return git_reference__is_remote(ref->name);
 }
 
+int git_reference__is_tag(const char *ref_name)
+{
+	return git__prefixcmp(ref_name, GIT_REFS_TAGS_DIR) == 0;
+}
+
+int git_reference_is_tag(git_reference *ref)
+{
+	assert(ref);
+	return git_reference__is_tag(ref->name);
+}
+
 static int peel_error(int error, git_reference *ref, const char* msg)
 {
 	giterr_set(
@@ -961,9 +1017,9 @@ static int peel_error(int error, git_reference *ref, const char* msg)
 }
 
 int git_reference_peel(
-		git_object **peeled,
-		git_reference *ref,
-		git_otype target_type)
+	git_object **peeled,
+	git_reference *ref,
+	git_otype target_type)
 {
 	git_reference *resolved = NULL;
 	git_object *target = NULL;
@@ -1005,24 +1061,19 @@ cleanup:
 	return error;
 }
 
-int git_reference__is_valid_name(
-	const char *refname,
-	unsigned int flags)
+int git_reference__is_valid_name(const char *refname, unsigned int flags)
 {
-	int error;
-
-	error = git_reference__normalize_name(NULL, refname, flags) == 0;
-	giterr_clear();
+	if (git_reference__normalize_name(NULL, refname, flags) < 0) {
+		giterr_clear();
+		return false;
+	}
 
-	return error;
+	return true;
 }
 
-int git_reference_is_valid_name(
-	const char *refname)
+int git_reference_is_valid_name(const char *refname)
 {
-	return git_reference__is_valid_name(
-		refname,
-		GIT_REF_FORMAT_ALLOW_ONELEVEL);
+	return git_reference__is_valid_name(refname, GIT_REF_FORMAT_ALLOW_ONELEVEL);
 }
 
 const char *git_reference_shorthand(git_reference *ref)
diff --git a/src/refs.h b/src/refs.h
index f487ee3fc..80c7703fc 100644
--- a/src/refs.h
+++ b/src/refs.h
@@ -46,6 +46,8 @@
 #define GIT_STASH_FILE "stash"
 #define GIT_REFS_STASH_FILE GIT_REFS_DIR GIT_STASH_FILE
 
+#define GIT_REF_FORMAT__PRECOMPOSE_UNICODE	(1u << 16)
+
 #define GIT_REFNAME_MAX 1024
 
 struct git_reference {
@@ -61,12 +63,15 @@ struct git_reference {
 	char name[0];
 };
 
+git_reference *git_reference__set_name(git_reference *ref, const char *name);
+
 int git_reference__normalize_name_lax(char *buffer_out, size_t out_size, const char *name);
 int git_reference__normalize_name(git_buf *buf, const char *name, unsigned int flags);
 int git_reference__update_terminal(git_repository *repo, const char *ref_name, const git_oid *oid);
 int git_reference__is_valid_name(const char *refname, unsigned int flags);
 int git_reference__is_branch(const char *ref_name);
 int git_reference__is_remote(const char *ref_name);
+int git_reference__is_tag(const char *ref_name);
 
 /**
  * Lookup a reference by name and try to resolve to an OID.
diff --git a/src/refspec.c b/src/refspec.c
index a907df84c..a97340071 100644
--- a/src/refspec.c
+++ b/src/refspec.c
@@ -12,6 +12,7 @@
 #include "util.h"
 #include "posix.h"
 #include "refs.h"
+#include "vector.h"
 
 int git_refspec__parse(git_refspec *refspec, const char *input, bool is_fetch)
 {
@@ -225,25 +226,31 @@ int git_refspec_rtransform(char *out, size_t outlen, const git_refspec *spec, co
 	return refspec_transform_internal(out, outlen, spec->dst, spec->src, name);
 }
 
-static int refspec_transform(git_buf *out, const char *from, const char *to, const char *name)
+static int refspec_transform(
+	git_buf *out, const char *from, const char *to, const char *name)
 {
-	if (git_buf_sets(out, to) < 0)
+	size_t to_len   = to   ? strlen(to)   : 0;
+	size_t from_len = from ? strlen(from) : 0;
+	size_t name_len = name ? strlen(name) : 0;
+
+	if (git_buf_set(out, to, to_len) < 0)
 		return -1;
 
-	/*
-	 * No '*' at the end means that it's mapped to one specific
-	 * branch, so no actual transformation is needed.
-	 */
-	if (git_buf_len(out) > 0 && out->ptr[git_buf_len(out) - 1] != '*')
-		return 0;
+	if (to_len > 0) {
+		/* No '*' at the end of 'to' means that refspec is mapped to one
+		 * specific branch, so no actual transformation is needed.
+		 */
+		if (out->ptr[to_len - 1] != '*')
+			return 0;
+		git_buf_shorten(out, 1); /* remove trailing '*' copied from 'to' */
+	}
 
-	git_buf_truncate(out, git_buf_len(out) - 1); /* remove trailing '*' */
-	git_buf_puts(out, name + strlen(from) - 1);
+	if (from_len > 0) /* ignore trailing '*' from 'from' */
+		from_len--;
+	if (from_len > name_len)
+		from_len = name_len;
 
-	if (git_buf_oom(out))
-		return -1;
-
-	return 0;
+	return git_buf_put(out, name + from_len, name_len - from_len);
 }
 
 int git_refspec_transform_r(git_buf *out, const git_refspec *spec, const char *name)
@@ -281,3 +288,70 @@ git_direction git_refspec_direction(const git_refspec *spec)
 
 	return spec->push;
 }
+
+int git_refspec__dwim_one(git_vector *out, git_refspec *spec, git_vector *refs)
+{
+	git_buf buf = GIT_BUF_INIT;
+	size_t j, pos;
+	git_remote_head key;
+
+	const char* formatters[] = {
+		GIT_REFS_DIR "%s",
+		GIT_REFS_TAGS_DIR "%s",
+		GIT_REFS_HEADS_DIR "%s",
+		NULL
+	};
+
+	git_refspec *cur = git__calloc(1, sizeof(git_refspec));
+	GITERR_CHECK_ALLOC(cur);
+
+	cur->force = spec->force;
+	cur->push = spec->push;
+	cur->pattern = spec->pattern;
+	cur->matching = spec->matching;
+	cur->string = git__strdup(spec->string);
+
+	/* shorthand on the lhs */
+	if (git__prefixcmp(spec->src, GIT_REFS_DIR)) {
+		for (j = 0; formatters[j]; j++) {
+			git_buf_clear(&buf);
+			if (git_buf_printf(&buf, formatters[j], spec->src) < 0)
+				return -1;
+
+			key.name = (char *) git_buf_cstr(&buf);
+			if (!git_vector_search(&pos, refs, &key)) {
+				/* we found something to match the shorthand, set src to that */
+				cur->src = git_buf_detach(&buf);
+			}
+		}
+	}
+
+	/* No shorthands found, copy over the name */
+	if (cur->src == NULL && spec->src != NULL) {
+		cur->src = git__strdup(spec->src);
+		GITERR_CHECK_ALLOC(cur->src);
+	}
+
+	if (spec->dst && git__prefixcmp(spec->dst, GIT_REFS_DIR)) {
+		/* if it starts with "remotes" then we just prepend "refs/" */
+		if (!git__prefixcmp(spec->dst, "remotes/")) {
+			git_buf_puts(&buf, GIT_REFS_DIR);
+		} else {
+			git_buf_puts(&buf, GIT_REFS_HEADS_DIR);
+		}
+
+		if (git_buf_puts(&buf, spec->dst) < 0)
+			return -1;
+
+		cur->dst = git_buf_detach(&buf);
+	}
+
+	git_buf_free(&buf);
+
+	if (cur->dst == NULL && spec->dst != NULL) {
+		cur->dst = git__strdup(spec->dst);
+		GITERR_CHECK_ALLOC(cur->dst);
+	}
+
+	return git_vector_insert(out, cur);
+}
diff --git a/src/refspec.h b/src/refspec.h
index 44d484c7b..51b7bfee9 100644
--- a/src/refspec.h
+++ b/src/refspec.h
@@ -9,6 +9,7 @@
 
 #include "git2/refspec.h"
 #include "buffer.h"
+#include "vector.h"
 
 struct git_refspec {
 	char *string;
@@ -17,7 +18,6 @@ struct git_refspec {
 	unsigned int force :1,
 		push : 1,
 		pattern :1,
-		dwim :1,
 		matching :1;
 };
 
@@ -63,4 +63,10 @@ int git_refspec__serialize(git_buf *out, const git_refspec *refspec);
  */
 int git_refspec_is_wildcard(const git_refspec *spec);
 
+/**
+ * DWIM `spec` with `refs` existing on the remote, append the dwim'ed
+ * result in `out`.
+ */
+int git_refspec__dwim_one(git_vector *out, git_refspec *spec, git_vector *refs);
+
 #endif
diff --git a/src/remote.c b/src/remote.c
index 0e8354a11..3d890a5f1 100644
--- a/src/remote.c
+++ b/src/remote.c
@@ -10,6 +10,7 @@
 #include "git2/oid.h"
 #include "git2/net.h"
 
+#include "common.h"
 #include "config.h"
 #include "repository.h"
 #include "remote.h"
@@ -18,7 +19,7 @@
 #include "refspec.h"
 #include "fetchhead.h"
 
-#include <regex.h>
+static int dwim_refspecs(git_vector *out, git_vector *refspecs, git_vector *refs);
 
 static int add_refspec(git_remote *remote, const char *string, bool is_fetch)
 {
@@ -81,6 +82,37 @@ static int ensure_remote_name_is_valid(const char *name)
 	return error;
 }
 
+static int get_check_cert(int *out, git_repository *repo)
+{
+	git_config *cfg;
+	const char *val;
+	int error = 0;
+
+	assert(out && repo);
+
+	/* By default, we *DO* want to verify the certificate. */
+	*out = 1;
+
+	/* Go through the possible sources for SSL verification settings, from
+	 * most specific to least specific. */
+
+	/* GIT_SSL_NO_VERIFY environment variable */
+	if ((val = getenv("GIT_SSL_NO_VERIFY")) != NULL)
+		return git_config_parse_bool(out, val);
+
+	/* http.sslVerify config setting */
+	if ((error = git_repository_config__weakptr(&cfg, repo)) < 0)
+		return error;
+
+	if ((error = git_config_get_bool(out, cfg, "http.sslVerify")) == 0)
+		return 0;
+	else if (error != GIT_ENOTFOUND)
+		return error;
+
+	giterr_clear();
+	return 0;
+}
+
 static int create_internal(git_remote **out, git_repository *repo, const char *name, const char *url, const char *fetch)
 {
 	git_remote *remote;
@@ -94,9 +126,11 @@ static int create_internal(git_remote **out, git_repository *repo, const char *n
 	GITERR_CHECK_ALLOC(remote);
 
 	remote->repo = repo;
-	remote->check_cert = 1;
 	remote->update_fetchhead = 1;
 
+	if (get_check_cert(&remote->check_cert, repo) < 0)
+		goto on_error;
+
 	if (git_vector_init(&remote->refs, 32, NULL) < 0)
 		goto on_error;
 
@@ -183,6 +217,32 @@ on_error:
 	return -1;
 }
 
+int git_remote_create_with_fetchspec(git_remote **out, git_repository *repo, const char *name, const char *url, const char *fetch)
+{
+	git_remote *remote = NULL;
+	int error;
+
+	if ((error = ensure_remote_name_is_valid(name)) < 0)
+		return error;
+
+	if ((error = ensure_remote_doesnot_exist(repo, name)) < 0)
+		return error;
+
+	if (create_internal(&remote, repo, name, url, fetch) < 0)
+		goto on_error;
+
+	if (git_remote_save(remote) < 0)
+		goto on_error;
+
+	*out = remote;
+
+	return 0;
+
+on_error:
+	git_remote_free(remote);
+	return -1;
+}
+
 int git_remote_create_inmemory(git_remote **out, git_repository *repo, const char *fetch, const char *url)
 {
 	int error;
@@ -208,7 +268,8 @@ static int refspec_cb(const git_config_entry *entry, void *payload)
 }
 
 static int get_optional_config(
-	git_config *config, git_buf *buf, git_config_foreach_cb cb, void *payload)
+	bool *found, git_config *config, git_buf *buf,
+	git_config_foreach_cb cb, void *payload)
 {
 	int error = 0;
 	const char *key = git_buf_cstr(buf);
@@ -217,10 +278,13 @@ static int get_optional_config(
 		return -1;
 
 	if (cb != NULL)
-		error = git_config_get_multivar(config, key, NULL, cb, payload);
+		error = git_config_get_multivar_foreach(config, key, NULL, cb, payload);
 	else
 		error = git_config_get_string(payload, config, key);
 
+	if (found)
+		*found = !error;
+
 	if (error == GIT_ENOTFOUND) {
 		giterr_clear();
 		error = 0;
@@ -240,6 +304,7 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	int error = 0;
 	git_config *config;
 	struct refspec_cb_data data;
+	bool optional_setting_found = false, found;
 
 	assert(out && repo && name);
 
@@ -253,13 +318,16 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	GITERR_CHECK_ALLOC(remote);
 
 	memset(remote, 0x0, sizeof(git_remote));
-	remote->check_cert = 1;
 	remote->update_fetchhead = 1;
 	remote->name = git__strdup(name);
 	GITERR_CHECK_ALLOC(remote->name);
 
+	if ((error = get_check_cert(&remote->check_cert, repo)) < 0)
+		goto cleanup;
+
 	if ((git_vector_init(&remote->refs, 32, NULL) < 0) ||
-	    (git_vector_init(&remote->refspecs, 2, NULL))) {
+	    (git_vector_init(&remote->refspecs, 2, NULL) < 0) ||
+	    (git_vector_init(&remote->active_refspecs, 2, NULL) < 0)) {
 		error = -1;
 		goto cleanup;
 	}
@@ -269,27 +337,33 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 		goto cleanup;
 	}
 
-	if ((error = git_config_get_string(&val, config, git_buf_cstr(&buf))) < 0)
+	if ((error = get_optional_config(&found, config, &buf, NULL, (void *)&val)) < 0)
 		goto cleanup;
 
-	if (strlen(val) == 0) {
-		giterr_set(GITERR_INVALID, "Malformed remote '%s' - missing URL", name);
-		error = -1;
-		goto cleanup;
-	}
+	optional_setting_found |= found;
 
 	remote->repo = repo;
-	remote->url = git__strdup(val);
-	GITERR_CHECK_ALLOC(remote->url);
+
+	if (found && strlen(val) > 0) {
+		remote->url = git__strdup(val);
+		GITERR_CHECK_ALLOC(remote->url);
+	}
 
 	val = NULL;
 	git_buf_clear(&buf);
 	git_buf_printf(&buf, "remote.%s.pushurl", name);
 
-	if ((error = get_optional_config(config, &buf, NULL, (void *)&val)) < 0)
+	if ((error = get_optional_config(&found, config, &buf, NULL, (void *)&val)) < 0)
 		goto cleanup;
 
-	if (val) {
+	optional_setting_found |= found;
+
+	if (!optional_setting_found) {
+		error = GIT_ENOTFOUND;
+		goto cleanup;
+	}
+
+	if (found && strlen(val) > 0) {
 		remote->pushurl = git__strdup(val);
 		GITERR_CHECK_ALLOC(remote->pushurl);
 	}
@@ -299,19 +373,23 @@ int git_remote_load(git_remote **out, git_repository *repo, const char *name)
 	git_buf_clear(&buf);
 	git_buf_printf(&buf, "remote.%s.fetch", name);
 
-	if ((error = get_optional_config(config, &buf, refspec_cb, &data)) < 0)
+	if ((error = get_optional_config(NULL, config, &buf, refspec_cb, &data)) < 0)
 		goto cleanup;
 
 	data.fetch = false;
 	git_buf_clear(&buf);
 	git_buf_printf(&buf, "remote.%s.push", name);
 
-	if ((error = get_optional_config(config, &buf, refspec_cb, &data)) < 0)
+	if ((error = get_optional_config(NULL, config, &buf, refspec_cb, &data)) < 0)
 		goto cleanup;
 
 	if (download_tags_value(remote, config) < 0)
 		goto cleanup;
 
+	/* Move the data over to where the matching functions can find them */
+	if (dwim_refspecs(&remote->active_refspecs, &remote->refspecs, &remote->refs) < 0)
+		goto cleanup;
+
 	*out = remote;
 
 cleanup:
@@ -326,20 +404,22 @@ cleanup:
 static int update_config_refspec(const git_remote *remote, git_config *config, int direction)
 {
 	git_buf name = GIT_BUF_INIT;
-	int push;
+	unsigned int push;
 	const char *dir;
 	size_t i;
 	int error = 0;
+	const char *cname;
 
 	push = direction == GIT_DIRECTION_PUSH;
 	dir = push ? "push" : "fetch";
 
 	if (git_buf_printf(&name, "remote.%s.%s", remote->name, dir) < 0)
 		return -1;
+	cname = git_buf_cstr(&name);
 
 	/* Clear out the existing config */
 	while (!error)
-		error = git_config_delete_entry(config, git_buf_cstr(&name));
+		error = git_config_delete_multivar(config, cname, ".*");
 
 	if (error != GIT_ENOTFOUND)
 		return error;
@@ -350,8 +430,11 @@ static int update_config_refspec(const git_remote *remote, git_config *config, i
 		if (spec->push != push)
 			continue;
 
+		// "$^" is a unmatcheable regexp: it will not match anything at all, so
+		// all values will be considered new and we will not replace any
+		// present value.
 		if ((error = git_config_set_multivar(
-				config, git_buf_cstr(&name), "", spec->string)) < 0) {
+				config, cname, "$^", spec->string)) < 0) {
 			goto cleanup;
 		}
 	}
@@ -467,6 +550,12 @@ const char *git_remote_name(const git_remote *remote)
 	return remote->name;
 }
 
+git_repository *git_remote_owner(const git_remote *remote)
+{
+	assert(remote);
+	return remote->repo;
+}
+
 const char *git_remote_url(const git_remote *remote)
 {
 	assert(remote);
@@ -509,6 +598,8 @@ const char* git_remote__urlfordirection(git_remote *remote, int direction)
 {
 	assert(remote);
 
+	assert(direction == GIT_DIRECTION_FETCH || direction == GIT_DIRECTION_PUSH);
+
 	if (direction == GIT_DIRECTION_FETCH) {
 		return remote->url;
 	}
@@ -525,28 +616,32 @@ int git_remote_connect(git_remote *remote, git_direction direction)
 	git_transport *t;
 	const char *url;
 	int flags = GIT_TRANSPORTFLAGS_NONE;
+	int error;
 
 	assert(remote);
 
 	t = remote->transport;
 
 	url = git_remote__urlfordirection(remote, direction);
-	if (url == NULL )
+	if (url == NULL) {
+		giterr_set(GITERR_INVALID,
+			"Malformed remote '%s' - missing URL", remote->name);
 		return -1;
+	}
 
 	/* A transport could have been supplied in advance with
 	 * git_remote_set_transport */
-	if (!t && git_transport_new(&t, remote, url) < 0)
-		return -1;
+	if (!t && (error = git_transport_new(&t, remote, url)) < 0)
+		return error;
 
 	if (t->set_callbacks &&
-		t->set_callbacks(t, remote->callbacks.progress, NULL, remote->callbacks.payload) < 0)
+		(error = t->set_callbacks(t, remote->callbacks.progress, NULL, remote->callbacks.payload)) < 0)
 		goto on_error;
 
 	if (!remote->check_cert)
 		flags |= GIT_TRANSPORTFLAGS_NO_CHECK_CERT;
 
-	if (t->connect(t, url, remote->cred_acquire_cb, remote->cred_acquire_payload, direction, flags) < 0)
+	if ((error = t->connect(t, url, remote->callbacks.credentials, remote->callbacks.payload, direction, flags)) < 0)
 		goto on_error;
 
 	remote->transport = t;
@@ -559,20 +654,21 @@ on_error:
 	if (t == remote->transport)
 		remote->transport = NULL;
 
-	return -1;
+	return error;
 }
 
-int git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void *payload)
+int git_remote_ls(const git_remote_head ***out, size_t *size, git_remote *remote)
 {
 	assert(remote);
 
-	return remote->transport->ls(remote->transport, list_cb, payload);
+	return remote->transport->ls(out, size, remote->transport);
 }
 
 int git_remote__get_http_proxy(git_remote *remote, bool use_ssl, char **proxy_url)
 {
 	git_config *cfg;
 	const char *val;
+	int error;
 
 	assert(remote);
 
@@ -581,8 +677,8 @@ int git_remote__get_http_proxy(git_remote *remote, bool use_ssl, char **proxy_ur
 
 	*proxy_url = NULL;
 
-	if (git_repository_config__weakptr(&cfg, remote->repo) < 0)
-		return -1;
+	if ((error = git_repository_config__weakptr(&cfg, remote->repo)) < 0)
+		return error;
 
 	/* Go through the possible sources for proxy configuration, from most specific
 	 * to least specific. */
@@ -591,28 +687,33 @@ int git_remote__get_http_proxy(git_remote *remote, bool use_ssl, char **proxy_ur
 	if (remote->name && 0 != *(remote->name)) {
 		git_buf buf = GIT_BUF_INIT;
 
-		if (git_buf_printf(&buf, "remote.%s.proxy", remote->name) < 0)
-			return -1;
+		if ((error = git_buf_printf(&buf, "remote.%s.proxy", remote->name)) < 0)
+			return error;
 
-		if (!git_config_get_string(&val, cfg, git_buf_cstr(&buf)) &&
+		if ((error = git_config_get_string(&val, cfg, git_buf_cstr(&buf))) == 0 &&
 			val && ('\0' != *val)) {
 			git_buf_free(&buf);
 
 			*proxy_url = git__strdup(val);
 			GITERR_CHECK_ALLOC(*proxy_url);
 			return 0;
-		}
+		} else if (error != GIT_ENOTFOUND)
+			return error;
 
+		giterr_clear();
 		git_buf_free(&buf);
 	}
 
 	/* http.proxy config setting */
-	if (!git_config_get_string(&val, cfg, "http.proxy") &&
+	if ((error = git_config_get_string(&val, cfg, "http.proxy")) == 0 &&
 		val && ('\0' != *val)) {
 		*proxy_url = git__strdup(val);
 		GITERR_CHECK_ALLOC(*proxy_url);
 		return 0;
-	}
+	} else if (error != GIT_ENOTFOUND)
+		return error;
+
+	giterr_clear();
 
 	/* HTTP_PROXY / HTTPS_PROXY environment variables */
 	val = use_ssl ? getenv("HTTPS_PROXY") : getenv("HTTP_PROXY");
@@ -626,67 +727,31 @@ int git_remote__get_http_proxy(git_remote *remote, bool use_ssl, char **proxy_ur
 	return 0;
 }
 
-static int store_refs(git_remote_head *head, void *payload)
+/* DWIM `refspecs` based on `refs` and append the output to `out` */
+static int dwim_refspecs(git_vector *out, git_vector *refspecs, git_vector *refs)
 {
-	git_vector *refs = (git_vector *)payload;
-
-	return git_vector_insert(refs, head);
-}
-
-static int dwim_refspecs(git_vector *refspecs, git_vector *refs)
-{
-	git_buf buf = GIT_BUF_INIT;
+	size_t i;
 	git_refspec *spec;
-	size_t i, j, pos;
-	git_remote_head key;
-
-	const char* formatters[] = {
-		GIT_REFS_DIR "%s",
-		GIT_REFS_TAGS_DIR "%s",
-		GIT_REFS_HEADS_DIR "%s",
-		NULL
-	};
 
 	git_vector_foreach(refspecs, i, spec) {
-		if (spec->dwim)
-			continue;
-
-		/* shorthand on the lhs */
-		if (git__prefixcmp(spec->src, GIT_REFS_DIR)) {
-			for (j = 0; formatters[j]; j++) {
-				git_buf_clear(&buf);
-				if (git_buf_printf(&buf, formatters[j], spec->src) < 0)
-					return -1;
-
-				key.name = (char *) git_buf_cstr(&buf);
-				if (!git_vector_search(&pos, refs, &key)) {
-					/* we found something to match the shorthand, set src to that */
-					git__free(spec->src);
-					spec->src = git_buf_detach(&buf);
-				}
-			}
-		}
-
-		if (spec->dst && git__prefixcmp(spec->dst, GIT_REFS_DIR)) {
-			/* if it starts with "remotes" then we just prepend "refs/" */
-			if (!git__prefixcmp(spec->dst, "remotes/")) {
-				git_buf_puts(&buf, GIT_REFS_DIR);
-			} else {
-				git_buf_puts(&buf, GIT_REFS_HEADS_DIR);
-			}
+		if (git_refspec__dwim_one(out, spec, refs) < 0)
+			return -1;
+	}
 
-			if (git_buf_puts(&buf, spec->dst) < 0)
-				return -1;
+	return 0;
+}
 
-			git__free(spec->dst);
-			spec->dst = git_buf_detach(&buf);
-		}
+static void free_refspecs(git_vector *vec)
+{
+	size_t i;
+	git_refspec *spec;
 
-		spec->dwim = 1;
+	git_vector_foreach(vec, i, spec) {
+		git_refspec__free(spec);
+		git__free(spec);
 	}
 
-	git_buf_free(&buf);
-	return 0;
+	git_vector_clear(vec);
 }
 
 static int remote_head_cmp(const void *_a, const void *_b)
@@ -697,32 +762,65 @@ static int remote_head_cmp(const void *_a, const void *_b)
 	return git__strcmp_cb(a->name, b->name);
 }
 
-int git_remote_download(
-		git_remote *remote,
-		git_transfer_progress_callback progress_cb,
-		void *progress_payload)
+static int ls_to_vector(git_vector *out, git_remote *remote)
+{
+	git_remote_head **heads;
+	size_t heads_len, i;
+
+	if (git_remote_ls((const git_remote_head ***)&heads, &heads_len, remote) < 0)
+		return -1;
+
+	if (git_vector_init(out, heads_len, remote_head_cmp) < 0)
+		return -1;
+
+	for (i = 0; i < heads_len; i++) {
+		if (git_vector_insert(out, heads[i]) < 0)
+			return -1;
+	}
+
+	return 0;
+}
+
+int git_remote_download(git_remote *remote)
 {
 	int error;
 	git_vector refs;
 
 	assert(remote);
 
-	if (git_vector_init(&refs, 16, remote_head_cmp) < 0)
+	if (ls_to_vector(&refs, remote) < 0)
 		return -1;
 
-	if (git_remote_ls(remote, store_refs, &refs) < 0) {
-		return -1;
-	}
+	free_refspecs(&remote->active_refspecs);
 
-	error = dwim_refspecs(&remote->refspecs, &refs);
+	error = dwim_refspecs(&remote->active_refspecs, &remote->refspecs, &refs);
 	git_vector_free(&refs);
+
 	if (error < 0)
 		return -1;
 
 	if ((error = git_fetch_negotiate(remote)) < 0)
 		return error;
 
-	return git_fetch_download_pack(remote, progress_cb, progress_payload);
+	return git_fetch_download_pack(remote);
+}
+
+int git_remote_fetch(git_remote *remote)
+{
+	int error;
+
+	/* Connect and download everything */
+	if ((error = git_remote_connect(remote, GIT_DIRECTION_FETCH)) < 0)
+		return error;
+
+	if ((error = git_remote_download(remote)) < 0)
+		return error;
+
+	/* We don't need to be connected anymore */
+	git_remote_disconnect(remote);
+
+	/* Create "remote/foo" branches for all remote branches */
+	return git_remote_update_tips(remote);
 }
 
 static int remote_head_for_fetchspec_src(git_remote_head **out, git_vector *update_heads, const char *fetchspec_src)
@@ -759,7 +857,7 @@ static int remote_head_for_ref(git_remote_head **out, git_refspec *spec, git_vec
 		(!git_reference_is_branch(resolved_ref)) ||
 		(error = git_branch_upstream(&tracking_ref, resolved_ref)) < 0 ||
 		(error = git_refspec_transform_l(&remote_name, spec, git_reference_name(tracking_ref))) < 0) {
-		/* Not an error if HEAD is orphaned or no tracking branch */
+		/* Not an error if HEAD is unborn or no tracking branch */
 		if (error == GIT_ENOTFOUND)
 			error = 0;
 
@@ -947,10 +1045,8 @@ int git_remote_update_tips(git_remote *remote)
 	if (git_refspec__parse(&tagspec, GIT_REFSPEC_TAGS, true) < 0)
 		return -1;
 
-	if (git_vector_init(&refs, 16, NULL) < 0)
-		return -1;
 
-	if ((error = git_remote_ls(remote, store_refs, &refs)) < 0)
+	if ((error = ls_to_vector(&refs, remote)) < 0)
 		goto out;
 
 	if (remote->download_tags == GIT_REMOTE_DOWNLOAD_TAGS_ALL) {
@@ -958,7 +1054,7 @@ int git_remote_update_tips(git_remote *remote)
 		goto out;
 	}
 
-	git_vector_foreach(&remote->refspecs, i, spec) {
+	git_vector_foreach(&remote->active_refspecs, i, spec) {
 		if (spec->push)
 			continue;
 
@@ -967,8 +1063,8 @@ int git_remote_update_tips(git_remote *remote)
 	}
 
 out:
-	git_refspec__free(&tagspec);
 	git_vector_free(&refs);
+	git_refspec__free(&tagspec);
 	return error;
 }
 
@@ -1001,9 +1097,6 @@ void git_remote_disconnect(git_remote *remote)
 
 void git_remote_free(git_remote *remote)
 {
-	git_refspec *spec;
-	size_t i;
-
 	if (remote == NULL)
 		return;
 
@@ -1016,67 +1109,55 @@ void git_remote_free(git_remote *remote)
 
 	git_vector_free(&remote->refs);
 
-	git_vector_foreach(&remote->refspecs, i, spec) {
-		git_refspec__free(spec);
-		git__free(spec);
-	}
+	free_refspecs(&remote->refspecs);
 	git_vector_free(&remote->refspecs);
 
+	free_refspecs(&remote->active_refspecs);
+	git_vector_free(&remote->active_refspecs);
+
 	git__free(remote->url);
 	git__free(remote->pushurl);
 	git__free(remote->name);
 	git__free(remote);
 }
 
-struct cb_data {
-	git_vector *list;
-	regex_t *preg;
-};
-
-static int remote_list_cb(const git_config_entry *entry, void *data_)
+static int remote_list_cb(const git_config_entry *entry, void *payload)
 {
-	struct cb_data *data = (struct cb_data *)data_;
-	size_t nmatch = 2;
-	regmatch_t pmatch[2];
-	const char *name = entry->name;
+	git_vector *list = payload;
+	const char *name = entry->name + strlen("remote.");
+	size_t namelen = strlen(name);
+	char *remote_name;
 
-	if (!regexec(data->preg, name, nmatch, pmatch, 0)) {
-		char *remote_name = git__strndup(&name[pmatch[1].rm_so], pmatch[1].rm_eo - pmatch[1].rm_so);
-		GITERR_CHECK_ALLOC(remote_name);
+	/* we know name matches "remote.<stuff>.(push)?url" */
 
-		if (git_vector_insert(data->list, remote_name) < 0)
-			return -1;
-	}
+	if (!strcmp(&name[namelen - 4], ".url"))
+		remote_name = git__strndup(name, namelen - 4); /* strip ".url" */
+	else
+		remote_name = git__strndup(name, namelen - 8); /* strip ".pushurl" */
+	GITERR_CHECK_ALLOC(remote_name);
 
-	return 0;
+	return git_vector_insert(list, remote_name);
 }
 
 int git_remote_list(git_strarray *remotes_list, git_repository *repo)
 {
 	git_config *cfg;
 	git_vector list;
-	regex_t preg;
-	struct cb_data data;
 	int error;
 
 	if (git_repository_config__weakptr(&cfg, repo) < 0)
 		return -1;
 
-	if (git_vector_init(&list, 4, NULL) < 0)
+	if (git_vector_init(&list, 4, git__strcmp_cb) < 0)
 		return -1;
 
-	if (regcomp(&preg, "^remote\\.(.*)\\.url$", REG_EXTENDED) < 0) {
-		giterr_set(GITERR_OS, "Remote catch regex failed to compile");
-		return -1;
-	}
+	error = git_config_foreach_match(
+		cfg, "^remote\\..*\\.(push)?url$", remote_list_cb, &list);
 
-	data.list = &list;
-	data.preg = &preg;
-	error = git_config_foreach(cfg, remote_list_cb, &data);
-	regfree(&preg);
 	if (error < 0) {
 		size_t i;
 		char *elem;
+
 		git_vector_foreach(&list, i, elem) {
 			git__free(elem);
 		}
@@ -1090,6 +1171,8 @@ int git_remote_list(git_strarray *remotes_list, git_repository *repo)
 		return error;
 	}
 
+	git_vector_uniq(&list, git__free);
+
 	remotes_list->strings = (char **)list.contents;
 	remotes_list->count = list.length;
 
@@ -1103,7 +1186,7 @@ void git_remote_check_cert(git_remote *remote, int check)
 	remote->check_cert = check;
 }
 
-int git_remote_set_callbacks(git_remote *remote, git_remote_callbacks *callbacks)
+int git_remote_set_callbacks(git_remote *remote, const git_remote_callbacks *callbacks)
 {
 	assert(remote && callbacks);
 
@@ -1112,7 +1195,7 @@ int git_remote_set_callbacks(git_remote *remote, git_remote_callbacks *callbacks
 	memcpy(&remote->callbacks, callbacks, sizeof(git_remote_callbacks));
 
 	if (remote->transport && remote->transport->set_callbacks)
-		remote->transport->set_callbacks(remote->transport,
+		return remote->transport->set_callbacks(remote->transport,
 			remote->callbacks.progress,
 			NULL,
 			remote->callbacks.payload);
@@ -1120,17 +1203,6 @@ int git_remote_set_callbacks(git_remote *remote, git_remote_callbacks *callbacks
 	return 0;
 }
 
-void git_remote_set_cred_acquire_cb(
-	git_remote *remote,
-	git_cred_acquire_cb cred_acquire_cb,
-	void *payload)
-{
-	assert(remote);
-
-	remote->cred_acquire_cb = cred_acquire_cb;
-	remote->cred_acquire_payload = payload;
-}
-
 int git_remote_set_transport(git_remote *remote, git_transport *transport)
 {
 	assert(remote && transport);
@@ -1418,12 +1490,12 @@ int git_remote_rename(
 
 int git_remote_update_fetchhead(git_remote *remote)
 {
-	return remote->update_fetchhead;
+	return (remote->update_fetchhead != 0);
 }
 
 void git_remote_set_update_fetchhead(git_remote *remote, int value)
 {
-	remote->update_fetchhead = value;
+	remote->update_fetchhead = (value != 0);
 }
 
 int git_remote_is_valid_name(
@@ -1451,7 +1523,7 @@ git_refspec *git_remote__matching_refspec(git_remote *remote, const char *refnam
 	git_refspec *spec;
 	size_t i;
 
-	git_vector_foreach(&remote->refspecs, i, spec) {
+	git_vector_foreach(&remote->active_refspecs, i, spec) {
 		if (spec->push)
 			continue;
 
@@ -1467,7 +1539,7 @@ git_refspec *git_remote__matching_dst_refspec(git_remote *remote, const char *re
 	git_refspec *spec;
 	size_t i;
 
-	git_vector_foreach(&remote->refspecs, i, spec) {
+	git_vector_foreach(&remote->active_refspecs, i, spec) {
 		if (spec->push)
 			continue;
 
@@ -1490,17 +1562,71 @@ void git_remote_clear_refspecs(git_remote *remote)
 	git_vector_clear(&remote->refspecs);
 }
 
+static int add_and_dwim(git_remote *remote, const char *str, int push)
+{
+	git_refspec *spec;
+	git_vector *vec;
+
+	if (add_refspec(remote, str, !push) < 0)
+		return -1;
+
+	vec = &remote->refspecs;
+	spec = git_vector_get(vec, vec->length - 1);
+	return git_refspec__dwim_one(&remote->active_refspecs, spec, &remote->refs);
+}
+
 int git_remote_add_fetch(git_remote *remote, const char *refspec)
 {
-	return add_refspec(remote, refspec, true);
+	return add_and_dwim(remote, refspec, false);
 }
 
 int git_remote_add_push(git_remote *remote, const char *refspec)
 {
-	return add_refspec(remote, refspec, false);
+	return add_and_dwim(remote, refspec, true);
 }
 
-static int copy_refspecs(git_strarray *array, git_remote *remote, int push)
+static int set_refspecs(git_remote *remote, git_strarray *array, int push)
+{
+	git_vector *vec = &remote->refspecs;
+	git_refspec *spec;
+	size_t i;
+
+	/* Start by removing any refspecs of the same type */
+	for (i = 0; i < vec->length; i++) {
+		spec = git_vector_get(vec, i);
+		if (spec->push != push)
+			continue;
+
+		git_refspec__free(spec);
+		git__free(spec);
+		git_vector_remove(vec, i);
+		i--;
+	}
+
+	/* And now we add the new ones */
+
+	for (i = 0; i < array->count; i++) {
+		if (add_refspec(remote, array->strings[i], !push) < 0)
+			return -1;
+	}
+
+	free_refspecs(&remote->active_refspecs);
+	git_vector_clear(&remote->active_refspecs);
+
+	return dwim_refspecs(&remote->active_refspecs, &remote->refspecs, &remote->refs);
+}
+
+int git_remote_set_fetch_refspecs(git_remote *remote, git_strarray *array)
+{
+	return set_refspecs(remote, array, false);
+}
+
+int git_remote_set_push_refspecs(git_remote *remote, git_strarray *array)
+{
+	return set_refspecs(remote, array, true);
+}
+
+static int copy_refspecs(git_strarray *array, git_remote *remote, unsigned int push)
 {
 	size_t i;
 	git_vector refspecs;
@@ -1555,18 +1681,3 @@ const git_refspec *git_remote_get_refspec(git_remote *remote, size_t n)
 {
 	return git_vector_get(&remote->refspecs, n);
 }
-
-int git_remote_remove_refspec(git_remote *remote, size_t n)
-{
-	git_refspec *spec;
-
-	assert(remote);
-
-	spec = git_vector_get(&remote->refspecs, n);
-	if (spec) {
-		git_refspec__free(spec);
-		git__free(spec);
-	}
-
-	return git_vector_remove(&remote->refspecs, n);
-}
diff --git a/src/remote.h b/src/remote.h
index dce4803ed..4164a14b3 100644
--- a/src/remote.h
+++ b/src/remote.h
@@ -21,16 +21,15 @@ struct git_remote {
 	char *pushurl;
 	git_vector refs;
 	git_vector refspecs;
-	git_cred_acquire_cb cred_acquire_cb;
-	void *cred_acquire_payload;
+	git_vector active_refspecs;
 	git_transport *transport;
 	git_repository *repo;
 	git_remote_callbacks callbacks;
 	git_transfer_progress stats;
 	unsigned int need_pack;
 	git_remote_autotag_option_t download_tags;
-	unsigned int check_cert;
-	unsigned int update_fetchhead;
+	int check_cert;
+	int update_fetchhead;
 };
 
 const char* git_remote__urlfordirection(struct git_remote *remote, int direction);
diff --git a/src/repository.c b/src/repository.c
index ed9469c59..dcc02e4fb 100644
--- a/src/repository.c
+++ b/src/repository.c
@@ -33,8 +33,6 @@
 
 #define GIT_REPO_VERSION 0
 
-#define GIT_TEMPLATE_DIR "/usr/share/git-core/templates"
-
 static void set_odb(git_repository *repo, git_odb *odb)
 {
 	if (odb) {
@@ -266,7 +264,7 @@ static int find_ceiling_dir_offset(
 			buf[--len] = '\0';
 
 		if (!strncmp(path, buf2, len) &&
-			path[len] == '/' &&
+			(path[len] == '/' || !path[len]) &&
 			len > max_len)
 		{
 			max_len = len;
@@ -322,17 +320,18 @@ static int find_repo(
 	git_buf path = GIT_BUF_INIT;
 	struct stat st;
 	dev_t initial_device = 0;
-	bool try_with_dot_git = false;
+	bool try_with_dot_git = ((flags & GIT_REPOSITORY_OPEN_BARE) != 0);
 	int ceiling_offset;
 
 	git_buf_free(repo_path);
 
-	if ((error = git_path_prettify_dir(&path, start_path, NULL)) < 0)
+	if ((error = git_path_prettify(&path, start_path, NULL)) < 0)
 		return error;
 
 	ceiling_offset = find_ceiling_dir_offset(path.ptr, ceiling_dirs);
 
-	if ((error = git_buf_joinpath(&path, path.ptr, DOT_GIT)) < 0)
+	if (!try_with_dot_git &&
+		(error = git_buf_joinpath(&path, path.ptr, DOT_GIT)) < 0)
 		return error;
 
 	while (!error && !git_buf_len(repo_path)) {
@@ -384,7 +383,7 @@ static int find_repo(
 		try_with_dot_git = !try_with_dot_git;
 	}
 
-	if (!error && parent_path != NULL) {
+	if (!error && parent_path && !(flags & GIT_REPOSITORY_OPEN_BARE)) {
 		if (!git_buf_len(repo_path))
 			git_buf_clear(parent_path);
 		else {
@@ -460,7 +459,9 @@ int git_repository_open_ext(
 	repo->path_repository = git_buf_detach(&path);
 	GITERR_CHECK_ALLOC(repo->path_repository);
 
-	if ((error = load_config_data(repo)) < 0 ||
+	if ((flags & GIT_REPOSITORY_OPEN_BARE) != 0)
+		repo->is_bare = 1;
+	else if ((error = load_config_data(repo)) < 0 ||
 		(error = load_workdir(repo, &parent)) < 0)
 	{
 		git_repository_free(repo);
@@ -815,7 +816,7 @@ static int repo_init_create_head(const char *git_dir, const char *ref_name)
 	const char *fmt;
 
 	if (git_buf_joinpath(&ref_path, git_dir, GIT_HEAD_FILE) < 0 ||
-		git_filebuf_open(&ref, ref_path.ptr, 0) < 0)
+		git_filebuf_open(&ref, ref_path.ptr, 0, GIT_REFS_FILE_MODE) < 0)
 		goto fail;
 
 	if (!ref_name)
@@ -827,7 +828,7 @@ static int repo_init_create_head(const char *git_dir, const char *ref_name)
 		fmt = "ref: " GIT_REFS_HEADS_DIR "%s\n";
 
 	if (git_filebuf_printf(&ref, fmt, ref_name) < 0 ||
-		git_filebuf_commit(&ref, GIT_REFS_FILE_MODE) < 0)
+		git_filebuf_commit(&ref) < 0)
 		goto fail;
 
 	git_buf_free(&ref_path);
@@ -842,10 +843,6 @@ fail:
 static bool is_chmod_supported(const char *file_path)
 {
 	struct stat st1, st2;
-	static int _is_supported = -1;
-
-	if (_is_supported > -1)
-		return _is_supported;
 
 	if (p_stat(file_path, &st1) < 0)
 		return false;
@@ -856,27 +853,19 @@ static bool is_chmod_supported(const char *file_path)
 	if (p_stat(file_path, &st2) < 0)
 		return false;
 
-	_is_supported = (st1.st_mode != st2.st_mode);
-
-	return _is_supported;
+	return (st1.st_mode != st2.st_mode);
 }
 
 static bool is_filesystem_case_insensitive(const char *gitdir_path)
 {
 	git_buf path = GIT_BUF_INIT;
-	static int _is_insensitive = -1;
-
-	if (_is_insensitive > -1)
-		return _is_insensitive;
+	int is_insensitive = -1;
 
-	if (git_buf_joinpath(&path, gitdir_path, "CoNfIg") < 0)
-		goto cleanup;
-
-	_is_insensitive = git_path_exists(git_buf_cstr(&path));
+	if (!git_buf_joinpath(&path, gitdir_path, "CoNfIg"))
+		is_insensitive = git_path_exists(git_buf_cstr(&path));
 
-cleanup:
 	git_buf_free(&path);
-	return _is_insensitive;
+	return is_insensitive;
 }
 
 static bool are_symlinks_supported(const char *wd_path)
@@ -884,26 +873,77 @@ static bool are_symlinks_supported(const char *wd_path)
 	git_buf path = GIT_BUF_INIT;
 	int fd;
 	struct stat st;
-	static int _symlinks_supported = -1;
-
-	if (_symlinks_supported > -1)
-		return _symlinks_supported;
+	int symlinks_supported = -1;
 
-	if ((fd = git_futils_mktmp(&path, wd_path)) < 0 ||
+	if ((fd = git_futils_mktmp(&path, wd_path, 0666)) < 0 ||
 		p_close(fd) < 0 ||
 		p_unlink(path.ptr) < 0 ||
 		p_symlink("testing", path.ptr) < 0 ||
 		p_lstat(path.ptr, &st) < 0)
-		_symlinks_supported = false;
+		symlinks_supported = false;
 	else
-		_symlinks_supported = (S_ISLNK(st.st_mode) != 0);
+		symlinks_supported = (S_ISLNK(st.st_mode) != 0);
 
 	(void)p_unlink(path.ptr);
 	git_buf_free(&path);
 
-	return _symlinks_supported;
+	return symlinks_supported;
+}
+
+#ifdef GIT_USE_ICONV
+
+static const char *nfc_file = "\xC3\x85\x73\x74\x72\xC3\xB6\x6D.XXXXXX";
+static const char *nfd_file = "\x41\xCC\x8A\x73\x74\x72\x6F\xCC\x88\x6D.XXXXXX";
+
+/* Check if the platform is decomposing unicode data for us.  We will
+ * emulate core Git and prefer to use precomposed unicode data internally
+ * on these platforms, composing the decomposed unicode on the fly.
+ *
+ * This mainly happens on the Mac where HDFS stores filenames as
+ * decomposed unicode.  Even on VFAT and SAMBA file systems, the Mac will
+ * return decomposed unicode from readdir() even when the actual
+ * filesystem is storing precomposed unicode.
+ */
+static bool does_fs_decompose_unicode_paths(const char *wd_path)
+{
+	git_buf path = GIT_BUF_INIT;
+	int fd;
+	bool found_decomposed = false;
+	char tmp[6];
+
+	/* Create a file using a precomposed path and then try to find it
+	 * using the decomposed name.  If the lookup fails, then we will mark
+	 * that we should precompose unicode for this repository.
+	 */
+	if (git_buf_joinpath(&path, wd_path, nfc_file) < 0 ||
+		(fd = p_mkstemp(path.ptr)) < 0)
+		goto done;
+	p_close(fd);
+
+	/* record trailing digits generated by mkstemp */
+	memcpy(tmp, path.ptr + path.size - sizeof(tmp), sizeof(tmp));
+
+	/* try to look up as NFD path */
+	if (git_buf_joinpath(&path, wd_path, nfd_file) < 0)
+		goto done;
+	memcpy(path.ptr + path.size - sizeof(tmp), tmp, sizeof(tmp));
+
+	found_decomposed = git_path_exists(path.ptr);
+
+	/* remove temporary file (using original precomposed path) */
+	if (git_buf_joinpath(&path, wd_path, nfc_file) < 0)
+		goto done;
+	memcpy(path.ptr + path.size - sizeof(tmp), tmp, sizeof(tmp));
+
+	(void)p_unlink(path.ptr);
+
+done:
+	git_buf_free(&path);
+	return found_decomposed;
 }
 
+#endif
+
 static int create_empty_file(const char *path, mode_t mode)
 {
 	int fd;
@@ -921,71 +961,131 @@ static int create_empty_file(const char *path, mode_t mode)
 	return 0;
 }
 
+static int repo_local_config(
+	git_config **out,
+	git_buf *config_dir,
+	git_repository *repo,
+	const char *repo_dir)
+{
+	int error = 0;
+	git_config *parent;
+	const char *cfg_path;
+
+	if (git_buf_joinpath(config_dir, repo_dir, GIT_CONFIG_FILENAME_INREPO) < 0)
+		return -1;
+	cfg_path = git_buf_cstr(config_dir);
+
+	/* make LOCAL config if missing */
+	if (!git_path_isfile(cfg_path) &&
+		(error = create_empty_file(cfg_path, GIT_CONFIG_FILE_MODE)) < 0)
+		return error;
+
+	/* if no repo, just open that file directly */
+	if (!repo)
+		return git_config_open_ondisk(out, cfg_path);
+
+	/* otherwise, open parent config and get that level */
+	if ((error = git_repository_config__weakptr(&parent, repo)) < 0)
+		return error;
+
+	if (git_config_open_level(out, parent, GIT_CONFIG_LEVEL_LOCAL) < 0) {
+		giterr_clear();
+
+		if (!(error = git_config_add_file_ondisk(
+				parent, cfg_path, GIT_CONFIG_LEVEL_LOCAL, false)))
+			error = git_config_open_level(out, parent, GIT_CONFIG_LEVEL_LOCAL);
+	}
+
+	git_config_free(parent);
+
+	return error;
+}
+
+static int repo_init_fs_configs(
+	git_config *cfg,
+	const char *cfg_path,
+	const char *repo_dir,
+	const char *work_dir,
+	bool update_ignorecase)
+{
+	int error = 0;
+
+	if (!work_dir)
+		work_dir = repo_dir;
+
+	if ((error = git_config_set_bool(
+			cfg, "core.filemode", is_chmod_supported(cfg_path))) < 0)
+		return error;
+
+	if (!are_symlinks_supported(work_dir)) {
+		if ((error = git_config_set_bool(cfg, "core.symlinks", false)) < 0)
+			return error;
+	} else if (git_config_delete_entry(cfg, "core.symlinks") < 0)
+		giterr_clear();
+
+	if (update_ignorecase) {
+		if (is_filesystem_case_insensitive(repo_dir)) {
+			if ((error = git_config_set_bool(cfg, "core.ignorecase", true)) < 0)
+				return error;
+		} else if (git_config_delete_entry(cfg, "core.ignorecase") < 0)
+			giterr_clear();
+	}
+
+#ifdef GIT_USE_ICONV
+	if ((error = git_config_set_bool(
+			cfg, "core.precomposeunicode",
+			does_fs_decompose_unicode_paths(work_dir))) < 0)
+		return error;
+#endif
+
+	return 0;
+}
+
 static int repo_init_config(
 	const char *repo_dir,
 	const char *work_dir,
-	git_repository_init_options *opts)
+	uint32_t flags,
+	uint32_t mode)
 {
 	int error = 0;
 	git_buf cfg_path = GIT_BUF_INIT;
 	git_config *config = NULL;
+	bool is_bare = ((flags & GIT_REPOSITORY_INIT_BARE) != 0);
+	bool is_reinit = ((flags & GIT_REPOSITORY_INIT__IS_REINIT) != 0);
 
-#define SET_REPO_CONFIG(TYPE, NAME, VAL) do {\
-	if ((error = git_config_set_##TYPE(config, NAME, VAL)) < 0) \
-		goto cleanup; } while (0)
+	if ((error = repo_local_config(&config, &cfg_path, NULL, repo_dir)) < 0)
+		goto cleanup;
 
-	if (git_buf_joinpath(&cfg_path, repo_dir, GIT_CONFIG_FILENAME_INREPO) < 0)
-		return -1;
+	if (is_reinit && (error = check_repositoryformatversion(config)) < 0)
+		goto cleanup;
 
-	if (!git_path_isfile(git_buf_cstr(&cfg_path)) &&
-		create_empty_file(git_buf_cstr(&cfg_path), GIT_CONFIG_FILE_MODE) < 0) {
-			git_buf_free(&cfg_path);
-			return -1;
-	}
+#define SET_REPO_CONFIG(TYPE, NAME, VAL) do { \
+	if ((error = git_config_set_##TYPE(config, NAME, VAL)) < 0) \
+		goto cleanup; } while (0)
 
-	if (git_config_open_ondisk(&config, git_buf_cstr(&cfg_path)) < 0) {
-		git_buf_free(&cfg_path);
-		return -1;
-	}
+	SET_REPO_CONFIG(bool, "core.bare", is_bare);
+	SET_REPO_CONFIG(int32, "core.repositoryformatversion", GIT_REPO_VERSION);
 
-	if ((opts->flags & GIT_REPOSITORY_INIT__IS_REINIT) != 0 &&
-		(error = check_repositoryformatversion(config)) < 0)
+	if ((error = repo_init_fs_configs(
+			config, cfg_path.ptr, repo_dir, work_dir, !is_reinit)) < 0)
 		goto cleanup;
 
-	SET_REPO_CONFIG(
-		bool, "core.bare", (opts->flags & GIT_REPOSITORY_INIT_BARE) != 0);
-	SET_REPO_CONFIG(
-		int32, "core.repositoryformatversion", GIT_REPO_VERSION);
-	SET_REPO_CONFIG(
-		bool, "core.filemode", is_chmod_supported(git_buf_cstr(&cfg_path)));
-
-	if (!(opts->flags & GIT_REPOSITORY_INIT_BARE)) {
+	if (!is_bare) {
 		SET_REPO_CONFIG(bool, "core.logallrefupdates", true);
 
-		if (!are_symlinks_supported(work_dir))
-			SET_REPO_CONFIG(bool, "core.symlinks", false);
-
-		if (!(opts->flags & GIT_REPOSITORY_INIT__NATURAL_WD)) {
+		if (!(flags & GIT_REPOSITORY_INIT__NATURAL_WD))
 			SET_REPO_CONFIG(string, "core.worktree", work_dir);
-		}
-		else if ((opts->flags & GIT_REPOSITORY_INIT__IS_REINIT) != 0) {
+		else if (is_reinit) {
 			if (git_config_delete_entry(config, "core.worktree") < 0)
 				giterr_clear();
 		}
-	} else {
-		if (!are_symlinks_supported(repo_dir))
-			SET_REPO_CONFIG(bool, "core.symlinks", false);
 	}
 
-	if (!(opts->flags & GIT_REPOSITORY_INIT__IS_REINIT) &&
-		is_filesystem_case_insensitive(repo_dir))
-		SET_REPO_CONFIG(bool, "core.ignorecase", true);
-
-	if (opts->mode == GIT_REPOSITORY_INIT_SHARED_GROUP) {
+	if (mode == GIT_REPOSITORY_INIT_SHARED_GROUP) {
 		SET_REPO_CONFIG(int32, "core.sharedrepository", 1);
 		SET_REPO_CONFIG(bool, "receive.denyNonFastforwards", true);
 	}
-	else if (opts->mode == GIT_REPOSITORY_INIT_SHARED_ALL) {
+	else if (mode == GIT_REPOSITORY_INIT_SHARED_ALL) {
 		SET_REPO_CONFIG(int32, "core.sharedrepository", 2);
 		SET_REPO_CONFIG(bool, "receive.denyNonFastforwards", true);
 	}
@@ -997,6 +1097,41 @@ cleanup:
 	return error;
 }
 
+static int repo_reinit_submodule_fs(git_submodule *sm, const char *n, void *p)
+{
+	git_repository *smrepo = NULL;
+	GIT_UNUSED(n); GIT_UNUSED(p);
+
+	if (git_submodule_open(&smrepo, sm) < 0 ||
+		git_repository_reinit_filesystem(smrepo, true) < 0)
+		giterr_clear();
+	git_repository_free(smrepo);
+
+	return 0;
+}
+
+int git_repository_reinit_filesystem(git_repository *repo, int recurse)
+{
+	int error = 0;
+	git_buf path = GIT_BUF_INIT;
+	git_config *config = NULL;
+	const char *repo_dir = git_repository_path(repo);
+
+	if (!(error = repo_local_config(&config, &path, repo, repo_dir)))
+		error = repo_init_fs_configs(
+			config, path.ptr, repo_dir, git_repository_workdir(repo), true);
+
+	git_config_free(config);
+	git_buf_free(&path);
+
+	git_repository__cvar_cache_clear(repo);
+
+	if (!repo->is_bare && recurse)
+		(void)git_submodule_foreach(repo, repo_reinit_submodule_fs, NULL);
+
+	return error;
+}
+
 static int repo_write_template(
 	const char *git_dir,
 	bool allow_overwrite,
@@ -1131,31 +1266,34 @@ static int repo_init_structure(
 
 	/* Copy external template if requested */
 	if (external_tpl) {
-		git_config *cfg;
-		const char *tdir;
+		git_config *cfg = NULL;
+		const char *tdir = NULL;
+		bool default_template = false;
+		git_buf template_buf = GIT_BUF_INIT;
 
 		if (opts->template_path)
 			tdir = opts->template_path;
-		else if ((error = git_config_open_default(&cfg)) < 0)
-			return error;
-		else {
+		else if ((error = git_config_open_default(&cfg)) >= 0) {
 			error = git_config_get_string(&tdir, cfg, "init.templatedir");
-
-			git_config_free(cfg);
-
-			if (error && error != GIT_ENOTFOUND)
-				return error;
-
 			giterr_clear();
-			tdir = GIT_TEMPLATE_DIR;
 		}
 
-		error = git_futils_cp_r(tdir, repo_dir,
-			GIT_CPDIR_COPY_SYMLINKS | GIT_CPDIR_CHMOD_DIRS |
-			GIT_CPDIR_SIMPLE_TO_MODE, dmode);
+		if (!tdir) {
+			if (!(error = git_futils_find_template_dir(&template_buf)))
+				tdir = template_buf.ptr;
+			default_template = true;
+		}
+
+		if (tdir)
+			error = git_futils_cp_r(tdir, repo_dir,
+				GIT_CPDIR_COPY_SYMLINKS | GIT_CPDIR_CHMOD_DIRS |
+				GIT_CPDIR_SIMPLE_TO_MODE, dmode);
+
+		git_buf_free(&template_buf);
+		git_config_free(cfg);
 
 		if (error < 0) {
-			if (strcmp(tdir, GIT_TEMPLATE_DIR) != 0)
+			if (!default_template)
 				return error;
 
 			/* if template was default, ignore error and use internal */
@@ -1382,22 +1520,22 @@ int git_repository_init_ext(
 		opts->flags |= GIT_REPOSITORY_INIT__IS_REINIT;
 
 		error = repo_init_config(
-			git_buf_cstr(&repo_path), git_buf_cstr(&wd_path), opts);
+			repo_path.ptr, wd_path.ptr, opts->flags, opts->mode);
 
 		/* TODO: reinitialize the templates */
 	}
 	else {
 		if (!(error = repo_init_structure(
-				git_buf_cstr(&repo_path), git_buf_cstr(&wd_path), opts)) &&
+				repo_path.ptr, wd_path.ptr, opts)) &&
 			!(error = repo_init_config(
-				git_buf_cstr(&repo_path), git_buf_cstr(&wd_path), opts)))
+				repo_path.ptr, wd_path.ptr, opts->flags, opts->mode)))
 			error = repo_init_create_head(
-				git_buf_cstr(&repo_path), opts->initial_head);
+				repo_path.ptr, opts->initial_head);
 	}
 	if (error < 0)
 		goto cleanup;
 
-	error = git_repository_open(out, git_buf_cstr(&repo_path));
+	error = git_repository_open(out, repo_path.ptr);
 
 	if (!error && opts->origin_url)
 		error = repo_init_create_origin(*out, opts->origin_url);
@@ -1448,10 +1586,10 @@ int git_repository_head(git_reference **head_out, git_repository *repo)
 	error = git_reference_lookup_resolved(head_out, repo, git_reference_symbolic_target(head), -1);
 	git_reference_free(head);
 
-	return error == GIT_ENOTFOUND ? GIT_EORPHANEDHEAD : error;
+	return error == GIT_ENOTFOUND ? GIT_EUNBORNBRANCH : error;
 }
 
-int git_repository_head_orphan(git_repository *repo)
+int git_repository_head_unborn(git_repository *repo)
 {
 	git_reference *ref = NULL;
 	int error;
@@ -1459,7 +1597,7 @@ int git_repository_head_orphan(git_repository *repo)
 	error = git_repository_head(&ref, repo);
 	git_reference_free(ref);
 
-	if (error == GIT_EORPHANEDHEAD)
+	if (error == GIT_EUNBORNBRANCH)
 		return 1;
 
 	if (error < 0)
@@ -1492,24 +1630,20 @@ static int repo_contains_no_reference(git_repository *repo)
 int git_repository_is_empty(git_repository *repo)
 {
 	git_reference *head = NULL;
-	int error;
+	int is_empty = 0;
 
 	if (git_reference_lookup(&head, repo, GIT_HEAD_FILE) < 0)
 		return -1;
 
-	if (!(error = git_reference_type(head) == GIT_REF_SYMBOLIC))
-		goto cleanup;
-
-	if (!(error = strcmp(
-		git_reference_symbolic_target(head),
-		GIT_REFS_HEADS_DIR "master") == 0))
-			goto cleanup;
-
-	error = repo_contains_no_reference(repo);
+	if (git_reference_type(head) == GIT_REF_SYMBOLIC)
+		is_empty =
+			(strcmp(git_reference_symbolic_target(head),
+					GIT_REFS_HEADS_DIR "master") == 0) &&
+			repo_contains_no_reference(repo);
 
-cleanup:
 	git_reference_free(head);
-	return error < 0 ? -1 : error;
+
+	return is_empty;
 }
 
 const char *git_repository_path(git_repository *repo)
@@ -1650,7 +1784,7 @@ int git_repository_hashfile(
 	const char *as_path)
 {
 	int error;
-	git_vector filters = GIT_VECTOR_INIT;
+	git_filter_list *fl = NULL;
 	git_file fd = -1;
 	git_off_t len;
 	git_buf full_path = GIT_BUF_INIT;
@@ -1672,7 +1806,8 @@ int git_repository_hashfile(
 
 	/* passing empty string for "as_path" indicated --no-filters */
 	if (strlen(as_path) > 0) {
-		error = git_filters_load(&filters, repo, as_path, GIT_FILTER_TO_ODB);
+		error = git_filter_list_load(
+			&fl, repo, NULL, as_path, GIT_FILTER_TO_ODB);
 		if (error < 0)
 			return error;
 	} else {
@@ -1699,12 +1834,12 @@ int git_repository_hashfile(
 		goto cleanup;
 	}
 
-	error = git_odb__hashfd_filtered(out, fd, (size_t)len, type, &filters);
+	error = git_odb__hashfd_filtered(out, fd, (size_t)len, type, fl);
 
 cleanup:
 	if (fd >= 0)
 		p_close(fd);
-	git_filters_free(&filters);
+	git_filter_list_free(fl);
 	git_buf_free(&full_path);
 
 	return error;
diff --git a/src/repository.h b/src/repository.h
index 12dc50d51..832df3bd2 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -37,6 +37,7 @@ typedef enum {
 	GIT_CVAR_IGNORESTAT,    /* core.ignorestat */
 	GIT_CVAR_TRUSTCTIME,    /* core.trustctime */
 	GIT_CVAR_ABBREV,        /* core.abbrev */
+	GIT_CVAR_PRECOMPOSE,    /* core.precomposeunicode */
 	GIT_CVAR_CACHE_MAX
 } git_cvar_cached;
 
@@ -86,6 +87,8 @@ typedef enum {
 	GIT_TRUSTCTIME_DEFAULT = GIT_CVAR_TRUE,
 	/* core.abbrev */
 	GIT_ABBREV_DEFAULT = 7,
+	/* core.precomposeunicode */
+	GIT_PRECOMPOSE_DEFAULT = GIT_CVAR_FALSE,
 
 } git_cvar_value;
 
diff --git a/src/reset.c b/src/reset.c
index cea212a93..a9780bfbc 100644
--- a/src/reset.c
+++ b/src/reset.c
@@ -24,10 +24,9 @@ int git_reset_default(
 {
 	git_object *commit = NULL;
 	git_tree *tree = NULL;
-	git_diff_list *diff = NULL;
+	git_diff *diff = NULL;
 	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
-	size_t i;
-	git_diff_delta *delta;
+	size_t i, max_i;
 	git_index_entry entry;
 	int error;
 	git_index *index = NULL;
@@ -58,7 +57,9 @@ int git_reset_default(
 		&diff, repo, tree, index, &opts)) < 0)
 			goto cleanup;
 
-	git_vector_foreach(&diff->deltas, i, delta) {
+	for (i = 0, max_i = git_diff_num_deltas(diff); i < max_i; ++i) {
+		const git_diff_delta *delta = git_diff_get_delta(diff, i);
+
 		if ((error = git_index_conflict_remove(index, delta->old_file.path)) < 0)
 			goto cleanup;
 
@@ -85,7 +86,7 @@ cleanup:
 	git_object_free(commit);
 	git_tree_free(tree);
 	git_index_free(index);
-	git_diff_list_free(diff);
+	git_diff_free(diff);
 
 	return error;
 }
@@ -135,7 +136,7 @@ int git_reset(
 
 	if (reset_type == GIT_RESET_HARD) {
 		/* overwrite working directory with HEAD */
-		opts.checkout_strategy = GIT_CHECKOUT_FORCE;
+		opts.checkout_strategy = GIT_CHECKOUT_FORCE | GIT_CHECKOUT_SKIP_UNMERGED;
 
 		if ((error = git_checkout_tree(repo, (git_object *)tree, &opts)) < 0)
 			goto cleanup;
diff --git a/src/revparse.c b/src/revparse.c
index bcfb0843f..c120b466f 100644
--- a/src/revparse.c
+++ b/src/revparse.c
@@ -93,11 +93,7 @@ static int revparse_lookup_object(
 	int error;
 	git_reference *ref;
 
-	error = maybe_sha(object_out, repo, spec);
-	if (!error)
-		return 0;
-
-	if (error < 0 && error != GIT_ENOTFOUND)
+	if ((error = maybe_sha(object_out, repo, spec)) != GIT_ENOTFOUND)
 		return error;
 
 	error = git_reference_dwim(&ref, repo, spec);
@@ -112,24 +108,17 @@ static int revparse_lookup_object(
 		return error;
 	}
 
-	if (error < 0 && error != GIT_ENOTFOUND)
-		return error;
-
-	error = maybe_abbrev(object_out, repo, spec);
-	if (!error)
-		return 0;
-
-	if (error < 0 && error != GIT_ENOTFOUND)
+	if (error != GIT_ENOTFOUND)
 		return error;
 
-	error = maybe_describe(object_out, repo, spec);
-	if (!error)
-		return 0;
+	if ((strlen(spec) < GIT_OID_HEXSZ) &&
+		((error = maybe_abbrev(object_out, repo, spec)) != GIT_ENOTFOUND))
+			return error;
 
-	if (error < 0 && error != GIT_ENOTFOUND)
+	if ((error = maybe_describe(object_out, repo, spec)) != GIT_ENOTFOUND)
 		return error;
 
-	giterr_set(GITERR_REFERENCE, "Refspec '%s' not found.", spec);
+	giterr_set(GITERR_REFERENCE, "Revspec '%s' not found.", spec);
 	return GIT_ENOTFOUND;
 }
 
@@ -171,7 +160,7 @@ static int retrieve_previously_checked_out_branch_or_revision(git_object **out,
 	if (git_reference_lookup(&ref, repo, GIT_HEAD_FILE) < 0)
 		goto cleanup;
 
-	if (git_reflog_read(&reflog, ref) < 0)
+	if (git_reflog_read(&reflog, repo, GIT_HEAD_FILE) < 0)
 		goto cleanup;
 
 	numentries  = git_reflog_entrycount(reflog);
@@ -219,7 +208,7 @@ static int retrieve_oid_from_reflog(git_oid *oid, git_reference *ref, size_t ide
 	const git_reflog_entry *entry;
 	bool search_by_pos = (identifier <= 100000000);
 
-	if (git_reflog_read(&reflog, ref) < 0)
+	if (git_reflog_read(&reflog, git_reference_owner(ref), git_reference_name(ref)) < 0)
 		return -1;
 
 	numentries = git_reflog_entrycount(reflog);
@@ -228,7 +217,7 @@ static int retrieve_oid_from_reflog(git_oid *oid, git_reference *ref, size_t ide
 		if (numentries < identifier + 1) {
 			giterr_set(
 				GITERR_REFERENCE,
-				"Reflog for '%s' has only "PRIuZ" entries, asked for "PRIuZ,
+				"Reflog for '%s' has only %"PRIuZ" entries, asked for %"PRIuZ,
 				git_reference_name(ref), numentries, identifier);
 
 			error = GIT_ENOTFOUND;
@@ -685,6 +674,8 @@ int revparse__ext(
 	git_reference *reference = NULL;
 	git_object *base_rev = NULL;
 
+	bool should_return_reference = true;
+
 	assert(object_out && reference_out && repo && spec);
 
 	*object_out = NULL;
@@ -693,6 +684,8 @@ int revparse__ext(
 	while (spec[pos]) {
 		switch (spec[pos]) {
 		case '^':
+			should_return_reference = false;
+
 			if ((error = ensure_base_rev_loaded(&base_rev, &reference, spec, identifier_len, repo, false)) < 0)
 				goto cleanup;
 
@@ -725,6 +718,8 @@ int revparse__ext(
 		{
 			git_object *temp_object = NULL;
 
+			should_return_reference = false;
+
 			if ((error = extract_how_many(&n, spec, &pos)) < 0)
 				goto cleanup;
 
@@ -743,6 +738,8 @@ int revparse__ext(
 		{
 			git_object *temp_object = NULL;
 
+			should_return_reference = false;
+
 			if ((error = extract_path(&buf, spec, &pos)) < 0)
 				goto cleanup;
 
@@ -807,6 +804,11 @@ int revparse__ext(
 	if ((error = ensure_base_rev_loaded(&base_rev, &reference, spec, identifier_len, repo, false)) < 0)
 		goto cleanup;
 
+	if (!should_return_reference) {
+		git_reference_free(reference);
+		reference = NULL;
+	}
+
 	*object_out = base_rev;
 	*reference_out = reference;
 	*identifier_len_out = identifier_len;
@@ -899,13 +901,9 @@ int git_revparse(
 			rstr++;
 		}
 
-		if ((error = git_revparse_single(&revspec->from, repo, lstr)) < 0) {
-			return error;
-		}
-
-		if ((error = git_revparse_single(&revspec->to, repo, rstr)) < 0) {
-			return error;
-		}
+		error = git_revparse_single(&revspec->from, repo, lstr);
+		if (!error)
+			error = git_revparse_single(&revspec->to, repo, rstr);
 
 		git__free((void*)lstr);
 	} else {
diff --git a/src/revwalk.c b/src/revwalk.c
index 528d02b20..3dd14b419 100644
--- a/src/revwalk.c
+++ b/src/revwalk.c
@@ -14,8 +14,6 @@
 #include "git2/revparse.h"
 #include "merge.h"
 
-#include <regex.h>
-
 git_commit_list_node *git_revwalk__commit_lookup(
 	git_revwalk *walk, const git_oid *oid)
 {
@@ -41,28 +39,50 @@ git_commit_list_node *git_revwalk__commit_lookup(
 	return commit;
 }
 
-static void mark_uninteresting(git_commit_list_node *commit)
+static int mark_uninteresting(git_commit_list_node *commit)
 {
 	unsigned short i;
+	git_array_t(git_commit_list_node *) pending = GIT_ARRAY_INIT;
+	git_commit_list_node **tmp;
+
 	assert(commit);
 
-	commit->uninteresting = 1;
+	git_array_alloc(pending);
+	GITERR_CHECK_ARRAY(pending);
 
-	/* This means we've reached a merge base, so there's no need to walk any more */
-	if ((commit->flags & (RESULT | STALE)) == RESULT)
-		return;
+	do {
+		commit->uninteresting = 1;
+
+		/* This means we've reached a merge base, so there's no need to walk any more */
+		if ((commit->flags & (RESULT | STALE)) == RESULT) {
+			tmp = git_array_pop(pending);
+			commit = tmp ? *tmp : NULL;
+			continue;
+		}
+
+		for (i = 0; i < commit->out_degree; ++i)
+			if (!commit->parents[i]->uninteresting) {
+				git_commit_list_node **node = git_array_alloc(pending);
+				GITERR_CHECK_ALLOC(node);
+				*node = commit->parents[i];
+			}
+
+		tmp = git_array_pop(pending);
+		commit = tmp ? *tmp : NULL;
 
-	for (i = 0; i < commit->out_degree; ++i)
-		if (!commit->parents[i]->uninteresting)
-			mark_uninteresting(commit->parents[i]);
+	} while (git_array_size(pending) > 0);
+
+	git_array_clear(pending);
+
+	return 0;
 }
 
 static int process_commit(git_revwalk *walk, git_commit_list_node *commit, int hide)
 {
 	int error;
 
-	if (hide)
-		mark_uninteresting(commit);
+	if (hide && mark_uninteresting(commit) < 0)
+		return -1;
 
 	if (commit->seen)
 		return 0;
@@ -77,10 +97,14 @@ static int process_commit(git_revwalk *walk, git_commit_list_node *commit, int h
 
 static int process_commit_parents(git_revwalk *walk, git_commit_list_node *commit)
 {
-	unsigned short i;
+	unsigned short i, max;
 	int error = 0;
 
-	for (i = 0; i < commit->out_degree && !error; ++i)
+	max = commit->out_degree;
+	if (walk->first_parent && commit->out_degree)
+		max = 1;
+
+	for (i = 0; i < max && !error; ++i)
 		error = process_commit(walk, commit->parents[i], commit->uninteresting);
 
 	return error;
@@ -155,48 +179,35 @@ static int push_glob_cb(const char *refname, void *data_)
 
 static int push_glob(git_revwalk *walk, const char *glob, int hide)
 {
+	int error = 0;
 	git_buf buf = GIT_BUF_INIT;
 	struct push_cb_data data;
-	regex_t preg;
+	size_t wildcard;
 
 	assert(walk && glob);
 
 	/* refs/ is implied if not given in the glob */
-	if (strncmp(glob, GIT_REFS_DIR, strlen(GIT_REFS_DIR))) {
-		git_buf_printf(&buf, GIT_REFS_DIR "%s", glob);
-	} else {
+	if (git__prefixcmp(glob, GIT_REFS_DIR) != 0)
+		git_buf_joinpath(&buf, GIT_REFS_DIR, glob);
+	else
 		git_buf_puts(&buf, glob);
-	}
 
 	/* If no '?', '*' or '[' exist, we append '/ *' to the glob */
-	memset(&preg, 0x0, sizeof(regex_t));
-	if (regcomp(&preg, "[?*[]", REG_EXTENDED)) {
-		giterr_set(GITERR_OS, "Regex failed to compile");
-		git_buf_free(&buf);
-		return -1;
-	}
-
-	if (regexec(&preg, glob, 0, NULL, 0))
-		git_buf_puts(&buf, "/*");
-
-	if (git_buf_oom(&buf))
-		goto on_error;
+	wildcard = strcspn(glob, "?*[");
+	if (!glob[wildcard])
+		git_buf_put(&buf, "/*", 2);
 
 	data.walk = walk;
 	data.hide = hide;
 
-	if (git_reference_foreach_glob(
-		walk->repo, git_buf_cstr(&buf), push_glob_cb, &data) < 0)
-		goto on_error;
-
-	regfree(&preg);
-	git_buf_free(&buf);
-	return 0;
+	if (git_buf_oom(&buf))
+		error = -1;
+	else
+		error = git_reference_foreach_glob(
+			walk->repo, git_buf_cstr(&buf), push_glob_cb, &data);
 
-on_error:
-	regfree(&preg);
 	git_buf_free(&buf);
-	return -1;
+	return error;
 }
 
 int git_revwalk_push_glob(git_revwalk *walk, const char *glob)
@@ -311,7 +322,7 @@ static int revwalk_next_unsorted(git_commit_list_node **object_out, git_revwalk
 static int revwalk_next_toposort(git_commit_list_node **object_out, git_revwalk *walk)
 {
 	git_commit_list_node *next;
-	unsigned short i;
+	unsigned short i, max;
 
 	for (;;) {
 		next = git_commit_list_pop(&walk->iterator_topo);
@@ -325,7 +336,12 @@ static int revwalk_next_toposort(git_commit_list_node **object_out, git_revwalk
 			continue;
 		}
 
-		for (i = 0; i < next->out_degree; ++i) {
+
+		max = next->out_degree;
+		if (walk->first_parent && next->out_degree)
+			max = 1;
+
+		for (i = 0; i < max; ++i) {
 			git_commit_list_node *parent = next->parents[i];
 
 			if (--parent->in_degree == 0 && parent->topo_delay) {
@@ -483,6 +499,11 @@ void git_revwalk_sorting(git_revwalk *walk, unsigned int sort_mode)
 	}
 }
 
+void git_revwalk_simplify_first_parent(git_revwalk *walk)
+{
+	walk->first_parent = 1;
+}
+
 int git_revwalk_next(git_oid *oid, git_revwalk *walk)
 {
 	int error;
diff --git a/src/revwalk.h b/src/revwalk.h
index 22696dfcd..8c821d098 100644
--- a/src/revwalk.h
+++ b/src/revwalk.h
@@ -31,7 +31,8 @@ struct git_revwalk {
 	int (*get_next)(git_commit_list_node **, git_revwalk *);
 	int (*enqueue)(git_revwalk *, git_commit_list_node *);
 
-	unsigned walking:1;
+	unsigned walking:1,
+		first_parent: 1;
 	unsigned int sorting;
 
 	/* merge base calculation */
diff --git a/src/sha1_lookup.c b/src/sha1_lookup.c
index b7e66cc69..c6b561340 100644
--- a/src/sha1_lookup.c
+++ b/src/sha1_lookup.c
@@ -9,6 +9,7 @@
 
 #include "sha1_lookup.h"
 #include "common.h"
+#include "oid.h"
 
 /*
  * Conventional binary search loop looks like this:
@@ -108,7 +109,54 @@ int sha1_entry_pos(const void *table,
 			 * byte 0 thru (ofs-1) are the same between
 			 * lo and hi; ofs is the first byte that is
 			 * different.
+			 *
+			 * If ofs==20, then no bytes are different,
+			 * meaning we have entries with duplicate
+			 * keys. We know that we are in a solid run
+			 * of this entry (because the entries are
+			 * sorted, and our lo and hi are the same,
+			 * there can be nothing but this single key
+			 * in between). So we can stop the search.
+			 * Either one of these entries is it (and
+			 * we do not care which), or we do not have
+			 * it.
+			 *
+			 * Furthermore, we know that one of our
+			 * endpoints must be the edge of the run of
+			 * duplicates. For example, given this
+			 * sequence:
+			 *
+			 *     idx 0 1 2 3 4 5
+			 *     key A C C C C D
+			 *
+			 * If we are searching for "B", we might
+			 * hit the duplicate run at lo=1, hi=3
+			 * (e.g., by first mi=3, then mi=0). But we
+			 * can never have lo > 1, because B < C.
+			 * That is, if our key is less than the
+			 * run, we know that "lo" is the edge, but
+			 * we can say nothing of "hi". Similarly,
+			 * if our key is greater than the run, we
+			 * know that "hi" is the edge, but we can
+			 * say nothing of "lo".
+			 *
+			 * Therefore if we do not find it, we also
+			 * know where it would go if it did exist:
+			 * just on the far side of the edge that we
+			 * know about.
 			 */
+			if (ofs == 20) {
+				mi = lo;
+				mi_key = base + elem_size * mi + key_offset;
+				cmp = memcmp(mi_key, key, 20);
+				if (!cmp)
+					return mi;
+				if (cmp < 0)
+					return -1 - hi;
+				else
+					return -1 - lo;
+			}
+
 			hiv = hi_key[ofs_0];
 			if (ofs_0 < 19)
 				hiv = (hiv << 8) | hi_key[ofs_0+1];
@@ -176,3 +224,26 @@ int sha1_entry_pos(const void *table,
 	} while (lo < hi);
 	return -((int)lo)-1;
 }
+
+int sha1_position(const void *table,
+			size_t stride,
+			unsigned lo, unsigned hi,
+			const unsigned char *key)
+{
+	const unsigned char *base = table;
+
+	do {
+		unsigned mi = (lo + hi) / 2;
+		int cmp = git_oid__hashcmp(base + mi * stride, key);
+
+		if (!cmp)
+			return mi;
+
+		if (cmp > 0)
+			hi = mi;
+		else
+			lo = mi+1;
+	} while (lo < hi);
+
+	return -((int)lo)-1;
+}
diff --git a/src/sha1_lookup.h b/src/sha1_lookup.h
index 9a3537273..3799620c7 100644
--- a/src/sha1_lookup.h
+++ b/src/sha1_lookup.h
@@ -15,4 +15,9 @@ int sha1_entry_pos(const void *table,
 			unsigned lo, unsigned hi, unsigned nr,
 			const unsigned char *key);
 
+int sha1_position(const void *table,
+			size_t stride,
+			unsigned lo, unsigned hi,
+			const unsigned char *key);
+
 #endif
diff --git a/src/signature.c b/src/signature.c
index 0a34ccfaa..ec51a42e9 100644
--- a/src/signature.c
+++ b/src/signature.c
@@ -74,7 +74,7 @@ int git_signature_new(git_signature **sig_out, const char *name, const char *ema
 		git_signature_free(p);
 		return signature_error("Signature cannot have an empty name");
 	}
-		
+
 	p->when.time = time;
 	p->when.offset = offset;
 
@@ -84,8 +84,12 @@ int git_signature_new(git_signature **sig_out, const char *name, const char *ema
 
 git_signature *git_signature_dup(const git_signature *sig)
 {
-	git_signature *new = git__calloc(1, sizeof(git_signature));
+	git_signature *new;
 
+	if (sig == NULL)
+		return NULL;
+
+	new = git__calloc(1, sizeof(git_signature));
 	if (new == NULL)
 		return NULL;
 
@@ -129,6 +133,23 @@ int git_signature_now(git_signature **sig_out, const char *name, const char *ema
 	return 0;
 }
 
+int git_signature_default(git_signature **out, git_repository *repo)
+{
+	int error;
+	git_config *cfg;
+	const char *user_name, *user_email;
+
+	if ((error = git_repository_config(&cfg, repo)) < 0)
+		return error;
+
+	if (!(error = git_config_get_string(&user_name, cfg, "user.name")) &&
+		!(error = git_config_get_string(&user_email, cfg, "user.email")))
+		error = git_signature_now(out, user_name, user_email);
+
+	git_config_free(cfg);
+	return error;
+}
+
 int git_signature__parse(git_signature *sig, const char **buffer_out,
 		const char *buffer_end, const char *header, char ender)
 {
diff --git a/src/sortedcache.c b/src/sortedcache.c
new file mode 100644
index 000000000..466e55dbe
--- /dev/null
+++ b/src/sortedcache.c
@@ -0,0 +1,380 @@
+#include "sortedcache.h"
+
+GIT__USE_STRMAP;
+
+int git_sortedcache_new(
+	git_sortedcache **out,
+	size_t item_path_offset,
+	git_sortedcache_free_item_fn free_item,
+	void *free_item_payload,
+	git_vector_cmp item_cmp,
+	const char *path)
+{
+	git_sortedcache *sc;
+	size_t pathlen;
+
+	pathlen = path ? strlen(path) : 0;
+
+	sc = git__calloc(sizeof(git_sortedcache) + pathlen + 1, 1);
+	GITERR_CHECK_ALLOC(sc);
+
+	if (git_pool_init(&sc->pool, 1, 0) < 0 ||
+		git_vector_init(&sc->items, 4, item_cmp) < 0 ||
+		(sc->map = git_strmap_alloc()) == NULL)
+		goto fail;
+
+	if (git_rwlock_init(&sc->lock)) {
+		giterr_set(GITERR_OS, "Failed to initialize lock");
+		goto fail;
+	}
+
+	sc->item_path_offset  = item_path_offset;
+	sc->free_item         = free_item;
+	sc->free_item_payload = free_item_payload;
+	GIT_REFCOUNT_INC(sc);
+	if (pathlen)
+		memcpy(sc->path, path, pathlen);
+
+	*out = sc;
+	return 0;
+
+fail:
+	if (sc->map)
+		git_strmap_free(sc->map);
+	git_vector_free(&sc->items);
+	git_pool_clear(&sc->pool);
+	git__free(sc);
+	return -1;
+}
+
+void git_sortedcache_incref(git_sortedcache *sc)
+{
+	GIT_REFCOUNT_INC(sc);
+}
+
+const char *git_sortedcache_path(git_sortedcache *sc)
+{
+	return sc->path;
+}
+
+static void sortedcache_clear(git_sortedcache *sc)
+{
+	git_strmap_clear(sc->map);
+
+	if (sc->free_item) {
+		size_t i;
+		void *item;
+
+		git_vector_foreach(&sc->items, i, item) {
+			sc->free_item(sc->free_item_payload, item);
+		}
+	}
+
+	git_vector_clear(&sc->items);
+
+	git_pool_clear(&sc->pool);
+}
+
+static void sortedcache_free(git_sortedcache *sc)
+{
+	/* acquire write lock to make sure everyone else is done */
+	if (git_sortedcache_wlock(sc) < 0)
+		return;
+
+	sortedcache_clear(sc);
+	git_vector_free(&sc->items);
+	git_strmap_free(sc->map);
+
+	git_sortedcache_wunlock(sc);
+
+	git_rwlock_free(&sc->lock);
+	git__free(sc);
+}
+
+void git_sortedcache_free(git_sortedcache *sc)
+{
+	if (!sc)
+		return;
+	GIT_REFCOUNT_DEC(sc, sortedcache_free);
+}
+
+static int sortedcache_copy_item(void *payload, void *tgt_item, void *src_item)
+{
+	git_sortedcache *sc = payload;
+	/* path will already have been copied by upsert */
+	memcpy(tgt_item, src_item, sc->item_path_offset);
+	return 0;
+}
+
+/* copy a sorted cache */
+int git_sortedcache_copy(
+	git_sortedcache **out,
+	git_sortedcache *src,
+	bool lock,
+	int (*copy_item)(void *payload, void *tgt_item, void *src_item),
+	void *payload)
+{
+	int error = 0;
+	git_sortedcache *tgt;
+	size_t i;
+	void *src_item, *tgt_item;
+
+	/* just use memcpy if no special copy fn is passed in */
+	if (!copy_item) {
+		copy_item = sortedcache_copy_item;
+		payload   = src;
+	}
+
+	if ((error = git_sortedcache_new(
+			&tgt, src->item_path_offset,
+			src->free_item, src->free_item_payload,
+			src->items._cmp, src->path)) < 0)
+		return error;
+
+	if (lock && git_sortedcache_rlock(src) < 0) {
+		git_sortedcache_free(tgt);
+		return -1;
+	}
+
+	git_vector_foreach(&src->items, i, src_item) {
+		char *path = ((char *)src_item) + src->item_path_offset;
+
+		if ((error = git_sortedcache_upsert(&tgt_item, tgt, path)) < 0 ||
+			(error = copy_item(payload, tgt_item, src_item)) < 0)
+			break;
+	}
+
+	if (lock)
+		git_sortedcache_runlock(src);
+	if (error)
+		git_sortedcache_free(tgt);
+
+	*out = !error ? tgt : NULL;
+
+	return error;
+}
+
+/* lock sortedcache while making modifications */
+int git_sortedcache_wlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+
+	if (git_rwlock_wrlock(&sc->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire write lock on cache");
+		return -1;
+	}
+	return 0;
+}
+
+/* unlock sorted cache when done with modifications */
+void git_sortedcache_wunlock(git_sortedcache *sc)
+{
+	git_vector_sort(&sc->items);
+	git_rwlock_wrunlock(&sc->lock);
+}
+
+/* lock sortedcache for read */
+int git_sortedcache_rlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+
+	if (git_rwlock_rdlock(&sc->lock) < 0) {
+		giterr_set(GITERR_OS, "Unable to acquire read lock on cache");
+		return -1;
+	}
+	return 0;
+}
+
+/* unlock sorted cache when done reading */
+void git_sortedcache_runlock(git_sortedcache *sc)
+{
+	GIT_UNUSED(sc); /* prevent warning when compiled w/o threads */
+	git_rwlock_rdunlock(&sc->lock);
+}
+
+/* if the file has changed, lock cache and load file contents into buf;
+ * returns <0 on error, >0 if file has not changed
+ */
+int git_sortedcache_lockandload(git_sortedcache *sc, git_buf *buf)
+{
+	int error, fd;
+
+	if ((error = git_sortedcache_wlock(sc)) < 0)
+		return error;
+
+	if ((error = git_futils_filestamp_check(&sc->stamp, sc->path)) <= 0)
+		goto unlock;
+
+	if (!git__is_sizet(sc->stamp.size)) {
+		giterr_set(GITERR_INVALID, "Unable to load file larger than size_t");
+		error = -1;
+		goto unlock;
+	}
+
+	if ((fd = git_futils_open_ro(sc->path)) < 0) {
+		error = fd;
+		goto unlock;
+	}
+
+	if (buf)
+		error = git_futils_readbuffer_fd(buf, fd, (size_t)sc->stamp.size);
+
+	(void)p_close(fd);
+
+	if (error < 0)
+		goto unlock;
+
+	return 1; /* return 1 -> file needs reload and was successfully loaded */
+
+unlock:
+	git_sortedcache_wunlock(sc);
+	return error;
+}
+
+void git_sortedcache_updated(git_sortedcache *sc)
+{
+	 /* update filestamp to latest value */
+	if (git_futils_filestamp_check(&sc->stamp, sc->path) < 0)
+		giterr_clear();
+}
+
+/* release all items in sorted cache */
+int git_sortedcache_clear(git_sortedcache *sc, bool wlock)
+{
+	if (wlock && git_sortedcache_wlock(sc) < 0)
+		return -1;
+
+	sortedcache_clear(sc);
+
+	if (wlock)
+		git_sortedcache_wunlock(sc);
+
+	return 0;
+}
+
+/* find and/or insert item, returning pointer to item data */
+int git_sortedcache_upsert(void **out, git_sortedcache *sc, const char *key)
+{
+	int error = 0;
+	khiter_t pos;
+	void *item;
+	size_t keylen, itemlen;
+	char *item_key;
+
+	pos = git_strmap_lookup_index(sc->map, key);
+	if (git_strmap_valid_index(sc->map, pos)) {
+		item = git_strmap_value_at(sc->map, pos);
+		goto done;
+	}
+
+	keylen  = strlen(key);
+	itemlen = sc->item_path_offset + keylen + 1;
+	itemlen = (itemlen + 7) & ~7;
+
+	if ((item = git_pool_mallocz(&sc->pool, (uint32_t)itemlen)) == NULL) {
+		/* don't use GITERR_CHECK_ALLOC b/c of lock */
+		error = -1;
+		goto done;
+	}
+
+	/* one strange thing is that even if the vector or hash table insert
+	 * fail, there is no way to free the pool item so we just abandon it
+	 */
+
+	item_key = ((char *)item) + sc->item_path_offset;
+	memcpy(item_key, key, keylen);
+
+	pos = kh_put(str, sc->map, item_key, &error);
+	if (error < 0)
+		goto done;
+
+	if (!error)
+		kh_key(sc->map, pos) = item_key;
+	kh_val(sc->map, pos) = item;
+
+	error = git_vector_insert(&sc->items, item);
+	if (error < 0)
+		git_strmap_delete_at(sc->map, pos);
+
+done:
+	if (out)
+		*out = !error ? item : NULL;
+	return error;
+}
+
+/* lookup item by key */
+void *git_sortedcache_lookup(const git_sortedcache *sc, const char *key)
+{
+	khiter_t pos = git_strmap_lookup_index(sc->map, key);
+	if (git_strmap_valid_index(sc->map, pos))
+		return git_strmap_value_at(sc->map, pos);
+	return NULL;
+}
+
+/* find out how many items are in the cache */
+size_t git_sortedcache_entrycount(const git_sortedcache *sc)
+{
+	return git_vector_length(&sc->items);
+}
+
+/* lookup item by index */
+void *git_sortedcache_entry(git_sortedcache *sc, size_t pos)
+{
+	/* make sure the items are sorted so this gets the correct item */
+	if (!sc->items.sorted)
+		git_vector_sort(&sc->items);
+
+	return git_vector_get(&sc->items, pos);
+}
+
+/* helper struct so bsearch callback can know offset + key value for cmp */
+struct sortedcache_magic_key {
+	size_t offset;
+	const char *key;
+};
+
+static int sortedcache_magic_cmp(const void *key, const void *value)
+{
+	const struct sortedcache_magic_key *magic = key;
+	const char *value_key = ((const char *)value) + magic->offset;
+	return strcmp(magic->key, value_key);
+}
+
+/* lookup index of item by key */
+int git_sortedcache_lookup_index(
+	size_t *out, git_sortedcache *sc, const char *key)
+{
+	struct sortedcache_magic_key magic;
+
+	magic.offset = sc->item_path_offset;
+	magic.key    = key;
+
+	return git_vector_bsearch2(out, &sc->items, sortedcache_magic_cmp, &magic);
+}
+
+/* remove entry from cache */
+int git_sortedcache_remove(git_sortedcache *sc, size_t pos)
+{
+	char *item;
+	khiter_t mappos;
+
+	/* because of pool allocation, this can't actually remove the item,
+	 * but we can remove it from the items vector and the hash table.
+	 */
+
+	if ((item = git_vector_get(&sc->items, pos)) == NULL) {
+		giterr_set(GITERR_INVALID, "Removing item out of range");
+		return GIT_ENOTFOUND;
+	}
+
+	(void)git_vector_remove(&sc->items, pos);
+
+	mappos = git_strmap_lookup_index(sc->map, item + sc->item_path_offset);
+	git_strmap_delete_at(sc->map, mappos);
+
+	if (sc->free_item)
+		sc->free_item(sc->free_item_payload, item);
+
+	return 0;
+}
+
diff --git a/src/sortedcache.h b/src/sortedcache.h
new file mode 100644
index 000000000..4cacad62b
--- /dev/null
+++ b/src/sortedcache.h
@@ -0,0 +1,178 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_sorted_cache_h__
+#define INCLUDE_sorted_cache_h__
+
+#include "util.h"
+#include "fileops.h"
+#include "vector.h"
+#include "thread-utils.h"
+#include "pool.h"
+#include "strmap.h"
+
+#include <stddef.h>
+
+/*
+ * The purpose of this data structure is to cache the parsed contents of a
+ * file (a.k.a. the backing file) where each item in the file can be
+ * identified by a key string and you want to both look them up by name
+ * and traverse them in sorted order.  Each item is assumed to itself end
+ * in a GIT_FLEX_ARRAY.
+ */
+
+typedef void (*git_sortedcache_free_item_fn)(void *payload, void *item);
+
+typedef struct {
+	git_refcount rc;
+	git_rwlock   lock;
+	size_t       item_path_offset;
+	git_sortedcache_free_item_fn free_item;
+	void         *free_item_payload;
+	git_pool     pool;
+	git_vector   items;
+	git_strmap   *map;
+	git_futils_filestamp stamp;
+	char         path[GIT_FLEX_ARRAY];
+} git_sortedcache;
+
+/* Create a new sortedcache
+ *
+ * Even though every sortedcache stores items with a GIT_FLEX_ARRAY at
+ * the end containing their key string, you have to provide the item_cmp
+ * sorting function because the sorting function doesn't get a payload
+ * and therefore can't know the offset to the item key string. :-(
+ *
+ * @param out The allocated git_sortedcache
+ * @param item_path_offset Offset to the GIT_FLEX_ARRAY item key in the
+ *        struct - use offsetof(struct mine, key-field) to get this
+ * @param free_item Optional callback to free each item
+ * @param free_item_payload Optional payload passed to free_item callback
+ * @param item_cmp Compare the keys of two items
+ * @param path The path to the backing store file for this cache; this
+ *        may be NULL.  The cache makes it easy to load this and check
+ *        if it has been modified since the last load and/or write.
+ */
+int git_sortedcache_new(
+	git_sortedcache **out,
+	size_t item_path_offset, /* use offsetof(struct, path-field) macro */
+	git_sortedcache_free_item_fn free_item,
+	void *free_item_payload,
+	git_vector_cmp item_cmp,
+	const char *path);
+
+/* Copy a sorted cache
+ *
+ * - `copy_item` can be NULL to just use memcpy
+ * - if `lock`, grabs read lock on `src` during copy and releases after
+ */
+int git_sortedcache_copy(
+	git_sortedcache **out,
+	git_sortedcache *src,
+	bool lock,
+	int (*copy_item)(void *payload, void *tgt_item, void *src_item),
+	void *payload);
+
+/* Free sorted cache (first calling `free_item` callbacks)
+ *
+ * Don't call on a locked collection - it may acquire a write lock
+ */
+void git_sortedcache_free(git_sortedcache *sc);
+
+/* Increment reference count - balance with call to free */
+void git_sortedcache_incref(git_sortedcache *sc);
+
+/* Get the pathname associated with this cache at creation time */
+const char *git_sortedcache_path(git_sortedcache *sc);
+
+/*
+ * CACHE WRITE FUNCTIONS
+ *
+ * The following functions require you to have a writer lock to make the
+ * modification.  Some of the functions take a `wlock` parameter and
+ * will optionally lock and unlock for you if that is passed as true.
+ *
+ */
+
+/* Lock sortedcache for write */
+int git_sortedcache_wlock(git_sortedcache *sc);
+
+/* Unlock sorted cache when done with write */
+void git_sortedcache_wunlock(git_sortedcache *sc);
+
+/* Lock cache and load backing file into a buffer.
+ *
+ * This grabs a write lock on the cache then looks at the modification
+ * time and size of the file on disk.
+ *
+ * If the file appears to have changed, this loads the file contents into
+ * the buffer and returns a positive value leaving the cache locked - the
+ * caller should parse the file content, update the cache as needed, then
+ * release the lock.  NOTE: In this case, the caller MUST unlock the cache.
+ *
+ * If the file appears to be unchanged, then this automatically releases
+ * the lock on the cache, clears the buffer, and returns 0.
+ *
+ * @return 0 if up-to-date, 1 if out-of-date, <0 on error
+ */
+int git_sortedcache_lockandload(git_sortedcache *sc, git_buf *buf);
+
+/* Refresh file timestamp after write completes
+ * You should already be holding the write lock when you call this.
+ */
+void git_sortedcache_updated(git_sortedcache *sc);
+
+/* Release all items in sorted cache
+ *
+ * If `wlock` is true, grabs write lock and releases when done, otherwise
+ * you should already be holding a write lock when you call this.
+ */
+int git_sortedcache_clear(git_sortedcache *sc, bool wlock);
+
+/* Find and/or insert item, returning pointer to item data.
+ * You should already be holding the write lock when you call this.
+ */
+int git_sortedcache_upsert(
+	void **out, git_sortedcache *sc, const char *key);
+
+/* Removes entry at pos from cache
+ * You should already be holding the write lock when you call this.
+ */
+int git_sortedcache_remove(git_sortedcache *sc, size_t pos);
+
+/*
+ * CACHE READ FUNCTIONS
+ *
+ * The following functions access items in the cache.  To prevent the
+ * results from being invalidated before they can be used, you should be
+ * holding either a read lock or a write lock when using these functions.
+ *
+ */
+
+/* Lock sortedcache for read */
+int git_sortedcache_rlock(git_sortedcache *sc);
+
+/* Unlock sorted cache when done with read */
+void git_sortedcache_runlock(git_sortedcache *sc);
+
+/* Lookup item by key - returns NULL if not found */
+void *git_sortedcache_lookup(const git_sortedcache *sc, const char *key);
+
+/* Get how many items are in the cache
+ *
+ * You can call this function without holding a lock, but be aware
+ * that it may change before you use it.
+ */
+size_t git_sortedcache_entrycount(const git_sortedcache *sc);
+
+/* Lookup item by index - returns NULL if out of range */
+void *git_sortedcache_entry(git_sortedcache *sc, size_t pos);
+
+/* Lookup index of item by key - returns GIT_ENOTFOUND if not found */
+int git_sortedcache_lookup_index(
+	size_t *out, git_sortedcache *sc, const char *key);
+
+#endif
diff --git a/src/stash.c b/src/stash.c
index 1222634d5..083c2a4cd 100644
--- a/src/stash.c
+++ b/src/stash.c
@@ -27,7 +27,7 @@ static int retrieve_head(git_reference **out, git_repository *repo)
 {
 	int error = git_repository_head(out, repo);
 
-	if (error == GIT_EORPHANEDHEAD)
+	if (error == GIT_EUNBORNBRANCH)
 		return create_error(error, "You do not have the initial commit yet.");
 
 	return error;
@@ -117,7 +117,7 @@ static int build_tree_from_index(git_tree **out, git_index *index)
 static int commit_index(
 	git_commit **i_commit,
 	git_index *index,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message,
 	const git_commit *parent)
 {
@@ -153,65 +153,61 @@ cleanup:
 	return error;
 }
 
-struct cb_data {
-	git_index *index;
-
-	int error;
-
+struct stash_update_rules {
 	bool include_changed;
 	bool include_untracked;
 	bool include_ignored;
 };
 
-static int update_index_cb(
-	const git_diff_delta *delta,
-	float progress,
-	void *payload)
+static int stash_update_index_from_diff(
+	git_index *index,
+	const git_diff *diff,
+	struct stash_update_rules *data)
 {
-	struct cb_data *data = (struct cb_data *)payload;
-	const char *add_path = NULL;
-
-	GIT_UNUSED(progress);
-
-	switch (delta->status) {
-	case GIT_DELTA_IGNORED:
-		if (data->include_ignored)
-			add_path = delta->new_file.path;
-		break;
-
-	case GIT_DELTA_UNTRACKED:
-		if (data->include_untracked)
-			add_path = delta->new_file.path;
-		break;
-
-	case GIT_DELTA_ADDED:
-	case GIT_DELTA_MODIFIED:
-		if (data->include_changed)
-			add_path = delta->new_file.path;
-		break;
-
-	case GIT_DELTA_DELETED:
-		if (!data->include_changed)
+	int error = 0;
+	size_t d, max_d = git_diff_num_deltas(diff);
+
+	for (d = 0; !error && d < max_d; ++d) {
+		const char *add_path = NULL;
+		const git_diff_delta *delta = git_diff_get_delta(diff, d);
+
+		switch (delta->status) {
+		case GIT_DELTA_IGNORED:
+			if (data->include_ignored)
+				add_path = delta->new_file.path;
 			break;
-		if (git_index_find(NULL, data->index, delta->old_file.path) == 0)
-			data->error = git_index_remove(
-				data->index, delta->old_file.path, 0);
-		break;
-
-	default:
-		/* Unimplemented */
-		giterr_set(
-			GITERR_INVALID,
-			"Cannot update index. Unimplemented status (%d)",
-			delta->status);
-		data->error = -1;
-		break;
-	}
 
-	if (add_path != NULL)
-		data->error = git_index_add_bypath(data->index, add_path);
+		case GIT_DELTA_UNTRACKED:
+			if (data->include_untracked)
+				add_path = delta->new_file.path;
+			break;
+
+		case GIT_DELTA_ADDED:
+		case GIT_DELTA_MODIFIED:
+			if (data->include_changed)
+				add_path = delta->new_file.path;
+			break;
+
+		case GIT_DELTA_DELETED:
+			if (data->include_changed &&
+				!git_index_find(NULL, index, delta->old_file.path))
+				error = git_index_remove(index, delta->old_file.path, 0);
+			break;
+
+		default:
+			/* Unimplemented */
+			giterr_set(
+				GITERR_INVALID,
+				"Cannot update index. Unimplemented status (%d)",
+				delta->status);
+			return -1;
+		}
+
+		if (add_path != NULL)
+			error = git_index_add_bypath(index, add_path);
+	}
 
-	return data->error;
+	return error;
 }
 
 static int build_untracked_tree(
@@ -221,15 +217,13 @@ static int build_untracked_tree(
 	uint32_t flags)
 {
 	git_tree *i_tree = NULL;
-	git_diff_list *diff = NULL;
+	git_diff *diff = NULL;
 	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
-	struct cb_data data = {0};
+	struct stash_update_rules data = {0};
 	int error;
 
 	git_index_clear(index);
 
-	data.index = index;
-
 	if (flags & GIT_STASH_INCLUDE_UNTRACKED) {
 		opts.flags |= GIT_DIFF_INCLUDE_UNTRACKED |
 			GIT_DIFF_RECURSE_UNTRACKED_DIRS;
@@ -248,18 +242,13 @@ static int build_untracked_tree(
 			&diff, git_index_owner(index), i_tree, &opts)) < 0)
 		goto cleanup;
 
-	if ((error = git_diff_foreach(
-			diff, update_index_cb, NULL, NULL, &data)) < 0)
-	{
-		if (error == GIT_EUSER)
-			error = data.error;
+	if ((error = stash_update_index_from_diff(index, diff, &data)) < 0)
 		goto cleanup;
-	}
 
 	error = build_tree_from_index(tree_out, index);
 
 cleanup:
-	git_diff_list_free(diff);
+	git_diff_free(diff);
 	git_tree_free(i_tree);
 	return error;
 }
@@ -267,7 +256,7 @@ cleanup:
 static int commit_untracked(
 	git_commit **u_commit,
 	git_index *index,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message,
 	git_commit *i_commit,
 	uint32_t flags)
@@ -311,41 +300,29 @@ static int build_workdir_tree(
 {
 	git_repository *repo = git_index_owner(index);
 	git_tree *b_tree = NULL;
-	git_diff_list *diff = NULL, *diff2 = NULL;
+	git_diff *diff = NULL;
 	git_diff_options opts = GIT_DIFF_OPTIONS_INIT;
-	struct cb_data data = {0};
+	struct stash_update_rules data = {0};
 	int error;
 
-	if ((error = git_commit_tree(&b_tree, b_commit)) < 0)
-		goto cleanup;
+	opts.flags = GIT_DIFF_IGNORE_SUBMODULES;
 
-	if ((error = git_diff_tree_to_index(&diff, repo, b_tree, NULL, &opts)) < 0)
-		goto cleanup;
-
-	if ((error = git_diff_index_to_workdir(&diff2, repo, NULL, &opts)) < 0)
+	if ((error = git_commit_tree(&b_tree, b_commit)) < 0)
 		goto cleanup;
 
-	if ((error = git_diff_merge(diff, diff2)) < 0)
+	if ((error = git_diff_tree_to_workdir_with_index(
+			&diff, repo, b_tree, &opts)) < 0)
 		goto cleanup;
 
-	data.index = index;
 	data.include_changed = true;
 
-	if ((error = git_diff_foreach(
-			diff, update_index_cb, NULL, NULL, &data)) < 0)
-	{
-		if (error == GIT_EUSER)
-			error = data.error;
+	if ((error = stash_update_index_from_diff(index, diff, &data)) < 0)
 		goto cleanup;
-	}
 
-
-	if ((error = build_tree_from_index(tree_out, index)) < 0)
-		goto cleanup;
+	error = build_tree_from_index(tree_out, index);
 
 cleanup:
-	git_diff_list_free(diff);
-	git_diff_list_free(diff2);
+	git_diff_free(diff);
 	git_tree_free(b_tree);
 
 	return error;
@@ -354,7 +331,7 @@ cleanup:
 static int commit_worktree(
 	git_oid *w_commit_oid,
 	git_index *index,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message,
 	git_commit *i_commit,
 	git_commit *b_commit,
@@ -431,17 +408,19 @@ cleanup:
 static int update_reflog(
 	git_oid *w_commit_oid,
 	git_repository *repo,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message)
 {
-	git_reference *stash = NULL;
+	git_reference *stash;
 	git_reflog *reflog = NULL;
 	int error;
 
 	if ((error = git_reference_create(&stash, repo, GIT_REFS_STASH_FILE, w_commit_oid, 1)) < 0)
 		goto cleanup;
 
-	if ((error = git_reflog_read(&reflog, stash)) < 0)
+	git_reference_free(stash);
+
+	if ((error = git_reflog_read(&reflog, repo, GIT_REFS_STASH_FILE) < 0))
 		goto cleanup;
 
 	if ((error = git_reflog_append(reflog, w_commit_oid, stasher, message)) < 0)
@@ -451,7 +430,6 @@ static int update_reflog(
 		goto cleanup;
 
 cleanup:
-	git_reference_free(stash);
 	git_reflog_free(reflog);
 	return error;
 }
@@ -474,12 +452,14 @@ static int ensure_there_are_changes_to_stash(
 	git_status_options opts = GIT_STATUS_OPTIONS_INIT;
 
 	opts.show  = GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
+	opts.flags = GIT_STATUS_OPT_EXCLUDE_SUBMODULES;
+
 	if (include_untracked_files)
-		opts.flags = GIT_STATUS_OPT_INCLUDE_UNTRACKED |
+		opts.flags |= GIT_STATUS_OPT_INCLUDE_UNTRACKED |
 		GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS;
 
 	if (include_ignored_files)
-		opts.flags = GIT_STATUS_OPT_INCLUDE_IGNORED;
+		opts.flags |= GIT_STATUS_OPT_INCLUDE_IGNORED;
 
 	error = git_status_foreach_ext(repo, &opts, is_dirty_cb, NULL);
 
@@ -510,7 +490,7 @@ static int reset_index_and_workdir(
 int git_stash_save(
 	git_oid *out,
 	git_repository *repo,
-	git_signature *stasher,
+	const git_signature *stasher,
 	const char *message,
 	uint32_t flags)
 {
@@ -595,7 +575,7 @@ int git_stash_foreach(
 	if (error < 0)
 		goto cleanup;
 
-	if ((error = git_reflog_read(&reflog, stash)) < 0)
+	if ((error = git_reflog_read(&reflog, repo, GIT_REFS_STASH_FILE)) < 0)
 		goto cleanup;
 
 	max = git_reflog_entrycount(reflog);
@@ -629,7 +609,7 @@ int git_stash_drop(
 	if ((error = git_reference_lookup(&stash, repo, GIT_REFS_STASH_FILE)) < 0)
 		return error;
 
-	if ((error = git_reflog_read(&reflog, stash)) < 0)
+	if ((error = git_reflog_read(&reflog, repo, GIT_REFS_STASH_FILE)) < 0)
 		goto cleanup;
 
 	max = git_reflog_entrycount(reflog);
diff --git a/src/status.c b/src/status.c
index e520c1017..07fdcb5c3 100644
--- a/src/status.c
+++ b/src/status.c
@@ -52,7 +52,7 @@ static unsigned int index_delta2status(const git_diff_delta *head2idx)
 }
 
 static unsigned int workdir_delta2status(
-	git_diff_list *diff, git_diff_delta *idx2wd)
+	git_diff *diff, git_diff_delta *idx2wd)
 {
 	git_status_t st = GIT_STATUS_CURRENT;
 
@@ -225,24 +225,6 @@ static git_status_list *git_status_list_alloc(git_index *index)
 	return status;
 }
 
-/*
-static int newfile_cmp(const void *a, const void *b)
-{
-	const git_diff_delta *delta_a = a;
-	const git_diff_delta *delta_b = b;
-
-	return git__strcmp(delta_a->new_file.path, delta_b->new_file.path);
-}
-
-static int newfile_casecmp(const void *a, const void *b)
-{
-	const git_diff_delta *delta_a = a;
-	const git_diff_delta *delta_b = b;
-
-	return git__strcasecmp(delta_a->new_file.path, delta_b->new_file.path);
-}
-*/
-
 int git_status_list_new(
 	git_status_list **out,
 	git_repository *repo,
@@ -251,14 +233,14 @@ int git_status_list_new(
 	git_index *index = NULL;
 	git_status_list *status = NULL;
 	git_diff_options diffopt = GIT_DIFF_OPTIONS_INIT;
-	git_diff_find_options findopts_i2w = GIT_DIFF_FIND_OPTIONS_INIT;
+	git_diff_find_options findopt = GIT_DIFF_FIND_OPTIONS_INIT;
 	git_tree *head = NULL;
 	git_status_show_t show =
 		opts ? opts->show : GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
 	int error = 0;
 	unsigned int flags = opts ? opts->flags : GIT_STATUS_OPT_DEFAULTS;
 
-	assert(show <= GIT_STATUS_SHOW_INDEX_THEN_WORKDIR);
+	assert(show <= GIT_STATUS_SHOW_WORKDIR_ONLY);
 
 	*out = NULL;
 
@@ -269,12 +251,17 @@ int git_status_list_new(
 		return error;
 
 	/* if there is no HEAD, that's okay - we'll make an empty iterator */
-	if (((error = git_repository_head_tree(&head, repo)) < 0) &&
-		error != GIT_ENOTFOUND && error != GIT_EORPHANEDHEAD) {
-		git_index_free(index); /* release index */
-		return error;
+	if ((error = git_repository_head_tree(&head, repo)) < 0) {
+		if (error != GIT_ENOTFOUND && error != GIT_EUNBORNBRANCH)
+			goto done;
+		giterr_clear();
 	}
 
+	/* refresh index from disk unless prevented */
+	if ((flags & GIT_STATUS_OPT_NO_REFRESH) == 0 &&
+		git_index_read(index, false) < 0)
+		giterr_clear();
+
 	status = git_status_list_alloc(index);
 	GITERR_CHECK_ALLOC(status);
 
@@ -284,6 +271,7 @@ int git_status_list_new(
 	}
 
 	diffopt.flags = GIT_DIFF_INCLUDE_TYPECHANGE;
+	findopt.flags = GIT_DIFF_FIND_FOR_UNTRACKED;
 
 	if ((flags & GIT_STATUS_OPT_INCLUDE_UNTRACKED) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_INCLUDE_UNTRACKED;
@@ -300,37 +288,32 @@ int git_status_list_new(
 	if ((flags & GIT_STATUS_OPT_EXCLUDE_SUBMODULES) != 0)
 		diffopt.flags = diffopt.flags | GIT_DIFF_IGNORE_SUBMODULES;
 
-	findopts_i2w.flags |= GIT_DIFF_FIND_FOR_UNTRACKED;
+	if ((flags & GIT_STATUS_OPT_RENAMES_FROM_REWRITES) != 0)
+		findopt.flags = findopt.flags |
+			GIT_DIFF_FIND_AND_BREAK_REWRITES |
+			GIT_DIFF_FIND_RENAMES_FROM_REWRITES |
+			GIT_DIFF_BREAK_REWRITES_FOR_RENAMES_ONLY;
 
 	if (show != GIT_STATUS_SHOW_WORKDIR_ONLY) {
 		if ((error = git_diff_tree_to_index(
-				&status->head2idx, repo, head, NULL, &diffopt)) < 0)
+				&status->head2idx, repo, head, index, &diffopt)) < 0)
 			goto done;
 
 		if ((flags & GIT_STATUS_OPT_RENAMES_HEAD_TO_INDEX) != 0 &&
-			(error = git_diff_find_similar(status->head2idx, NULL)) < 0)
+			(error = git_diff_find_similar(status->head2idx, &findopt)) < 0)
 			goto done;
 	}
 
 	if (show != GIT_STATUS_SHOW_INDEX_ONLY) {
 		if ((error = git_diff_index_to_workdir(
-				&status->idx2wd, repo, NULL, &diffopt)) < 0)
+				&status->idx2wd, repo, index, &diffopt)) < 0)
 			goto done;
 
 		if ((flags & GIT_STATUS_OPT_RENAMES_INDEX_TO_WORKDIR) != 0 &&
-			(error = git_diff_find_similar(status->idx2wd, &findopts_i2w)) < 0)
+			(error = git_diff_find_similar(status->idx2wd, &findopt)) < 0)
 			goto done;
 	}
 
-	if (show == GIT_STATUS_SHOW_INDEX_THEN_WORKDIR) {
-		if ((error = git_diff__paired_foreach(
-				status->head2idx, NULL, status_collect, status)) < 0)
-			goto done;
-
-		git_diff_list_free(status->head2idx);
-		status->head2idx = NULL;
-	}
-
 	if ((error = git_diff__paired_foreach(
 			status->head2idx, status->idx2wd, status_collect, status)) < 0)
 		goto done;
@@ -383,8 +366,8 @@ void git_status_list_free(git_status_list *status)
 	if (status == NULL)
 		return;
 
-	git_diff_list_free(status->head2idx);
-	git_diff_list_free(status->idx2wd);
+	git_diff_free(status->head2idx);
+	git_diff_free(status->idx2wd);
 
 	git_vector_foreach(&status->paired, i, status_entry)
 		git__free(status_entry);
diff --git a/src/status.h b/src/status.h
index b58e0ebd6..33008b89c 100644
--- a/src/status.h
+++ b/src/status.h
@@ -14,8 +14,8 @@
 struct git_status_list {
 	git_status_options opts;
 
-	git_diff_list *head2idx;
-	git_diff_list *idx2wd;
+	git_diff *head2idx;
+	git_diff *idx2wd;
 
 	git_vector paired;
 };
diff --git a/src/strmap.c b/src/strmap.c
new file mode 100644
index 000000000..b26a13d1f
--- /dev/null
+++ b/src/strmap.c
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "strmap.h"
+
+int git_strmap_next(
+	void **data,
+	git_strmap_iter* iter,
+	git_strmap *map)
+{
+	if (!map)
+		return GIT_ERROR;
+
+	while (*iter != git_strmap_end(map)) {
+		if (!(git_strmap_has_data(map, *iter))) {
+			++(*iter);
+			continue;
+		}
+
+		*data = git_strmap_value_at(map, *iter);
+
+		++(*iter);
+
+		return GIT_OK;
+	}
+
+	return GIT_ITEROVER;
+}
diff --git a/src/strmap.h b/src/strmap.h
index 44176a0fc..8276ab468 100644
--- a/src/strmap.h
+++ b/src/strmap.h
@@ -17,6 +17,7 @@
 
 __KHASH_TYPE(str, const char *, void *);
 typedef khash_t(str) git_strmap;
+typedef khiter_t git_strmap_iter;
 
 #define GIT__USE_STRMAP \
 	__KHASH_IMPL(str, static kh_inline, const char *, void *, 1, kh_str_hash_func, kh_str_hash_equal)
@@ -31,7 +32,9 @@ typedef khash_t(str) git_strmap;
 #define git_strmap_valid_index(h, idx) (idx != kh_end(h))
 
 #define git_strmap_exists(h, k) (kh_get(str, h, k) != kh_end(h))
+#define git_strmap_has_data(h, idx) kh_exist(h, idx)
 
+#define git_strmap_key(h, idx)             kh_key(h, idx)
 #define git_strmap_value_at(h, idx)        kh_val(h, idx)
 #define git_strmap_set_value_at(h, idx, v) kh_val(h, idx) = v
 #define git_strmap_delete_at(h, idx)       kh_del(str, h, idx)
@@ -61,4 +64,12 @@ typedef khash_t(str) git_strmap;
 #define git_strmap_foreach		kh_foreach
 #define git_strmap_foreach_value	kh_foreach_value
 
+#define git_strmap_begin		kh_begin
+#define git_strmap_end		kh_end
+
+int git_strmap_next(
+	void **data,
+	git_strmap_iter* iter,
+	git_strmap *map);
+
 #endif
diff --git a/src/submodule.c b/src/submodule.c
index 89eba2aa4..586494fed 100644
--- a/src/submodule.c
+++ b/src/submodule.c
@@ -9,9 +9,7 @@
 #include "git2/config.h"
 #include "git2/sys/config.h"
 #include "git2/types.h"
-#include "git2/repository.h"
 #include "git2/index.h"
-#include "git2/submodule.h"
 #include "buffer.h"
 #include "buf_text.h"
 #include "vector.h"
@@ -32,6 +30,8 @@ static git_cvar_map _sm_update_map[] = {
 	{GIT_CVAR_STRING, "rebase", GIT_SUBMODULE_UPDATE_REBASE},
 	{GIT_CVAR_STRING, "merge", GIT_SUBMODULE_UPDATE_MERGE},
 	{GIT_CVAR_STRING, "none", GIT_SUBMODULE_UPDATE_NONE},
+	{GIT_CVAR_FALSE, NULL, GIT_SUBMODULE_UPDATE_NONE},
+	{GIT_CVAR_TRUE, NULL, GIT_SUBMODULE_UPDATE_CHECKOUT},
 };
 
 static git_cvar_map _sm_ignore_map[] = {
@@ -39,6 +39,8 @@ static git_cvar_map _sm_ignore_map[] = {
 	{GIT_CVAR_STRING, "untracked", GIT_SUBMODULE_IGNORE_UNTRACKED},
 	{GIT_CVAR_STRING, "dirty", GIT_SUBMODULE_IGNORE_DIRTY},
 	{GIT_CVAR_STRING, "all", GIT_SUBMODULE_IGNORE_ALL},
+	{GIT_CVAR_FALSE, NULL, GIT_SUBMODULE_IGNORE_NONE},
+	{GIT_CVAR_TRUE, NULL, GIT_SUBMODULE_IGNORE_ALL},
 };
 
 static kh_inline khint_t str_hash_no_trailing_slash(const char *s)
@@ -73,15 +75,11 @@ static int load_submodule_config(git_repository *repo);
 static git_config_backend *open_gitmodules(git_repository *, bool, const git_oid *);
 static int lookup_head_remote(git_buf *url, git_repository *repo);
 static int submodule_get(git_submodule **, git_repository *, const char *, const char *);
-static void submodule_release(git_submodule *sm, int decr);
-static int submodule_load_from_index(git_repository *, const git_index_entry *);
-static int submodule_load_from_head(git_repository*, const char*, const git_oid*);
 static int submodule_load_from_config(const git_config_entry *, void *);
 static int submodule_load_from_wd_lite(git_submodule *, const char *, void *);
 static int submodule_update_config(git_submodule *, const char *, const char *, bool, bool);
-static void submodule_mode_mismatch(git_repository *, const char *, unsigned int);
-static int submodule_index_status(unsigned int *status, git_submodule *sm);
-static int submodule_wd_status(unsigned int *status, git_submodule *sm);
+static void submodule_get_index_status(unsigned int *, git_submodule *);
+static void submodule_get_wd_status(unsigned int *, git_submodule *, git_repository *, git_submodule_ignore_t);
 
 static int submodule_cmp(const void *a, const void *b)
 {
@@ -163,7 +161,7 @@ int git_submodule_foreach(
 		 * us from issuing a callback twice for a submodule where the name
 		 * and path are not the same.
 		 */
-		if (sm->refcount > 1) {
+		if (GIT_REFCOUNT_VAL(sm) > 1) {
 			if (git_vector_bsearch(NULL, &seen, sm) != GIT_ENOTFOUND)
 				continue;
 			if ((error = git_vector_insert(&seen, sm)) < 0)
@@ -195,9 +193,7 @@ void git_submodule_config_free(git_repository *repo)
 	if (smcfg == NULL)
 		return;
 
-	git_strmap_foreach_value(smcfg, sm, {
-		submodule_release(sm,1);
-	});
+	git_strmap_foreach_value(smcfg, sm, { git_submodule_free(sm); });
 	git_strmap_free(smcfg);
 }
 
@@ -338,7 +334,7 @@ int git_submodule_add_finalize(git_submodule *sm)
 
 	assert(sm);
 
-	if ((error = git_repository_index__weakptr(&index, sm->owner)) < 0 ||
+	if ((error = git_repository_index__weakptr(&index, sm->repo)) < 0 ||
 		(error = git_index_add_bypath(index, GIT_MODULES_FILE)) < 0)
 		return error;
 
@@ -348,7 +344,7 @@ int git_submodule_add_finalize(git_submodule *sm)
 int git_submodule_add_to_index(git_submodule *sm, int write_index)
 {
 	int error;
-	git_repository *repo, *sm_repo = NULL;
+	git_repository *sm_repo = NULL;
 	git_index *index;
 	git_buf path = GIT_BUF_INIT;
 	git_commit *head;
@@ -357,14 +353,12 @@ int git_submodule_add_to_index(git_submodule *sm, int write_index)
 
 	assert(sm);
 
-	repo = sm->owner;
-
 	/* force reload of wd OID by git_submodule_open */
 	sm->flags = sm->flags & ~GIT_SUBMODULE_STATUS__WD_OID_VALID;
 
-	if ((error = git_repository_index__weakptr(&index, repo)) < 0 ||
+	if ((error = git_repository_index__weakptr(&index, sm->repo)) < 0 ||
 		(error = git_buf_joinpath(
-			&path, git_repository_workdir(repo), sm->path)) < 0 ||
+			&path, git_repository_workdir(sm->repo), sm->path)) < 0 ||
 		(error = git_submodule_open(&sm_repo, sm)) < 0)
 		goto cleanup;
 
@@ -378,7 +372,8 @@ int git_submodule_add_to_index(git_submodule *sm, int write_index)
 
 	memset(&entry, 0, sizeof(entry));
 	entry.path = sm->path;
-	git_index_entry__init_from_stat(&entry, &st);
+	git_index_entry__init_from_stat(
+		&entry, &st, !(git_index_caps(index) & GIT_INDEXCAP_NO_FILEMODE));
 
 	/* calling git_submodule_open will have set sm->wd_oid if possible */
 	if ((sm->flags & GIT_SUBMODULE_STATUS__WD_OID_VALID) == 0) {
@@ -416,15 +411,34 @@ cleanup:
 	return error;
 }
 
+const char *git_submodule_ignore_to_str(git_submodule_ignore_t ignore)
+{
+	int i;
+	for (i = 0; i < (int)ARRAY_SIZE(_sm_ignore_map); ++i)
+		if (_sm_ignore_map[i].map_value == ignore)
+			return _sm_ignore_map[i].str_match;
+	return NULL;
+}
+
+const char *git_submodule_update_to_str(git_submodule_update_t update)
+{
+	int i;
+	for (i = 0; i < (int)ARRAY_SIZE(_sm_update_map); ++i)
+		if (_sm_update_map[i].map_value == update)
+			return _sm_update_map[i].str_match;
+	return NULL;
+}
+
 int git_submodule_save(git_submodule *submodule)
 {
 	int error = 0;
 	git_config_backend *mods;
 	git_buf key = GIT_BUF_INIT;
+	const char *val;
 
 	assert(submodule);
 
-	mods = open_gitmodules(submodule->owner, true, NULL);
+	mods = open_gitmodules(submodule->repo, true, NULL);
 	if (!mods) {
 		giterr_set(GITERR_SUBMODULE,
 			"Adding submodules to a bare repository is not supported (for now)");
@@ -445,22 +459,14 @@ int git_submodule_save(git_submodule *submodule)
 		goto cleanup;
 
 	if (!(error = submodule_config_key_trunc_puts(&key, "update")) &&
-		submodule->update != GIT_SUBMODULE_UPDATE_DEFAULT)
-	{
-		const char *val = (submodule->update == GIT_SUBMODULE_UPDATE_CHECKOUT) ?
-			NULL : _sm_update_map[submodule->update].str_match;
+		(val = git_submodule_update_to_str(submodule->update)) != NULL)
 		error = git_config_file_set_string(mods, key.ptr, val);
-	}
 	if (error < 0)
 		goto cleanup;
 
 	if (!(error = submodule_config_key_trunc_puts(&key, "ignore")) &&
-		submodule->ignore != GIT_SUBMODULE_IGNORE_DEFAULT)
-	{
-		const char *val = (submodule->ignore == GIT_SUBMODULE_IGNORE_NONE) ?
-			NULL : _sm_ignore_map[submodule->ignore].str_match;
+		(val = git_submodule_ignore_to_str(submodule->ignore)) != NULL)
 		error = git_config_file_set_string(mods, key.ptr, val);
-	}
 	if (error < 0)
 		goto cleanup;
 
@@ -487,7 +493,7 @@ cleanup:
 git_repository *git_submodule_owner(git_submodule *submodule)
 {
 	assert(submodule);
-	return submodule->owner;
+	return submodule->repo;
 }
 
 const char *git_submodule_name(git_submodule *submodule)
@@ -544,11 +550,12 @@ const git_oid *git_submodule_wd_id(git_submodule *submodule)
 {
 	assert(submodule);
 
+	/* load unless we think we have a valid oid */
 	if (!(submodule->flags & GIT_SUBMODULE_STATUS__WD_OID_VALID)) {
 		git_repository *subrepo;
 
 		/* calling submodule open grabs the HEAD OID if possible */
-		if (!git_submodule_open(&subrepo, submodule))
+		if (!git_submodule_open_bare(&subrepo, submodule))
 			git_repository_free(subrepo);
 		else
 			giterr_clear();
@@ -563,7 +570,8 @@ const git_oid *git_submodule_wd_id(git_submodule *submodule)
 git_submodule_ignore_t git_submodule_ignore(git_submodule *submodule)
 {
 	assert(submodule);
-	return submodule->ignore;
+	return (submodule->ignore < GIT_SUBMODULE_IGNORE_NONE) ?
+		GIT_SUBMODULE_IGNORE_NONE : submodule->ignore;
 }
 
 git_submodule_ignore_t git_submodule_set_ignore(
@@ -573,7 +581,7 @@ git_submodule_ignore_t git_submodule_set_ignore(
 
 	assert(submodule);
 
-	if (ignore == GIT_SUBMODULE_IGNORE_DEFAULT)
+	if (ignore == GIT_SUBMODULE_IGNORE_RESET)
 		ignore = submodule->ignore_default;
 
 	old = submodule->ignore;
@@ -584,7 +592,8 @@ git_submodule_ignore_t git_submodule_set_ignore(
 git_submodule_update_t git_submodule_update(git_submodule *submodule)
 {
 	assert(submodule);
-	return submodule->update;
+	return (submodule->update < GIT_SUBMODULE_UPDATE_CHECKOUT) ?
+		GIT_SUBMODULE_UPDATE_CHECKOUT : submodule->update;
 }
 
 git_submodule_update_t git_submodule_set_update(
@@ -594,7 +603,7 @@ git_submodule_update_t git_submodule_set_update(
 
 	assert(submodule);
 
-	if (update == GIT_SUBMODULE_UPDATE_DEFAULT)
+	if (update == GIT_SUBMODULE_UPDATE_RESET)
 		update = submodule->update_default;
 
 	old = submodule->update;
@@ -625,6 +634,7 @@ int git_submodule_set_fetch_recurse_submodules(
 int git_submodule_init(git_submodule *submodule, int overwrite)
 {
 	int error;
+	const char *val;
 
 	/* write "submodule.NAME.url" */
 
@@ -641,14 +651,10 @@ int git_submodule_init(git_submodule *submodule, int overwrite)
 
 	/* write "submodule.NAME.update" if not default */
 
-	if (submodule->update == GIT_SUBMODULE_UPDATE_CHECKOUT)
-		error = submodule_update_config(
-			submodule, "update", NULL, (overwrite != 0), false);
-	else if (submodule->update != GIT_SUBMODULE_UPDATE_DEFAULT)
-		error = submodule_update_config(
-			submodule, "update",
-			_sm_update_map[submodule->update].str_match,
-			(overwrite != 0), false);
+	val = (submodule->update == GIT_SUBMODULE_UPDATE_CHECKOUT) ?
+		NULL : git_submodule_update_to_str(submodule->update);
+	error = submodule_update_config(
+		submodule, "update", val, (overwrite != 0), false);
 
 	return error;
 }
@@ -667,51 +673,70 @@ int git_submodule_sync(git_submodule *submodule)
 		submodule, "url", submodule->url, true, true);
 }
 
-int git_submodule_open(
-	git_repository **subrepo,
-	git_submodule *submodule)
+static int git_submodule__open(
+	git_repository **subrepo, git_submodule *sm, bool bare)
 {
 	int error;
 	git_buf path = GIT_BUF_INIT;
-	git_repository *repo;
-	const char *workdir;
+	unsigned int flags = GIT_REPOSITORY_OPEN_NO_SEARCH;
+	const char *wd;
 
-	assert(submodule && subrepo);
+	assert(sm && subrepo);
 
-	repo = submodule->owner;
-	workdir = git_repository_workdir(repo);
+	if (git_repository__ensure_not_bare(
+			sm->repo, "open submodule repository") < 0)
+		return GIT_EBAREREPO;
 
-	if (!workdir) {
-		giterr_set(GITERR_REPOSITORY,
-			"Cannot open submodule repository in a bare repo");
-		return GIT_ENOTFOUND;
-	}
-
-	if ((submodule->flags & GIT_SUBMODULE_STATUS_IN_WD) == 0) {
-		giterr_set(GITERR_REPOSITORY,
-			"Cannot open submodule repository that is not checked out");
-		return GIT_ENOTFOUND;
-	}
+	wd = git_repository_workdir(sm->repo);
 
-	if (git_buf_joinpath(&path, workdir, submodule->path) < 0)
+	if (git_buf_joinpath(&path, wd, sm->path) < 0 ||
+		git_buf_joinpath(&path, path.ptr, DOT_GIT) < 0)
 		return -1;
 
-	error = git_repository_open(subrepo, path.ptr);
+	sm->flags = sm->flags &
+		~(GIT_SUBMODULE_STATUS_IN_WD |
+		  GIT_SUBMODULE_STATUS__WD_OID_VALID |
+		  GIT_SUBMODULE_STATUS__WD_SCANNED);
 
-	git_buf_free(&path);
+	if (bare)
+		flags |= GIT_REPOSITORY_OPEN_BARE;
+
+	error = git_repository_open_ext(subrepo, path.ptr, flags, wd);
 
-	/* if we have opened the submodule successfully, let's grab the HEAD OID */
+	/* if we opened the submodule successfully, grab HEAD OID, etc. */
 	if (!error) {
-		if (!git_reference_name_to_id(
-				&submodule->wd_oid, *subrepo, GIT_HEAD_FILE))
-			submodule->flags |= GIT_SUBMODULE_STATUS__WD_OID_VALID;
+		sm->flags |= GIT_SUBMODULE_STATUS_IN_WD |
+			GIT_SUBMODULE_STATUS__WD_SCANNED;
+
+		if (!git_reference_name_to_id(&sm->wd_oid, *subrepo, GIT_HEAD_FILE))
+			sm->flags |= GIT_SUBMODULE_STATUS__WD_OID_VALID;
 		else
 			giterr_clear();
+	} else if (git_path_exists(path.ptr)) {
+		sm->flags |= GIT_SUBMODULE_STATUS__WD_SCANNED |
+			GIT_SUBMODULE_STATUS_IN_WD;
+	} else {
+		git_buf_rtruncate_at_char(&path, '/'); /* remove "/.git" */
+
+		if (git_path_isdir(path.ptr))
+			sm->flags |= GIT_SUBMODULE_STATUS__WD_SCANNED;
 	}
 
+	git_buf_free(&path);
+
 	return error;
 }
 
+int git_submodule_open_bare(git_repository **subrepo, git_submodule *sm)
+{
+	return git_submodule__open(subrepo, sm, true);
+}
+
+int git_submodule_open(git_repository **subrepo, git_submodule *sm)
+{
+	return git_submodule__open(subrepo, sm, false);
+}
+
 int git_submodule_reload_all(git_repository *repo)
 {
 	assert(repo);
@@ -719,74 +744,100 @@ int git_submodule_reload_all(git_repository *repo)
 	return load_submodule_config(repo);
 }
 
-int git_submodule_reload(git_submodule *submodule)
+static void submodule_update_from_index_entry(
+	git_submodule *sm, const git_index_entry *ie)
 {
-	git_repository *repo;
-	git_index *index;
-	int error;
-	size_t pos;
-	git_tree *head;
-	git_config_backend *mods;
+	bool already_found = (sm->flags & GIT_SUBMODULE_STATUS_IN_INDEX) != 0;
 
-	assert(submodule);
+	if (!S_ISGITLINK(ie->mode)) {
+		if (!already_found)
+			sm->flags |= GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE;
+	} else {
+		if (already_found)
+			sm->flags |= GIT_SUBMODULE_STATUS__INDEX_MULTIPLE_ENTRIES;
+		else
+			git_oid_cpy(&sm->index_oid, &ie->oid);
 
-	/* refresh index data */
+		sm->flags |= GIT_SUBMODULE_STATUS_IN_INDEX |
+			GIT_SUBMODULE_STATUS__INDEX_OID_VALID;
+	}
+}
+
+static int submodule_update_index(git_submodule *sm)
+{
+	git_index *index;
+	const git_index_entry *ie;
 
-	repo = submodule->owner;
-	if (git_repository_index__weakptr(&index, repo) < 0)
+	if (git_repository_index__weakptr(&index, sm->repo) < 0)
 		return -1;
 
-	submodule->flags = submodule->flags &
+	sm->flags = sm->flags &
 		~(GIT_SUBMODULE_STATUS_IN_INDEX |
 		  GIT_SUBMODULE_STATUS__INDEX_OID_VALID);
 
-	if (!git_index_find(&pos, index, submodule->path)) {
-		const git_index_entry *entry = git_index_get_byindex(index, pos);
+	if (!(ie = git_index_get_bypath(index, sm->path, 0)))
+		return 0;
 
-		if (S_ISGITLINK(entry->mode)) {
-			if ((error = submodule_load_from_index(repo, entry)) < 0)
-				return error;
-		} else {
-			submodule_mode_mismatch(
-				repo, entry->path, GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE);
-		}
+	submodule_update_from_index_entry(sm, ie);
+
+	return 0;
+}
+
+static void submodule_update_from_head_data(
+	git_submodule *sm, mode_t mode, const git_oid *id)
+{
+	if (!S_ISGITLINK(mode))
+		sm->flags |= GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE;
+	else {
+		git_oid_cpy(&sm->head_oid, id);
+
+		sm->flags |= GIT_SUBMODULE_STATUS_IN_HEAD |
+			GIT_SUBMODULE_STATUS__HEAD_OID_VALID;
 	}
+}
 
-	/* refresh HEAD tree data */
+static int submodule_update_head(git_submodule *submodule)
+{
+	git_tree *head = NULL;
+	git_tree_entry *te = NULL;
 
-	if (!(error = git_repository_head_tree(&head, repo))) {
-		git_tree_entry *te;
+	submodule->flags = submodule->flags &
+		~(GIT_SUBMODULE_STATUS_IN_HEAD |
+		  GIT_SUBMODULE_STATUS__HEAD_OID_VALID);
 
-		submodule->flags = submodule->flags &
-			~(GIT_SUBMODULE_STATUS_IN_HEAD |
-			  GIT_SUBMODULE_STATUS__HEAD_OID_VALID);
+	/* if we can't look up file in current head, then done */
+	if (git_repository_head_tree(&head, submodule->repo) < 0 ||
+		git_tree_entry_bypath(&te, head, submodule->path) < 0)
+		giterr_clear();
+	else
+		submodule_update_from_head_data(submodule, te->attr, &te->oid);
 
-		if (!(error = git_tree_entry_bypath(&te, head, submodule->path))) {
+	git_tree_entry_free(te);
+	git_tree_free(head);
+	return 0;
+}
 
-			if (S_ISGITLINK(te->attr)) {
-				error = submodule_load_from_head(repo, submodule->path, &te->oid);
-			} else {
-				submodule_mode_mismatch(
-					repo, submodule->path,
-					GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE);
-			}
+int git_submodule_reload(git_submodule *submodule)
+{
+	int error = 0;
+	git_config_backend *mods;
 
-			git_tree_entry_free(te);
-		}
-		else if (error == GIT_ENOTFOUND) {
-			giterr_clear();
-			error = 0;
-		}
+	assert(submodule);
 
-		git_tree_free(head);
-	}
+	/* refresh index data */
 
-	if (error < 0)
-		return error;
+	if (submodule_update_index(submodule) < 0)
+		return -1;
+
+	/* refresh HEAD tree data */
+
+	if (submodule_update_head(submodule) < 0)
+		return -1;
 
 	/* refresh config data */
 
-	if ((mods = open_gitmodules(repo, false, NULL)) != NULL) {
+	mods = open_gitmodules(submodule->repo, false, NULL);
+	if (mods != NULL) {
 		git_buf path = GIT_BUF_INIT;
 
 		git_buf_sets(&path, "submodule\\.");
@@ -797,7 +848,7 @@ int git_submodule_reload(git_submodule *submodule)
 			error = -1;
 		else
 			error = git_config_file_foreach_match(
-				mods, path.ptr, submodule_load_from_config, repo);
+				mods, path.ptr, submodule_load_from_config, submodule->repo);
 
 		git_buf_free(&path);
 		git_config_file_free(mods);
@@ -816,38 +867,90 @@ int git_submodule_reload(git_submodule *submodule)
 	return error;
 }
 
-int git_submodule_status(
-	unsigned int *status,
-	git_submodule *submodule)
+static void submodule_copy_oid_maybe(
+	git_oid *tgt, const git_oid *src, bool valid)
 {
-	int error = 0;
-	unsigned int status_val;
+	if (tgt) {
+		if (valid)
+			memcpy(tgt, src, sizeof(*tgt));
+		else
+			memset(tgt, 0, sizeof(*tgt));
+	}
+}
+
+int git_submodule__status(
+	unsigned int *out_status,
+	git_oid *out_head_id,
+	git_oid *out_index_id,
+	git_oid *out_wd_id,
+	git_submodule *sm,
+	git_submodule_ignore_t ign)
+{
+	unsigned int status;
+	git_repository *smrepo = NULL;
+
+	if (ign < GIT_SUBMODULE_IGNORE_NONE)
+		ign = sm->ignore;
 
-	assert(status && submodule);
+	/* only return location info if ignore == all */
+	if (ign == GIT_SUBMODULE_IGNORE_ALL) {
+		*out_status = (sm->flags & GIT_SUBMODULE_STATUS__IN_FLAGS);
+		return 0;
+	}
 
-	status_val = GIT_SUBMODULE_STATUS__CLEAR_INTERNAL(submodule->flags);
+	/* refresh the index OID */
+	if (submodule_update_index(sm) < 0)
+		return -1;
+
+	/* refresh the HEAD OID */
+	if (submodule_update_head(sm) < 0)
+		return -1;
 
-	if (submodule->ignore != GIT_SUBMODULE_IGNORE_ALL) {
-		if (!(error = submodule_index_status(&status_val, submodule)))
-			error = submodule_wd_status(&status_val, submodule);
+	/* for ignore == dirty, don't scan the working directory */
+	if (ign == GIT_SUBMODULE_IGNORE_DIRTY) {
+		/* git_submodule_open_bare will load WD OID data */
+		if (git_submodule_open_bare(&smrepo, sm) < 0)
+			giterr_clear();
+		else
+			git_repository_free(smrepo);
+		smrepo = NULL;
+	} else if (git_submodule_open(&smrepo, sm) < 0) {
+		giterr_clear();
+		smrepo = NULL;
 	}
 
-	*status = status_val;
+	status = GIT_SUBMODULE_STATUS__CLEAR_INTERNAL(sm->flags);
 
-	return error;
+	submodule_get_index_status(&status, sm);
+	submodule_get_wd_status(&status, sm, smrepo, ign);
+
+	git_repository_free(smrepo);
+
+	*out_status = status;
+
+	submodule_copy_oid_maybe(out_head_id, &sm->head_oid,
+		(sm->flags & GIT_SUBMODULE_STATUS__HEAD_OID_VALID) != 0);
+	submodule_copy_oid_maybe(out_index_id, &sm->index_oid,
+		(sm->flags & GIT_SUBMODULE_STATUS__INDEX_OID_VALID) != 0);
+	submodule_copy_oid_maybe(out_wd_id, &sm->wd_oid,
+		(sm->flags & GIT_SUBMODULE_STATUS__WD_OID_VALID) != 0);
+
+	return 0;
 }
 
-int git_submodule_location(
-	unsigned int *location_status,
-	git_submodule *submodule)
+int git_submodule_status(unsigned int *status, git_submodule *sm)
 {
-	assert(location_status && submodule);
+	assert(status && sm);
 
-	*location_status = submodule->flags &
-		(GIT_SUBMODULE_STATUS_IN_HEAD | GIT_SUBMODULE_STATUS_IN_INDEX |
-		 GIT_SUBMODULE_STATUS_IN_CONFIG | GIT_SUBMODULE_STATUS_IN_WD);
+	return git_submodule__status(status, NULL, NULL, NULL, sm, 0);
+}
 
-	return 0;
+int git_submodule_location(unsigned int *location, git_submodule *sm)
+{
+	assert(location && sm);
+
+	return git_submodule__status(
+		location, NULL, NULL, NULL, sm, GIT_SUBMODULE_IGNORE_ALL);
 }
 
 
@@ -857,54 +960,50 @@ int git_submodule_location(
 
 static git_submodule *submodule_alloc(git_repository *repo, const char *name)
 {
+	size_t namelen;
 	git_submodule *sm;
 
-	if (!name || !strlen(name)) {
+	if (!name || !(namelen = strlen(name))) {
 		giterr_set(GITERR_SUBMODULE, "Invalid submodule name");
 		return NULL;
 	}
 
 	sm = git__calloc(1, sizeof(git_submodule));
 	if (sm == NULL)
-		goto fail;
+		return NULL;
 
-	sm->path = sm->name = git__strdup(name);
-	if (!sm->name)
-		goto fail;
+	sm->name = sm->path = git__strdup(name);
+	if (!sm->name) {
+		git__free(sm);
+		return NULL;
+	}
 
-	sm->owner = repo;
-	sm->refcount = 1;
+	GIT_REFCOUNT_INC(sm);
+	sm->ignore = sm->ignore_default = GIT_SUBMODULE_IGNORE_NONE;
+	sm->update = sm->update_default = GIT_SUBMODULE_UPDATE_CHECKOUT;
+	sm->repo   = repo;
 
 	return sm;
-
-fail:
-	submodule_release(sm, 0);
-	return NULL;
 }
 
-static void submodule_release(git_submodule *sm, int decr)
+static void submodule_release(git_submodule *sm)
 {
 	if (!sm)
 		return;
 
-	sm->refcount -= decr;
-
-	if (sm->refcount == 0) {
-		if (sm->name != sm->path) {
-			git__free(sm->path);
-			sm->path = NULL;
-		}
-
-		git__free(sm->name);
-		sm->name = NULL;
-
-		git__free(sm->url);
-		sm->url = NULL;
-
-		sm->owner = NULL;
+	if (sm->path != sm->name)
+		git__free(sm->path);
+	git__free(sm->name);
+	git__free(sm->url);
+	git__memzero(sm, sizeof(*sm));
+	git__free(sm);
+}
 
-		git__free(sm);
-	}
+void git_submodule_free(git_submodule *sm)
+{
+	if (!sm)
+		return;
+	GIT_REFCOUNT_DEC(sm, submodule_release);
 }
 
 static int submodule_get(
@@ -927,6 +1026,7 @@ static int submodule_get(
 
 	if (!git_strmap_valid_index(smcfg, pos)) {
 		sm = submodule_alloc(repo, name);
+		GITERR_CHECK_ALLOC(sm);
 
 		/* insert value at name - if another thread beats us to it, then use
 		 * their record and release our own.
@@ -934,10 +1034,10 @@ static int submodule_get(
 		pos = kh_put(str, smcfg, sm->name, &error);
 
 		if (error < 0) {
-			submodule_release(sm, 1);
+			git_submodule_free(sm);
 			sm = NULL;
 		} else if (error == 0) {
-			submodule_release(sm, 1);
+			git_submodule_free(sm);
 			sm = git_strmap_value_at(smcfg, pos);
 		} else {
 			git_strmap_set_value_at(smcfg, pos, sm);
@@ -951,50 +1051,41 @@ static int submodule_get(
 	return (sm != NULL) ? 0 : -1;
 }
 
-static int submodule_load_from_index(
-	git_repository *repo, const git_index_entry *entry)
+static int submodule_config_error(const char *property, const char *value)
 {
-	git_submodule *sm;
+	giterr_set(GITERR_INVALID,
+		"Invalid value for submodule '%s' property: '%s'", property, value);
+	return -1;
+}
 
-	if (submodule_get(&sm, repo, entry->path, NULL) < 0)
-		return -1;
+int git_submodule_parse_ignore(git_submodule_ignore_t *out, const char *value)
+{
+	int val;
 
-	if (sm->flags & GIT_SUBMODULE_STATUS_IN_INDEX) {
-		sm->flags |= GIT_SUBMODULE_STATUS__INDEX_MULTIPLE_ENTRIES;
-		return 0;
+	if (git_config_lookup_map_value(
+			&val, _sm_ignore_map, ARRAY_SIZE(_sm_ignore_map), value) < 0) {
+		*out = GIT_SUBMODULE_IGNORE_NONE;
+		return submodule_config_error("ignore", value);
 	}
 
-	sm->flags |= GIT_SUBMODULE_STATUS_IN_INDEX;
-
-	git_oid_cpy(&sm->index_oid, &entry->oid);
-	sm->flags |= GIT_SUBMODULE_STATUS__INDEX_OID_VALID;
-
+	*out = (git_submodule_ignore_t)val;
 	return 0;
 }
 
-static int submodule_load_from_head(
-	git_repository *repo, const char *path, const git_oid *oid)
+int git_submodule_parse_update(git_submodule_update_t *out, const char *value)
 {
-	git_submodule *sm;
-
-	if (submodule_get(&sm, repo, path, NULL) < 0)
-		return -1;
-
-	sm->flags |= GIT_SUBMODULE_STATUS_IN_HEAD;
+	int val;
 
-	git_oid_cpy(&sm->head_oid, oid);
-	sm->flags |= GIT_SUBMODULE_STATUS__HEAD_OID_VALID;
+	if (git_config_lookup_map_value(
+			&val, _sm_update_map, ARRAY_SIZE(_sm_update_map), value) < 0) {
+		*out = GIT_SUBMODULE_UPDATE_CHECKOUT;
+		return submodule_config_error("update", value);
+	}
 
+	*out = (git_submodule_update_t)val;
 	return 0;
 }
 
-static int submodule_config_error(const char *property, const char *value)
-{
-	giterr_set(GITERR_INVALID,
-		"Invalid value for submodule '%s' property: '%s'", property, value);
-	return -1;
-}
-
 static int submodule_load_from_config(
 	const git_config_entry *entry, void *data)
 {
@@ -1012,8 +1103,10 @@ static int submodule_load_from_config(
 
 	namestart = key + strlen("submodule.");
 	property  = strrchr(namestart, '.');
-	if (property == NULL)
+
+	if (!property || (property == namestart))
 		return 0;
+
 	property++;
 	is_path = (strcasecmp(property, "path") == 0);
 
@@ -1047,11 +1140,11 @@ static int submodule_load_from_config(
 		git_strmap_insert2(smcfg, alternate, sm, old_sm, error);
 
 		if (error >= 0)
-			sm->refcount++; /* inserted under a new key */
+			GIT_REFCOUNT_INC(sm); /* inserted under a new key */
 
 		/* if we replaced an old module under this key, release the old one */
 		if (old_sm && ((git_submodule *)old_sm) != sm) {
-			submodule_release(old_sm, 1);
+			git_submodule_free(old_sm);
 			/* TODO: log warning about multiple submodules with same path */
 		}
 	}
@@ -1076,22 +1169,18 @@ static int submodule_load_from_config(
 			return -1;
 	}
 	else if (strcasecmp(property, "update") == 0) {
-		int val;
-		if (git_config_lookup_map_value(
-			&val, _sm_update_map, ARRAY_SIZE(_sm_update_map), value) < 0)
-			return submodule_config_error("update", value);
-		sm->update_default = sm->update = (git_submodule_update_t)val;
+		if (git_submodule_parse_update(&sm->update, value) < 0)
+			return -1;
+		sm->update_default = sm->update;
 	}
 	else if (strcasecmp(property, "fetchRecurseSubmodules") == 0) {
 		if (git__parse_bool(&sm->fetch_recurse, value) < 0)
 			return submodule_config_error("fetchRecurseSubmodules", value);
 	}
 	else if (strcasecmp(property, "ignore") == 0) {
-		int val;
-		if (git_config_lookup_map_value(
-			&val, _sm_ignore_map, ARRAY_SIZE(_sm_ignore_map), value) < 0)
-			return submodule_config_error("ignore", value);
-		sm->ignore_default = sm->ignore = (git_submodule_ignore_t)val;
+		if (git_submodule_parse_ignore(&sm->ignore, value) < 0)
+			return -1;
+		sm->ignore_default = sm->ignore;
 	}
 	/* ignore other unknown submodule properties */
 
@@ -1101,13 +1190,15 @@ static int submodule_load_from_config(
 static int submodule_load_from_wd_lite(
 	git_submodule *sm, const char *name, void *payload)
 {
-	git_repository *repo = git_submodule_owner(sm);
 	git_buf path = GIT_BUF_INIT;
 
 	GIT_UNUSED(name);
 	GIT_UNUSED(payload);
 
-	if (git_buf_joinpath(&path, git_repository_workdir(repo), sm->path) < 0)
+	if (git_repository_is_bare(sm->repo))
+		return 0;
+
+	if (git_buf_joinpath(&path, git_repository_workdir(sm->repo), sm->path) < 0)
 		return -1;
 
 	if (git_path_isdir(path.ptr))
@@ -1121,18 +1212,6 @@ static int submodule_load_from_wd_lite(
 	return 0;
 }
 
-static void submodule_mode_mismatch(
-	git_repository *repo, const char *path, unsigned int flag)
-{
-	khiter_t pos = git_strmap_lookup_index(repo->submodules, path);
-
-	if (git_strmap_valid_index(repo->submodules, pos)) {
-		git_submodule *sm = git_strmap_value_at(repo->submodules, pos);
-
-		sm->flags |= flag;
-	}
-}
-
 static int load_submodule_config_from_index(
 	git_repository *repo, git_oid *gitmodules_oid)
 {
@@ -1146,18 +1225,21 @@ static int load_submodule_config_from_index(
 		return error;
 
 	while (!(error = git_iterator_advance(&entry, i))) {
-
-		if (S_ISGITLINK(entry->mode)) {
-			error = submodule_load_from_index(repo, entry);
-			if (error < 0)
-				break;
-		} else {
-			submodule_mode_mismatch(
-				repo, entry->path, GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE);
-
-			if (strcmp(entry->path, GIT_MODULES_FILE) == 0)
-				git_oid_cpy(gitmodules_oid, &entry->oid);
-		}
+		khiter_t pos = git_strmap_lookup_index(repo->submodules, entry->path);
+		git_submodule *sm;
+
+		if (git_strmap_valid_index(repo->submodules, pos)) {
+			sm = git_strmap_value_at(repo->submodules, pos);
+
+			if (S_ISGITLINK(entry->mode))
+				submodule_update_from_index_entry(sm, entry);
+			else
+				sm->flags |= GIT_SUBMODULE_STATUS__INDEX_NOT_SUBMODULE;
+		} else if (S_ISGITLINK(entry->mode)) {
+			if (!submodule_get(&sm, repo, entry->path, NULL))
+				submodule_update_from_index_entry(sm, entry);
+		} else if (strcmp(entry->path, GIT_MODULES_FILE) == 0)
+			git_oid_cpy(gitmodules_oid, &entry->oid);
 	}
 
 	if (error == GIT_ITEROVER)
@@ -1176,8 +1258,11 @@ static int load_submodule_config_from_head(
 	git_iterator *i;
 	const git_index_entry *entry;
 
-	if ((error = git_repository_head_tree(&head, repo)) < 0)
-		return error;
+	/* if we can't look up current head, then there's no submodule in it */
+	if (git_repository_head_tree(&head, repo) < 0) {
+		giterr_clear();
+		return 0;
+	}
 
 	if ((error = git_iterator_for_tree(&i, head, 0, NULL, NULL)) < 0) {
 		git_tree_free(head);
@@ -1185,18 +1270,24 @@ static int load_submodule_config_from_head(
 	}
 
 	while (!(error = git_iterator_advance(&entry, i))) {
-
-		if (S_ISGITLINK(entry->mode)) {
-			error = submodule_load_from_head(repo, entry->path, &entry->oid);
-			if (error < 0)
-				break;
-		} else {
-			submodule_mode_mismatch(
-				repo, entry->path, GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE);
-
-			if (strcmp(entry->path, GIT_MODULES_FILE) == 0 &&
-				git_oid_iszero(gitmodules_oid))
-				git_oid_cpy(gitmodules_oid, &entry->oid);
+		khiter_t pos = git_strmap_lookup_index(repo->submodules, entry->path);
+		git_submodule *sm;
+
+		if (git_strmap_valid_index(repo->submodules, pos)) {
+			sm = git_strmap_value_at(repo->submodules, pos);
+
+			if (S_ISGITLINK(entry->mode))
+				submodule_update_from_head_data(
+					sm, entry->mode, &entry->oid);
+			else
+				sm->flags |= GIT_SUBMODULE_STATUS__HEAD_NOT_SUBMODULE;
+		} else if (S_ISGITLINK(entry->mode)) {
+			if (!submodule_get(&sm, repo, entry->path, NULL))
+				submodule_update_from_head_data(
+					sm, entry->mode, &entry->oid);
+		} else if (strcmp(entry->path, GIT_MODULES_FILE) == 0 &&
+				   git_oid_iszero(gitmodules_oid)) {
+			git_oid_cpy(gitmodules_oid, &entry->oid);
 		}
 	}
 
@@ -1381,7 +1472,7 @@ static int submodule_update_config(
 
 	assert(submodule);
 
-	error = git_repository_config__weakptr(&config, submodule->owner);
+	error = git_repository_config__weakptr(&config, submodule->repo);
 	if (error < 0)
 		return error;
 
@@ -1409,11 +1500,13 @@ cleanup:
 	return error;
 }
 
-static int submodule_index_status(unsigned int *status, git_submodule *sm)
+static void submodule_get_index_status(unsigned int *status, git_submodule *sm)
 {
 	const git_oid *head_oid  = git_submodule_head_id(sm);
 	const git_oid *index_oid = git_submodule_index_id(sm);
 
+	*status = *status & ~GIT_SUBMODULE_STATUS__INDEX_FLAGS;
+
 	if (!head_oid) {
 		if (index_oid)
 			*status |= GIT_SUBMODULE_STATUS_INDEX_ADDED;
@@ -1422,27 +1515,23 @@ static int submodule_index_status(unsigned int *status, git_submodule *sm)
 		*status |= GIT_SUBMODULE_STATUS_INDEX_DELETED;
 	else if (!git_oid_equal(head_oid, index_oid))
 		*status |= GIT_SUBMODULE_STATUS_INDEX_MODIFIED;
-
-	return 0;
 }
 
-static int submodule_wd_status(unsigned int *status, git_submodule *sm)
+static void submodule_get_wd_status(
+	unsigned int *status,
+	git_submodule *sm,
+	git_repository *sm_repo,
+	git_submodule_ignore_t ign)
 {
-	int error = 0;
-	const git_oid *wd_oid, *index_oid;
-	git_repository *sm_repo = NULL;
-
-	/* open repo now if we need it (so wd_id() call won't reopen) */
-	if ((sm->ignore == GIT_SUBMODULE_IGNORE_NONE ||
-		 sm->ignore == GIT_SUBMODULE_IGNORE_UNTRACKED) &&
-		(sm->flags & GIT_SUBMODULE_STATUS_IN_WD) != 0)
-	{
-		if ((error = git_submodule_open(&sm_repo, sm)) < 0)
-			return error;
-	}
+	const git_oid *index_oid = git_submodule_index_id(sm);
+	const git_oid *wd_oid =
+		(sm->flags & GIT_SUBMODULE_STATUS__WD_OID_VALID) ? &sm->wd_oid : NULL;
+	git_tree *sm_head = NULL;
+	git_index *index = NULL;
+	git_diff_options opt = GIT_DIFF_OPTIONS_INIT;
+	git_diff *diff;
 
-	index_oid = git_submodule_index_id(sm);
-	wd_oid    = git_submodule_wd_id(sm);
+	*status = *status & ~GIT_SUBMODULE_STATUS__WD_FLAGS;
 
 	if (!index_oid) {
 		if (wd_oid)
@@ -1458,59 +1547,51 @@ static int submodule_wd_status(unsigned int *status, git_submodule *sm)
 	else if (!git_oid_equal(index_oid, wd_oid))
 		*status |= GIT_SUBMODULE_STATUS_WD_MODIFIED;
 
-	if (sm_repo != NULL) {
-		git_tree *sm_head;
-		git_diff_options opt = GIT_DIFF_OPTIONS_INIT;
-		git_diff_list *diff;
-
-		/* the diffs below could be optimized with an early termination
-		 * option to the git_diff functions, but for now this is sufficient
-		 * (and certainly no worse that what core git does).
-		 */
-
-		/* perform head-to-index diff on submodule */
+	/* if we have no repo, then we're done */
+	if (!sm_repo)
+		return;
 
-		if ((error = git_repository_head_tree(&sm_head, sm_repo)) < 0)
-			return error;
+	/* the diffs below could be optimized with an early termination
+	 * option to the git_diff functions, but for now this is sufficient
+	 * (and certainly no worse that what core git does).
+	 */
 
-		if (sm->ignore == GIT_SUBMODULE_IGNORE_NONE)
-			opt.flags |= GIT_DIFF_INCLUDE_UNTRACKED;
+	if (ign == GIT_SUBMODULE_IGNORE_NONE)
+		opt.flags |= GIT_DIFF_INCLUDE_UNTRACKED;
 
-		error = git_diff_tree_to_index(&diff, sm_repo, sm_head, NULL, &opt);
+	(void)git_repository_index__weakptr(&index, sm_repo);
 
-		if (!error) {
+	/* if we don't have an unborn head, check diff with index */
+	if (git_repository_head_tree(&sm_head, sm_repo) < 0)
+		giterr_clear();
+	else {
+		/* perform head to index diff on submodule */
+		if (git_diff_tree_to_index(&diff, sm_repo, sm_head, index, &opt) < 0)
+			giterr_clear();
+		else {
 			if (git_diff_num_deltas(diff) > 0)
 				*status |= GIT_SUBMODULE_STATUS_WD_INDEX_MODIFIED;
-
-			git_diff_list_free(diff);
+			git_diff_free(diff);
 			diff = NULL;
 		}
 
 		git_tree_free(sm_head);
+	}
 
-		if (error < 0)
-			return error;
-
-		/* perform index-to-workdir diff on submodule */
-
-		error = git_diff_index_to_workdir(&diff, sm_repo, NULL, &opt);
-
-		if (!error) {
-			size_t untracked =
-				git_diff_num_deltas_of_type(diff, GIT_DELTA_UNTRACKED);
-
-			if (untracked > 0)
-				*status |= GIT_SUBMODULE_STATUS_WD_UNTRACKED;
+	/* perform index-to-workdir diff on submodule */
+	if (git_diff_index_to_workdir(&diff, sm_repo, index, &opt) < 0)
+		giterr_clear();
+	else {
+		size_t untracked =
+			git_diff_num_deltas_of_type(diff, GIT_DELTA_UNTRACKED);
 
-			if (git_diff_num_deltas(diff) != untracked)
-				*status |= GIT_SUBMODULE_STATUS_WD_WD_MODIFIED;
+		if (untracked > 0)
+			*status |= GIT_SUBMODULE_STATUS_WD_UNTRACKED;
 
-			git_diff_list_free(diff);
-			diff = NULL;
-		}
+		if (git_diff_num_deltas(diff) != untracked)
+			*status |= GIT_SUBMODULE_STATUS_WD_WD_MODIFIED;
 
-		git_repository_free(sm_repo);
+		git_diff_free(diff);
+		diff = NULL;
 	}
-
-	return error;
 }
diff --git a/src/submodule.h b/src/submodule.h
index ba8e2518e..b05937503 100644
--- a/src/submodule.h
+++ b/src/submodule.h
@@ -7,6 +7,10 @@
 #ifndef INCLUDE_submodule_h__
 #define INCLUDE_submodule_h__
 
+#include "git2/submodule.h"
+#include "git2/repository.h"
+#include "fileops.h"
+
 /* Notes:
  *
  * Submodule information can be in four places: the index, the config files
@@ -44,44 +48,51 @@
  * an entry for every submodule found in the HEAD and index, and for every
  * submodule described in .gitmodules.  The fields are as follows:
  *
- * - `owner` is the git_repository containing this submodule
+ * - `rc` tracks the refcount of how many hash table entries in the
+ *   git_submodule_cache there are for this submodule.  It only comes into
+ *   play if the name and path of the submodule differ.
+ *
  * - `name` is the name of the submodule from .gitmodules.
  * - `path` is the path to the submodule from the repo root.  It is almost
  *    always the same as `name`.
  * - `url` is the url for the submodule.
- * - `tree_oid` is the SHA1 for the submodule path in the repo HEAD.
- * - `index_oid` is the SHA1 for the submodule recorded in the index.
- * - `workdir_oid` is the SHA1 for the HEAD of the checked out submodule.
  * - `update` is a git_submodule_update_t value - see gitmodules(5) update.
+ * - `update_default` is the update value from the config
  * - `ignore` is a git_submodule_ignore_t value - see gitmodules(5) ignore.
+ * - `ignore_default` is the ignore value from the config
  * - `fetch_recurse` is 0 or 1 - see gitmodules(5) fetchRecurseSubmodules.
- * - `refcount` tracks how many hashmap entries there are for this submodule.
- *   It only comes into play if the name and path of the submodule differ.
- * - `flags` is for internal use, tracking where this submodule has been
- *   found (head, index, config, workdir) and other misc info about it.
+ *
+ * - `repo` is the parent repository that contains this submodule.
+ * - `flags` after for internal use, tracking where this submodule has been
+ *   found (head, index, config, workdir) and known status info, etc.
+ * - `head_oid` is the SHA1 for the submodule path in the repo HEAD.
+ * - `index_oid` is the SHA1 for the submodule recorded in the index.
+ * - `wd_oid` is the SHA1 for the HEAD of the checked out submodule.
  *
  * If the submodule has been added to .gitmodules but not yet git added,
- * then the `index_oid` will be valid and zero.  If the submodule has been
- * deleted, but the delete has not been committed yet, then the `index_oid`
- * will be set, but the `url` will be NULL.
+ * then the `index_oid` will be zero but still marked valid.  If the
+ * submodule has been deleted, but the delete has not been committed yet,
+ * then the `index_oid` will be set, but the `url` will be NULL.
  */
 struct git_submodule {
-	git_repository *owner;
+	git_refcount rc;
+
+	/* information from config */
 	char *name;
-	char *path; /* important: may point to same string data as "name" */
+	char *path; /* important: may just point to "name" string */
 	char *url;
-	uint32_t flags;
-	git_oid head_oid;
-	git_oid index_oid;
-	git_oid wd_oid;
-	/* information from config */
 	git_submodule_update_t update;
 	git_submodule_update_t update_default;
 	git_submodule_ignore_t ignore;
 	git_submodule_ignore_t ignore_default;
 	int fetch_recurse;
+
 	/* internal information */
-	int refcount;
+	git_repository *repo;
+	uint32_t flags;
+	git_oid head_oid;
+	git_oid index_oid;
+	git_oid wd_oid;
 };
 
 /* Additional flags on top of public GIT_SUBMODULE_STATUS values */
@@ -99,4 +110,29 @@ enum {
 #define GIT_SUBMODULE_STATUS__CLEAR_INTERNAL(S) \
 	((S) & ~(0xFFFFFFFFu << 20))
 
+/* Internal status fn returns status and optionally the various OIDs */
+extern int git_submodule__status(
+	unsigned int *out_status,
+	git_oid *out_head_id,
+	git_oid *out_index_id,
+	git_oid *out_wd_id,
+	git_submodule *sm,
+	git_submodule_ignore_t ign);
+
+/* Open submodule repository as bare repo for quick HEAD check, etc. */
+extern int git_submodule_open_bare(
+	git_repository **repo,
+	git_submodule *submodule);
+
+/* Release reference to submodule object - not currently for external use */
+extern void git_submodule_free(git_submodule *sm);
+
+extern int git_submodule_parse_ignore(
+	git_submodule_ignore_t *out, const char *value);
+extern int git_submodule_parse_update(
+	git_submodule_update_t *out, const char *value);
+
+extern const char *git_submodule_ignore_to_str(git_submodule_ignore_t);
+extern const char *git_submodule_update_to_str(git_submodule_update_t);
+
 #endif
diff --git a/src/tag.c b/src/tag.c
index 71f4c1eb1..31a3c8b80 100644
--- a/src/tag.c
+++ b/src/tag.c
@@ -366,10 +366,10 @@ int git_tag_create_frombuffer(git_oid *oid, git_repository *repo, const char *bu
 	if (git_odb_open_wstream(&stream, odb, strlen(buffer), GIT_OBJ_TAG) < 0)
 		return -1;
 
-	stream->write(stream, buffer, strlen(buffer));
+	git_odb_stream_write(stream, buffer, strlen(buffer));
 
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
+	error = git_odb_stream_finalize_write(oid, stream);
+	git_odb_stream_free(stream);
 
 	if (error < 0) {
 		git_buf_free(&ref_name);
diff --git a/src/thread-utils.h b/src/thread-utils.h
index 83148188d..914c1357d 100644
--- a/src/thread-utils.h
+++ b/src/thread-utils.h
@@ -38,15 +38,11 @@ typedef git_atomic git_atomic_ssize;
 
 #endif
 
-GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
-{
-	a->val = val;
-}
-
 #ifdef GIT_THREADS
 
 #define git_thread pthread_t
-#define git_thread_create(thread, attr, start_routine, arg) pthread_create(thread, attr, start_routine, arg)
+#define git_thread_create(thread, attr, start_routine, arg) \
+	pthread_create(thread, attr, start_routine, arg)
 #define git_thread_kill(thread) pthread_cancel(thread)
 #define git_thread_exit(status)	pthread_exit(status)
 #define git_thread_join(id, status) pthread_join(id, status)
@@ -66,6 +62,41 @@ GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
 #define git_cond_signal(c)	pthread_cond_signal(c)
 #define git_cond_broadcast(c)	pthread_cond_broadcast(c)
 
+/* Pthread (-ish) rwlock
+ *
+ * This differs from normal pthreads rwlocks in two ways:
+ * 1. Separate APIs for releasing read locks and write locks (as
+ *    opposed to the pure POSIX API which only has one unlock fn)
+ * 2. You should not use recursive read locks (i.e. grabbing a read
+ *    lock in a thread that already holds a read lock) because the
+ *    Windows implementation doesn't support it
+ */
+#define git_rwlock pthread_rwlock_t
+#define git_rwlock_init(a)		pthread_rwlock_init(a, NULL)
+#define git_rwlock_rdlock(a)	pthread_rwlock_rdlock(a)
+#define git_rwlock_rdunlock(a)	pthread_rwlock_rdunlock(a)
+#define git_rwlock_wrlock(a)	pthread_rwlock_wrlock(a)
+#define git_rwlock_wrunlock(a)	pthread_rwlock_wrunlock(a)
+#define git_rwlock_free(a)		pthread_rwlock_destroy(a)
+#define GIT_RWLOCK_STATIC_INIT	PTHREAD_RWLOCK_INITIALIZER
+
+#ifndef GIT_WIN32
+#define pthread_rwlock_rdunlock pthread_rwlock_unlock
+#define pthread_rwlock_wrunlock pthread_rwlock_unlock
+#endif
+
+
+GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
+{
+#if defined(GIT_WIN32)
+	InterlockedExchange(&a->val, (LONG)val);
+#elif defined(__GNUC__)
+	__sync_lock_test_and_set(&a->val, val);
+#else
+#	error "Unsupported architecture for atomic operations"
+#endif
+}
+
 GIT_INLINE(int) git_atomic_inc(git_atomic *a)
 {
 #if defined(GIT_WIN32)
@@ -100,11 +131,11 @@ GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 }
 
 GIT_INLINE(void *) git___compare_and_swap(
-	volatile void **ptr, void *oldval, void *newval)
+	void * volatile *ptr, void *oldval, void *newval)
 {
 	volatile void *foundval;
 #if defined(GIT_WIN32)
-	foundval = InterlockedCompareExchangePointer(ptr, newval, oldval);
+	foundval = InterlockedCompareExchangePointer((volatile PVOID *)ptr, newval, oldval);
 #elif defined(__GNUC__)
 	foundval = __sync_val_compare_and_swap(ptr, oldval, newval);
 #else
@@ -113,6 +144,16 @@ GIT_INLINE(void *) git___compare_and_swap(
 	return (foundval == oldval) ? oldval : newval;
 }
 
+GIT_INLINE(volatile void *) git___swap(
+	void * volatile *ptr, void *newval)
+{
+#if defined(GIT_WIN32)
+	return InterlockedExchangePointer(ptr, newval);
+#else
+	return __sync_lock_test_and_set(ptr, newval);
+#endif
+}
+
 #ifdef GIT_ARCH_64
 
 GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
@@ -131,7 +172,7 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 #else
 
 #define git_thread unsigned int
-#define git_thread_create(thread, attr, start_routine, arg) (void)0
+#define git_thread_create(thread, attr, start_routine, arg) 0
 #define git_thread_kill(thread) (void)0
 #define git_thread_exit(status) (void)0
 #define git_thread_join(id, status) (void)0
@@ -151,6 +192,22 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 #define git_cond_signal(c) (void)0
 #define git_cond_broadcast(c) (void)0
 
+/* Pthreads rwlock */
+#define git_rwlock unsigned int
+#define git_rwlock_init(a)		0
+#define git_rwlock_rdlock(a)	0
+#define git_rwlock_rdunlock(a)	(void)0
+#define git_rwlock_wrlock(a)	0
+#define git_rwlock_wrunlock(a)	(void)0
+#define git_rwlock_free(a)		(void)0
+#define GIT_RWLOCK_STATIC_INIT	0
+
+
+GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
+{
+	a->val = val;
+}
+
 GIT_INLINE(int) git_atomic_inc(git_atomic *a)
 {
 	return ++a->val;
@@ -168,7 +225,7 @@ GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 }
 
 GIT_INLINE(void *) git___compare_and_swap(
-	volatile void **ptr, void *oldval, void *newval)
+	void * volatile *ptr, void *oldval, void *newval)
 {
 	if (*ptr == oldval)
 		*ptr = newval;
@@ -177,6 +234,14 @@ GIT_INLINE(void *) git___compare_and_swap(
 	return oldval;
 }
 
+GIT_INLINE(volatile void *) git___swap(
+	void * volatile *ptr, void *newval)
+{
+	volatile void *old = *ptr;
+	*ptr = newval;
+	return old;
+}
+
 #ifdef GIT_ARCH_64
 
 GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
@@ -189,13 +254,18 @@ GIT_INLINE(int64_t) git_atomic64_add(git_atomic64 *a, int64_t addend)
 
 #endif
 
+GIT_INLINE(int) git_atomic_get(git_atomic *a)
+{
+	return (int)a->val;
+}
+
 /* Atomically replace oldval with newval
  * @return oldval if it was replaced or newval if it was not
  */
 #define git__compare_and_swap(P,O,N) \
-	git___compare_and_swap((volatile void **)P, O, N)
+	git___compare_and_swap((void * volatile *)P, O, N)
 
-#define git__swap(ptr, val) git__compare_and_swap(&ptr, ptr, val)
+#define git__swap(ptr, val) (void *)git___swap((void * volatile *)&ptr, val)
 
 extern int git_online_cpus(void);
 
diff --git a/src/transport.c b/src/transport.c
index 37c244c97..ff926b1be 100644
--- a/src/transport.c
+++ b/src/transport.c
@@ -42,6 +42,8 @@ static transport_definition transports[] = {
 	{NULL, 0, 0}
 };
 
+static git_vector additional_transports = GIT_VECTOR_INIT;
+
 #define GIT_TRANSPORT_COUNT (sizeof(transports)/sizeof(transports[0])) - 1
 
 static int transport_find_fn(const char *url, git_transport_cb *callback, void **param)
@@ -61,6 +63,14 @@ static int transport_find_fn(const char *url, git_transport_cb *callback, void *
 			definition = definition_iter;
 	}
 
+	git_vector_foreach(&additional_transports, i, definition_iter) {
+		if (strncasecmp(url, definition_iter->prefix, strlen(definition_iter->prefix)))
+			continue;
+
+		if (definition_iter->priority > priority)
+			definition = definition_iter;
+	}
+
 #ifdef GIT_WIN32
 	/* On Windows, it might not be possible to discern between absolute local
 	 * and ssh paths - first check if this is a valid local path that points
@@ -73,7 +83,7 @@ static int transport_find_fn(const char *url, git_transport_cb *callback, void *
 	/* It could be a SSH remote path. Check to see if there's a :
 	 * SSH is an unsupported transport mechanism in this version of libgit2 */
 	if (!definition && strrchr(url, ':'))
-		definition = &dummy_transport_definition; 
+		definition = &dummy_transport_definition;
 #else
 	/* For other systems, perform the SSH check first, to avoid going to the
 	 * filesystem if it is not necessary */
@@ -97,7 +107,7 @@ static int transport_find_fn(const char *url, git_transport_cb *callback, void *
 
 	*callback = definition->fn;
 	*param = definition->param;
-	
+
 	return 0;
 }
 
@@ -135,6 +145,62 @@ int git_transport_new(git_transport **out, git_remote *owner, const char *url)
 	return 0;
 }
 
+int git_transport_register(
+	const char *prefix,
+	unsigned priority,
+	git_transport_cb cb,
+	void *param)
+{
+	transport_definition *d;
+
+	d = git__calloc(sizeof(transport_definition), 1);
+	GITERR_CHECK_ALLOC(d);
+
+	d->prefix = git__strdup(prefix);
+
+	if (!d->prefix)
+		goto on_error;
+
+	d->priority = priority;
+	d->fn = cb;
+	d->param = param;
+
+	if (git_vector_insert(&additional_transports, d) < 0)
+		goto on_error;
+
+	return 0;
+
+on_error:
+	git__free(d->prefix);
+	git__free(d);
+	return -1;
+}
+
+int git_transport_unregister(
+	const char *prefix,
+	unsigned priority)
+{
+	transport_definition *d;
+	unsigned i;
+
+	git_vector_foreach(&additional_transports, i, d) {
+		if (d->priority == priority && !strcasecmp(d->prefix, prefix)) {
+			if (git_vector_remove(&additional_transports, i) < 0)
+				return -1;
+
+			git__free(d->prefix);
+			git__free(d);
+
+			if (!additional_transports.length)
+				git_vector_free(&additional_transports);
+
+			return 0;
+		}
+	}
+
+	return GIT_ENOTFOUND;
+}
+
 /* from remote.h */
 int git_remote_valid_url(const char *url)
 {
diff --git a/src/transports/cred.c b/src/transports/cred.c
index ba5de6e93..05d2c8dc6 100644
--- a/src/transports/cred.c
+++ b/src/transports/cred.c
@@ -9,19 +9,49 @@
 #include "smart.h"
 #include "git2/cred_helpers.h"
 
+int git_cred_has_username(git_cred *cred)
+{
+	int ret = 0;
+
+	switch (cred->credtype) {
+	case GIT_CREDTYPE_USERPASS_PLAINTEXT: {
+		git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
+		ret = !!c->username;
+		break;
+	}
+	case GIT_CREDTYPE_SSH_KEY: {
+		git_cred_ssh_key *c = (git_cred_ssh_key *)cred;
+		ret = !!c->username;
+		break;
+	}
+	case GIT_CREDTYPE_SSH_CUSTOM: {
+		git_cred_ssh_custom *c = (git_cred_ssh_custom *)cred;
+		ret = !!c->username;
+		break;
+	}
+	case GIT_CREDTYPE_DEFAULT: {
+		ret = 0;
+		break;
+	}
+	}
+
+	return ret;
+}
+
 static void plaintext_free(struct git_cred *cred)
 {
 	git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
-	size_t pass_len = strlen(c->password);
 
 	git__free(c->username);
 
 	/* Zero the memory which previously held the password */
-	git__memzero(c->password, pass_len);
-	git__free(c->password);
-
-	memset(c, 0, sizeof(*c));
+	if (c->password) {
+		size_t pass_len = strlen(c->password);
+		git__memzero(c->password, pass_len);
+		git__free(c->password);
+	}
 
+	git__memzero(c, sizeof(*c));
 	git__free(c);
 }
 
@@ -32,8 +62,7 @@ int git_cred_userpass_plaintext_new(
 {
 	git_cred_userpass_plaintext *c;
 
-	if (!cred)
-		return -1;
+	assert(cred && username && password);
 
 	c = git__malloc(sizeof(git_cred_userpass_plaintext));
 	GITERR_CHECK_ALLOC(c);
@@ -59,53 +88,74 @@ int git_cred_userpass_plaintext_new(
 	return 0;
 }
 
-#ifdef GIT_SSH
-static void ssh_keyfile_passphrase_free(struct git_cred *cred)
+static void ssh_key_free(struct git_cred *cred)
 {
-	git_cred_ssh_keyfile_passphrase *c = (git_cred_ssh_keyfile_passphrase *)cred;
-	size_t pass_len = strlen(c->passphrase);
+	git_cred_ssh_key *c =
+		(git_cred_ssh_key *)cred;
 
-    if (c->publickey) {
-        git__free(c->publickey);
-    }
-    
+	git__free(c->username);
+	git__free(c->publickey);
 	git__free(c->privatekey);
 
-    if (c->passphrase) {
-        /* Zero the memory which previously held the passphrase */
-        git__memzero(c->passphrase, pass_len);
-        git__free(c->passphrase);
-    }
+	if (c->passphrase) {
+		/* Zero the memory which previously held the passphrase */
+		size_t pass_len = strlen(c->passphrase);
+		git__memzero(c->passphrase, pass_len);
+		git__free(c->passphrase);
+	}
+
+	git__memzero(c, sizeof(*c));
+	git__free(c);
+}
+
+static void ssh_custom_free(struct git_cred *cred)
+{
+	git_cred_ssh_custom *c = (git_cred_ssh_custom *)cred;
 
-	memset(c, 0, sizeof(*c));
+	git__free(c->username);
+	git__free(c->publickey);
+
+	git__memzero(c, sizeof(*c));
+	git__free(c);
+}
+
+static void default_free(struct git_cred *cred)
+{
+	git_cred_default *c = (git_cred_default *)cred;
 
 	git__free(c);
 }
 
-int git_cred_ssh_keyfile_passphrase_new(
+int git_cred_ssh_key_new(
 	git_cred **cred,
+	const char *username,
 	const char *publickey,
 	const char *privatekey,
 	const char *passphrase)
 {
-	git_cred_ssh_keyfile_passphrase *c;
+	git_cred_ssh_key *c;
 
 	assert(cred && privatekey);
 
-	c = git__calloc(1, sizeof(git_cred_ssh_keyfile_passphrase));
+	c = git__calloc(1, sizeof(git_cred_ssh_key));
 	GITERR_CHECK_ALLOC(c);
 
-	c->parent.credtype = GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE;
-	c->parent.free = ssh_keyfile_passphrase_free;
-    
-    c->privatekey = git__strdup(privatekey);
+	c->parent.credtype = GIT_CREDTYPE_SSH_KEY;
+	c->parent.free = ssh_key_free;
+
+	if (username) {
+		c->username = git__strdup(username);
+		GITERR_CHECK_ALLOC(c->username);
+	}
+
+	c->privatekey = git__strdup(privatekey);
 	GITERR_CHECK_ALLOC(c->privatekey);
-    
-    if (publickey) {
+
+	if (publickey) {
 		c->publickey = git__strdup(publickey);
 		GITERR_CHECK_ALLOC(c->publickey);
 	}
-	
+
 	if (passphrase) {
 		c->passphrase = git__strdup(passphrase);
 		GITERR_CHECK_ALLOC(c->passphrase);
@@ -115,48 +165,56 @@ int git_cred_ssh_keyfile_passphrase_new(
 	return 0;
 }
 
-static void ssh_publickey_free(struct git_cred *cred)
+int git_cred_ssh_custom_new(
+	git_cred **cred,
+	const char *username,
+	const char *publickey,
+	size_t publickey_len,
+	git_cred_sign_callback sign_callback,
+	void *sign_data)
 {
-	git_cred_ssh_publickey *c = (git_cred_ssh_publickey *)cred;
+	git_cred_ssh_custom *c;
+
+	assert(cred);
 
-    git__free(c->publickey);
+	c = git__calloc(1, sizeof(git_cred_ssh_custom));
+	GITERR_CHECK_ALLOC(c);
 
-    c->sign_callback = NULL;
-    c->sign_data = NULL;
-    
-	memset(c, 0, sizeof(*c));
+	c->parent.credtype = GIT_CREDTYPE_SSH_CUSTOM;
+	c->parent.free = ssh_custom_free;
 
-	git__free(c);
+	if (username) {
+		c->username = git__strdup(username);
+		GITERR_CHECK_ALLOC(c->username);
+	}
+
+	if (publickey_len > 0) {
+		c->publickey = git__malloc(publickey_len);
+		GITERR_CHECK_ALLOC(c->publickey);
+
+		memcpy(c->publickey, publickey, publickey_len);
+	}
+
+	c->publickey_len = publickey_len;
+	c->sign_callback = sign_callback;
+	c->sign_data = sign_data;
+
+	*cred = &c->parent;
+	return 0;
 }
 
-int git_cred_ssh_publickey_new(
-	git_cred **cred,
-	const char *publickey,
-    size_t publickey_len,
-	LIBSSH2_USERAUTH_PUBLICKEY_SIGN_FUNC((*sign_callback)),
-    void *sign_data)
+int git_cred_default_new(git_cred **cred)
 {
-	git_cred_ssh_publickey *c;
+	git_cred_default *c;
 
-	if (!cred)
-		return -1;
+	assert(cred);
 
-	c = git__malloc(sizeof(git_cred_ssh_publickey));
+	c = git__calloc(1, sizeof(git_cred_default));
 	GITERR_CHECK_ALLOC(c);
 
-	c->parent.credtype = GIT_CREDTYPE_SSH_PUBLICKEY;
-	c->parent.free = ssh_publickey_free;
-    
-    c->publickey = git__malloc(publickey_len);
-	GITERR_CHECK_ALLOC(c->publickey);
-	
-    memcpy(c->publickey, publickey, publickey_len);
-    
-    c->publickey_len = publickey_len;
-    c->sign_callback = sign_callback;
-    c->sign_data = sign_data;
+	c->credtype = GIT_CREDTYPE_DEFAULT;
+	c->free = default_free;
 
-	*cred = &c->parent;
+	*cred = c;
 	return 0;
 }
-#endif
diff --git a/src/transports/git.c b/src/transports/git.c
index 3a0b86345..5dcd4eff7 100644
--- a/src/transports/git.c
+++ b/src/transports/git.c
@@ -179,39 +179,33 @@ static int _git_uploadpack_ls(
 	const char *url,
 	git_smart_subtransport_stream **stream)
 {
-	char *host, *port, *user=NULL, *pass=NULL;
+	char *host=NULL, *port=NULL, *path=NULL, *user=NULL, *pass=NULL;
+	const char *stream_url = url;
 	git_stream *s;
+	int error = -1;
 
 	*stream = NULL;
-
 	if (!git__prefixcmp(url, prefix_git))
-		url += strlen(prefix_git);
+		stream_url += strlen(prefix_git);
 
-	if (git_stream_alloc(t, url, cmd_uploadpack, stream) < 0)
+	if (git_stream_alloc(t, stream_url, cmd_uploadpack, stream) < 0)
 		return -1;
 
 	s = (git_stream *)*stream;
 
-	if (gitno_extract_url_parts(&host, &port, &user, &pass, url, GIT_DEFAULT_PORT) < 0)
-		goto on_error;
-
-	if (gitno_connect(&s->socket, host, port, 0) < 0)
-		goto on_error;
-
-	t->current_stream = s;
-	git__free(host);
-	git__free(port);
-	git__free(user);
-	git__free(pass);
-	return 0;
+	if (!(error = gitno_extract_url_parts(&host, &port, &path, &user, &pass, url, GIT_DEFAULT_PORT))) {
+		if (!(error = gitno_connect(&s->socket, host, port, 0)))
+			t->current_stream = s;
 
-on_error:
-	if (*stream)
+		git__free(host);
+		git__free(port);
+		git__free(path);
+		git__free(user);
+		git__free(pass);
+	} else if (*stream)
 		git_stream_free(*stream);
 
-	git__free(host);
-	git__free(port);
-	return -1;
+	return error;
 }
 
 static int _git_uploadpack(
@@ -235,39 +229,33 @@ static int _git_receivepack_ls(
 	const char *url,
 	git_smart_subtransport_stream **stream)
 {
-	char *host, *port, *user=NULL, *pass=NULL;
+	char *host=NULL, *port=NULL, *path=NULL, *user=NULL, *pass=NULL;
+	const char *stream_url = url;
 	git_stream *s;
+	int error;
 
 	*stream = NULL;
-
 	if (!git__prefixcmp(url, prefix_git))
-		url += strlen(prefix_git);
+		stream_url += strlen(prefix_git);
 
-	if (git_stream_alloc(t, url, cmd_receivepack, stream) < 0)
+	if (git_stream_alloc(t, stream_url, cmd_receivepack, stream) < 0)
 		return -1;
 
 	s = (git_stream *)*stream;
 
-	if (gitno_extract_url_parts(&host, &port, &user, &pass, url, GIT_DEFAULT_PORT) < 0)
-		goto on_error;
-
-	if (gitno_connect(&s->socket, host, port, 0) < 0)
-		goto on_error;
-
-	t->current_stream = s;
-	git__free(host);
-	git__free(port);
-	git__free(user);
-	git__free(pass);
-	return 0;
+	if (!(error = gitno_extract_url_parts(&host, &port, &path, &user, &pass, url, GIT_DEFAULT_PORT))) {
+		if (!(error = gitno_connect(&s->socket, host, port, 0)))
+			t->current_stream = s;
 
-on_error:
-	if (*stream)
+		git__free(host);
+		git__free(port);
+		git__free(path);
+		git__free(user);
+		git__free(pass);
+	} else if (*stream)
 		git_stream_free(*stream);
 
-	git__free(host);
-	git__free(port);
-	return -1;
+	return error;
 }
 
 static int _git_receivepack(
diff --git a/src/transports/http.c b/src/transports/http.c
index eca06ead2..ace0d97d0 100644
--- a/src/transports/http.c
+++ b/src/transports/http.c
@@ -12,8 +12,6 @@
 #include "netops.h"
 #include "smart.h"
 
-static const char *prefix_http = "http://";
-static const char *prefix_https = "https://";
 static const char *upload_pack_service = "upload-pack";
 static const char *upload_pack_ls_service_url = "/info/refs?service=git-upload-pack";
 static const char *upload_pack_service_url = "/git-upload-pack";
@@ -59,16 +57,11 @@ typedef struct {
 	git_smart_subtransport parent;
 	transport_smart *owner;
 	gitno_socket socket;
-	const char *path;
-	char *host;
-	char *port;
-	char *user_from_url;
-	char *pass_from_url;
+	gitno_connection_data connection_data;
 	git_cred *cred;
 	git_cred *url_cred;
 	http_authmechanism_t auth_mechanism;
-	unsigned connected : 1,
-		use_ssl : 1;
+	bool connected;
 
 	/* Parser structures */
 	http_parser parser;
@@ -125,18 +118,12 @@ static int gen_request(
 	size_t content_length)
 {
 	http_subtransport *t = OWNING_SUBTRANSPORT(s);
+	const char *path = t->connection_data.path ? t->connection_data.path : "/";
 
-	if (!t->path)
-		t->path = "/";
-
-	/* If we were redirected, make sure to respect that here */
-	if (s->redirect_url)
-		git_buf_printf(buf, "%s %s HTTP/1.1\r\n", s->verb, s->redirect_url);
-	else
-		git_buf_printf(buf, "%s %s%s HTTP/1.1\r\n", s->verb, t->path, s->service_url);
+	git_buf_printf(buf, "%s %s%s HTTP/1.1\r\n", s->verb, path, s->service_url);
 
 	git_buf_puts(buf, "User-Agent: git/1.0 (libgit2 " LIBGIT2_VERSION ")\r\n");
-	git_buf_printf(buf, "Host: %s\r\n", t->host);
+	git_buf_printf(buf, "Host: %s\r\n", t->connection_data.host);
 
 	if (s->chunked || content_length > 0) {
 		git_buf_printf(buf, "Accept: application/x-git-%s-result\r\n", s->service);
@@ -156,9 +143,9 @@ static int gen_request(
 		return -1;
 
 	/* Use url-parsed basic auth if username and password are both provided */
-	if (!t->cred && t->user_from_url && t->pass_from_url) {
-		if (!t->url_cred &&
-			 git_cred_userpass_plaintext_new(&t->url_cred, t->user_from_url, t->pass_from_url) < 0)
+	if (!t->cred && t->connection_data.user && t->connection_data.pass) {
+		if (!t->url_cred && git_cred_userpass_plaintext_new(&t->url_cred,
+					t->connection_data.user, t->connection_data.pass) < 0)
 			return -1;
 		if (apply_basic_credential(buf, t->url_cred) < 0) return -1;
 	}
@@ -209,7 +196,7 @@ static int on_header_ready(http_subtransport *t)
 	}
 	else if (!strcasecmp("Location", git_buf_cstr(name))) {
 		if (!t->location) {
-			t->location= git__strdup(git_buf_cstr(value));
+			t->location = git__strdup(git_buf_cstr(value));
 			GITERR_CHECK_ALLOC(t->location);
 		}
 	}
@@ -283,7 +270,7 @@ static int on_headers_complete(http_parser *parser)
 
 			if (t->owner->cred_acquire_cb(&t->cred,
 					t->owner->url,
-					t->user_from_url,
+					t->connection_data.user,
 					allowed_types,
 					t->owner->cred_acquire_payload) < 0)
 				return PARSE_ERROR_GENERIC;
@@ -298,20 +285,18 @@ static int on_headers_complete(http_parser *parser)
 	/* Check for a redirect.
 	 * Right now we only permit a redirect to the same hostname. */
 	if ((parser->status_code == 301 ||
-		 parser->status_code == 302 ||
-		 (parser->status_code == 303 && get_verb == s->verb) ||
-		 parser->status_code == 307) &&
-		t->location) {
+	     parser->status_code == 302 ||
+	     (parser->status_code == 303 && get_verb == s->verb) ||
+	     parser->status_code == 307) &&
+	    t->location) {
 
 		if (s->redirect_count >= 7) {
 			giterr_set(GITERR_NET, "Too many redirects");
 			return t->parse_error = PARSE_ERROR_GENERIC;
 		}
 
-		if (t->location[0] != '/') {
-			giterr_set(GITERR_NET, "Only relative redirects are supported");
+		if (gitno_connection_data_from_url(&t->connection_data, t->location, s->service_url) < 0)
 			return t->parse_error = PARSE_ERROR_GENERIC;
-		}
 
 		/* Set the redirect URL on the stream. This is a transfer of
 		 * ownership of the memory. */
@@ -468,7 +453,7 @@ static int http_connect(http_subtransport *t)
 	if (t->socket.socket)
 		gitno_close(&t->socket);
 
-	if (t->use_ssl) {
+	if (t->connection_data.use_ssl) {
 		int tflags;
 
 		if (t->owner->parent.read_flags(&t->owner->parent, &tflags) < 0)
@@ -480,7 +465,7 @@ static int http_connect(http_subtransport *t)
 			flags |= GITNO_CONNECT_SSL_NO_CHECK_CERT;
 	}
 
-	if (gitno_connect(&t->socket, t->host, t->port, flags) < 0)
+	if (gitno_connect(&t->socket, t->connection_data.host, t->connection_data.port, flags) < 0)
 		return -1;
 
 	t->connected = 1;
@@ -822,50 +807,30 @@ static int http_action(
 	git_smart_service_t action)
 {
 	http_subtransport *t = (http_subtransport *)subtransport;
-	const char *default_port = NULL;
 	int ret;
 
 	if (!stream)
 		return -1;
 
-	if (!t->host || !t->port || !t->path) {
-		if (!git__prefixcmp(url, prefix_http)) {
-			url = url + strlen(prefix_http);
-			default_port = "80";
-		}
-
-		if (!git__prefixcmp(url, prefix_https)) {
-			url += strlen(prefix_https);
-			default_port = "443";
-			t->use_ssl = 1;
-		}
-
-		if (!default_port)
-			return -1;
-
-		if ((ret = gitno_extract_url_parts(&t->host, &t->port,
-						&t->user_from_url, &t->pass_from_url, url, default_port)) < 0)
-			return ret;
-
-		t->path = strchr(url, '/');
-	}
+	if ((!t->connection_data.host || !t->connection_data.port || !t->connection_data.path) &&
+		 (ret = gitno_connection_data_from_url(&t->connection_data, url, NULL)) < 0)
+		return ret;
 
 	if (http_connect(t) < 0)
 		return -1;
 
-	switch (action)
-	{
-		case GIT_SERVICE_UPLOADPACK_LS:
-			return http_uploadpack_ls(t, stream);
+	switch (action) {
+	case GIT_SERVICE_UPLOADPACK_LS:
+		return http_uploadpack_ls(t, stream);
 
-		case GIT_SERVICE_UPLOADPACK:
-			return http_uploadpack(t, stream);
+	case GIT_SERVICE_UPLOADPACK:
+		return http_uploadpack(t, stream);
 
-		case GIT_SERVICE_RECEIVEPACK_LS:
-			return http_receivepack_ls(t, stream);
+	case GIT_SERVICE_RECEIVEPACK_LS:
+		return http_receivepack_ls(t, stream);
 
-		case GIT_SERVICE_RECEIVEPACK:
-			return http_receivepack(t, stream);
+	case GIT_SERVICE_RECEIVEPACK:
+		return http_receivepack(t, stream);
 	}
 
 	*stream = NULL;
@@ -893,25 +858,7 @@ static int http_close(git_smart_subtransport *subtransport)
 		t->url_cred = NULL;
 	}
 
-	if (t->host) {
-		git__free(t->host);
-		t->host = NULL;
-	}
-
-	if (t->port) {
-		git__free(t->port);
-		t->port = NULL;
-	}
-
-	if (t->user_from_url) {
-		git__free(t->user_from_url);
-		t->user_from_url = NULL;
-	}
-
-	if (t->pass_from_url) {
-		git__free(t->pass_from_url);
-		t->pass_from_url = NULL;
-	}
+	gitno_connection_data_free_ptrs(&t->connection_data);
 
 	return 0;
 }
diff --git a/src/transports/local.c b/src/transports/local.c
index 550060958..4502f0202 100644
--- a/src/transports/local.c
+++ b/src/transports/local.c
@@ -119,15 +119,24 @@ on_error:
 
 static int store_refs(transport_local *t)
 {
-	unsigned int i;
+	size_t i;
+	git_remote_head *head;
 	git_strarray ref_names = {0};
 
 	assert(t);
 
-	if (git_reference_list(&ref_names, t->repo) < 0 ||
-		git_vector_init(&t->refs, ref_names.count, NULL) < 0)
+	if (git_reference_list(&ref_names, t->repo) < 0)
 		goto on_error;
 
+	/* Clear all heads we might have fetched in a previous connect */
+	git_vector_foreach(&t->refs, i, head) {
+		git__free(head->name);
+		git__free(head);
+	}
+
+	/* Clear the vector so we can reuse it */
+	git_vector_clear(&t->refs);
+
 	/* Sort the references first */
 	git__tsort((void **)ref_names.strings, ref_names.count, &git__strcmp_cb);
 
@@ -204,21 +213,17 @@ static int local_connect(
 	return 0;
 }
 
-static int local_ls(git_transport *transport, git_headlist_cb list_cb, void *payload)
+static int local_ls(const git_remote_head ***out, size_t *size, git_transport *transport)
 {
 	transport_local *t = (transport_local *)transport;
-	unsigned int i;
-	git_remote_head *head = NULL;
 
 	if (!t->have_refs) {
 		giterr_set(GITERR_NET, "The transport has not yet loaded the refs");
 		return -1;
 	}
 
-	git_vector_foreach(&t->refs, i, head) {
-		if (list_cb(head, payload))
-			return GIT_EUSER;
-	}
+	*out = (const git_remote_head **) t->refs.contents;
+	*size = t->refs.length;
 
 	return 0;
 }
@@ -278,9 +283,9 @@ static int local_push_copy_object(
 		odb_obj_size, odb_obj_type)) < 0)
 		goto on_error;
 
-	if (odb_stream->write(odb_stream, (char *)git_odb_object_data(odb_obj),
+	if (git_odb_stream_write(odb_stream, (char *)git_odb_object_data(odb_obj),
 		odb_obj_size) < 0 ||
-		odb_stream->finalize_write(&remote_odb_obj_oid, odb_stream) < 0) {
+		git_odb_stream_finalize_write(&remote_odb_obj_oid, odb_stream) < 0) {
 		error = -1;
 	} else if (git_oid__cmp(&obj->id, &remote_odb_obj_oid) != 0) {
 		giterr_set(GITERR_ODB, "Error when writing object to remote odb "
@@ -289,7 +294,7 @@ static int local_push_copy_object(
 		error = -1;
 	}
 
-	odb_stream->free(odb_stream);
+	git_odb_stream_free(odb_stream);
 
 on_error:
 	git_odb_object_free(odb_obj);
@@ -352,7 +357,8 @@ static int local_push(
 	   non-bare repo push support would require checking configs to see if
 	   we should override the default 'don't let this happen' behavior */
 	if (!remote_repo->is_bare) {
-		error = -1;
+		error = GIT_EBAREREPO;
+		giterr_set(GITERR_INVALID, "Local push doesn't (yet) support pushing to non-bare repos.");
 		goto on_error;
 	}
 
@@ -424,7 +430,7 @@ static int local_push(
 
 		if (!url || t->parent.close(&t->parent) < 0 ||
 			t->parent.connect(&t->parent, url,
-			push->remote->cred_acquire_cb, NULL, GIT_DIRECTION_PUSH, flags))
+			push->remote->callbacks.credentials, NULL, GIT_DIRECTION_PUSH, flags))
 			goto on_error;
 	}
 
@@ -449,7 +455,7 @@ static int foreach_cb(void *buf, size_t len, void *payload)
 	foreach_data *data = (foreach_data*)payload;
 
 	data->stats->received_bytes += len;
-	return data->writepack->add(data->writepack, buf, len, data->stats);
+	return data->writepack->append(data->writepack, buf, len, data->stats);
 }
 
 static int local_download_pack(
@@ -571,8 +577,6 @@ static void local_cancel(git_transport *transport)
 static int local_close(git_transport *transport)
 {
 	transport_local *t = (transport_local *)transport;
-	size_t i;
-	git_remote_head *head;
 
 	t->connected = 0;
 
@@ -586,19 +590,21 @@ static int local_close(git_transport *transport)
 		t->url = NULL;
 	}
 
-	git_vector_foreach(&t->refs, i, head) {
-		git__free(head->name);
-		git__free(head);
-	}
-
-	git_vector_free(&t->refs);
-
 	return 0;
 }
 
 static void local_free(git_transport *transport)
 {
 	transport_local *t = (transport_local *)transport;
+	size_t i;
+	git_remote_head *head;
+
+	git_vector_foreach(&t->refs, i, head) {
+		git__free(head->name);
+		git__free(head);
+	}
+
+	git_vector_free(&t->refs);
 
 	/* Close the transport, if it's still open. */
 	local_close(transport);
@@ -632,6 +638,7 @@ int git_transport_local(git_transport **out, git_remote *owner, void *param)
 	t->parent.read_flags = local_read_flags;
 	t->parent.cancel = local_cancel;
 
+	git_vector_init(&t->refs, 0, NULL);
 	t->owner = owner;
 
 	*out = (git_transport *) t;
diff --git a/src/transports/smart.c b/src/transports/smart.c
index 416eb221f..5242beb65 100644
--- a/src/transports/smart.c
+++ b/src/transports/smart.c
@@ -23,8 +23,13 @@ static int git_smart__recv_cb(gitno_buffer *buf)
 
 	buf->offset += bytes_read;
 
-	if (t->packetsize_cb)
-		t->packetsize_cb(bytes_read, t->packetsize_payload);
+	if (t->packetsize_cb && !t->cancelled.val)
+		if (t->packetsize_cb(bytes_read, t->packetsize_payload)) {
+			git_atomic_set(&t->cancelled, 1);
+
+			giterr_clear();
+			return GIT_EUSER;
+		}
 
 	return (int)(buf->offset - old_len);
 }
@@ -58,6 +63,24 @@ static int git_smart__set_callbacks(
 	return 0;
 }
 
+int git_smart__update_heads(transport_smart *t)
+{
+	size_t i;
+	git_pkt *pkt;
+
+	git_vector_clear(&t->heads);
+	git_vector_foreach(&t->refs, i, pkt) {
+		git_pkt_ref *ref = (git_pkt_ref *) pkt;
+		if (pkt->type != GIT_PKT_REF)
+			continue;
+
+		if (git_vector_insert(&t->heads, &ref->head) < 0)
+			return -1;
+	}
+
+	return 0;
+}
+
 static int git_smart__connect(
 	git_transport *transport,
 	const char *url,
@@ -135,6 +158,9 @@ static int git_smart__connect(
 		git_pkt_free((git_pkt *)first);
 	}
 
+	/* Keep a list of heads for _ls */
+	git_smart__update_heads(t);
+
 	if (t->rpc && git_smart__reset_stream(t, false) < 0)
 		return -1;
 
@@ -144,28 +170,17 @@ static int git_smart__connect(
 	return 0;
 }
 
-static int git_smart__ls(git_transport *transport, git_headlist_cb list_cb, void *payload)
+static int git_smart__ls(const git_remote_head ***out, size_t *size, git_transport *transport)
 {
 	transport_smart *t = (transport_smart *)transport;
-	unsigned int i;
-	git_pkt *p = NULL;
 
 	if (!t->have_refs) {
 		giterr_set(GITERR_NET, "The transport has not yet loaded the refs");
 		return -1;
 	}
 
-	git_vector_foreach(&t->refs, i, p) {
-		git_pkt_ref *pkt = NULL;
-
-		if (p->type != GIT_PKT_REF)
-			continue;
-
-		pkt = (git_pkt_ref *)p;
-
-		if (list_cb(&pkt->head, payload))
-			return GIT_EUSER;
-	}
+	*out = (const git_remote_head **) t->heads.contents;
+	*size = t->heads.length;
 
 	return 0;
 }
@@ -288,6 +303,7 @@ static void git_smart__free(git_transport *transport)
 	/* Free the subtransport */
 	t->wrapped->free(t->wrapped);
 
+	git_vector_free(&t->heads);
 	git_vector_foreach(refs, i, p)
 		git_pkt_free(p);
 
@@ -335,6 +351,11 @@ int git_transport_smart(git_transport **out, git_remote *owner, void *param)
 		return -1;
 	}
 
+	if (git_vector_init(&t->heads, 16, ref_name_cmp) < 0) {
+		git__free(t);
+		return -1;
+	}
+
 	if (definition->callback(&t->wrapped, &t->parent) < 0) {
 		git__free(t);
 		return -1;
diff --git a/src/transports/smart.h b/src/transports/smart.h
index c52401a3c..32f0be7f2 100644
--- a/src/transports/smart.h
+++ b/src/transports/smart.h
@@ -16,11 +16,13 @@
 
 #define GIT_CAP_OFS_DELTA "ofs-delta"
 #define GIT_CAP_MULTI_ACK "multi_ack"
+#define GIT_CAP_MULTI_ACK_DETAILED "multi_ack_detailed"
 #define GIT_CAP_SIDE_BAND "side-band"
 #define GIT_CAP_SIDE_BAND_64K "side-band-64k"
 #define GIT_CAP_INCLUDE_TAG "include-tag"
 #define GIT_CAP_DELETE_REFS "delete-refs"
 #define GIT_CAP_REPORT_STATUS "report-status"
+#define GIT_CAP_THIN_PACK "thin-pack"
 
 enum git_pkt_type {
 	GIT_PKT_CMD,
@@ -39,7 +41,7 @@ enum git_pkt_type {
 	GIT_PKT_UNPACK,
 };
 
-/* Used for multi-ack */
+/* Used for multi_ack and mutli_ack_detailed */
 enum git_ack_status {
 	GIT_ACK_NONE,
 	GIT_ACK_CONTINUE,
@@ -112,14 +114,16 @@ typedef struct transport_smart_caps {
 	int common:1,
 		ofs_delta:1,
 		multi_ack: 1,
+		multi_ack_detailed: 1,
 		side_band:1,
 		side_band_64k:1,
 		include_tag:1,
 		delete_refs:1,
-		report_status:1;
+		report_status:1,
+		thin_pack:1;
 } transport_smart_caps;
 
-typedef void (*packetsize_cb)(size_t received, void *payload);
+typedef int (*packetsize_cb)(size_t received, void *payload);
 
 typedef struct {
 	git_transport parent;
@@ -136,6 +140,7 @@ typedef struct {
 	git_smart_subtransport_stream *current_stream;
 	transport_smart_caps caps;
 	git_vector refs;
+	git_vector heads;
 	git_vector common;
 	git_atomic cancelled;
 	packetsize_cb packetsize_cb;
@@ -169,6 +174,8 @@ int git_smart__download_pack(
 int git_smart__negotiation_step(git_transport *transport, void *data, size_t len);
 int git_smart__get_push_stream(transport_smart *t, git_smart_subtransport_stream **out);
 
+int git_smart__update_heads(transport_smart *t);
+
 /* smart_pkt.c */
 int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_t len);
 int git_pkt_buffer_flush(git_buf *buf);
diff --git a/src/transports/smart_pkt.c b/src/transports/smart_pkt.c
index 99da37567..2bb09c750 100644
--- a/src/transports/smart_pkt.c
+++ b/src/transports/smart_pkt.c
@@ -39,7 +39,7 @@ static int flush_pkt(git_pkt **out)
 	return 0;
 }
 
-/* the rest of the line will be useful for multi_ack */
+/* the rest of the line will be useful for multi_ack and multi_ack_detailed */
 static int ack_pkt(git_pkt **out, const char *line, size_t len)
 {
 	git_pkt_ack *pkt;
@@ -62,6 +62,10 @@ static int ack_pkt(git_pkt **out, const char *line, size_t len)
 	if (len >= 7) {
 		if (!git__prefixcmp(line + 1, "continue"))
 			pkt->status = GIT_ACK_CONTINUE;
+		if (!git__prefixcmp(line + 1, "common"))
+			pkt->status = GIT_ACK_COMMON;
+		if (!git__prefixcmp(line + 1, "ready"))
+			pkt->status = GIT_ACK_READY;
 	}
 
 	*out = (git_pkt *) pkt;
@@ -456,22 +460,27 @@ static int buffer_want_with_caps(const git_remote_head *head, transport_smart_ca
 	char oid[GIT_OID_HEXSZ +1] = {0};
 	unsigned int len;
 
-	/* Prefer side-band-64k if the server supports both */
-	if (caps->side_band) {
-		if (caps->side_band_64k)
-			git_buf_printf(&str, "%s ", GIT_CAP_SIDE_BAND_64K);
-		else
-			git_buf_printf(&str, "%s ", GIT_CAP_SIDE_BAND);
-	}
-	if (caps->ofs_delta)
-		git_buf_puts(&str, GIT_CAP_OFS_DELTA " ");
-
-	if (caps->multi_ack)
+	/* Prefer multi_ack_detailed */
+	if (caps->multi_ack_detailed)
+		git_buf_puts(&str, GIT_CAP_MULTI_ACK_DETAILED " ");
+	else if (caps->multi_ack)
 		git_buf_puts(&str, GIT_CAP_MULTI_ACK " ");
 
+	/* Prefer side-band-64k if the server supports both */
+	if (caps->side_band_64k)
+		git_buf_printf(&str, "%s ", GIT_CAP_SIDE_BAND_64K);
+	else if (caps->side_band)
+		git_buf_printf(&str, "%s ", GIT_CAP_SIDE_BAND);
+
 	if (caps->include_tag)
 		git_buf_puts(&str, GIT_CAP_INCLUDE_TAG " ");
 
+	if (caps->thin_pack)
+		git_buf_puts(&str, GIT_CAP_THIN_PACK " ");
+
+	if (caps->ofs_delta)
+		git_buf_puts(&str, GIT_CAP_OFS_DELTA " ");
+
 	if (git_buf_oom(&str))
 		return -1;
 
diff --git a/src/transports/smart_protocol.c b/src/transports/smart_protocol.c
index 636616717..3bf1f9329 100644
--- a/src/transports/smart_protocol.c
+++ b/src/transports/smart_protocol.c
@@ -13,8 +13,11 @@
 #include "push.h"
 #include "pack-objects.h"
 #include "remote.h"
+#include "util.h"
 
 #define NETWORK_XFER_THRESHOLD (100*1024)
+/* The minimal interval between progress updates (in seconds). */
+#define MIN_PROGRESS_UPDATE_INTERVAL 0.5
 
 int git_smart__store_refs(transport_smart *t, int flushes)
 {
@@ -46,7 +49,7 @@ int git_smart__store_refs(transport_smart *t, int flushes)
 
 		if (error == GIT_EBUFS) {
 			if ((recvd = gitno_recv(buf)) < 0)
-				return -1;
+				return recvd;
 
 			if (recvd == 0 && !flush) {
 				giterr_set(GITERR_NET, "Early EOF");
@@ -94,6 +97,13 @@ int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps)
 			continue;
 		}
 
+		/* Keep multi_ack_detailed before multi_ack */
+		if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK_DETAILED)) {
+			caps->common = caps->multi_ack_detailed = 1;
+			ptr += strlen(GIT_CAP_MULTI_ACK_DETAILED);
+			continue;
+		}
+
 		if (!git__prefixcmp(ptr, GIT_CAP_MULTI_ACK)) {
 			caps->common = caps->multi_ack = 1;
 			ptr += strlen(GIT_CAP_MULTI_ACK);
@@ -125,6 +135,12 @@ int git_smart__detect_caps(git_pkt_ref *pkt, transport_smart_caps *caps)
 			continue;
 		}
 
+		if (!git__prefixcmp(ptr, GIT_CAP_THIN_PACK)) {
+			caps->common = caps->thin_pack = 1;
+			ptr += strlen(GIT_CAP_THIN_PACK);
+			continue;
+		}
+
 		/* We don't know this capability, so skip it */
 		ptr = strchr(ptr, ' ');
 	}
@@ -148,10 +164,10 @@ static int recv_pkt(git_pkt **out, gitno_buffer *buf)
 			break; /* return the pkt */
 
 		if (error < 0 && error != GIT_EBUFS)
-			return -1;
+			return error;
 
 		if ((ret = gitno_recv(buf)) < 0)
-			return -1;
+			return ret;
 	} while (error);
 
 	gitno_consume(buf, line_end);
@@ -168,10 +184,11 @@ static int store_common(transport_smart *t)
 {
 	git_pkt *pkt = NULL;
 	gitno_buffer *buf = &t->buffer;
+	int error;
 
 	do {
-		if (recv_pkt(&pkt, buf) < 0)
-			return -1;
+		if ((error = recv_pkt(&pkt, buf)) < 0)
+			return error;
 
 		if (pkt->type == GIT_PKT_ACK) {
 			if (git_vector_insert(&t->common, pkt) < 0)
@@ -211,6 +228,7 @@ static int fetch_setup_walk(git_revwalk **out, git_repository *repo)
 
 		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
 			continue;
+
 		if (git_revwalk_push(walk, git_reference_target(ref)) < 0)
 			goto on_error;
 
@@ -227,7 +245,33 @@ on_error:
 	return -1;
 }
 
-int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_remote_head * const *refs, size_t count)
+static int wait_while_ack(gitno_buffer *buf)
+{
+	int error;
+	git_pkt_ack *pkt = NULL;
+
+	while (1) {
+		git__free(pkt);
+
+		if ((error = recv_pkt((git_pkt **)&pkt, buf)) < 0)
+			return error;
+
+		if (pkt->type == GIT_PKT_NAK)
+			break;
+
+		if (pkt->type == GIT_PKT_ACK &&
+		    (pkt->status != GIT_ACK_CONTINUE ||
+		     pkt->status != GIT_ACK_COMMON)) {
+			git__free(pkt);
+			return 0;
+		}
+	}
+
+	git__free(pkt);
+	return 0;
+}
+
+int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, const git_remote_head * const *wants, size_t count)
 {
 	transport_smart *t = (transport_smart *)transport;
 	gitno_buffer *buf = &t->buffer;
@@ -237,19 +281,20 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 	unsigned int i;
 	git_oid oid;
 
-	/* No own logic, do our thing */
-	if ((error = git_pkt_buffer_wants(refs, count, &t->caps, &data)) < 0)
+	if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 		return error;
 
 	if ((error = fetch_setup_walk(&walk, repo)) < 0)
 		goto on_error;
+
 	/*
-	 * We don't support any kind of ACK extensions, so the negotiation
-	 * boils down to sending what we have and listening for an ACK
-	 * every once in a while.
+	 * Our support for ACK extensions is simply to parse them. On
+	 * the first ACK we will accept that as enough common
+	 * objects. We give up if we haven't found an answer in the
+	 * first 256 we send.
 	 */
 	i = 0;
-	while (true) {
+	while (i < 256) {
 		error = git_revwalk_next(&oid, walk);
 
 		if (error < 0) {
@@ -278,7 +323,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 				goto on_error;
 
 			git_buf_clear(&data);
-			if (t->caps.multi_ack) {
+			if (t->caps.multi_ack || t->caps.multi_ack_detailed) {
 				if ((error = store_common(t)) < 0)
 					goto on_error;
 			} else {
@@ -307,7 +352,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 			git_pkt_ack *pkt;
 			unsigned int i;
 
-			if ((error = git_pkt_buffer_wants(refs, count, &t->caps, &data)) < 0)
+			if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 				goto on_error;
 
 			git_vector_foreach(&t->common, i, pkt) {
@@ -327,7 +372,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 		git_pkt_ack *pkt;
 		unsigned int i;
 
-		if ((error = git_pkt_buffer_wants(refs, count, &t->caps, &data)) < 0)
+		if ((error = git_pkt_buffer_wants(wants, count, &t->caps, &data)) < 0)
 			goto on_error;
 
 		git_vector_foreach(&t->common, i, pkt) {
@@ -356,7 +401,7 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 	git_revwalk_free(walk);
 
 	/* Now let's eat up whatever the server gives us */
-	if (!t->caps.multi_ack) {
+	if (!t->caps.multi_ack && !t->caps.multi_ack_detailed) {
 		pkt_type = recv_pkt(NULL, buf);
 
 		if (pkt_type < 0) {
@@ -366,22 +411,10 @@ int git_smart__negotiate_fetch(git_transport *transport, git_repository *repo, c
 			return -1;
 		}
 	} else {
-		git_pkt_ack *pkt;
-		do {
-			if ((error = recv_pkt((git_pkt **)&pkt, buf)) < 0)
-				return error;
-
-			if (pkt->type == GIT_PKT_NAK ||
-			    (pkt->type == GIT_PKT_ACK && pkt->status != GIT_ACK_CONTINUE)) {
-				git__free(pkt);
-				break;
-			}
-
-			git__free(pkt);
-		} while (1);
+		error = wait_while_ack(buf);
 	}
 
-	return 0;
+	return error;
 
 on_error:
 	git_revwalk_free(walk);
@@ -399,16 +432,16 @@ static int no_sideband(transport_smart *t, struct git_odb_writepack *writepack,
 			return GIT_EUSER;
 		}
 
-		if (writepack->add(writepack, buf->data, buf->offset, stats) < 0)
+		if (writepack->append(writepack, buf->data, buf->offset, stats) < 0)
 			return -1;
 
 		gitno_consume_n(buf, buf->offset);
 
 		if ((recvd = gitno_recv(buf)) < 0)
-			return -1;
+			return recvd;
 	} while(recvd > 0);
 
-	if (writepack->commit(writepack, stats))
+	if (writepack->commit(writepack, stats) < 0)
 		return -1;
 
 	return 0;
@@ -422,7 +455,7 @@ struct network_packetsize_payload
 	size_t last_fired_bytes;
 };
 
-static void network_packetsize(size_t received, void *payload)
+static int network_packetsize(size_t received, void *payload)
 {
 	struct network_packetsize_payload *npp = (struct network_packetsize_payload*)payload;
 
@@ -432,8 +465,12 @@ static void network_packetsize(size_t received, void *payload)
 	/* Fire notification if the threshold is reached */
 	if ((npp->stats->received_bytes - npp->last_fired_bytes) > NETWORK_XFER_THRESHOLD) {
 		npp->last_fired_bytes = npp->stats->received_bytes;
-		npp->callback(npp->stats, npp->payload);
+
+		if (npp->callback(npp->stats, npp->payload))
+			return GIT_EUSER;
 	}
+
+	return 0;
 }
 
 int git_smart__download_pack(
@@ -447,7 +484,7 @@ int git_smart__download_pack(
 	gitno_buffer *buf = &t->buffer;
 	git_odb *odb;
 	struct git_odb_writepack *writepack = NULL;
-	int error = -1;
+	int error = 0;
 	struct network_packetsize_payload npp = {0};
 
 	memset(stats, 0, sizeof(git_transfer_progress));
@@ -460,13 +497,14 @@ int git_smart__download_pack(
 		t->packetsize_payload = &npp;
 
 		/* We might have something in the buffer already from negotiate_fetch */
-		if (t->buffer.offset > 0)
-			t->packetsize_cb(t->buffer.offset, t->packetsize_payload);
+		if (t->buffer.offset > 0 && !t->cancelled.val)
+			if (t->packetsize_cb(t->buffer.offset, t->packetsize_payload))
+				git_atomic_set(&t->cancelled, 1);
 	}
 
 	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
 		((error = git_odb_write_pack(&writepack, odb, progress_cb, progress_payload)) < 0))
-		goto on_error;
+		goto done;
 
 	/*
 	 * If the remote doesn't support the side-band, we can feed
@@ -474,37 +512,46 @@ int git_smart__download_pack(
 	 * check which one belongs there.
 	 */
 	if (!t->caps.side_band && !t->caps.side_band_64k) {
-		if (no_sideband(t, writepack, buf, stats) < 0)
-			goto on_error;
-
-		goto on_success;
+		error = no_sideband(t, writepack, buf, stats);
+		goto done;
 	}
 
 	do {
 		git_pkt *pkt;
 
+		/* Check cancellation before network call */
 		if (t->cancelled.val) {
 			giterr_set(GITERR_NET, "The fetch was cancelled by the user");
 			error = GIT_EUSER;
-			goto on_error;
+			goto done;
 		}
 
-		if (recv_pkt(&pkt, buf) < 0)
-			goto on_error;
+		if ((error = recv_pkt(&pkt, buf)) < 0)
+			goto done;
+
+		/* Check cancellation after network call */
+		if (t->cancelled.val) {
+			giterr_set(GITERR_NET, "The fetch was cancelled by the user");
+			error = GIT_EUSER;
+			goto done;
+		}
 
 		if (pkt->type == GIT_PKT_PROGRESS) {
 			if (t->progress_cb) {
 				git_pkt_progress *p = (git_pkt_progress *) pkt;
-				t->progress_cb(p->data, p->len, t->message_cb_payload);
+				if (t->progress_cb(p->data, p->len, t->message_cb_payload)) {
+					giterr_set(GITERR_NET, "The fetch was cancelled by the user");
+					return GIT_EUSER;
+				}
 			}
 			git__free(pkt);
 		} else if (pkt->type == GIT_PKT_DATA) {
 			git_pkt_data *p = (git_pkt_data *) pkt;
-			error = writepack->add(writepack, p->data, p->len, stats);
+			error = writepack->append(writepack, p->data, p->len, stats);
 
 			git__free(pkt);
 			if (error < 0)
-				goto on_error;
+				goto done;
 		} else if (pkt->type == GIT_PKT_FLUSH) {
 			/* A flush indicates the end of the packfile */
 			git__free(pkt);
@@ -512,13 +559,9 @@ int git_smart__download_pack(
 		}
 	} while (1);
 
-	if (writepack->commit(writepack, stats) < 0)
-		goto on_error;
-
-on_success:
-	error = 0;
+	error = writepack->commit(writepack, stats);
 
-on_error:
+done:
 	if (writepack)
 		writepack->free(writepack);
 
@@ -656,7 +699,7 @@ static int parse_report(gitno_buffer *buf, git_push *push)
 
 		if (error == GIT_EBUFS) {
 			if ((recvd = gitno_recv(buf)) < 0)
-				return -1;
+				return recvd;
 
 			if (recvd == 0) {
 				giterr_set(GITERR_NET, "Early EOF");
@@ -801,22 +844,56 @@ static int update_refs_from_report(
 	return 0;
 }
 
+struct push_packbuilder_payload
+{
+	git_smart_subtransport_stream *stream;
+	git_packbuilder *pb;
+	git_push_transfer_progress cb;
+	void *cb_payload;
+	size_t last_bytes;
+	double last_progress_report_time;
+};
+
 static int stream_thunk(void *buf, size_t size, void *data)
 {
-	git_smart_subtransport_stream *s = (git_smart_subtransport_stream *)data;
+	int error = 0;
+	struct push_packbuilder_payload *payload = data;
 
-	return s->write(s, (const char *)buf, size);
+	if ((error = payload->stream->write(payload->stream, (const char *)buf, size)) < 0)
+		return error;
+
+	if (payload->cb) {
+		double current_time = git__timer();
+		payload->last_bytes += size;
+
+		if ((current_time - payload->last_progress_report_time) >= MIN_PROGRESS_UPDATE_INTERVAL) {
+			payload->last_progress_report_time = current_time;
+			if (payload->cb(payload->pb->nr_written, payload->pb->nr_objects, payload->last_bytes, payload->cb_payload)) {
+				giterr_clear();
+				error = GIT_EUSER;
+			}
+		}
+	}
+
+	return error;
 }
 
 int git_smart__push(git_transport *transport, git_push *push)
 {
 	transport_smart *t = (transport_smart *)transport;
-	git_smart_subtransport_stream *s;
+	struct push_packbuilder_payload packbuilder_payload = {0};
 	git_buf pktline = GIT_BUF_INIT;
-	int error = -1, need_pack = 0;
+	int error = 0, need_pack = 0;
 	push_spec *spec;
 	unsigned int i;
 
+	packbuilder_payload.pb = push->pb;
+
+	if (push->transfer_progress_cb) {
+		packbuilder_payload.cb = push->transfer_progress_cb;
+		packbuilder_payload.cb_payload = push->transfer_progress_cb_payload;
+	}
+
 #ifdef PUSH_DEBUG
 {
 	git_remote_head *head;
@@ -848,29 +925,36 @@ int git_smart__push(git_transport *transport, git_push *push)
 		}
 	}
 
-	if (git_smart__get_push_stream(t, &s) < 0 ||
-		gen_pktline(&pktline, push) < 0 ||
-		s->write(s, git_buf_cstr(&pktline), git_buf_len(&pktline)) < 0)
-		goto on_error;
+	if ((error = git_smart__get_push_stream(t, &packbuilder_payload.stream)) < 0 ||
+		(error = gen_pktline(&pktline, push)) < 0 ||
+		(error = packbuilder_payload.stream->write(packbuilder_payload.stream, git_buf_cstr(&pktline), git_buf_len(&pktline))) < 0)
+		goto done;
 
-	if (need_pack && git_packbuilder_foreach(push->pb, &stream_thunk, s) < 0)
-		goto on_error;
+	if (need_pack &&
+		(error = git_packbuilder_foreach(push->pb, &stream_thunk, &packbuilder_payload)) < 0)
+		goto done;
 
 	/* If we sent nothing or the server doesn't support report-status, then
 	 * we consider the pack to have been unpacked successfully */
 	if (!push->specs.length || !push->report_status)
 		push->unpack_ok = 1;
-	else if (parse_report(&t->buffer, push) < 0)
-		goto on_error;
+	else if ((error = parse_report(&t->buffer, push)) < 0)
+		goto done;
 
-	if (push->status.length &&
-		update_refs_from_report(&t->refs, &push->specs, &push->status) < 0)
-		goto on_error;
+	/* If progress is being reported write the final report */
+	if (push->transfer_progress_cb) {
+		push->transfer_progress_cb(push->pb->nr_written, push->pb->nr_objects, packbuilder_payload.last_bytes, push->transfer_progress_cb_payload);
+	}
 
-	error = 0;
+	if (push->status.length) {
+		error = update_refs_from_report(&t->refs, &push->specs, &push->status);
+		if (error < 0)
+			goto done;
 
-on_error:
-	git_buf_free(&pktline);
+		error = git_smart__update_heads(t);
+	}
 
+done:
+	git_buf_free(&pktline);
 	return error;
 }
diff --git a/src/transports/ssh.c b/src/transports/ssh.c
index a312c8d08..4a905e3c9 100644
--- a/src/transports/ssh.c
+++ b/src/transports/ssh.c
@@ -5,19 +5,18 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 
-#ifdef GIT_SSH
-
 #include "git2.h"
 #include "buffer.h"
 #include "netops.h"
 #include "smart.h"
 
+#ifdef GIT_SSH
+
 #include <libssh2.h>
 
 #define OWNING_SUBTRANSPORT(s) ((ssh_subtransport *)(s)->parent.subtransport)
 
 static const char prefix_ssh[] = "ssh://";
-static const char default_user[] = "git";
 static const char cmd_uploadpack[] = "git-upload-pack";
 static const char cmd_receivepack[] = "git-receive-pack";
 
@@ -38,6 +37,14 @@ typedef struct {
 	git_cred *cred;
 } ssh_subtransport;
 
+static void ssh_error(LIBSSH2_SESSION *session, const char *errmsg)
+{
+	char *ssherr;
+	libssh2_session_last_error(session, &ssherr, NULL, 0);
+
+	giterr_set(GITERR_SSH, "%s: %s", errmsg, ssherr);
+}
+
 /*
  * Create a git protocol request.
  *
@@ -46,27 +53,29 @@ typedef struct {
 static int gen_proto(git_buf *request, const char *cmd, const char *url)
 {
 	char *repo;
-	
+
 	if (!git__prefixcmp(url, prefix_ssh)) {
 		url = url + strlen(prefix_ssh);
 		repo = strchr(url, '/');
 	} else {
 		repo = strchr(url, ':');
+		if (repo) repo++;
 	}
-	
+
 	if (!repo) {
+		giterr_set(GITERR_NET, "Malformed git protocol URL");
 		return -1;
 	}
-	
+
 	int len = strlen(cmd) + 1 /* Space */ + 1 /* Quote */ + strlen(repo) + 1 /* Quote */ + 1;
-	
+
 	git_buf_grow(request, len);
 	git_buf_printf(request, "%s '%s'", cmd, repo);
 	git_buf_putc(request, '\0');
-	
+
 	if (git_buf_oom(request))
 		return -1;
-	
+
 	return 0;
 }
 
@@ -74,21 +83,19 @@ static int send_command(ssh_stream *s)
 {
 	int error;
 	git_buf request = GIT_BUF_INIT;
-	
+
 	error = gen_proto(&request, s->cmd, s->url);
 	if (error < 0)
 		goto cleanup;
-	
-	error = libssh2_channel_exec(
-		s->channel,
-		request.ptr
-	);
 
-	if (0 != error)
+	error = libssh2_channel_exec(s->channel, request.ptr);
+	if (error < LIBSSH2_ERROR_NONE) {
+		ssh_error(s->session, "SSH could not execute request");
 		goto cleanup;
-	
+	}
+
 	s->sent_command = 1;
-	
+
 cleanup:
 	git_buf_free(&request);
 	return error;
@@ -100,19 +107,21 @@ static int ssh_stream_read(
 	size_t buf_size,
 	size_t *bytes_read)
 {
+	int rc;
 	ssh_stream *s = (ssh_stream *)stream;
-	
+
 	*bytes_read = 0;
-	
+
 	if (!s->sent_command && send_command(s) < 0)
 		return -1;
-	
-	int rc = libssh2_channel_read(s->channel, buffer, buf_size);
-	if (rc < 0)
+
+	if ((rc = libssh2_channel_read(s->channel, buffer, buf_size)) < LIBSSH2_ERROR_NONE) {
+		ssh_error(s->session, "SSH could not read data");;
 		return -1;
-	
+	}
+
 	*bytes_read = rc;
-	
+
 	return 0;
 }
 
@@ -122,16 +131,16 @@ static int ssh_stream_write(
 	size_t len)
 {
 	ssh_stream *s = (ssh_stream *)stream;
-	
+
 	if (!s->sent_command && send_command(s) < 0)
 		return -1;
-	
-	int rc = libssh2_channel_write(s->channel, buffer, len);
-	if (rc < 0) {
+
+	if (libssh2_channel_write(s->channel, buffer, len) < LIBSSH2_ERROR_NONE) {
+		ssh_error(s->session, "SSH could not write data");
 		return -1;
 	}
-	
-	return rc;
+
+	return 0;
 }
 
 static void ssh_stream_free(git_smart_subtransport_stream *stream)
@@ -139,26 +148,27 @@ static void ssh_stream_free(git_smart_subtransport_stream *stream)
 	ssh_stream *s = (ssh_stream *)stream;
 	ssh_subtransport *t = OWNING_SUBTRANSPORT(s);
 	int ret;
-	
+
 	GIT_UNUSED(ret);
-	
+
 	t->current_stream = NULL;
-	
+
 	if (s->channel) {
 		libssh2_channel_close(s->channel);
-        libssh2_channel_free(s->channel);
-        s->channel = NULL;
+		libssh2_channel_free(s->channel);
+		s->channel = NULL;
 	}
-	
+
 	if (s->session) {
-		libssh2_session_free(s->session), s->session = NULL;
+		libssh2_session_free(s->session);
+		s->session = NULL;
 	}
-	
+
 	if (s->socket.socket) {
-		ret = gitno_close(&s->socket);
-		assert(!ret);
+		(void)gitno_close(&s->socket);
+		/* can't do anything here with error return value */
 	}
-	
+
 	git__free(s->url);
 	git__free(s);
 }
@@ -170,26 +180,25 @@ static int ssh_stream_alloc(
 	git_smart_subtransport_stream **stream)
 {
 	ssh_stream *s;
-	
-	if (!stream)
-		return -1;
-	
+
+	assert(stream);
+
 	s = git__calloc(sizeof(ssh_stream), 1);
 	GITERR_CHECK_ALLOC(s);
-	
+
 	s->parent.subtransport = &t->parent;
 	s->parent.read = ssh_stream_read;
 	s->parent.write = ssh_stream_write;
 	s->parent.free = ssh_stream_free;
-	
+
 	s->cmd = cmd;
+
 	s->url = git__strdup(url);
-	
 	if (!s->url) {
 		git__free(s);
 		return -1;
 	}
-	
+
 	*stream = &s->parent;
 	return 0;
 }
@@ -201,136 +210,127 @@ static int git_ssh_extract_url_parts(
 {
 	char *colon, *at;
 	const char *start;
-    
-    colon = strchr(url, ':');
-	
-	if (colon == NULL) {
-		giterr_set(GITERR_NET, "Malformed URL: missing :");
-		return -1;
-	}
-	
+
+	colon = strchr(url, ':');
+
+
 	at = strchr(url, '@');
 	if (at) {
-		start = at+1;
+		start = at + 1;
 		*username = git__substrdup(url, at - url);
+		GITERR_CHECK_ALLOC(*username);
 	} else {
 		start = url;
-		*username = git__strdup(default_user);
+		*username = NULL;
+	}
+
+	if (colon == NULL || (colon < start)) {
+		giterr_set(GITERR_NET, "Malformed URL");
+		return -1;
 	}
-	
+
 	*host = git__substrdup(start, colon - start);
-	
+	GITERR_CHECK_ALLOC(*host);
+
 	return 0;
 }
 
 static int _git_ssh_authenticate_session(
 	LIBSSH2_SESSION* session,
 	const char *user,
-	git_cred* cred
-)
+	git_cred* cred)
 {
 	int rc;
+
 	do {
 		switch (cred->credtype) {
-			case GIT_CREDTYPE_USERPASS_PLAINTEXT: {
-				git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
-				rc = libssh2_userauth_password(
-					session, 
-					c->username,
-					c->password
-				);
-				break;
-			}
-			case GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE: {
-				git_cred_ssh_keyfile_passphrase *c = (git_cred_ssh_keyfile_passphrase *)cred;
-				rc = libssh2_userauth_publickey_fromfile(
-					session, 
-					user,
-					c->publickey,
-					c->privatekey,
-					c->passphrase
-				);
-				break;
-			}
-			case GIT_CREDTYPE_SSH_PUBLICKEY: {
-				git_cred_ssh_publickey *c = (git_cred_ssh_publickey *)cred;
-				rc = libssh2_userauth_publickey(
-					session,
-					user,
-					(const unsigned char *)c->publickey,
-					c->publickey_len,
-					c->sign_callback,
-					&c->sign_data
-				);
-				break;
-			}
-			default:
-				rc = LIBSSH2_ERROR_AUTHENTICATION_FAILED;
+		case GIT_CREDTYPE_USERPASS_PLAINTEXT: {
+			git_cred_userpass_plaintext *c = (git_cred_userpass_plaintext *)cred;
+			user = c->username ? c->username : user;
+			rc = libssh2_userauth_password(session, user, c->password);
+			break;
+		}
+		case GIT_CREDTYPE_SSH_KEY: {
+			git_cred_ssh_key *c = (git_cred_ssh_key *)cred;
+			user = c->username ? c->username : user;
+			rc = libssh2_userauth_publickey_fromfile(
+				session, c->username, c->publickey, c->privatekey, c->passphrase);
+			break;
+		}
+		case GIT_CREDTYPE_SSH_CUSTOM: {
+			git_cred_ssh_custom *c = (git_cred_ssh_custom *)cred;
+
+			user = c->username ? c->username : user;
+			rc = libssh2_userauth_publickey(
+				session, c->username, (const unsigned char *)c->publickey,
+				c->publickey_len, c->sign_callback, &c->sign_data);
+			break;
+		}
+		default:
+			rc = LIBSSH2_ERROR_AUTHENTICATION_FAILED;
 		}
-    } while (LIBSSH2_ERROR_EAGAIN == rc || LIBSSH2_ERROR_TIMEOUT == rc);
-	
-    return rc;
+	} while (LIBSSH2_ERROR_EAGAIN == rc || LIBSSH2_ERROR_TIMEOUT == rc);
+
+	if (rc != LIBSSH2_ERROR_NONE) {
+		ssh_error(session, "Failed to authenticate SSH session");
+		return -1;
+	}
+
+	return 0;
 }
 
-static int _git_ssh_session_create
-(
+static int _git_ssh_session_create(
 	LIBSSH2_SESSION** session,
-	gitno_socket socket
-)
+	gitno_socket socket)
 {
-	if (!session) {
+	int rc = 0;
+	LIBSSH2_SESSION* s;
+
+	assert(session);
+
+	s = libssh2_session_init();
+	if (!s) {
+		giterr_set(GITERR_NET, "Failed to initialize SSH session");
+		return -1;
+	}
+
+	do {
+		rc = libssh2_session_startup(s, socket.socket);
+	} while (LIBSSH2_ERROR_EAGAIN == rc || LIBSSH2_ERROR_TIMEOUT == rc);
+
+	if (rc != LIBSSH2_ERROR_NONE) {
+		ssh_error(s, "Failed to start SSH session");
+		libssh2_session_free(s);
 		return -1;
 	}
-	
-	LIBSSH2_SESSION* s = libssh2_session_init();
-    if (!s)
-        return -1;
-	
-    int rc = 0;
-    do {
-        rc = libssh2_session_startup(s, socket.socket);
-    } while (LIBSSH2_ERROR_EAGAIN == rc || LIBSSH2_ERROR_TIMEOUT == rc);
-	
-	if (0 != rc) {
-        goto on_error;
-    }
-	
+
 	libssh2_session_set_blocking(s, 1);
-	
+
 	*session = s;
-	
+
 	return 0;
-	
-on_error:
-	if (s) {
-		libssh2_session_free(s), s = NULL;
-	}
-	
-	return -1;
 }
 
 static int _git_ssh_setup_conn(
 	ssh_subtransport *t,
 	const char *url,
 	const char *cmd,
-	git_smart_subtransport_stream **stream
-)
+	git_smart_subtransport_stream **stream)
 {
-	char *host, *port=NULL, *user=NULL, *pass=NULL;
+	char *host=NULL, *port=NULL, *path=NULL, *user=NULL, *pass=NULL;
 	const char *default_port="22";
 	ssh_stream *s;
 	LIBSSH2_SESSION* session=NULL;
 	LIBSSH2_CHANNEL* channel=NULL;
-	
+
 	*stream = NULL;
 	if (ssh_stream_alloc(t, url, cmd, stream) < 0)
 		return -1;
-	
+
 	s = (ssh_stream *)*stream;
-	
+
 	if (!git__prefixcmp(url, prefix_ssh)) {
-		url = url + strlen(prefix_ssh);
-		if (gitno_extract_url_parts(&host, &port, &user, &pass, url, default_port) < 0)
+		if (gitno_extract_url_parts(&host, &port, &path, &user, &pass, url, default_port) < 0)
 			goto on_error;
 	} else {
 		if (git_ssh_extract_url_parts(&host, &user, url) < 0)
@@ -338,61 +338,78 @@ static int _git_ssh_setup_conn(
 		port = git__strdup(default_port);
 		GITERR_CHECK_ALLOC(port);
 	}
-	
+
 	if (gitno_connect(&s->socket, host, port, 0) < 0)
 		goto on_error;
-	
+
 	if (user && pass) {
 		if (git_cred_userpass_plaintext_new(&t->cred, user, pass) < 0)
 			goto on_error;
-	} else {
-		if (t->owner->cred_acquire_cb(&t->cred,
-				t->owner->url,
-				user,
-				GIT_CREDTYPE_USERPASS_PLAINTEXT | GIT_CREDTYPE_SSH_KEYFILE_PASSPHRASE,
+	} else if (t->owner->cred_acquire_cb) {
+		if (t->owner->cred_acquire_cb(
+				&t->cred, t->owner->url, user,
+				GIT_CREDTYPE_USERPASS_PLAINTEXT |
+				GIT_CREDTYPE_SSH_KEY |
+				GIT_CREDTYPE_SSH_CUSTOM,
 				t->owner->cred_acquire_payload) < 0)
-			return -1;
+			goto on_error;
+
+		if (!t->cred) {
+			giterr_set(GITERR_SSH, "Callback failed to initialize SSH credentials");
+			goto on_error;
+		}
+	} else {
+		giterr_set(GITERR_SSH, "Cannot set up SSH connection without credentials");
+		goto on_error;
 	}
 	assert(t->cred);
-	
-	if (!user) {
-		user = git__strdup(default_user);
+
+	if (!user && !git_cred_has_username(t->cred)) {
+		giterr_set_str(GITERR_NET, "Cannot authenticate without a username");
+		goto on_error;
 	}
-	
+
 	if (_git_ssh_session_create(&session, s->socket) < 0)
 		goto on_error;
-	
-    if (_git_ssh_authenticate_session(session, user, t->cred) < 0)
+
+	if (_git_ssh_authenticate_session(session, user, t->cred) < 0)
 		goto on_error;
-	
+
 	channel = libssh2_channel_open_session(session);
-	if (!channel)
-        goto on_error;
-	
+	if (!channel) {
+		ssh_error(session, "Failed to open SSH channel");
+		goto on_error;
+	}
+
 	libssh2_channel_set_blocking(channel, 1);
-	
+
 	s->session = session;
 	s->channel = channel;
-	
+
 	t->current_stream = s;
 	git__free(host);
 	git__free(port);
+	git__free(path);
 	git__free(user);
 	git__free(pass);
 
 	return 0;
-	
+
 on_error:
+	s->session = NULL;
+	s->channel = NULL;
+	t->current_stream = NULL;
+
 	if (*stream)
 		ssh_stream_free(*stream);
-	
+
 	git__free(host);
 	git__free(port);
 	git__free(user);
 	git__free(pass);
 
 	if (session)
-		libssh2_session_free(session), session = NULL;
+		libssh2_session_free(session);
 
 	return -1;
 }
@@ -404,7 +421,7 @@ static int ssh_uploadpack_ls(
 {
 	if (_git_ssh_setup_conn(t, url, cmd_uploadpack, stream) < 0)
 		return -1;
-	
+
 	return 0;
 }
 
@@ -414,12 +431,12 @@ static int ssh_uploadpack(
 	git_smart_subtransport_stream **stream)
 {
 	GIT_UNUSED(url);
-	
+
 	if (t->current_stream) {
 		*stream = &t->current_stream->parent;
 		return 0;
 	}
-	
+
 	giterr_set(GITERR_NET, "Must call UPLOADPACK_LS before UPLOADPACK");
 	return -1;
 }
@@ -431,7 +448,7 @@ static int ssh_receivepack_ls(
 {
 	if (_git_ssh_setup_conn(t, url, cmd_receivepack, stream) < 0)
 		return -1;
-	
+
 	return 0;
 }
 
@@ -441,12 +458,12 @@ static int ssh_receivepack(
 	git_smart_subtransport_stream **stream)
 {
 	GIT_UNUSED(url);
-	
+
 	if (t->current_stream) {
 		*stream = &t->current_stream->parent;
 		return 0;
 	}
-	
+
 	giterr_set(GITERR_NET, "Must call RECEIVEPACK_LS before RECEIVEPACK");
 	return -1;
 }
@@ -458,21 +475,21 @@ static int _ssh_action(
 	git_smart_service_t action)
 {
 	ssh_subtransport *t = (ssh_subtransport *) subtransport;
-	
+
 	switch (action) {
 		case GIT_SERVICE_UPLOADPACK_LS:
 			return ssh_uploadpack_ls(t, url, stream);
-			
+
 		case GIT_SERVICE_UPLOADPACK:
 			return ssh_uploadpack(t, url, stream);
-			
+
 		case GIT_SERVICE_RECEIVEPACK_LS:
 			return ssh_receivepack_ls(t, url, stream);
-			
+
 		case GIT_SERVICE_RECEIVEPACK:
 			return ssh_receivepack(t, url, stream);
 	}
-	
+
 	*stream = NULL;
 	return -1;
 }
@@ -480,40 +497,49 @@ static int _ssh_action(
 static int _ssh_close(git_smart_subtransport *subtransport)
 {
 	ssh_subtransport *t = (ssh_subtransport *) subtransport;
-	
+
 	assert(!t->current_stream);
-	
+
 	GIT_UNUSED(t);
-	
+
 	return 0;
 }
 
 static void _ssh_free(git_smart_subtransport *subtransport)
 {
 	ssh_subtransport *t = (ssh_subtransport *) subtransport;
-	
+
 	assert(!t->current_stream);
-	
+
 	git__free(t);
 }
+#endif
 
-int git_smart_subtransport_ssh(git_smart_subtransport **out, git_transport *owner)
+int git_smart_subtransport_ssh(
+	git_smart_subtransport **out, git_transport *owner)
 {
+#ifdef GIT_SSH
 	ssh_subtransport *t;
-	
-	if (!out)
-		return -1;
-	
+
+	assert(out);
+
 	t = git__calloc(sizeof(ssh_subtransport), 1);
 	GITERR_CHECK_ALLOC(t);
-	
+
 	t->owner = (transport_smart *)owner;
 	t->parent.action = _ssh_action;
 	t->parent.close = _ssh_close;
 	t->parent.free = _ssh_free;
-	
+
 	*out = (git_smart_subtransport *) t;
 	return 0;
-}
+#else
+	GIT_UNUSED(owner);
+
+	assert(out);
+	*out = NULL;
 
+	giterr_set(GITERR_INVALID, "Cannot create SSH transport. Library was built without SSH support");
+	return -1;
 #endif
+}
diff --git a/src/transports/winhttp.c b/src/transports/winhttp.c
index 95e422dc0..673cd0faf 100644
--- a/src/transports/winhttp.c
+++ b/src/transports/winhttp.c
@@ -52,6 +52,7 @@ static const int no_check_cert_flags = SECURITY_FLAG_IGNORE_CERT_CN_INVALID |
 
 typedef enum {
 	GIT_WINHTTP_AUTH_BASIC = 1,
+	GIT_WINHTTP_AUTH_NEGOTIATE = 2,
 } winhttp_authmechanism_t;
 
 typedef struct {
@@ -73,17 +74,12 @@ typedef struct {
 typedef struct {
 	git_smart_subtransport parent;
 	transport_smart *owner;
-	const char *path;
-	char *host;
-	char *port;
-	char *user_from_url;
-	char *pass_from_url;
+	gitno_connection_data connection_data;
 	git_cred *cred;
 	git_cred *url_cred;
 	int auth_mechanism;
 	HINTERNET session;
 	HINTERNET connection;
-	unsigned use_ssl : 1;
 } winhttp_subtransport;
 
 static int apply_basic_credential(HINTERNET request, git_cred *cred)
@@ -143,6 +139,22 @@ on_error:
 	return error;
 }
 
+static int apply_default_credentials(HINTERNET request)
+{
+	/* If we are explicitly asked to deliver default credentials, turn set
+	 * the security level to low which will guarantee they are delivered.
+	 * The default is "medium" which applies to the intranet and sounds
+	 * like it would correspond to Internet Explorer security zones, but
+	 * in fact does not.
+	 */
+	DWORD data = WINHTTP_AUTOLOGON_SECURITY_LEVEL_LOW;
+
+	if (!WinHttpSetOption(request, WINHTTP_OPTION_AUTOLOGON_POLICY, &data, sizeof(DWORD)))
+		return -1;
+
+	return 0;
+}
+
 static int winhttp_stream_connect(winhttp_stream *s)
 {
 	winhttp_subtransport *t = OWNING_SUBTRANSPORT(s);
@@ -152,9 +164,10 @@ static int winhttp_stream_connect(winhttp_stream *s)
 	wchar_t *types[] = { L"*/*", NULL };
 	BOOL peerdist = FALSE;
 	int error = -1, wide_len;
+	unsigned long disable_redirects = WINHTTP_DISABLE_REDIRECTS;
 
 	/* Prepare URL */
-	git_buf_printf(&buf, "%s%s", t->path, s->service_url);
+	git_buf_printf(&buf, "%s%s", t->connection_data.path, s->service_url);
 
 	if (git_buf_oom(&buf))
 		return -1;
@@ -187,7 +200,7 @@ static int winhttp_stream_connect(winhttp_stream *s)
 			NULL,
 			WINHTTP_NO_REFERER,
 			types,
-			t->use_ssl ? WINHTTP_FLAG_SECURE : 0);
+			t->connection_data.use_ssl ? WINHTTP_FLAG_SECURE : 0);
 
 	if (!s->request) {
 		giterr_set(GITERR_OS, "Failed to open request");
@@ -195,7 +208,7 @@ static int winhttp_stream_connect(winhttp_stream *s)
 	}
 
 	/* Set proxy if necessary */
-	if (git_remote__get_http_proxy(t->owner->owner, t->use_ssl, &proxy_url) < 0)
+	if (git_remote__get_http_proxy(t->owner->owner, !!t->connection_data.use_ssl, &proxy_url) < 0)
 		goto on_error;
 
 	if (proxy_url) {
@@ -244,6 +257,17 @@ static int winhttp_stream_connect(winhttp_stream *s)
 		git__free(proxy_wide);
 	}
 
+	/* Disable WinHTTP redirects so we can handle them manually. Why, you ask?
+	 * http://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/b2ff8879-ab9f-4218-8f09-16d25dff87ae
+	 */
+	if (!WinHttpSetOption(s->request,
+		WINHTTP_OPTION_DISABLE_FEATURE,
+		&disable_redirects,
+		sizeof(disable_redirects))) {
+			giterr_set(GITERR_OS, "Failed to disable redirects");
+			goto on_error;
+	}
+
 	/* Strip unwanted headers (X-P2P-PeerDist, X-P2P-PeerDistEx) that WinHTTP
 	 * adds itself. This option may not be supported by the underlying
 	 * platform, so we do not error-check it */
@@ -258,22 +282,39 @@ static int winhttp_stream_connect(winhttp_stream *s)
 		goto on_error;
 	}
 
-	/* Send Content-Type header -- only necessary on a POST */
 	if (post_verb == s->verb) {
+		/* Send Content-Type and Accept headers -- only necessary on a POST */
 		git_buf_clear(&buf);
-		if (git_buf_printf(&buf, "Content-Type: application/x-git-%s-request", s->service) < 0)
+		if (git_buf_printf(&buf,
+			"Content-Type: application/x-git-%s-request",
+			s->service) < 0)
 			goto on_error;
 
 		git__utf8_to_16(ct, MAX_CONTENT_TYPE_LEN, git_buf_cstr(&buf));
 
-		if (!WinHttpAddRequestHeaders(s->request, ct, (ULONG) -1L, WINHTTP_ADDREQ_FLAG_ADD)) {
+		if (!WinHttpAddRequestHeaders(s->request, ct, (ULONG)-1L,
+			WINHTTP_ADDREQ_FLAG_ADD | WINHTTP_ADDREQ_FLAG_REPLACE)) {
+			giterr_set(GITERR_OS, "Failed to add a header to the request");
+			goto on_error;
+		}
+
+		git_buf_clear(&buf);
+		if (git_buf_printf(&buf,
+			"Accept: application/x-git-%s-result",
+			s->service) < 0)
+			goto on_error;
+
+		git__utf8_to_16(ct, MAX_CONTENT_TYPE_LEN, git_buf_cstr(&buf));
+
+		if (!WinHttpAddRequestHeaders(s->request, ct, (ULONG)-1L,
+			WINHTTP_ADDREQ_FLAG_ADD | WINHTTP_ADDREQ_FLAG_REPLACE)) {
 			giterr_set(GITERR_OS, "Failed to add a header to the request");
 			goto on_error;
 		}
 	}
 
 	/* If requested, disable certificate validation */
-	if (t->use_ssl) {
+	if (t->connection_data.use_ssl) {
 		int flags;
 
 		if (t->owner->parent.read_flags(&t->owner->parent, &flags) < 0)
@@ -293,12 +334,17 @@ static int winhttp_stream_connect(winhttp_stream *s)
 		t->auth_mechanism == GIT_WINHTTP_AUTH_BASIC &&
 		apply_basic_credential(s->request, t->cred) < 0)
 		goto on_error;
+	else if (t->cred &&
+		t->cred->credtype == GIT_CREDTYPE_DEFAULT &&
+		t->auth_mechanism == GIT_WINHTTP_AUTH_NEGOTIATE &&
+		apply_default_credentials(s->request) < 0)
+		goto on_error;
 
 	/* If no other credentials have been applied and the URL has username and
 	 * password, use those */
-	if (!t->cred && t->user_from_url && t->pass_from_url) {
+	if (!t->cred && t->connection_data.user && t->connection_data.pass) {
 		if (!t->url_cred &&
-			 git_cred_userpass_plaintext_new(&t->url_cred, t->user_from_url, t->pass_from_url) < 0)
+			git_cred_userpass_plaintext_new(&t->url_cred, t->connection_data.user, t->connection_data.pass) < 0)
 			goto on_error;
 		if (apply_basic_credential(s->request, t->url_cred) < 0)
 			goto on_error;
@@ -337,6 +383,12 @@ static int parse_unauthorized_response(
 		*auth_mechanism = GIT_WINHTTP_AUTH_BASIC;
 	}
 
+	if ((WINHTTP_AUTH_SCHEME_NTLM & supported) ||
+		(WINHTTP_AUTH_SCHEME_NEGOTIATE & supported)) {
+		*allowed_types |= GIT_CREDTYPE_DEFAULT;
+		*auth_mechanism = GIT_WINHTTP_AUTH_NEGOTIATE;
+	}
+
 	return 0;
 }
 
@@ -380,6 +432,50 @@ static int write_chunk(HINTERNET request, const char *buffer, size_t len)
 	return 0;
 }
 
+static int winhttp_connect(
+	winhttp_subtransport *t,
+	const char *url)
+{
+	wchar_t *ua = L"git/1.0 (libgit2 " WIDEN(LIBGIT2_VERSION) L")";
+	git_win32_path host;
+	int32_t port;
+	const char *default_port = "80";
+
+	/* Prepare port */
+	if (git__strtol32(&port, t->connection_data.port, NULL, 10) < 0)
+		return -1;
+
+	/* Prepare host */
+	git_win32_path_from_c(host, t->connection_data.host);
+
+	/* Establish session */
+	t->session = WinHttpOpen(
+		ua,
+		WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
+		WINHTTP_NO_PROXY_NAME,
+		WINHTTP_NO_PROXY_BYPASS,
+		0);
+
+	if (!t->session) {
+		giterr_set(GITERR_OS, "Failed to init WinHTTP");
+		return -1;
+	}
+
+	/* Establish connection */
+	t->connection = WinHttpConnect(
+		t->session,
+		host,
+		(INTERNET_PORT) port,
+		0);
+
+	if (!t->connection) {
+		giterr_set(GITERR_OS, "Failed to connect to host");
+		return -1;
+	}
+
+	return 0;
+}
+
 static int winhttp_stream_read(
 	git_smart_subtransport_stream *stream,
 	char *buffer,
@@ -511,50 +607,53 @@ replay:
 
 			/* Check for Windows 7. This workaround is only necessary on
 			 * Windows Vista and earlier. Windows 7 is version 6.1. */
-			if (!git_has_win32_version(6, 1)) {
-				wchar_t *location;
-				DWORD location_length;
-				int redirect_cmp;
-
-				/* OK, fetch the Location header from the redirect. */
-				if (WinHttpQueryHeaders(s->request,
-					WINHTTP_QUERY_LOCATION,
-					WINHTTP_HEADER_NAME_BY_INDEX,
-					WINHTTP_NO_OUTPUT_BUFFER,
-					&location_length,
-					WINHTTP_NO_HEADER_INDEX) ||
-					GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
-					giterr_set(GITERR_OS, "Failed to read Location header");
-					return -1;
-				}
-
-				location = git__malloc(location_length);
-				GITERR_CHECK_ALLOC(location);
-
-				if (!WinHttpQueryHeaders(s->request,
-					WINHTTP_QUERY_LOCATION,
-					WINHTTP_HEADER_NAME_BY_INDEX,
-					location,
-					&location_length,
-					WINHTTP_NO_HEADER_INDEX)) {
-					giterr_set(GITERR_OS, "Failed to read Location header");
-					git__free(location);
-					return -1;
-				}
+			wchar_t *location;
+			DWORD location_length;
+			char *location8;
+
+			/* OK, fetch the Location header from the redirect. */
+			if (WinHttpQueryHeaders(s->request,
+				WINHTTP_QUERY_LOCATION,
+				WINHTTP_HEADER_NAME_BY_INDEX,
+				WINHTTP_NO_OUTPUT_BUFFER,
+				&location_length,
+				WINHTTP_NO_HEADER_INDEX) ||
+				GetLastError() != ERROR_INSUFFICIENT_BUFFER) {
+				giterr_set(GITERR_OS, "Failed to read Location header");
+				return -1;
+			}
 
-				/* Compare the Location header with the request URI */
-				redirect_cmp = wcscmp(location, s->request_uri);
+			location = git__malloc(location_length);
+			location8 = git__malloc(location_length);
+			GITERR_CHECK_ALLOC(location);
+
+			if (!WinHttpQueryHeaders(s->request,
+				WINHTTP_QUERY_LOCATION,
+				WINHTTP_HEADER_NAME_BY_INDEX,
+				location,
+				&location_length,
+				WINHTTP_NO_HEADER_INDEX)) {
+				giterr_set(GITERR_OS, "Failed to read Location header");
 				git__free(location);
+				return -1;
+			}
+			git__utf16_to_8(location8, location_length, location);
+			git__free(location);
 
-				if (!redirect_cmp) {
-					/* Replay the request */
-					WinHttpCloseHandle(s->request);
-					s->request = NULL;
-					s->sent_request = 0;
+			/* Replay the request */
+			WinHttpCloseHandle(s->request);
+			s->request = NULL;
+			s->sent_request = 0;
 
-					goto replay;
-				}
+			if (!git__prefixcmp_icase(location8, prefix_https)) {
+				/* Upgrade to secure connection; disconnect and start over */
+				if (gitno_connection_data_from_url(&t->connection_data, location8, s->service_url) < 0)
+					return -1;
+				winhttp_connect(t, location8);
 			}
+
+			git__free(location8);
+			goto replay;
 		}
 
 		/* Handle authentication failures */
@@ -568,8 +667,9 @@ replay:
 			if (allowed_types &&
 				(!t->cred || 0 == (t->cred->credtype & allowed_types))) {
 
-				if (t->owner->cred_acquire_cb(&t->cred, t->owner->url, t->user_from_url, allowed_types, t->owner->cred_acquire_payload) < 0)
-					return -1;
+				if (t->owner->cred_acquire_cb(&t->cred, t->owner->url, t->connection_data.user, allowed_types, 
+					t->owner->cred_acquire_payload) < 0)
+					return GIT_EUSER;
 
 				assert(t->cred);
 
@@ -888,68 +988,6 @@ static int winhttp_stream_alloc(winhttp_subtransport *t, winhttp_stream **stream
 	return 0;
 }
 
-static int winhttp_connect(
-	winhttp_subtransport *t,
-	const char *url)
-{
-	wchar_t *ua = L"git/1.0 (libgit2 " WIDEN(LIBGIT2_VERSION) L")";
-	wchar_t host[GIT_WIN_PATH];
-	int32_t port;
-	const char *default_port = "80";
-	int ret;
-
-	if (!git__prefixcmp(url, prefix_http)) {
-		url = url + strlen(prefix_http);
-		default_port = "80";
-	}
-
-	if (!git__prefixcmp(url, prefix_https)) {
-		url += strlen(prefix_https);
-		default_port = "443";
-		t->use_ssl = 1;
-	}
-
-	if ((ret = gitno_extract_url_parts(&t->host, &t->port, &t->user_from_url,
-					&t->pass_from_url, url, default_port)) < 0)
-		return ret;
-
-	t->path = strchr(url, '/');
-
-	/* Prepare port */
-	if (git__strtol32(&port, t->port, NULL, 10) < 0)
-		return -1;
-
-	/* Prepare host */
-	git__utf8_to_16(host, GIT_WIN_PATH, t->host);
-
-	/* Establish session */
-	t->session = WinHttpOpen(
-			ua,
-			WINHTTP_ACCESS_TYPE_DEFAULT_PROXY,
-			WINHTTP_NO_PROXY_NAME,
-			WINHTTP_NO_PROXY_BYPASS,
-			0);
-
-	if (!t->session) {
-		giterr_set(GITERR_OS, "Failed to init WinHTTP");
-		return -1;
-	}
-	
-	/* Establish connection */
-	t->connection = WinHttpConnect(
-			t->session,
-			host,
-			port,
-			0);
-
-	if (!t->connection) {
-		giterr_set(GITERR_OS, "Failed to connect to host");
-		return -1;
-	}
-
-	return 0;
-}
-
 static int winhttp_uploadpack_ls(
 	winhttp_subtransport *t,
 	winhttp_stream *s)
@@ -989,7 +1027,7 @@ static int winhttp_receivepack(
 {
 	/* WinHTTP only supports Transfer-Encoding: chunked
 	 * on Windows Vista (NT 6.0) and higher. */
-	s->chunked = git_has_win32_version(6, 0);
+	s->chunked = git_has_win32_version(6, 0, 0);
 
 	if (s->chunked)
 		s->parent.write = winhttp_stream_write_chunked;
@@ -1013,9 +1051,10 @@ static int winhttp_action(
 	winhttp_stream *s;
 	int ret = -1;
 
-	if (!t->connection &&
-		winhttp_connect(t, url) < 0)
-		return -1;
+	if (!t->connection)
+		if (gitno_connection_data_from_url(&t->connection_data, url, NULL) < 0 ||
+			 winhttp_connect(t, url) < 0)
+			return -1;
 
 	if (winhttp_stream_alloc(t, &s) < 0)
 		return -1;
@@ -1056,25 +1095,7 @@ static int winhttp_close(git_smart_subtransport *subtransport)
 	winhttp_subtransport *t = (winhttp_subtransport *)subtransport;
 	int ret = 0;
 
-	if (t->host) {
-		git__free(t->host);
-		t->host = NULL;
-	}
-
-	if (t->port) {
-		git__free(t->port);
-		t->port = NULL;
-	}
-
-	if (t->user_from_url) {
-		git__free(t->user_from_url);
-		t->user_from_url = NULL;
-	}
-
-	if (t->pass_from_url) {
-		git__free(t->pass_from_url);
-		t->pass_from_url = NULL;
-	}
+	gitno_connection_data_free_ptrs(&t->connection_data);
 
 	if (t->cred) {
 		t->cred->free(t->cred);
diff --git a/src/tree-cache.c b/src/tree-cache.c
index 97ffc2acf..1d3997154 100644
--- a/src/tree-cache.c
+++ b/src/tree-cache.c
@@ -138,9 +138,11 @@ static int read_tree_internal(git_tree_cache **out,
 		tree->children = git__malloc(tree->children_count * sizeof(git_tree_cache *));
 		GITERR_CHECK_ALLOC(tree->children);
 
+		memset(tree->children, 0x0, tree->children_count * sizeof(git_tree_cache *));
+
 		for (i = 0; i < tree->children_count; ++i) {
 			if (read_tree_internal(&tree->children[i], &buffer, buffer_end, tree) < 0)
-				return -1;
+				goto corrupted;
 		}
 	}
 
@@ -150,7 +152,7 @@ static int read_tree_internal(git_tree_cache **out,
 
  corrupted:
 	git_tree_cache_free(tree);
-	giterr_set(GITERR_INDEX, "Corruped TREE extension in index");
+	giterr_set(GITERR_INDEX, "Corrupted TREE extension in index");
 	return -1;
 }
 
@@ -162,7 +164,7 @@ int git_tree_cache_read(git_tree_cache **tree, const char *buffer, size_t buffer
 		return -1;
 
 	if (buffer < buffer_end) {
-		giterr_set(GITERR_INDEX, "Corruped TREE extension in index (unexpected trailing data)");
+		giterr_set(GITERR_INDEX, "Corrupted TREE extension in index (unexpected trailing data)");
 		return -1;
 	}
 
@@ -176,9 +178,12 @@ void git_tree_cache_free(git_tree_cache *tree)
 	if (tree == NULL)
 		return;
 
-	for (i = 0; i < tree->children_count; ++i)
-		git_tree_cache_free(tree->children[i]);
+	if (tree->children != NULL) {
+		for (i = 0; i < tree->children_count; ++i)
+			git_tree_cache_free(tree->children[i]);
+
+		git__free(tree->children);
+	}
 
-	git__free(tree->children);
 	git__free(tree);
 }
diff --git a/src/tree.c b/src/tree.c
index 65d01b4d5..bb59ff82b 100644
--- a/src/tree.c
+++ b/src/tree.c
@@ -10,7 +10,7 @@
 #include "tree.h"
 #include "git2/repository.h"
 #include "git2/object.h"
-#include "path.h"
+#include "fileops.h"
 #include "tree-cache.h"
 #include "index.h"
 
@@ -29,19 +29,19 @@ static bool valid_filemode(const int filemode)
 GIT_INLINE(git_filemode_t) normalize_filemode(git_filemode_t filemode)
 {
 	/* Tree bits set, but it's not a commit */
-	if (filemode & GIT_FILEMODE_TREE && !(filemode & 0100000))
+	if (GIT_MODE_TYPE(filemode) == GIT_FILEMODE_TREE)
 		return GIT_FILEMODE_TREE;
 
-	/* If any of the x bits is set */
-	if (filemode & 0111)
+	/* If any of the x bits are set */
+	if (GIT_PERMS_IS_EXEC(filemode))
 		return GIT_FILEMODE_BLOB_EXECUTABLE;
 
 	/* 16XXXX means commit */
-	if ((filemode & GIT_FILEMODE_COMMIT) == GIT_FILEMODE_COMMIT)
+	if (GIT_MODE_TYPE(filemode) == GIT_FILEMODE_COMMIT)
 		return GIT_FILEMODE_COMMIT;
 
 	/* 12XXXX means commit */
-	if ((filemode & GIT_FILEMODE_LINK) == GIT_FILEMODE_LINK)
+	if (GIT_MODE_TYPE(filemode) == GIT_FILEMODE_LINK)
 		return GIT_FILEMODE_LINK;
 
 	/* Otherwise, return a blob */
@@ -237,7 +237,12 @@ void git_tree__free(void *_tree)
 
 git_filemode_t git_tree_entry_filemode(const git_tree_entry *entry)
 {
-	return (git_filemode_t)entry->attr;
+	return normalize_filemode(entry->attr);
+}
+
+git_filemode_t git_tree_entry_filemode_raw(const git_tree_entry *entry)
+{
+	return entry->attr;
 }
 
 const char *git_tree_entry_name(const git_tree_entry *entry)
@@ -386,8 +391,6 @@ int git_tree__parse(void *_tree, git_odb_object *odb_obj)
 		if (git__strtol32(&attr, buffer, &buffer, 8) < 0 || !buffer)
 			return tree_error("Failed to parse tree. Can't parse filemode", NULL);
 
-		attr = normalize_filemode(attr); /* make sure to normalize the filemode */
-
 		if (*buffer++ != ' ')
 			return tree_error("Failed to parse tree. Object is corrupted", NULL);
 
@@ -881,8 +884,10 @@ static int tree_walk(
 	git_vector_foreach(&tree->entries, i, entry) {
 		if (preorder) {
 			error = callback(path->ptr, entry, payload);
-			if (error > 0)
+			if (error > 0) {
+				error = 0;
 				continue;
+			}
 			if (error < 0) {
 				giterr_clear();
 				return GIT_EUSER;
@@ -905,11 +910,12 @@ static int tree_walk(
 				return -1;
 
 			error = tree_walk(subtree, callback, path, payload, preorder);
+			git_tree_free(subtree);
+
 			if (error != 0)
 				break;
 
 			git_buf_truncate(path, path_len);
-			git_tree_free(subtree);
 		}
 
 		if (!preorder && callback(path->ptr, entry, payload) < 0) {
diff --git a/src/util.c b/src/util.c
index c543a3d21..47516a8f7 100644
--- a/src/util.c
+++ b/src/util.c
@@ -33,6 +33,9 @@ int git_libgit2_capabilities()
 #if defined(GIT_SSL) || defined(GIT_WINHTTP)
 		| GIT_CAP_HTTPS
 #endif
+#if defined(GIT_SSH)
+		| GIT_CAP_SSH
+#endif
 	;
 }
 
@@ -114,6 +117,19 @@ int git_libgit2_opts(int key, ...)
 		*(va_arg(ap, ssize_t *)) = git_cache__current_storage.val;
 		*(va_arg(ap, ssize_t *)) = git_cache__max_storage;
 		break;
+
+	case GIT_OPT_GET_TEMPLATE_PATH:
+		{
+			char *out = va_arg(ap, char *);
+			size_t outlen = va_arg(ap, size_t);
+
+			error = git_futils_dirs_get_str(out, outlen, GIT_FUTILS_DIR_TEMPLATE);
+		}
+		break;
+
+	case GIT_OPT_SET_TEMPLATE_PATH:
+		error = git_futils_dirs_set(GIT_FUTILS_DIR_TEMPLATE, va_arg(ap, const char *));
+		break;
 	}
 
 	va_end(ap);
@@ -676,6 +692,9 @@ size_t git__unescape(char *str)
 {
 	char *scan, *pos = str;
 
+	if (!str)
+		return 0;
+
 	for (scan = str; *scan; pos++, scan++) {
 		if (*scan == '\\' && *(scan + 1) != '\0')
 			scan++; /* skip '\' but include next char */
@@ -707,8 +726,9 @@ static int GIT_STDLIB_CALL git__qsort_r_glue_cmp(
 void git__qsort_r(
 	void *els, size_t nel, size_t elsize, git__sort_r_cmp cmp, void *payload)
 {
-#if defined(__MINGW32__) || defined(__OpenBSD__) || defined(AMIGA) || \
-	defined(__gnu_hurd__) || \
+#if defined(__MINGW32__) || defined(AMIGA) || \
+	defined(__OpenBSD__) || defined(__NetBSD__) || \
+	defined(__gnu_hurd__) || defined(__ANDROID_API__) || \
 	(__GLIBC__ == 2 && __GLIBC_MINOR__ < 8)
 	git__insertsort_r(els, nel, elsize, NULL, cmp, payload);
 #elif defined(GIT_WIN32)
diff --git a/src/util.h b/src/util.h
index e0088399c..f9de909e9 100644
--- a/src/util.h
+++ b/src/util.h
@@ -55,6 +55,9 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n)
 
 	ptr = (char*)git__malloc(length + 1);
 
+	if (!ptr)
+		return NULL;
+
 	if (length)
 		memcpy(ptr, str, length);
 
@@ -79,7 +82,10 @@ GIT_INLINE(void *) git__realloc(void *ptr, size_t size)
 	return new_ptr;
 }
 
-#define git__free(ptr) free(ptr)
+GIT_INLINE(void) git__free(void *ptr)
+{
+	free(ptr);
+}
 
 #define STRCMP_CASESELECT(IGNORE_CASE, STR1, STR2) \
 	((IGNORE_CASE) ? strcasecmp((STR1), (STR2)) : strcmp((STR1), (STR2)))
@@ -221,6 +227,9 @@ typedef void (*git_refcount_freeptr)(void *r);
 
 #define GIT_REFCOUNT_OWNER(r) (((git_refcount *)(r))->owner)
 
+#define GIT_REFCOUNT_VAL(r) git_atomic_get(&((git_refcount *)(r))->refcount)
+
+
 static signed char from_hex[] = {
 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 00 */
 -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 10 */
@@ -291,6 +300,11 @@ GIT_INLINE(bool) git__isspace(int c)
 	return (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == '\v' || c == 0x85 /* Unicode CR+LF */);
 }
 
+GIT_INLINE(bool) git__isspace_nonlf(int c)
+{
+	return (c == ' ' || c == '\t' || c == '\f' || c == '\r' || c == '\v' || c == 0x85 /* Unicode CR+LF */);
+}
+
 GIT_INLINE(bool) git__iswildcard(int c)
 {
 	return (c == '*' || c == '?' || c == '[');
@@ -339,4 +353,65 @@ GIT_INLINE(void) git__memzero(void *data, size_t size)
 #endif
 }
 
+#ifdef GIT_WIN32
+
+GIT_INLINE(double) git__timer(void)
+{
+	/* We need the initial tick count to detect if the tick
+	 * count has rolled over. */
+	static DWORD initial_tick_count = 0;
+
+	/* GetTickCount returns the number of milliseconds that have
+	 * elapsed since the system was started. */
+	DWORD count = GetTickCount();
+
+	if(initial_tick_count == 0) {
+		initial_tick_count = count;
+	} else if (count < initial_tick_count) {
+		/* The tick count has rolled over - adjust for it. */
+		count = (0xFFFFFFFF - initial_tick_count) + count;
+	}
+
+	return (double) count / (double) 1000;
+}
+
+#elif __APPLE__
+
+#include <mach/mach_time.h>
+
+GIT_INLINE(double) git__timer(void)
+{
+   uint64_t time = mach_absolute_time();
+   static double scaling_factor = 0;
+
+   if (scaling_factor == 0) {
+       mach_timebase_info_data_t info;
+       (void)mach_timebase_info(&info);
+       scaling_factor = (double)info.numer / (double)info.denom;
+   }
+
+   return (double)time * scaling_factor / 1.0E-9;
+}
+
+#else
+
+#include <sys/time.h>
+
+GIT_INLINE(double) git__timer(void)
+{
+	struct timespec tp;
+
+	if (clock_gettime(CLOCK_MONOTONIC, &tp) == 0) {
+		return (double) tp.tv_sec + (double) tp.tv_nsec / 1E-9;
+	} else {
+		/* Fall back to using gettimeofday */
+		struct timeval tv;
+		struct timezone tz;
+		gettimeofday(&tv, &tz);
+		return (double)tv.tv_sec + (double)tv.tv_usec / 1E-6;
+	}
+}
+
+#endif
+
 #endif /* INCLUDE_util_h__ */
diff --git a/src/vector.c b/src/vector.c
index 5ba2fab18..362e7b0c0 100644
--- a/src/vector.c
+++ b/src/vector.c
@@ -220,7 +220,7 @@ void git_vector_pop(git_vector *v)
 		v->length--;
 }
 
-void git_vector_uniq(git_vector *v)
+void git_vector_uniq(git_vector *v, void  (*git_free_cb)(void *))
 {
 	git_vector_cmp cmp;
 	size_t i, j;
@@ -232,9 +232,12 @@ void git_vector_uniq(git_vector *v)
 	cmp = v->_cmp ? v->_cmp : strict_comparison;
 
 	for (i = 0, j = 1 ; j < v->length; ++j)
-		if (!cmp(v->contents[i], v->contents[j]))
+		if (!cmp(v->contents[i], v->contents[j])) {
+			if (git_free_cb)
+				git_free_cb(v->contents[i]);
+
 			v->contents[i] = v->contents[j];
-		else
+		} else
 			v->contents[++i] = v->contents[j];
 
 	v->length -= j - i - 1;
diff --git a/src/vector.h b/src/vector.h
index 1bda9c93d..279f5c6ee 100644
--- a/src/vector.h
+++ b/src/vector.h
@@ -55,6 +55,11 @@ GIT_INLINE(void *) git_vector_get(const git_vector *v, size_t position)
 
 #define GIT_VECTOR_GET(V,I) ((I) < (V)->length ? (V)->contents[(I)] : NULL)
 
+GIT_INLINE(size_t) git_vector_length(const git_vector *v)
+{
+	return v->length;
+}
+
 GIT_INLINE(void *) git_vector_last(const git_vector *v)
 {
 	return (v->length > 0) ? git_vector_get(v, v->length - 1) : NULL;
@@ -71,7 +76,7 @@ int git_vector_insert_sorted(git_vector *v, void *element,
 	int (*on_dup)(void **old, void *new));
 int git_vector_remove(git_vector *v, size_t idx);
 void git_vector_pop(git_vector *v);
-void git_vector_uniq(git_vector *v);
+void git_vector_uniq(git_vector *v, void  (*git_free_cb)(void *));
 void git_vector_remove_matching(
 	git_vector *v, int (*match)(const git_vector *v, size_t idx));
 
diff --git a/src/win32/dir.c b/src/win32/dir.c
index 8c51d8378..f7859b73f 100644
--- a/src/win32/dir.c
+++ b/src/win32/dir.c
@@ -5,8 +5,7 @@
  * a Linking Exception. For full terms see the included COPYING file.
  */
 #define GIT__WIN32_NO_WRAP_DIR
-#include "dir.h"
-#include "utf-conv.h"
+#include "posix.h"
 
 static int init_filter(char *filter, size_t n, const char *dir)
 {
@@ -25,36 +24,32 @@ static int init_filter(char *filter, size_t n, const char *dir)
 
 git__DIR *git__opendir(const char *dir)
 {
-	char filter[GIT_WIN_PATH];
-	wchar_t filter_w[GIT_WIN_PATH];
+	git_win32_path_as_utf8 filter;
+	git_win32_path filter_w;
 	git__DIR *new = NULL;
+	size_t dirlen;
 
 	if (!dir || !init_filter(filter, sizeof(filter), dir))
 		return NULL;
 
-	new = git__calloc(1, sizeof(*new));
+	dirlen = strlen(dir);
+
+	new = git__calloc(sizeof(*new) + dirlen + 1, 1);
 	if (!new)
 		return NULL;
+	memcpy(new->dir, dir, dirlen);
 
-	new->dir = git__strdup(dir);
-	if (!new->dir)
-		goto fail;
-
-	git__utf8_to_16(filter_w, GIT_WIN_PATH, filter);
+	git_win32_path_from_c(filter_w, filter);
 	new->h = FindFirstFileW(filter_w, &new->f);
 
 	if (new->h == INVALID_HANDLE_VALUE) {
 		giterr_set(GITERR_OS, "Could not open directory '%s'", dir);
-		goto fail;
+		git__free(new);
+		return NULL;
 	}
 
 	new->first = 1;
 	return new;
-
-fail:
-	git__free(new->dir);
-	git__free(new);
-	return NULL;
 }
 
 int git__readdir_ext(
@@ -80,7 +75,7 @@ int git__readdir_ext(
 	if (wcslen(d->f.cFileName) >= sizeof(entry->d_name))
 		return -1;
 
-	git__utf16_to_8(entry->d_name, d->f.cFileName);
+	git_win32_path_to_c(entry->d_name, d->f.cFileName);
 	entry->d_ino = 0;
 
 	*result = entry;
@@ -101,8 +96,8 @@ struct git__dirent *git__readdir(git__DIR *d)
 
 void git__rewinddir(git__DIR *d)
 {
-	char filter[GIT_WIN_PATH];
-	wchar_t filter_w[GIT_WIN_PATH];
+	git_win32_path_as_utf8 filter;
+	git_win32_path filter_w;
 
 	if (!d)
 		return;
@@ -116,7 +111,7 @@ void git__rewinddir(git__DIR *d)
 	if (!init_filter(filter, sizeof(filter), d->dir))
 		return;
 
-	git__utf8_to_16(filter_w, GIT_WIN_PATH, filter);
+	git_win32_path_from_c(filter_w, filter);
 	d->h = FindFirstFileW(filter_w, &d->f);
 
 	if (d->h == INVALID_HANDLE_VALUE)
@@ -134,8 +129,7 @@ int git__closedir(git__DIR *d)
 		FindClose(d->h);
 		d->h = INVALID_HANDLE_VALUE;
 	}
-	git__free(d->dir);
-	d->dir = NULL;
+
 	git__free(d);
 	return 0;
 }
diff --git a/src/win32/dir.h b/src/win32/dir.h
index 7696d468e..24d48f6ba 100644
--- a/src/win32/dir.h
+++ b/src/win32/dir.h
@@ -11,15 +11,15 @@
 
 struct git__dirent {
 	int d_ino;
-	char d_name[261];
+	git_win32_path_as_utf8 d_name;
 };
 
 typedef struct {
 	HANDLE h;
 	WIN32_FIND_DATAW f;
 	struct git__dirent entry;
-	char *dir;
 	int first;
+	char dir[GIT_FLEX_ARRAY];
 } git__DIR;
 
 extern git__DIR *git__opendir(const char *);
diff --git a/src/win32/error.c b/src/win32/error.c
index a62a07e82..bc598ae32 100644
--- a/src/win32/error.c
+++ b/src/win32/error.c
@@ -47,7 +47,7 @@ char *git_win32_get_error_message(DWORD error_code)
 		(LPWSTR)&lpMsgBuf, 0, NULL)) {
 
 		/* Invalid code point check supported on Vista+ only */
-		if (git_has_win32_version(6, 0))
+		if (git_has_win32_version(6, 0, 0))
 			dwFlags = WC_ERR_INVALID_CHARS;
 		else
 			dwFlags = 0;
diff --git a/src/win32/findfile.c b/src/win32/findfile.c
index 5dd3de13d..a9e812e28 100644
--- a/src/win32/findfile.c
+++ b/src/win32/findfile.c
@@ -23,11 +23,11 @@ int git_win32__expand_path(struct git_win32__path *s_root, const wchar_t *templ)
 	return s_root->len ? 0 : -1;
 }
 
-static int win32_path_utf16_to_8(git_buf *path_utf8, const wchar_t *path_utf16)
+static int win32_path_to_8(git_buf *path_utf8, const wchar_t *path)
 {
 	char temp_utf8[GIT_PATH_MAX];
 
-	git__utf16_to_8(temp_utf8, path_utf16);
+	git__utf16_to_8(temp_utf8, GIT_PATH_MAX, path);
 	git_path_mkposix(temp_utf8);
 
 	return git_buf_sets(path_utf8, temp_utf8);
@@ -53,7 +53,7 @@ int git_win32__find_file(
 	if (*filename == '/' || *filename == '\\')
 		filename++;
 
-	git__utf8_to_16(file_utf16 + root->len - 1, alloc_len, filename);
+	git__utf8_to_16(file_utf16 + root->len - 1, alloc_len - root->len, filename);
 
 	/* check access */
 	if (_waccess(file_utf16, F_OK) < 0) {
@@ -61,7 +61,7 @@ int git_win32__find_file(
 		return GIT_ENOTFOUND;
 	}
 
-	win32_path_utf16_to_8(path, file_utf16);
+	win32_path_to_8(path, file_utf16);
 	git__free(file_utf16);
 
 	return 0;
@@ -86,7 +86,7 @@ static wchar_t* win32_walkpath(wchar_t *path, wchar_t *buf, size_t buflen)
 	return (path != base) ? path : NULL;
 }
 
-static int win32_find_git_in_path(git_buf *buf, const wchar_t *gitexe)
+static int win32_find_git_in_path(git_buf *buf, const wchar_t *gitexe, const wchar_t *subdir)
 {
 	wchar_t *env = _wgetenv(L"PATH"), lastch;
 	struct git_win32__path root;
@@ -110,10 +110,10 @@ static int win32_find_git_in_path(git_buf *buf, const wchar_t *gitexe)
 		wcscpy(&root.path[root.len], gitexe);
 
 		if (_waccess(root.path, F_OK) == 0 && root.len > 5) {
-			/* replace "bin\\" or "cmd\\" with "etc\\" */
-			wcscpy(&root.path[root.len - 4], L"etc\\");
+			/* replace "bin\\" or "cmd\\" with subdir */
+			wcscpy(&root.path[root.len - 4], subdir);
 
-			win32_path_utf16_to_8(buf, root.path);
+			win32_path_to_8(buf, root.path);
 			return 0;
 		}
 	}
@@ -122,7 +122,7 @@ static int win32_find_git_in_path(git_buf *buf, const wchar_t *gitexe)
 }
 
 static int win32_find_git_in_registry(
-	git_buf *buf, const HKEY hieve, const wchar_t *key)
+	git_buf *buf, const HKEY hieve, const wchar_t *key, const wchar_t *subdir)
 {
 	HKEY hKey;
 	DWORD dwType = REG_SZ;
@@ -130,9 +130,9 @@ static int win32_find_git_in_registry(
 
 	assert(buf);
 
-	path16.len = 0;
+	path16.len = MAX_PATH;
 
-	if (RegOpenKeyExW(hieve, key, 0, KEY_ALL_ACCESS, &hKey) == ERROR_SUCCESS) {
+	if (RegOpenKeyExW(hieve, key, 0, KEY_READ, &hKey) == ERROR_SUCCESS) {
 		if (RegQueryValueExW(hKey, L"InstallLocation", NULL, &dwType,
 			(LPBYTE)&path16.path, &path16.len) == ERROR_SUCCESS)
 		{
@@ -143,10 +143,10 @@ static int win32_find_git_in_registry(
 				return -1;
 			}
 
-			wcscat(path16.path, L"etc\\");
+			wcscat(path16.path, subdir);
 			path16.len += 4;
 
-			win32_path_utf16_to_8(buf, path16.path);
+			win32_path_to_8(buf, path16.path);
 		}
 
 		RegCloseKey(hKey);
@@ -156,7 +156,7 @@ static int win32_find_git_in_registry(
 }
 
 static int win32_find_existing_dirs(
-	git_buf *out, const wchar_t *tmpl[], char *temp[])
+	git_buf *out, const wchar_t *tmpl[])
 {
 	struct git_win32__path path16;
 	git_buf buf = GIT_BUF_INIT;
@@ -168,7 +168,7 @@ static int win32_find_existing_dirs(
 			path16.path[0] != L'%' &&
 			!_waccess(path16.path, F_OK))
 		{
-			win32_path_utf16_to_8(&buf, path16.path);
+			win32_path_to_8(&buf, path16.path);
 
 			if (buf.size)
 				git_buf_join(out, GIT_PATH_LIST_SEPARATOR, out->ptr, buf.ptr);
@@ -180,26 +180,26 @@ static int win32_find_existing_dirs(
 	return (git_buf_oom(out) ? -1 : 0);
 }
 
-int git_win32__find_system_dirs(git_buf *out)
+int git_win32__find_system_dirs(git_buf *out, const wchar_t *subdir)
 {
 	git_buf buf = GIT_BUF_INIT;
 
 	/* directories where git.exe & git.cmd are found */
-	if (!win32_find_git_in_path(&buf, L"git.exe") && buf.size)
+	if (!win32_find_git_in_path(&buf, L"git.exe", subdir) && buf.size)
 		git_buf_set(out, buf.ptr, buf.size);
 	else
 		git_buf_clear(out);
 
-	if (!win32_find_git_in_path(&buf, L"git.cmd") && buf.size)
+	if (!win32_find_git_in_path(&buf, L"git.cmd", subdir) && buf.size)
 		git_buf_join(out, GIT_PATH_LIST_SEPARATOR, out->ptr, buf.ptr);
 
 	/* directories where git is installed according to registry */
 	if (!win32_find_git_in_registry(
-			&buf, HKEY_CURRENT_USER, REG_MSYSGIT_INSTALL_LOCAL) && buf.size)
+			&buf, HKEY_CURRENT_USER, REG_MSYSGIT_INSTALL_LOCAL, subdir) && buf.size)
 		git_buf_join(out, GIT_PATH_LIST_SEPARATOR, out->ptr, buf.ptr);
 
 	if (!win32_find_git_in_registry(
-			&buf, HKEY_LOCAL_MACHINE, REG_MSYSGIT_INSTALL) && buf.size)
+			&buf, HKEY_LOCAL_MACHINE, REG_MSYSGIT_INSTALL, subdir) && buf.size)
 		git_buf_join(out, GIT_PATH_LIST_SEPARATOR, out->ptr, buf.ptr);
 
 	git_buf_free(&buf);
@@ -209,7 +209,6 @@ int git_win32__find_system_dirs(git_buf *out)
 
 int git_win32__find_global_dirs(git_buf *out)
 {
-	char *temp[3];
 	static const wchar_t *global_tmpls[4] = {
 		L"%HOME%\\",
 		L"%HOMEDRIVE%%HOMEPATH%\\",
@@ -217,12 +216,11 @@ int git_win32__find_global_dirs(git_buf *out)
 		NULL,
 	};
 
-	return win32_find_existing_dirs(out, global_tmpls, temp);
+	return win32_find_existing_dirs(out, global_tmpls);
 }
 
 int git_win32__find_xdg_dirs(git_buf *out)
 {
-	char *temp[6];
 	static const wchar_t *global_tmpls[7] = {
 		L"%XDG_CONFIG_HOME%\\git",
 		L"%APPDATA%\\git",
@@ -233,5 +231,5 @@ int git_win32__find_xdg_dirs(git_buf *out)
 		NULL,
 	};
 
-	return win32_find_existing_dirs(out, global_tmpls, temp);
+	return win32_find_existing_dirs(out, global_tmpls);
 }
diff --git a/src/win32/findfile.h b/src/win32/findfile.h
index fc79e1b72..11bf7e620 100644
--- a/src/win32/findfile.h
+++ b/src/win32/findfile.h
@@ -19,7 +19,7 @@ extern int git_win32__expand_path(
 extern int git_win32__find_file(
 	git_buf *path, const struct git_win32__path *root, const char *filename);
 
-extern int git_win32__find_system_dirs(git_buf *out);
+extern int git_win32__find_system_dirs(git_buf *out, const wchar_t *subpath);
 extern int git_win32__find_global_dirs(git_buf *out);
 extern int git_win32__find_xdg_dirs(git_buf *out);
 
diff --git a/src/win32/mingw-compat.h b/src/win32/mingw-compat.h
index 7b97b48db..fe0abfb54 100644
--- a/src/win32/mingw-compat.h
+++ b/src/win32/mingw-compat.h
@@ -19,6 +19,11 @@
 # define S_IFLNK _S_IFLNK
 # define S_ISLNK(m) (((m) & _S_IFMT) == _S_IFLNK)
 
+GIT_INLINE(size_t) p_strnlen(const char *s, size_t maxlen) {
+	const char *end = memchr(s, 0, maxlen);
+	return end ? (size_t)(end - s) : maxlen;
+}
+
 #endif
 
 #endif /* INCLUDE_mingw_compat__ */
diff --git a/src/win32/posix.h b/src/win32/posix.h
index c49c2175c..24cba23e0 100644
--- a/src/win32/posix.h
+++ b/src/win32/posix.h
@@ -8,7 +8,16 @@
 #define INCLUDE_posix__w32_h__
 
 #include "common.h"
+#include "../posix.h"
 #include "utf-conv.h"
+#include "dir.h"
+
+/* define some standard errnos that the runtime may be missing.  for example,
+ * mingw lacks EAFNOSUPPORT. */
+
+#ifndef EAFNOSUPPORT
+# define EAFNOSUPPORT (INT_MAX-1)
+#endif
 
 GIT_INLINE(int) p_link(const char *old, const char *new)
 {
@@ -20,9 +29,9 @@ GIT_INLINE(int) p_link(const char *old, const char *new)
 
 GIT_INLINE(int) p_mkdir(const char *path, mode_t mode)
 {
-	wchar_t buf[GIT_WIN_PATH];
+	git_win32_path buf;
 	GIT_UNUSED(mode);
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path_from_c(buf, path);
 	return _wmkdir(buf);
 }
 
diff --git a/src/win32/posix_w32.c b/src/win32/posix_w32.c
index f04974428..18f717b0f 100644
--- a/src/win32/posix_w32.c
+++ b/src/win32/posix_w32.c
@@ -16,8 +16,8 @@
 
 int p_unlink(const char *path)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	_wchmod(buf, 0666);
 	return _wunlink(buf);
 }
@@ -59,10 +59,11 @@ static int do_lstat(
 	const char *file_name, struct stat *buf, int posix_enotdir)
 {
 	WIN32_FILE_ATTRIBUTE_DATA fdata;
-	wchar_t fbuf[GIT_WIN_PATH], lastch;
+	git_win32_path fbuf;
+	wchar_t lastch;
 	int flen;
 
-	flen = git__utf8_to_16(fbuf, GIT_WIN_PATH, file_name);
+	flen = git_win32_path_from_c(fbuf, file_name);
 
 	/* truncate trailing slashes */
 	for (; flen > 0; --flen) {
@@ -90,6 +91,9 @@ static int do_lstat(
 		if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
 			fMode |= S_IFLNK;
 
+		if ((fMode & (S_IFDIR | S_IFLNK)) == (S_IFDIR | S_IFLNK)) // junction
+			fMode ^= S_IFLNK;
+
 		buf->st_ino = 0;
 		buf->st_gid = 0;
 		buf->st_uid = 0;
@@ -105,10 +109,10 @@ static int do_lstat(
 		 * the length of the path pointed to, which we expect everywhere else
 		 */
 		if (S_ISLNK(fMode)) {
-			char target[GIT_WIN_PATH];
+			git_win32_path_as_utf8 target;
 			int readlink_result;
 
-			readlink_result = p_readlink(file_name, target, GIT_WIN_PATH);
+			readlink_result = p_readlink(file_name, target, sizeof(target));
 
 			if (readlink_result == -1)
 				return -1;
@@ -121,8 +125,8 @@ static int do_lstat(
 
 	errno = ENOENT;
 
-	/* We need POSIX behavior, then ENOTDIR must set when any of the folders in the
-	 * file path is a regular file,otherwise ENOENT must be set.
+	/* To match POSIX behavior, set ENOTDIR when any of the folders in the
+	 * file path is a regular file, otherwise set ENOENT.
 	 */
 	if (posix_enotdir) {
 		/* scan up path until we find an existing item */
@@ -156,13 +160,22 @@ int p_lstat_posixly(const char *filename, struct stat *buf)
 	return do_lstat(filename, buf, 1);
 }
 
+
+/*
+ * Parts of the The p_readlink function are heavily inspired by the php 
+ * readlink function in link_win32.c
+ *
+ * Copyright (c) 1999 - 2012 The PHP Group. All rights reserved.
+ *
+ * For details of the PHP license see http://www.php.net/license/3_01.txt
+ */
 int p_readlink(const char *link, char *target, size_t target_len)
 {
 	typedef DWORD (WINAPI *fpath_func)(HANDLE, LPWSTR, DWORD, DWORD);
 	static fpath_func pGetFinalPath = NULL;
 	HANDLE hFile;
 	DWORD dwRet;
-	wchar_t link_w[GIT_WIN_PATH];
+	git_win32_path link_w;
 	wchar_t* target_w;
 	int error = 0;
 
@@ -185,7 +198,7 @@ int p_readlink(const char *link, char *target, size_t target_len)
 		}
 	}
 
-	git__utf8_to_16(link_w, GIT_WIN_PATH, link);
+	git_win32_path_from_c(link_w, link);
 
 	hFile = CreateFileW(link_w,			// file to open
 			GENERIC_READ,			// open for reading
@@ -251,10 +264,10 @@ int p_symlink(const char *old, const char *new)
 
 int p_open(const char *path, int flags, ...)
 {
-	wchar_t buf[GIT_WIN_PATH];
+	git_win32_path buf;
 	mode_t mode = 0;
 
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path_from_c(buf, path);
 
 	if (flags & O_CREAT) {
 		va_list arg_list;
@@ -269,8 +282,8 @@ int p_open(const char *path, int flags, ...)
 
 int p_creat(const char *path, mode_t mode)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return _wopen(buf, _O_WRONLY | _O_CREAT | _O_TRUNC | _O_BINARY, mode);
 }
 
@@ -296,7 +309,7 @@ int p_getcwd(char *buffer_out, size_t size)
 
 int p_stat(const char* path, struct stat* buf)
 {
-	char target[GIT_WIN_PATH];
+	git_win32_path_as_utf8 target;
 	int error = 0;
 
 	error = do_lstat(path, buf, 0);
@@ -304,7 +317,7 @@ int p_stat(const char* path, struct stat* buf)
 	/* We need not do this in a loop to unwind chains of symlinks since
 	 * p_readlink calls GetFinalPathNameByHandle which does it for us. */
 	if (error >= 0 && S_ISLNK(buf->st_mode) &&
-		(error = p_readlink(path, target, GIT_WIN_PATH)) >= 0)
+		(error = p_readlink(path, target, sizeof(target))) >= 0)
 		error = do_lstat(target, buf, 0);
 
 	return error;
@@ -312,23 +325,23 @@ int p_stat(const char* path, struct stat* buf)
 
 int p_chdir(const char* path)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return _wchdir(buf);
 }
 
 int p_chmod(const char* path, mode_t mode)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return _wchmod(buf, mode);
 }
 
 int p_rmdir(const char* path)
 {
 	int error;
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 
 	error = _wrmdir(buf);
 
@@ -344,24 +357,24 @@ int p_rmdir(const char* path)
 
 int p_hide_directory__w32(const char *path)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return (SetFileAttributesW(buf, FILE_ATTRIBUTE_HIDDEN) != 0) ? 0 : -1;
 }
 
 char *p_realpath(const char *orig_path, char *buffer)
 {
 	int ret;
-	wchar_t orig_path_w[GIT_WIN_PATH];
-	wchar_t buffer_w[GIT_WIN_PATH];
+	git_win32_path orig_path_w;
+	git_win32_path buffer_w;
 
-	git__utf8_to_16(orig_path_w, GIT_WIN_PATH, orig_path);
+	git_win32_path_from_c(orig_path_w, orig_path);
 
 	/* Implicitly use GetCurrentDirectory which can be a threading issue */
-	ret = GetFullPathNameW(orig_path_w, GIT_WIN_PATH, buffer_w, NULL);
+	ret = GetFullPathNameW(orig_path_w, GIT_WIN_PATH_UTF16, buffer_w, NULL);
 
 	/* According to MSDN, a return value equals to zero means a failure. */
-	if (ret == 0 || ret > GIT_WIN_PATH)
+	if (ret == 0 || ret > GIT_WIN_PATH_UTF16)
 		buffer = NULL;
 
 	else if (GetFileAttributesW(buffer_w) == INVALID_FILE_ATTRIBUTES) {
@@ -445,18 +458,18 @@ int p_setenv(const char* name, const char* value, int overwrite)
 
 int p_access(const char* path, mode_t mode)
 {
-	wchar_t buf[GIT_WIN_PATH];
-	git__utf8_to_16(buf, GIT_WIN_PATH, path);
+	git_win32_path buf;
+	git_win32_path_from_c(buf, path);
 	return _waccess(buf, mode);
 }
 
 int p_rename(const char *from, const char *to)
 {
-	wchar_t wfrom[GIT_WIN_PATH];
-	wchar_t wto[GIT_WIN_PATH];
+	git_win32_path wfrom;
+	git_win32_path wto;
 
-	git__utf8_to_16(wfrom, GIT_WIN_PATH, from);
-	git__utf8_to_16(wto, GIT_WIN_PATH, to);
+	git_win32_path_from_c(wfrom, from);
+	git_win32_path_from_c(wto, to);
 	return MoveFileExW(wfrom, wto, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) ? 0 : -1;
 }
 
@@ -505,94 +518,40 @@ p_gmtime_r (const time_t *timer, struct tm *result)
 	return result;
 }
 
-#if defined(_MSC_VER) || defined(_MSC_EXTENSIONS)
-#define DELTA_EPOCH_IN_MICROSECS  11644473600000000Ui64
-#else
-#define DELTA_EPOCH_IN_MICROSECS  11644473600000000ULL
-#endif
- 
-#ifndef _TIMEZONE_DEFINED
-#define _TIMEZONE_DEFINED
-struct timezone 
-{
-	int  tz_minuteswest; /* minutes W of Greenwich */
-	int  tz_dsttime;     /* type of dst correction */
-};
-#endif
-
-int p_gettimeofday(struct timeval *tv, struct timezone *tz)
+int p_inet_pton(int af, const char *src, void *dst)
 {
-	FILETIME ft;
-	unsigned __int64 tmpres = 0;
-	static int tzflag;
-
-	if (NULL != tv)
-	{
-		GetSystemTimeAsFileTime(&ft);
-
-		tmpres |= ft.dwHighDateTime;
-		tmpres <<= 32;
-		tmpres |= ft.dwLowDateTime;
-
-		/*converting file time to unix epoch*/
-		tmpres /= 10;  /*convert into microseconds*/
-		tmpres -= DELTA_EPOCH_IN_MICROSECS;
-		tv->tv_sec = (long)(tmpres / 1000000UL);
-		tv->tv_usec = (long)(tmpres % 1000000UL);
-	}
+	struct sockaddr_storage sin;
+	void *addr;
+	int sin_len = sizeof(struct sockaddr_storage), addr_len;
+	int error = 0;
 
-	if (NULL != tz)
-	{
-		 if (!tzflag)
-		{
-			_tzset();
-			tzflag++;
-		}
-		tz->tz_minuteswest = _timezone / 60;
-		tz->tz_dsttime = _daylight;
+	if (af == AF_INET) {
+		addr = &((struct sockaddr_in *)&sin)->sin_addr;
+		addr_len = sizeof(struct in_addr);
+	} else if (af == AF_INET6) {
+		addr = &((struct sockaddr_in6 *)&sin)->sin6_addr;
+		addr_len = sizeof(struct in6_addr);
+	} else {
+		errno = EAFNOSUPPORT;
+		return -1;
 	}
 
-	return 0;
-}
-
-int p_inet_pton(int af, const char* src, void* dst)
-{
-	union {
-		struct sockaddr_in6 sin6;
-		struct sockaddr_in sin;
-	} sa;
-	int srcsize;
-
-	switch(af)
-	{
-		case AF_INET:
-			sa.sin.sin_family = AF_INET;
-			srcsize = (int)sizeof(sa.sin);
-		break;
-		case AF_INET6:
-			sa.sin6.sin6_family = AF_INET6;
-			srcsize = (int)sizeof(sa.sin6);
-		break;
-		default:
-			errno = WSAEPFNOSUPPORT;
-			return -1;
+	if ((error = WSAStringToAddressA((LPSTR)src, af, NULL, (LPSOCKADDR)&sin, &sin_len)) == 0) {
+		memcpy(dst, addr, addr_len);
+		return 1;
 	}
 
-	if (WSAStringToAddress((LPSTR)src, af, NULL, (struct sockaddr *) &sa, &srcsize) != 0)
-	{
-		errno = WSAGetLastError();
+	switch(WSAGetLastError()) {
+	case WSAEINVAL:
+		return 0;
+	case WSAEFAULT:
+		errno = ENOSPC;
+		return -1;
+	case WSA_NOT_ENOUGH_MEMORY:
+		errno = ENOMEM;
 		return -1;
 	}
 
-	switch(af)
-	{
-		case AF_INET:
-			memcpy(dst, &sa.sin.sin_addr, sizeof(sa.sin.sin_addr));
-		break;
-		case AF_INET6:
-			memcpy(dst, &sa.sin6.sin6_addr, sizeof(sa.sin6.sin6_addr));
-		break;
-	}
-
-	return 1;
+	errno = EINVAL;
+	return -1;
 }
diff --git a/src/win32/precompiled.h b/src/win32/precompiled.h
index 5de7e6f34..cbfe98812 100644
--- a/src/win32/precompiled.h
+++ b/src/win32/precompiled.h
@@ -1,4 +1,5 @@
 #include "git2.h"
+#include "common.h"
 
 #include <assert.h>
 #include <errno.h>
@@ -6,6 +7,8 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
+#include <fcntl.h>
+#include <time.h>
 
 #include <sys/types.h>
 #include <sys/stat.h>
diff --git a/src/win32/pthread.c b/src/win32/pthread.c
index 2f263b3e0..db8927471 100644
--- a/src/win32/pthread.c
+++ b/src/win32/pthread.c
@@ -6,6 +6,7 @@
  */
 
 #include "pthread.h"
+#include "../global.h"
 
 int pthread_create(
 	pthread_t *GIT_RESTRICT thread,
@@ -127,9 +128,10 @@ int pthread_cond_signal(pthread_cond_t *cond)
 	return 0;
 }
 
-/* pthread_cond_broadcast is not implemented because doing so with just Win32 events
- * is quite complicated, and no caller in libgit2 uses it yet. */
-
+/* pthread_cond_broadcast is not implemented because doing so with just
+ * Win32 events is quite complicated, and no caller in libgit2 uses it
+ * yet.
+ */
 int pthread_num_processors_np(void)
 {
 	DWORD_PTR p, s;
@@ -142,3 +144,111 @@ int pthread_num_processors_np(void)
 	return n ? n : 1;
 }
 
+
+static HINSTANCE win32_kernel32_dll;
+
+typedef void (WINAPI *win32_srwlock_fn)(GIT_SRWLOCK *);
+
+static win32_srwlock_fn win32_srwlock_initialize;
+static win32_srwlock_fn win32_srwlock_acquire_shared;
+static win32_srwlock_fn win32_srwlock_release_shared;
+static win32_srwlock_fn win32_srwlock_acquire_exclusive;
+static win32_srwlock_fn win32_srwlock_release_exclusive;
+
+int pthread_rwlock_init(
+	pthread_rwlock_t *GIT_RESTRICT lock,
+	const pthread_rwlockattr_t *GIT_RESTRICT attr)
+{
+	(void)attr;
+
+	if (win32_srwlock_initialize)
+		win32_srwlock_initialize(&lock->native.srwl);
+	else
+		InitializeCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_rdlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_acquire_shared)
+		win32_srwlock_acquire_shared(&lock->native.srwl);
+	else
+		EnterCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_rdunlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_release_shared)
+		win32_srwlock_release_shared(&lock->native.srwl);
+	else
+		LeaveCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_wrlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_acquire_exclusive)
+		win32_srwlock_acquire_exclusive(&lock->native.srwl);
+	else
+		EnterCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_wrunlock(pthread_rwlock_t *lock)
+{
+	if (win32_srwlock_release_exclusive)
+		win32_srwlock_release_exclusive(&lock->native.srwl);
+	else
+		LeaveCriticalSection(&lock->native.csec);
+
+	return 0;
+}
+
+int pthread_rwlock_destroy(pthread_rwlock_t *lock)
+{
+	if (!win32_srwlock_initialize)
+		DeleteCriticalSection(&lock->native.csec);
+	git__memzero(lock, sizeof(*lock));
+	return 0;
+}
+
+
+static void win32_pthread_shutdown(void)
+{
+	if (win32_kernel32_dll) {
+		FreeLibrary(win32_kernel32_dll);
+		win32_kernel32_dll = NULL;
+	}
+}
+
+int win32_pthread_initialize(void)
+{
+	if (win32_kernel32_dll)
+		return 0;
+
+	win32_kernel32_dll = LoadLibrary("Kernel32.dll");
+	if (!win32_kernel32_dll) {
+		giterr_set(GITERR_OS, "Could not load Kernel32.dll!");
+		return -1;
+	}
+
+	win32_srwlock_initialize = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "InitializeSRWLock");
+	win32_srwlock_acquire_shared = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "AcquireSRWLockShared");
+	win32_srwlock_release_shared = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockShared");
+	win32_srwlock_acquire_exclusive = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "AcquireSRWLockExclusive");
+	win32_srwlock_release_exclusive = (win32_srwlock_fn)
+		GetProcAddress(win32_kernel32_dll, "ReleaseSRWLockExclusive");
+
+	git__on_shutdown(win32_pthread_shutdown);
+
+	return 0;
+}
diff --git a/src/win32/pthread.h b/src/win32/pthread.h
index 8277ecf6e..af5b121f0 100644
--- a/src/win32/pthread.h
+++ b/src/win32/pthread.h
@@ -19,22 +19,34 @@
 typedef int pthread_mutexattr_t;
 typedef int pthread_condattr_t;
 typedef int pthread_attr_t;
+typedef int pthread_rwlockattr_t;
+
 typedef CRITICAL_SECTION pthread_mutex_t;
 typedef HANDLE pthread_t;
 typedef HANDLE pthread_cond_t;
 
-#define PTHREAD_MUTEX_INITIALIZER {(void*)-1};
+typedef struct { void *Ptr; } GIT_SRWLOCK;
+
+typedef struct {
+	union {
+		GIT_SRWLOCK srwl;
+		CRITICAL_SECTION csec;
+	} native;
+} pthread_rwlock_t;
+
+#define PTHREAD_MUTEX_INITIALIZER  {(void*)-1}
 
 int pthread_create(
-	pthread_t *GIT_RESTRICT,
-	const pthread_attr_t *GIT_RESTRICT,
+	pthread_t *GIT_RESTRICT thread,
+	const pthread_attr_t *GIT_RESTRICT attr,
 	void *(*start_routine)(void*),
-	void *__restrict);
+	void *GIT_RESTRICT arg);
 
 int pthread_join(pthread_t, void **);
 
 int pthread_mutex_init(
-	pthread_mutex_t *GIT_RESTRICT, const pthread_mutexattr_t *GIT_RESTRICT);
+	pthread_mutex_t *GIT_RESTRICT mutex,
+	const pthread_mutexattr_t *GIT_RESTRICT mutexattr);
 int pthread_mutex_destroy(pthread_mutex_t *);
 int pthread_mutex_lock(pthread_mutex_t *);
 int pthread_mutex_unlock(pthread_mutex_t *);
@@ -47,4 +59,15 @@ int pthread_cond_signal(pthread_cond_t *);
 
 int pthread_num_processors_np(void);
 
+int pthread_rwlock_init(
+	pthread_rwlock_t *GIT_RESTRICT lock,
+	const pthread_rwlockattr_t *GIT_RESTRICT attr);
+int pthread_rwlock_rdlock(pthread_rwlock_t *);
+int pthread_rwlock_rdunlock(pthread_rwlock_t *);
+int pthread_rwlock_wrlock(pthread_rwlock_t *);
+int pthread_rwlock_wrunlock(pthread_rwlock_t *);
+int pthread_rwlock_destroy(pthread_rwlock_t *);
+
+extern int win32_pthread_initialize(void);
+
 #endif
diff --git a/src/win32/utf-conv.c b/src/win32/utf-conv.c
index c06f3a8c2..d4dbfbab9 100644
--- a/src/win32/utf-conv.c
+++ b/src/win32/utf-conv.c
@@ -70,12 +70,12 @@ void git__utf8_to_16(wchar_t *dest, size_t length, const char *src)
 }
 #endif
 
-int git__utf8_to_16(wchar_t *dest, size_t length, const char *src)
+int git__utf8_to_16(wchar_t * dest, size_t dest_size, const char *src)
 {
-	return MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)length);
+	return MultiByteToWideChar(CP_UTF8, 0, src, -1, dest, (int)dest_size);
 }
 
-int git__utf16_to_8(char *out, const wchar_t *input)
+int git__utf16_to_8(char *dest, size_t dest_size, const wchar_t *src)
 {
-	return WideCharToMultiByte(CP_UTF8, 0, input, -1, out, GIT_WIN_PATH, NULL, NULL);
+	return WideCharToMultiByte(CP_UTF8, 0, src, -1, dest, (int)dest_size, NULL, NULL);
 }
diff --git a/src/win32/utf-conv.h b/src/win32/utf-conv.h
index 6cc9205f7..3af77580e 100644
--- a/src/win32/utf-conv.h
+++ b/src/win32/utf-conv.h
@@ -4,16 +4,35 @@
  * This file is part of libgit2, distributed under the GNU GPL v2 with
  * a Linking Exception. For full terms see the included COPYING file.
  */
+#ifndef INCLUDE_git_utfconv_h__
+#define INCLUDE_git_utfconv_h__
 
 #include <wchar.h>
+#include "common.h"
 
-#ifndef INCLUDE_git_utfconv_h__
-#define INCLUDE_git_utfconv_h__
+/* Maximum characters in a Windows path plus one for NUL byte */
+#define GIT_WIN_PATH_UTF16 (260 + 1)
 
-#define GIT_WIN_PATH (260 + 1)
+/* Maximum bytes necessary to convert a full-length UTF16 path to UTF8 */
+#define GIT_WIN_PATH_UTF8  (260 * 4 + 1)
 
-int git__utf8_to_16(wchar_t *dest, size_t length, const char *src);
-int git__utf16_to_8(char *dest, const wchar_t *src);
+typedef wchar_t git_win32_path[GIT_WIN_PATH_UTF16];
 
-#endif
+typedef char git_win32_path_as_utf8[GIT_WIN_PATH_UTF8];
 
+/* dest_size is the size of dest in wchar_t's */
+int git__utf8_to_16(wchar_t * dest, size_t dest_size, const char *src);
+/* dest_size is the size of dest in char's */
+int git__utf16_to_8(char *dest, size_t dest_size, const wchar_t *src);
+
+GIT_INLINE(int) git_win32_path_from_c(git_win32_path dest, const char *src)
+{
+	return git__utf8_to_16(dest, GIT_WIN_PATH_UTF16, src);
+}
+
+GIT_INLINE(int) git_win32_path_to_c(git_win32_path_as_utf8 dest, const wchar_t *src)
+{
+	return git__utf16_to_8(dest, GIT_WIN_PATH_UTF8, src);
+}
+
+#endif
diff --git a/src/win32/version.h b/src/win32/version.h
index 483962b57..79667697f 100644
--- a/src/win32/version.h
+++ b/src/win32/version.h
@@ -9,12 +9,29 @@
 
 #include <windows.h>
 
-GIT_INLINE(int) git_has_win32_version(int major, int minor)
+GIT_INLINE(int) git_has_win32_version(int major, int minor, int service_pack)
 {
-	WORD wVersion = LOWORD(GetVersion());
+	OSVERSIONINFOEX version_test = {0};
+	DWORD version_test_mask;
+	DWORDLONG version_condition_mask = 0;
+	
+	version_test.dwOSVersionInfoSize = sizeof(OSVERSIONINFOEX);
+	version_test.dwMajorVersion = major;
+	version_test.dwMinorVersion = minor;
+	version_test.wServicePackMajor = (WORD)service_pack;
+	version_test.wServicePackMinor = 0;
 
-	return LOBYTE(wVersion) > major ||
-		(LOBYTE(wVersion) == major && HIBYTE(wVersion) >= minor);
+	version_test_mask = (VER_MAJORVERSION | VER_MINORVERSION | VER_SERVICEPACKMAJOR | VER_SERVICEPACKMINOR);
+
+	VER_SET_CONDITION(version_condition_mask, VER_MAJORVERSION, VER_GREATER_EQUAL);
+	VER_SET_CONDITION(version_condition_mask, VER_MINORVERSION, VER_GREATER_EQUAL);
+	VER_SET_CONDITION(version_condition_mask, VER_SERVICEPACKMAJOR, VER_GREATER_EQUAL);
+	VER_SET_CONDITION(version_condition_mask, VER_SERVICEPACKMINOR, VER_GREATER_EQUAL);
+
+	if (!VerifyVersionInfo(&version_test, version_test_mask, version_condition_mask))
+		return 0;
+
+	return 1;
 }
 
 #endif