Merge remote-tracking branch 'source/development' into update-test

Merging main libgit2!
Conflicts:
	CMakeLists.txt
	src/unix/map.c

160 files changed, 28966 insertions, 8420 deletions

diff --git a/src/attr.c b/src/attr.c
new file mode 100644
index 000000000..fb6651196
--- /dev/null
+++ b/src/attr.c
@@ -0,0 +1,677 @@
+#include "repository.h"
+#include "fileops.h"
+#include "config.h"
+#include <ctype.h>
+
+GIT__USE_STRMAP;
+
+static int collect_attr_files(
+	git_repository *repo,
+	uint32_t flags,
+	const char *path,
+	git_vector *files);
+
+
+int git_attr_get(
+	const char **value,
+    git_repository *repo,
+	uint32_t flags,
+	const char *pathname,
+	const char *name)
+{
+	int error;
+	git_attr_path path;
+	git_vector files = GIT_VECTOR_INIT;
+	unsigned int i, j;
+	git_attr_file *file;
+	git_attr_name attr;
+	git_attr_rule *rule;
+
+	*value = NULL;
+
+	if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0)
+		return -1;
+
+	if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0)
+		goto cleanup;
+
+	attr.name = name;
+	attr.name_hash = git_attr_file__name_hash(name);
+
+	git_vector_foreach(&files, i, file) {
+
+		git_attr_file__foreach_matching_rule(file, &path, j, rule) {
+			int pos = git_vector_bsearch(&rule->assigns, &attr);
+			if (pos >= 0) {
+				*value = ((git_attr_assignment *)git_vector_get(
+							  &rule->assigns, pos))->value;
+				goto cleanup;
+			}
+		}
+	}
+
+cleanup:
+	git_vector_free(&files);
+	git_attr_path__free(&path);
+
+	return error;
+}
+
+
+typedef struct {
+	git_attr_name name;
+	git_attr_assignment *found;
+} attr_get_many_info;
+
+int git_attr_get_many(
+	const char **values,
+    git_repository *repo,
+	uint32_t flags,
+	const char *pathname,
+    size_t num_attr,
+	const char **names)
+{
+	int error;
+	git_attr_path path;
+	git_vector files = GIT_VECTOR_INIT;
+	unsigned int i, j, k;
+	git_attr_file *file;
+	git_attr_rule *rule;
+	attr_get_many_info *info = NULL;
+	size_t num_found = 0;
+
+	memset((void *)values, 0, sizeof(const char *) * num_attr);
+
+	if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0)
+		return -1;
+
+	if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0)
+		goto cleanup;
+
+	info = git__calloc(num_attr, sizeof(attr_get_many_info));
+	GITERR_CHECK_ALLOC(info);
+
+	git_vector_foreach(&files, i, file) {
+
+		git_attr_file__foreach_matching_rule(file, &path, j, rule) {
+
+			for (k = 0; k < num_attr; k++) {
+				int pos;
+
+				if (info[k].found != NULL) /* already found assignment */
+					continue;
+
+				if (!info[k].name.name) {
+					info[k].name.name = names[k];
+					info[k].name.name_hash = git_attr_file__name_hash(names[k]);
+				}
+
+				pos = git_vector_bsearch(&rule->assigns, &info[k].name);
+				if (pos >= 0) {
+					info[k].found = (git_attr_assignment *)
+						git_vector_get(&rule->assigns, pos);
+					values[k] = info[k].found->value;
+
+					if (++num_found == num_attr)
+						goto cleanup;
+				}
+			}
+		}
+	}
+
+cleanup:
+	git_vector_free(&files);
+	git_attr_path__free(&path);
+	git__free(info);
+
+	return error;
+}
+
+
+int git_attr_foreach(
+    git_repository *repo,
+	uint32_t flags,
+	const char *pathname,
+	int (*callback)(const char *name, const char *value, void *payload),
+	void *payload)
+{
+	int error;
+	git_attr_path path;
+	git_vector files = GIT_VECTOR_INIT;
+	unsigned int i, j, k;
+	git_attr_file *file;
+	git_attr_rule *rule;
+	git_attr_assignment *assign;
+	git_strmap *seen = NULL;
+
+	if (git_attr_path__init(&path, pathname, git_repository_workdir(repo)) < 0)
+		return -1;
+
+	if ((error = collect_attr_files(repo, flags, pathname, &files)) < 0)
+		goto cleanup;
+
+	seen = git_strmap_alloc();
+	GITERR_CHECK_ALLOC(seen);
+
+	git_vector_foreach(&files, i, file) {
+
+		git_attr_file__foreach_matching_rule(file, &path, j, rule) {
+
+			git_vector_foreach(&rule->assigns, k, assign) {
+				/* skip if higher priority assignment was already seen */
+				if (git_strmap_exists(seen, assign->name))
+					continue;
+
+				git_strmap_insert(seen, assign->name, assign, error);
+				if (error >= 0)
+					error = callback(assign->name, assign->value, payload);
+
+				if (error != 0)
+					goto cleanup;
+			}
+		}
+	}
+
+cleanup:
+	git_strmap_free(seen);
+	git_vector_free(&files);
+	git_attr_path__free(&path);
+
+	return error;
+}
+
+
+int git_attr_add_macro(
+	git_repository *repo,
+	const char *name,
+	const char *values)
+{
+	int error;
+	git_attr_rule *macro = NULL;
+	git_pool *pool;
+
+	if (git_attr_cache__init(repo) < 0)
+		return -1;
+
+	macro = git__calloc(1, sizeof(git_attr_rule));
+	GITERR_CHECK_ALLOC(macro);
+
+	pool = &git_repository_attr_cache(repo)->pool;
+
+	macro->match.pattern = git_pool_strdup(pool, name);
+	GITERR_CHECK_ALLOC(macro->match.pattern);
+
+	macro->match.length = strlen(macro->match.pattern);
+	macro->match.flags = GIT_ATTR_FNMATCH_MACRO;
+
+	error = git_attr_assignment__parse(repo, pool, &macro->assigns, &values);
+
+	if (!error)
+		error = git_attr_cache__insert_macro(repo, macro);
+
+	if (error < 0)
+		git_attr_rule__free(macro);
+
+	return error;
+}
+
+bool git_attr_cache__is_cached(
+	git_repository *repo, git_attr_file_source source, const char *path)
+{
+	git_buf cache_key = GIT_BUF_INIT;
+	git_strmap *files = git_repository_attr_cache(repo)->files;
+	const char *workdir = git_repository_workdir(repo);
+	bool rval;
+
+	if (workdir && git__prefixcmp(path, workdir) == 0)
+		path += strlen(workdir);
+	if (git_buf_printf(&cache_key, "%d#%s", (int)source, path) < 0)
+		return false;
+
+	rval = git_strmap_exists(files, git_buf_cstr(&cache_key));
+
+	git_buf_free(&cache_key);
+
+	return rval;
+}
+
+static int load_attr_file(
+	const char **data,
+	git_attr_file_stat_sig *sig,
+	const char *filename)
+{
+	int error;
+	git_buf content = GIT_BUF_INIT;
+	struct stat st;
+
+	if (p_stat(filename, &st) < 0)
+		return GIT_ENOTFOUND;
+
+	if (sig != NULL &&
+		(git_time_t)st.st_mtime == sig->seconds &&
+		(git_off_t)st.st_size == sig->size &&
+		(unsigned int)st.st_ino == sig->ino)
+		return GIT_ENOTFOUND;
+
+	error = git_futils_readbuffer_updated(&content, filename, NULL, NULL);
+	if (error < 0)
+		return error;
+
+	if (sig != NULL) {
+		sig->seconds = (git_time_t)st.st_mtime;
+		sig->size    = (git_off_t)st.st_size;
+		sig->ino     = (unsigned int)st.st_ino;
+	}
+
+	*data = git_buf_detach(&content);
+
+	return 0;
+}
+
+static int load_attr_blob_from_index(
+	const char **content,
+	git_blob **blob,
+	git_repository *repo,
+	const git_oid *old_oid,
+	const char *relfile)
+{
+	int error;
+	git_index *index;
+	git_index_entry *entry;
+
+	if ((error = git_repository_index__weakptr(&index, repo)) < 0 ||
+		(error = git_index_find(index, relfile)) < 0)
+		return error;
+
+	entry = git_index_get(index, error);
+
+	if (old_oid && git_oid_cmp(old_oid, &entry->oid) == 0)
+		return GIT_ENOTFOUND;
+
+	if ((error = git_blob_lookup(blob, repo, &entry->oid)) < 0)
+		return error;
+
+	*content = git_blob_rawcontent(*blob);
+	return 0;
+}
+
+static int load_attr_from_cache(
+	git_attr_file **file,
+	git_attr_cache *cache,
+	git_attr_file_source source,
+	const char *relative_path)
+{
+	git_buf  cache_key = GIT_BUF_INIT;
+	khiter_t cache_pos;
+
+	*file = NULL;
+
+	if (!cache || !cache->files)
+		return 0;
+
+	if (git_buf_printf(&cache_key, "%d#%s", (int)source, relative_path) < 0)
+		return -1;
+
+	cache_pos = git_strmap_lookup_index(cache->files, cache_key.ptr);
+
+	git_buf_free(&cache_key);
+
+	if (git_strmap_valid_index(cache->files, cache_pos))
+		*file = git_strmap_value_at(cache->files, cache_pos);
+
+	return 0;
+}
+
+int git_attr_cache__internal_file(
+	git_repository *repo,
+	const char *filename,
+	git_attr_file **file)
+{
+	int error = 0;
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	khiter_t cache_pos = git_strmap_lookup_index(cache->files, filename);
+
+	if (git_strmap_valid_index(cache->files, cache_pos)) {
+		*file = git_strmap_value_at(cache->files, cache_pos);
+		return 0;
+	}
+
+	if (git_attr_file__new(file, 0, filename, &cache->pool) < 0)
+		return -1;
+
+	git_strmap_insert(cache->files, (*file)->key + 2, *file, error);
+	if (error > 0)
+		error = 0;
+
+	return error;
+}
+
+int git_attr_cache__push_file(
+	git_repository *repo,
+	const char *base,
+	const char *filename,
+	git_attr_file_source source,
+	git_attr_file_parser parse,
+	git_vector *stack)
+{
+	int error = 0;
+	git_buf path = GIT_BUF_INIT;
+	const char *workdir = git_repository_workdir(repo);
+	const char *relfile, *content = NULL;
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	git_attr_file *file = NULL;
+	git_blob *blob = NULL;
+	git_attr_file_stat_sig st;
+
+	assert(filename && stack);
+
+	/* join base and path as needed */
+	if (base != NULL && git_path_root(filename) < 0) {
+		if (git_buf_joinpath(&path, base, filename) < 0)
+			return -1;
+		filename = path.ptr;
+	}
+
+	relfile = filename;
+	if (workdir && git__prefixcmp(relfile, workdir) == 0)
+		relfile += strlen(workdir);
+
+	/* check cache */
+	if (load_attr_from_cache(&file, cache, source, relfile) < 0)
+		return -1;
+
+	/* if not in cache, load data, parse, and cache */
+
+	if (source == GIT_ATTR_FILE_FROM_FILE) {
+		if (file)
+			memcpy(&st, &file->cache_data.st, sizeof(st));
+		else
+			memset(&st, 0, sizeof(st));
+
+		error = load_attr_file(&content, &st, filename);
+	} else {
+		error = load_attr_blob_from_index(&content, &blob,
+			repo, file ? &file->cache_data.oid : NULL, relfile);
+	}
+
+	if (error) {
+		/* not finding a file is not an error for this function */
+		if (error == GIT_ENOTFOUND) {
+			giterr_clear();
+			error = 0;
+		}
+		goto finish;
+	}
+
+	/* if we got here, we have to parse and/or reparse the file */
+	if (file)
+		git_attr_file__clear_rules(file);
+	else {
+		error = git_attr_file__new(&file, source, relfile, &cache->pool);
+		if (error < 0)
+			goto finish;
+	}
+
+	if (parse && (error = parse(repo, content, file)) < 0)
+		goto finish;
+
+	git_strmap_insert(cache->files, file->key, file, error);
+	if (error > 0)
+		error = 0;
+
+	/* remember "cache buster" file signature */
+	if (blob)
+		git_oid_cpy(&file->cache_data.oid, git_object_id((git_object *)blob));
+	else
+		memcpy(&file->cache_data.st, &st, sizeof(st));
+
+finish:
+	/* push file onto vector if we found one*/
+	if (!error && file != NULL)
+		error = git_vector_insert(stack, file);
+
+	if (error != 0)
+		git_attr_file__free(file);
+
+	if (blob)
+		git_blob_free(blob);
+	else
+		git__free((void *)content);
+
+	git_buf_free(&path);
+
+	return error;
+}
+
+#define push_attr_file(R,S,B,F) \
+	git_attr_cache__push_file((R),(B),(F),GIT_ATTR_FILE_FROM_FILE,git_attr_file__parse_buffer,(S))
+
+typedef struct {
+	git_repository *repo;
+	uint32_t flags;
+	const char *workdir;
+	git_index *index;
+	git_vector *files;
+} attr_walk_up_info;
+
+int git_attr_cache__decide_sources(
+	uint32_t flags, bool has_wd, bool has_index, git_attr_file_source *srcs)
+{
+	int count = 0;
+
+	switch (flags & 0x03) {
+	case GIT_ATTR_CHECK_FILE_THEN_INDEX:
+		if (has_wd)
+			srcs[count++] = GIT_ATTR_FILE_FROM_FILE;
+		if (has_index)
+			srcs[count++] = GIT_ATTR_FILE_FROM_INDEX;
+		break;
+	case GIT_ATTR_CHECK_INDEX_THEN_FILE:
+		if (has_index)
+			srcs[count++] = GIT_ATTR_FILE_FROM_INDEX;
+		if (has_wd)
+			srcs[count++] = GIT_ATTR_FILE_FROM_FILE;
+		break;
+	case GIT_ATTR_CHECK_INDEX_ONLY:
+		if (has_index)
+			srcs[count++] = GIT_ATTR_FILE_FROM_INDEX;
+		break;
+	}
+
+	return count;
+}
+
+static int push_one_attr(void *ref, git_buf *path)
+{
+	int error = 0, n_src, i;
+	attr_walk_up_info *info = (attr_walk_up_info *)ref;
+	git_attr_file_source src[2];
+
+	n_src = git_attr_cache__decide_sources(
+		info->flags, info->workdir != NULL, info->index != NULL, src);
+
+	for (i = 0; !error && i < n_src; ++i)
+		error = git_attr_cache__push_file(
+			info->repo, path->ptr, GIT_ATTR_FILE, src[i],
+			git_attr_file__parse_buffer, info->files);
+
+	return error;
+}
+
+static int collect_attr_files(
+	git_repository *repo,
+	uint32_t flags,
+	const char *path,
+	git_vector *files)
+{
+	int error;
+	git_buf dir = GIT_BUF_INIT;
+	const char *workdir = git_repository_workdir(repo);
+	attr_walk_up_info info;
+
+	if (git_attr_cache__init(repo) < 0 ||
+		git_vector_init(files, 4, NULL) < 0)
+		return -1;
+
+	/* Resolve path in a non-bare repo */
+	if (workdir != NULL)
+		error = git_path_find_dir(&dir, path, workdir);
+	else
+		error = git_path_dirname_r(&dir, path);
+	if (error < 0)
+		goto cleanup;
+
+	/* in precendence order highest to lowest:
+	 * - $GIT_DIR/info/attributes
+	 * - path components with .gitattributes
+	 * - config core.attributesfile
+	 * - $GIT_PREFIX/etc/gitattributes
+	 */
+
+	error = push_attr_file(
+		repo, files, git_repository_path(repo), GIT_ATTR_FILE_INREPO);
+	if (error < 0)
+		goto cleanup;
+
+	info.repo  = repo;
+	info.flags = flags;
+	info.workdir = workdir;
+	if (git_repository_index__weakptr(&info.index, repo) < 0)
+		giterr_clear(); /* no error even if there is no index */
+	info.files = files;
+
+	error = git_path_walk_up(&dir, workdir, push_one_attr, &info);
+	if (error < 0)
+		goto cleanup;
+
+	if (git_repository_attr_cache(repo)->cfg_attr_file != NULL) {
+		error = push_attr_file(
+			repo, files, NULL, git_repository_attr_cache(repo)->cfg_attr_file);
+		if (error < 0)
+			goto cleanup;
+	}
+
+	if ((flags & GIT_ATTR_CHECK_NO_SYSTEM) == 0) {
+		error = git_futils_find_system_file(&dir, GIT_ATTR_FILE_SYSTEM);
+		if (!error)
+			error = push_attr_file(repo, files, NULL, dir.ptr);
+		else if (error == GIT_ENOTFOUND)
+			error = 0;
+	}
+
+ cleanup:
+	if (error < 0)
+		git_vector_free(files);
+	git_buf_free(&dir);
+
+	return error;
+}
+
+
+int git_attr_cache__init(git_repository *repo)
+{
+	int ret;
+	git_attr_cache *cache = git_repository_attr_cache(repo);
+	git_config *cfg;
+
+	if (cache->initialized)
+		return 0;
+
+	/* cache config settings for attributes and ignores */
+	if (git_repository_config__weakptr(&cfg, repo) < 0)
+		return -1;
+
+	ret = git_config_get_string(&cache->cfg_attr_file, cfg, GIT_ATTR_CONFIG);
+	if (ret < 0 && ret != GIT_ENOTFOUND)
+		return ret;
+
+	ret = git_config_get_string(&cache->cfg_excl_file, cfg, GIT_IGNORE_CONFIG);
+	if (ret < 0 && ret != GIT_ENOTFOUND)
+		return ret;
+
+	giterr_clear();
+
+	/* allocate hashtable for attribute and ignore file contents */
+	if (cache->files == NULL) {
+		cache->files = git_strmap_alloc();
+		GITERR_CHECK_ALLOC(cache->files);
+	}
+
+	/* allocate hashtable for attribute macros */
+	if (cache->macros == NULL) {
+		cache->macros = git_strmap_alloc();
+		GITERR_CHECK_ALLOC(cache->macros);
+	}
+
+	/* allocate string pool */
+	if (git_pool_init(&cache->pool, 1, 0) < 0)
+		return -1;
+
+	cache->initialized = 1;
+
+	/* insert default macros */
+	return git_attr_add_macro(repo, "binary", "-diff -crlf -text");
+}
+
+void git_attr_cache_flush(
+	git_repository *repo)
+{
+	git_attr_cache *cache;
+
+	if (!repo)
+		return;
+
+	cache = git_repository_attr_cache(repo);
+
+	if (cache->files != NULL) {
+		git_attr_file *file;
+
+		git_strmap_foreach_value(cache->files, file, {
+			git_attr_file__free(file);
+		});
+
+		git_strmap_free(cache->files);
+	}
+
+	if (cache->macros != NULL) {
+		git_attr_rule *rule;
+
+		git_strmap_foreach_value(cache->macros, rule, {
+			git_attr_rule__free(rule);
+		});
+
+		git_strmap_free(cache->macros);
+	}
+
+	git_pool_clear(&cache->pool);
+
+	cache->initialized = 0;
+}
+
+int git_attr_cache__insert_macro(git_repository *repo, git_attr_rule *macro)
+{
+	git_strmap *macros = git_repository_attr_cache(repo)->macros;
+	int error;
+
+	/* TODO: generate warning log if (macro->assigns.length == 0) */
+	if (macro->assigns.length == 0)
+		return 0;
+
+	git_strmap_insert(macros, macro->match.pattern, macro, error);
+	return (error < 0) ? -1 : 0;
+}
+
+git_attr_rule *git_attr_cache__lookup_macro(
+	git_repository *repo, const char *name)
+{
+	git_strmap *macros = git_repository_attr_cache(repo)->macros;
+	khiter_t pos;
+
+	pos = git_strmap_lookup_index(macros, name);
+
+	if (!git_strmap_valid_index(macros, pos))
+		return NULL;
+
+	return (git_attr_rule *)git_strmap_value_at(macros, pos);
+}
+
diff --git a/src/attr.h b/src/attr.h
new file mode 100644
index 000000000..a35b1160f
--- /dev/null
+++ b/src/attr.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_attr_h__
+#define INCLUDE_attr_h__
+
+#include "attr_file.h"
+#include "strmap.h"
+
+#define GIT_ATTR_CONFIG   "core.attributesfile"
+#define GIT_IGNORE_CONFIG "core.excludesfile"
+
+typedef struct {
+	int initialized;
+	git_pool pool;
+	git_strmap *files;	/* hash path to git_attr_file of rules */
+	git_strmap *macros;	/* hash name to vector<git_attr_assignment> */
+	const char *cfg_attr_file; /* cached value of core.attributesfile */
+	const char *cfg_excl_file; /* cached value of core.excludesfile */
+} git_attr_cache;
+
+typedef int (*git_attr_file_parser)(
+	git_repository *, const char *, git_attr_file *);
+
+extern int git_attr_cache__init(git_repository *repo);
+
+extern int git_attr_cache__insert_macro(
+	git_repository *repo, git_attr_rule *macro);
+
+extern git_attr_rule *git_attr_cache__lookup_macro(
+	git_repository *repo, const char *name);
+
+extern int git_attr_cache__push_file(
+	git_repository *repo,
+	const char *base,
+	const char *filename,
+	git_attr_file_source source,
+	git_attr_file_parser parse,
+	git_vector *stack);
+
+extern int git_attr_cache__internal_file(
+	git_repository *repo,
+	const char *key,
+	git_attr_file **file_ptr);
+
+/* returns true if path is in cache */
+extern bool git_attr_cache__is_cached(
+	git_repository *repo, git_attr_file_source source, const char *path);
+
+extern int git_attr_cache__decide_sources(
+	uint32_t flags, bool has_wd, bool has_index, git_attr_file_source *srcs);
+
+#endif
diff --git a/src/attr_file.c b/src/attr_file.c
new file mode 100644
index 000000000..ca2f8fb58
--- /dev/null
+++ b/src/attr_file.c
@@ -0,0 +1,609 @@
+#include "common.h"
+#include "repository.h"
+#include "filebuf.h"
+#include "git2/blob.h"
+#include "git2/tree.h"
+#include <ctype.h>
+
+const char *git_attr__true  = "[internal]__TRUE__";
+const char *git_attr__false = "[internal]__FALSE__";
+const char *git_attr__unset = "[internal]__UNSET__";
+
+static int sort_by_hash_and_name(const void *a_raw, const void *b_raw);
+static void git_attr_rule__clear(git_attr_rule *rule);
+
+int git_attr_file__new(
+	git_attr_file **attrs_ptr,
+	git_attr_file_source from,
+	const char *path,
+	git_pool *pool)
+{
+	git_attr_file *attrs = NULL;
+
+	attrs = git__calloc(1, sizeof(git_attr_file));
+	GITERR_CHECK_ALLOC(attrs);
+
+	if (pool)
+		attrs->pool = pool;
+	else {
+		attrs->pool = git__calloc(1, sizeof(git_pool));
+		if (!attrs->pool || git_pool_init(attrs->pool, 1, 0) < 0)
+			goto fail;
+		attrs->pool_is_allocated = true;
+	}
+
+	if (path) {
+		size_t len = strlen(path);
+
+		attrs->key = git_pool_malloc(attrs->pool, (uint32_t)len + 3);
+		GITERR_CHECK_ALLOC(attrs->key);
+
+		attrs->key[0] = '0' + from;
+		attrs->key[1] = '#';
+		memcpy(&attrs->key[2], path, len);
+		attrs->key[len + 2] = '\0';
+	}
+
+	if (git_vector_init(&attrs->rules, 4, NULL) < 0)
+		goto fail;
+
+	*attrs_ptr = attrs;
+	return 0;
+
+fail:
+	git_attr_file__free(attrs);
+	attrs_ptr = NULL;
+	return -1;
+}
+
+int git_attr_file__parse_buffer(
+	git_repository *repo, const char *buffer, git_attr_file *attrs)
+{
+	int error = 0;
+	const char *scan = NULL;
+	char *context = NULL;
+	git_attr_rule *rule = NULL;
+
+	assert(buffer && attrs);
+
+	scan = buffer;
+
+	/* if subdir file path, convert context for file paths */
+	if (attrs->key && git__suffixcmp(attrs->key, "/" GIT_ATTR_FILE) == 0) {
+		context = attrs->key + 2;
+		context[strlen(context) - strlen(GIT_ATTR_FILE)] = '\0';
+	}
+
+	while (!error && *scan) {
+		/* allocate rule if needed */
+		if (!rule && !(rule = git__calloc(1, sizeof(git_attr_rule)))) {
+			error = -1;
+			break;
+		}
+
+		/* parse the next "pattern attr attr attr" line */
+		if (!(error = git_attr_fnmatch__parse(
+				&rule->match, attrs->pool, context, &scan)) &&
+			!(error = git_attr_assignment__parse(
+				repo, attrs->pool, &rule->assigns, &scan)))
+		{
+			if (rule->match.flags & GIT_ATTR_FNMATCH_MACRO)
+				/* should generate error/warning if this is coming from any
+				 * file other than .gitattributes at repo root.
+				 */
+				error = git_attr_cache__insert_macro(repo, rule);
+			else
+				error = git_vector_insert(&attrs->rules, rule);
+		}
+
+		/* if the rule wasn't a pattern, on to the next */
+		if (error < 0) {
+			git_attr_rule__clear(rule); /* reset rule contents */
+			if (error == GIT_ENOTFOUND)
+				error = 0;
+		} else {
+			rule = NULL; /* vector now "owns" the rule */
+		}
+	}
+
+	git_attr_rule__free(rule);
+
+	/* restore file path used for context */
+	if (context)
+		context[strlen(context)] = '.'; /* first char of GIT_ATTR_FILE */
+
+	return error;
+}
+
+int git_attr_file__new_and_load(
+	git_attr_file **attrs_ptr,
+	const char *path)
+{
+	int error;
+	git_buf content = GIT_BUF_INIT;
+
+	if ((error = git_attr_file__new(attrs_ptr, 0, path, NULL)) < 0)
+		return error;
+
+	if (!(error = git_futils_readbuffer(&content, path)))
+		error = git_attr_file__parse_buffer(
+			NULL, git_buf_cstr(&content), *attrs_ptr);
+
+	git_buf_free(&content);
+
+	if (error) {
+		git_attr_file__free(*attrs_ptr);
+		*attrs_ptr = NULL;
+	}
+
+	return error;
+}
+
+void git_attr_file__clear_rules(git_attr_file *file)
+{
+	unsigned int i;
+	git_attr_rule *rule;
+
+	git_vector_foreach(&file->rules, i, rule)
+		git_attr_rule__free(rule);
+
+	git_vector_free(&file->rules);
+}
+
+void git_attr_file__free(git_attr_file *file)
+{
+	if (!file)
+		return;
+
+	git_attr_file__clear_rules(file);
+
+	if (file->pool_is_allocated) {
+		git_pool_clear(file->pool);
+		git__free(file->pool);
+	}
+	file->pool = NULL;
+
+	git__free(file);
+}
+
+uint32_t git_attr_file__name_hash(const char *name)
+{
+	uint32_t h = 5381;
+	int c;
+	assert(name);
+	while ((c = (int)*name++) != 0)
+		h = ((h << 5) + h) + c;
+	return h;
+}
+
+
+int git_attr_file__lookup_one(
+	git_attr_file *file,
+	const git_attr_path *path,
+	const char *attr,
+	const char **value)
+{
+	unsigned int i;
+	git_attr_name name;
+	git_attr_rule *rule;
+
+	*value = NULL;
+
+	name.name = attr;
+	name.name_hash = git_attr_file__name_hash(attr);
+
+	git_attr_file__foreach_matching_rule(file, path, i, rule) {
+		int pos = git_vector_bsearch(&rule->assigns, &name);
+
+		if (pos >= 0) {
+			*value = ((git_attr_assignment *)
+					  git_vector_get(&rule->assigns, pos))->value;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+
+bool git_attr_fnmatch__match(
+	git_attr_fnmatch *match,
+	const git_attr_path *path)
+{
+	int fnm;
+
+	if (match->flags & GIT_ATTR_FNMATCH_DIRECTORY && !path->is_dir)
+		return false;
+
+	if (match->flags & GIT_ATTR_FNMATCH_FULLPATH)
+		fnm = p_fnmatch(match->pattern, path->path, FNM_PATHNAME);
+	else if (path->is_dir)
+		fnm = p_fnmatch(match->pattern, path->basename, FNM_LEADING_DIR);
+	else
+		fnm = p_fnmatch(match->pattern, path->basename, 0);
+
+	return (fnm == FNM_NOMATCH) ? false : true;
+}
+
+bool git_attr_rule__match(
+	git_attr_rule *rule,
+	const git_attr_path *path)
+{
+	bool matched = git_attr_fnmatch__match(&rule->match, path);
+
+	if (rule->match.flags & GIT_ATTR_FNMATCH_NEGATIVE)
+		matched = !matched;
+
+	return matched;
+}
+
+
+git_attr_assignment *git_attr_rule__lookup_assignment(
+	git_attr_rule *rule, const char *name)
+{
+	int pos;
+	git_attr_name key;
+	key.name = name;
+	key.name_hash = git_attr_file__name_hash(name);
+
+	pos = git_vector_bsearch(&rule->assigns, &key);
+
+	return (pos >= 0) ? git_vector_get(&rule->assigns, pos) : NULL;
+}
+
+int git_attr_path__init(
+	git_attr_path *info, const char *path, const char *base)
+{
+	/* build full path as best we can */
+	git_buf_init(&info->full, 0);
+
+	if (base != NULL && git_path_root(path) < 0) {
+		if (git_buf_joinpath(&info->full, base, path) < 0)
+			return -1;
+		info->path = info->full.ptr + strlen(base);
+	} else {
+		if (git_buf_sets(&info->full, path) < 0)
+			return -1;
+		info->path = info->full.ptr;
+	}
+
+	/* remove trailing slashes */
+	while (info->full.size > 0) {
+		if (info->full.ptr[info->full.size - 1] != '/')
+			break;
+		info->full.size--;
+	}
+	info->full.ptr[info->full.size] = '\0';
+
+	/* skip leading slashes in path */
+	while (*info->path == '/')
+		info->path++;
+
+	/* find trailing basename component */
+	info->basename = strrchr(info->path, '/');
+	if (info->basename)
+		info->basename++;
+	if (!info->basename || !*info->basename)
+		info->basename = info->path;
+
+	info->is_dir = (int)git_path_isdir(info->full.ptr);
+
+	return 0;
+}
+
+void git_attr_path__free(git_attr_path *info)
+{
+	git_buf_free(&info->full);
+	info->path = NULL;
+	info->basename = NULL;
+}
+
+
+/*
+ * From gitattributes(5):
+ *
+ * Patterns have the following format:
+ *
+ * - A blank line matches no files, so it can serve as a separator for
+ *   readability.
+ *
+ * - A line starting with # serves as a comment.
+ *
+ * - An optional prefix ! which negates the pattern; any matching file
+ *   excluded by a previous pattern will become included again. If a negated
+ *   pattern matches, this will override lower precedence patterns sources.
+ *
+ * - If the pattern ends with a slash, it is removed for the purpose of the
+ *   following description, but it would only find a match with a directory. In
+ *   other words, foo/ will match a directory foo and paths underneath it, but
+ *   will not match a regular file or a symbolic link foo (this is consistent
+ *   with the way how pathspec works in general in git).
+ *
+ * - If the pattern does not contain a slash /, git treats it as a shell glob
+ *   pattern and checks for a match against the pathname without leading
+ *   directories.
+ *
+ * - Otherwise, git treats the pattern as a shell glob suitable for consumption
+ *   by fnmatch(3) with the FNM_PATHNAME flag: wildcards in the pattern will
+ *   not match a / in the pathname. For example, "Documentation/\*.html" matches
+ *   "Documentation/git.html" but not "Documentation/ppc/ppc.html". A leading
+ *   slash matches the beginning of the pathname; for example, "/\*.c" matches
+ *   "cat-file.c" but not "mozilla-sha1/sha1.c".
+ */
+
+/*
+ * This will return 0 if the spec was filled out,
+ * GIT_ENOTFOUND if the fnmatch does not require matching, or
+ * another error code there was an actual problem.
+ */
+int git_attr_fnmatch__parse(
+	git_attr_fnmatch *spec,
+	git_pool *pool,
+	const char *source,
+	const char **base)
+{
+	const char *pattern, *scan;
+	int slash_count, allow_space;
+
+	assert(spec && base && *base);
+
+	spec->flags = (spec->flags & GIT_ATTR_FNMATCH_ALLOWSPACE);
+	allow_space = (spec->flags != 0);
+
+	pattern = *base;
+
+	while (git__isspace(*pattern)) pattern++;
+	if (!*pattern || *pattern == '#') {
+		*base = git__next_line(pattern);
+		return GIT_ENOTFOUND;
+	}
+
+	if (*pattern == '[') {
+		if (strncmp(pattern, "[attr]", 6) == 0) {
+			spec->flags = spec->flags | GIT_ATTR_FNMATCH_MACRO;
+			pattern += 6;
+		}
+		/* else a character range like [a-e]* which is accepted */
+	}
+
+	if (*pattern == '!') {
+		spec->flags = spec->flags | GIT_ATTR_FNMATCH_NEGATIVE;
+		pattern++;
+	}
+
+	slash_count = 0;
+	for (scan = pattern; *scan != '\0'; ++scan) {
+		/* scan until (non-escaped) white space */
+		if (git__isspace(*scan) && *(scan - 1) != '\\') {
+			if (!allow_space || (*scan != ' ' && *scan != '\t'))
+				break;
+		}
+
+		if (*scan == '/') {
+			spec->flags = spec->flags | GIT_ATTR_FNMATCH_FULLPATH;
+			slash_count++;
+			if (pattern == scan)
+				pattern++;
+		}
+		/* remember if we see an unescaped wildcard in pattern */
+		else if (git__iswildcard(*scan) &&
+			(scan == pattern || (*(scan - 1) != '\\')))
+			spec->flags = spec->flags | GIT_ATTR_FNMATCH_HASWILD;
+	}
+
+	*base = scan;
+
+	spec->length = scan - pattern;
+
+	if (pattern[spec->length - 1] == '/') {
+		spec->length--;
+		spec->flags = spec->flags | GIT_ATTR_FNMATCH_DIRECTORY;
+		if (--slash_count <= 0)
+			spec->flags = spec->flags & ~GIT_ATTR_FNMATCH_FULLPATH;
+	}
+
+	if ((spec->flags & GIT_ATTR_FNMATCH_FULLPATH) != 0 &&
+		source != NULL && git_path_root(pattern) < 0)
+	{
+		size_t sourcelen = strlen(source);
+		/* given an unrooted fullpath match from a file inside a repo,
+		 * prefix the pattern with the relative directory of the source file
+		 */
+		spec->pattern = git_pool_malloc(
+			pool, (uint32_t)(sourcelen + spec->length + 1));
+		if (spec->pattern) {
+			memcpy(spec->pattern, source, sourcelen);
+			memcpy(spec->pattern + sourcelen, pattern, spec->length);
+			spec->length += sourcelen;
+			spec->pattern[spec->length] = '\0';
+		}
+	} else {
+		spec->pattern = git_pool_strndup(pool, pattern, spec->length);
+	}
+
+	if (!spec->pattern) {
+		*base = git__next_line(pattern);
+		return -1;
+	} else {
+		/* strip '\' that might have be used for internal whitespace */
+		char *to = spec->pattern;
+		for (scan = spec->pattern; *scan; to++, scan++) {
+			if (*scan == '\\')
+				scan++; /* skip '\' but include next char */
+			if (to != scan)
+				*to = *scan;
+		}
+		if (to != scan) {
+			*to = '\0';
+			spec->length = (to - spec->pattern);
+		}
+	}
+
+	return 0;
+}
+
+static int sort_by_hash_and_name(const void *a_raw, const void *b_raw)
+{
+	const git_attr_name *a = a_raw;
+	const git_attr_name *b = b_raw;
+
+	if (b->name_hash < a->name_hash)
+		return 1;
+	else if (b->name_hash > a->name_hash)
+		return -1;
+	else
+		return strcmp(b->name, a->name);
+}
+
+static void git_attr_assignment__free(git_attr_assignment *assign)
+{
+	/* name and value are stored in a git_pool associated with the
+	 * git_attr_file, so they do not need to be freed here
+	 */
+	assign->name = NULL;
+	assign->value = NULL;
+	git__free(assign);
+}
+
+static int merge_assignments(void **old_raw, void *new_raw)
+{
+	git_attr_assignment **old = (git_attr_assignment **)old_raw;
+	git_attr_assignment *new = (git_attr_assignment *)new_raw;
+
+	GIT_REFCOUNT_DEC(*old, git_attr_assignment__free);
+	*old = new;
+	return GIT_EEXISTS;
+}
+
+int git_attr_assignment__parse(
+	git_repository *repo,
+	git_pool *pool,
+	git_vector *assigns,
+	const char **base)
+{
+	int error;
+	const char *scan = *base;
+	git_attr_assignment *assign = NULL;
+
+	assert(assigns && !assigns->length);
+
+	assigns->_cmp = sort_by_hash_and_name;
+
+	while (*scan && *scan != '\n') {
+		const char *name_start, *value_start;
+
+		/* skip leading blanks */
+		while (git__isspace(*scan) && *scan != '\n') scan++;
+
+		/* allocate assign if needed */
+		if (!assign) {
+			assign = git__calloc(1, sizeof(git_attr_assignment));
+			GITERR_CHECK_ALLOC(assign);
+			GIT_REFCOUNT_INC(assign);
+		}
+
+		assign->name_hash = 5381;
+		assign->value = git_attr__true;
+
+		/* look for magic name prefixes */
+		if (*scan == '-') {
+			assign->value = git_attr__false;
+			scan++;
+		} else if (*scan == '!') {
+			assign->value = git_attr__unset; /* explicit unspecified state */
+			scan++;
+		} else if (*scan == '#') /* comment rest of line */
+			break;
+
+		/* find the name */
+		name_start = scan;
+		while (*scan && !git__isspace(*scan) && *scan != '=') {
+			assign->name_hash =
+				((assign->name_hash << 5) + assign->name_hash) + *scan;
+			scan++;
+		}
+		if (scan == name_start) {
+			/* must have found lone prefix (" - ") or leading = ("=foo")
+			 * or end of buffer -- advance until whitespace and continue
+			 */
+			while (*scan && !git__isspace(*scan)) scan++;
+			continue;
+		}
+
+		/* allocate permanent storage for name */
+		assign->name = git_pool_strndup(pool, name_start, scan - name_start);
+		GITERR_CHECK_ALLOC(assign->name);
+
+		/* if there is an equals sign, find the value */
+		if (*scan == '=') {
+			for (value_start = ++scan; *scan && !git__isspace(*scan); ++scan);
+
+			/* if we found a value, allocate permanent storage for it */
+			if (scan > value_start) {
+				assign->value = git_pool_strndup(pool, value_start, scan - value_start);
+				GITERR_CHECK_ALLOC(assign->value);
+			}
+		}
+
+		/* expand macros (if given a repo with a macro cache) */
+		if (repo != NULL && assign->value == git_attr__true) {
+			git_attr_rule *macro =
+				git_attr_cache__lookup_macro(repo, assign->name);
+
+			if (macro != NULL) {
+				unsigned int i;
+				git_attr_assignment *massign;
+
+				git_vector_foreach(&macro->assigns, i, massign) {
+					GIT_REFCOUNT_INC(massign);
+
+					error = git_vector_insert_sorted(
+						assigns, massign, &merge_assignments);
+					if (error < 0 && error != GIT_EEXISTS)
+						return error;
+				}
+			}
+		}
+
+		/* insert allocated assign into vector */
+		error = git_vector_insert_sorted(assigns, assign, &merge_assignments);
+		if (error < 0 && error != GIT_EEXISTS)
+			return error;
+
+		/* clear assign since it is now "owned" by the vector */
+		assign = NULL;
+	}
+
+	if (assign != NULL)
+		git_attr_assignment__free(assign);
+
+	*base = git__next_line(scan);
+
+	return (assigns->length == 0) ? GIT_ENOTFOUND : 0;
+}
+
+static void git_attr_rule__clear(git_attr_rule *rule)
+{
+	unsigned int i;
+	git_attr_assignment *assign;
+
+	if (!rule)
+		return;
+
+	if (!(rule->match.flags & GIT_ATTR_FNMATCH_IGNORE)) {
+		git_vector_foreach(&rule->assigns, i, assign)
+			GIT_REFCOUNT_DEC(assign, git_attr_assignment__free);
+		git_vector_free(&rule->assigns);
+	}
+
+	/* match.pattern is stored in a git_pool, so no need to free */
+	rule->match.pattern = NULL;
+	rule->match.length = 0;
+}
+
+void git_attr_rule__free(git_attr_rule *rule)
+{
+	git_attr_rule__clear(rule);
+	git__free(rule);
+}
+
diff --git a/src/attr_file.h b/src/attr_file.h
new file mode 100644
index 000000000..7939f838a
--- /dev/null
+++ b/src/attr_file.h
@@ -0,0 +1,145 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_attr_file_h__
+#define INCLUDE_attr_file_h__
+
+#include "git2/attr.h"
+#include "vector.h"
+#include "pool.h"
+#include "buffer.h"
+
+#define GIT_ATTR_FILE			".gitattributes"
+#define GIT_ATTR_FILE_INREPO	"info/attributes"
+#define GIT_ATTR_FILE_SYSTEM	"gitattributes"
+
+#define GIT_ATTR_FNMATCH_NEGATIVE	(1U << 0)
+#define GIT_ATTR_FNMATCH_DIRECTORY	(1U << 1)
+#define GIT_ATTR_FNMATCH_FULLPATH	(1U << 2)
+#define GIT_ATTR_FNMATCH_MACRO		(1U << 3)
+#define GIT_ATTR_FNMATCH_IGNORE		(1U << 4)
+#define GIT_ATTR_FNMATCH_HASWILD	(1U << 5)
+#define GIT_ATTR_FNMATCH_ALLOWSPACE	(1U << 6)
+
+typedef struct {
+	char *pattern;
+	size_t length;
+	unsigned int flags;
+} git_attr_fnmatch;
+
+typedef struct {
+	git_attr_fnmatch match;
+	git_vector assigns;		/* vector of <git_attr_assignment*> */
+} git_attr_rule;
+
+typedef struct {
+	git_refcount unused;
+	const char *name;
+    uint32_t name_hash;
+} git_attr_name;
+
+typedef struct {
+	git_refcount rc;		/* for macros */
+	char *name;
+    uint32_t name_hash;
+    const char *value;
+} git_attr_assignment;
+
+typedef struct {
+	git_time_t seconds;
+	git_off_t  size;
+	unsigned int ino;
+} git_attr_file_stat_sig;
+
+typedef struct {
+	char *key;				/* cache "source#path" this was loaded from */
+	git_vector rules;		/* vector of <rule*> or <fnmatch*> */
+	git_pool *pool;
+	bool pool_is_allocated;
+	union {
+		git_oid oid;
+		git_attr_file_stat_sig st;
+	} cache_data;
+} git_attr_file;
+
+typedef struct {
+	git_buf     full;
+	const char *path;
+	const char *basename;
+	int         is_dir;
+} git_attr_path;
+
+typedef enum {
+	GIT_ATTR_FILE_FROM_FILE = 0,
+	GIT_ATTR_FILE_FROM_INDEX = 1
+} git_attr_file_source;
+
+/*
+ * git_attr_file API
+ */
+
+extern int git_attr_file__new(
+	git_attr_file **attrs_ptr, git_attr_file_source src, const char *path, git_pool *pool);
+
+extern int git_attr_file__new_and_load(
+	git_attr_file **attrs_ptr, const char *path);
+
+extern void git_attr_file__free(git_attr_file *file);
+
+extern void git_attr_file__clear_rules(git_attr_file *file);
+
+extern int git_attr_file__parse_buffer(
+	git_repository *repo, const char *buf, git_attr_file *file);
+
+extern int git_attr_file__lookup_one(
+	git_attr_file *file,
+	const git_attr_path *path,
+	const char *attr,
+	const char **value);
+
+/* loop over rules in file from bottom to top */
+#define git_attr_file__foreach_matching_rule(file, path, iter, rule)	\
+	git_vector_rforeach(&(file)->rules, (iter), (rule)) \
+		if (git_attr_rule__match((rule), (path)))
+
+extern uint32_t git_attr_file__name_hash(const char *name);
+
+
+/*
+ * other utilities
+ */
+
+extern int git_attr_fnmatch__parse(
+	git_attr_fnmatch *spec,
+	git_pool *pool,
+	const char *source,
+	const char **base);
+
+extern bool git_attr_fnmatch__match(
+	git_attr_fnmatch *rule,
+	const git_attr_path *path);
+
+extern void git_attr_rule__free(git_attr_rule *rule);
+
+extern bool git_attr_rule__match(
+	git_attr_rule *rule,
+	const git_attr_path *path);
+
+extern git_attr_assignment *git_attr_rule__lookup_assignment(
+	git_attr_rule *rule, const char *name);
+
+extern int git_attr_path__init(
+	git_attr_path *info, const char *path, const char *base);
+
+extern void git_attr_path__free(git_attr_path *info);
+
+extern int git_attr_assignment__parse(
+	git_repository *repo, /* needed to expand macros */
+	git_pool *pool,
+	git_vector *assigns,
+	const char **scan);
+
+#endif
diff --git a/src/backends/hiredis.c b/src/backends/hiredis.c
deleted file mode 100644
index 111abf7d3..000000000
--- a/src/backends/hiredis.c
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
- *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- */
-
-#include "common.h"
-#include "git2/object.h"
-#include "hash.h"
-#include "odb.h"
-
-#include "git2/odb_backend.h"
-
-#ifdef GIT2_HIREDIS_BACKEND
-
-#include <hiredis/hiredis.h>
-
-typedef struct {
-    git_odb_backend parent;
-
-    redisContext *db;
-} hiredis_backend;
-
-int hiredis_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *_backend, const git_oid *oid) {
-    hiredis_backend *backend;
-    int error;
-    redisReply *reply;
-
-    assert(len_p && type_p && _backend && oid);
-
-    backend = (hiredis_backend *) _backend;
-    error = GIT_ERROR;
-
-    reply = redisCommand(backend->db, "HMGET %b %s %s", oid->id, GIT_OID_RAWSZ,
-            "type", "size");
-
-    if (reply && reply->type == REDIS_REPLY_ARRAY) {
-        if (reply->element[0]->type != REDIS_REPLY_NIL &&
-                reply->element[0]->type != REDIS_REPLY_NIL) {
-            *type_p = (git_otype) atoi(reply->element[0]->str);
-            *len_p = (size_t) atoi(reply->element[1]->str);
-            error = GIT_SUCCESS;
-        } else {
-            error = GIT_ENOTFOUND;
-        }
-    } else {
-        error = GIT_ERROR;
-    }
-
-    freeReplyObject(reply);
-    return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to read header");
-}
-
-int hiredis_backend__read(void **data_p, size_t *len_p, git_otype *type_p, git_odb_backend *_backend, const git_oid *oid) {
-    hiredis_backend *backend;
-    int error;
-    redisReply *reply;
-
-    assert(data_p && len_p && type_p && _backend && oid);
-
-    backend = (hiredis_backend *) _backend;
-    error = GIT_ERROR;
-
-    reply = redisCommand(backend->db, "HMGET %b %s %s %s", oid->id, GIT_OID_RAWSZ,
-            "type", "size", "data");
-
-    if (reply && reply->type == REDIS_REPLY_ARRAY) {
-        if (reply->element[0]->type != REDIS_REPLY_NIL &&
-                reply->element[1]->type != REDIS_REPLY_NIL &&
-                reply->element[2]->type != REDIS_REPLY_NIL) {
-            *type_p = (git_otype) atoi(reply->element[0]->str);
-            *len_p = (size_t) atoi(reply->element[1]->str);
-            *data_p = git__malloc(*len_p);
-            if (*data_p == NULL) {
-                error = GIT_ENOMEM;
-            } else {
-                memcpy(*data_p, reply->element[2]->str, *len_p);
-                error = GIT_SUCCESS;
-            }
-        } else {
-            error = GIT_ENOTFOUND;
-        }
-    } else {
-        error = GIT_ERROR;
-    }
-
-    freeReplyObject(reply);
-    return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to read backend");
-}
-
-int hiredis_backend__read_prefix(git_oid *out_oid, void **data_p, size_t *len_p, git_otype *type_p, git_odb_backend *_backend,
-					const git_oid *short_oid, unsigned int len) {
-	if (len >= GIT_OID_HEXSZ) {
-		/* Just match the full identifier */
-		int error = hiredis_backend__read(data_p, len_p, type_p, backend, short_oid);
-		if (error == GIT_SUCCESS)
-			git_oid_cpy(out_oid, short_oid);
-
-		return error;
-	} else if (len < GIT_OID_HEXSZ) {
-		/* TODO */
-		return git__throw(GIT_ENOTIMPLEMENTED, "Hiredis backend cannot search objects from short oid");
-	}
-}
-
-int hiredis_backend__exists(git_odb_backend *_backend, const git_oid *oid) {
-    hiredis_backend *backend;
-    int found;
-    redisReply *reply;
-
-    assert(_backend && oid);
-
-    backend = (hiredis_backend *) _backend;
-    found = 0;
-
-    reply = redisCommand(backend->db, "exists %b", oid->id, GIT_OID_RAWSZ);
-    if (reply && reply->type != REDIS_REPLY_NIL && reply->type != REDIS_REPLY_ERROR)
-        found = 1;
-
-
-    freeReplyObject(reply);
-    return found;
-}
-
-int hiredis_backend__write(git_oid *id, git_odb_backend *_backend, const void *data, size_t len, git_otype type) {
-    hiredis_backend *backend;
-    int error;
-    redisReply *reply;
-
-    assert(id && _backend && data);
-
-    backend = (hiredis_backend *) _backend;
-    error = GIT_ERROR;
-
-    if ((error = git_odb_hash(id, data, len, type)) < 0)
-        return git__rethrow(error, "Failed to write backend");
-
-    reply = redisCommand(backend->db, "HMSET %b "
-            "type %d "
-            "size %d "
-            "data %b ", id->id, GIT_OID_RAWSZ,
-            (int) type, len, data, len);
-
-    error = (reply == NULL || reply->type == REDIS_REPLY_ERROR) ? GIT_ERROR : GIT_SUCCESS;
-
-    freeReplyObject(reply);
-    return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to write backend");
-}
-
-void hiredis_backend__free(git_odb_backend *_backend) {
-    hiredis_backend *backend;
-    assert(_backend);
-    backend = (hiredis_backend *) _backend;
-
-    redisFree(backend->db);
-
-    free(backend);
-}
-
-int git_odb_backend_hiredis(git_odb_backend **backend_out, const char *host, int port) {
-    hiredis_backend *backend;
-
-    backend = git__calloc(1, sizeof (hiredis_backend));
-    if (backend == NULL)
-        return GIT_ENOMEM;
-
-
-    backend->db = redisConnect(host, port);
-    if (backend->db->err)
-        goto cleanup;
-
-    backend->parent.read = &hiredis_backend__read;
-    backend->parent.read_prefix = &hiredis_backend__read_prefix;
-    backend->parent.read_header = &hiredis_backend__read_header;
-    backend->parent.write = &hiredis_backend__write;
-    backend->parent.exists = &hiredis_backend__exists;
-    backend->parent.free = &hiredis_backend__free;
-
-    *backend_out = (git_odb_backend *) backend;
-
-    return GIT_SUCCESS;
-cleanup:
-    free(backend);
-    return git__throw(GIT_ERROR, "Failed to get ODB backend");
-}
-
-#else
-
-int git_odb_backend_hiredis(git_odb_backend ** GIT_UNUSED(backend_out),
-        const char *GIT_UNUSED(host), int GIT_UNUSED(port)) {
-    GIT_UNUSED_ARG(backend_out);
-    GIT_UNUSED_ARG(host);
-    GIT_UNUSED_ARG(port);
-    return git__throw(GIT_ENOTIMPLEMENTED, "Failed to get ODB backend. Feature not yet implemented");
-}
-
-
-#endif /* HAVE_HIREDIS */
diff --git a/src/backends/sqlite.c b/src/backends/sqlite.c
deleted file mode 100644
index 8626349ec..000000000
--- a/src/backends/sqlite.c
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
- *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- */
-
-#include "common.h"
-#include "git2/object.h"
-#include "hash.h"
-#include "odb.h"
-
-#include "git2/odb_backend.h"
-
-#ifdef GIT2_SQLITE_BACKEND
-
-#include <sqlite3.h>
-
-#define GIT2_TABLE_NAME "git2_odb"
-
-typedef struct {
-	git_odb_backend parent;
-	sqlite3 *db;
-	sqlite3_stmt *st_read;
-	sqlite3_stmt *st_write;
-	sqlite3_stmt *st_read_header;
-} sqlite_backend;
-
-int sqlite_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *_backend, const git_oid *oid)
-{
-	sqlite_backend *backend;
-	int error;
-
-	assert(len_p && type_p && _backend && oid);
-
-	backend = (sqlite_backend *)_backend;
-	error = GIT_ERROR;
-
-	if (sqlite3_bind_text(backend->st_read_header, 1, (char *)oid->id, 20, SQLITE_TRANSIENT) == SQLITE_OK) {
-		if (sqlite3_step(backend->st_read_header) == SQLITE_ROW) {
-			*type_p = (git_otype)sqlite3_column_int(backend->st_read_header, 0);
-			*len_p = (size_t)sqlite3_column_int(backend->st_read_header, 1);
-			assert(sqlite3_step(backend->st_read_header) == SQLITE_DONE);
-			error = GIT_SUCCESS;
-		} else {
-			error = GIT_ENOTFOUND;
-		}
-	}
-
-	sqlite3_reset(backend->st_read_header);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "SQLite backend: Failed to read header");
-}
-
-
-int sqlite_backend__read(void **data_p, size_t *len_p, git_otype *type_p, git_odb_backend *_backend, const git_oid *oid)
-{
-	sqlite_backend *backend;
-	int error;
-
-	assert(data_p && len_p && type_p && _backend && oid);
-
-	backend = (sqlite_backend *)_backend;
-	error = GIT_ERROR;
-
-	if (sqlite3_bind_text(backend->st_read, 1, (char *)oid->id, 20, SQLITE_TRANSIENT) == SQLITE_OK) {
-		if (sqlite3_step(backend->st_read) == SQLITE_ROW) {
-			*type_p = (git_otype)sqlite3_column_int(backend->st_read, 0);
-			*len_p = (size_t)sqlite3_column_int(backend->st_read, 1);
-			*data_p = git__malloc(*len_p);
-
-			if (*data_p == NULL) {
-				error = GIT_ENOMEM;
-			} else {
-				memcpy(*data_p, sqlite3_column_blob(backend->st_read, 2), *len_p);
-				error = GIT_SUCCESS;
-			}
-
-			assert(sqlite3_step(backend->st_read) == SQLITE_DONE);
-		} else {
-			error = GIT_ENOTFOUND;
-		}
-	}
-
-	sqlite3_reset(backend->st_read);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "SQLite backend: Failed to read");
-}
-
-int sqlite_backend__read_prefix(git_oid *out_oid, void **data_p, size_t *len_p, git_otype *type_p, git_odb_backend *_backend,
-					const git_oid *short_oid, unsigned int len) {
-	if (len >= GIT_OID_HEXSZ) {
-		/* Just match the full identifier */
-		int error = sqlite_backend__read(data_p, len_p, type_p, _backend, short_oid);
-		if (error == GIT_SUCCESS)
-			git_oid_cpy(out_oid, short_oid);
-
-		return error;
-	} else if (len < GIT_OID_HEXSZ) {
-		/* TODO */
-		return git__throw(GIT_ENOTIMPLEMENTED, "Sqlite backend cannot search objects from short oid");
-	}
-}
-
-int sqlite_backend__exists(git_odb_backend *_backend, const git_oid *oid)
-{
-	sqlite_backend *backend;
-	int found;
-
-	assert(_backend && oid);
-
-	backend = (sqlite_backend *)_backend;
-	found = 0;
-
-	if (sqlite3_bind_text(backend->st_read_header, 1, (char *)oid->id, 20, SQLITE_TRANSIENT) == SQLITE_OK) {
-		if (sqlite3_step(backend->st_read_header) == SQLITE_ROW) {
-			found = 1;
-			assert(sqlite3_step(backend->st_read_header) == SQLITE_DONE);
-		}
-	}
-
-	sqlite3_reset(backend->st_read_header);
-	return found;
-}
-
-
-int sqlite_backend__write(git_oid *id, git_odb_backend *_backend, const void *data, size_t len, git_otype type)
-{
-	int error;
-	sqlite_backend *backend;
-
-	assert(id && _backend && data);
-
-	backend = (sqlite_backend *)_backend;
-
-	if ((error = git_odb_hash(id, data, len, type)) < 0)
-		return git__rethrow(error, "SQLite backend: Failed to write");
-
-	error = SQLITE_ERROR;
-
-	if (sqlite3_bind_text(backend->st_write, 1, (char *)id->id, 20, SQLITE_TRANSIENT) == SQLITE_OK &&
-		sqlite3_bind_int(backend->st_write, 2, (int)type) == SQLITE_OK &&
-		sqlite3_bind_int(backend->st_write, 3, len) == SQLITE_OK &&
-		sqlite3_bind_blob(backend->st_write, 4, data, len, SQLITE_TRANSIENT) == SQLITE_OK) {
-		error = sqlite3_step(backend->st_write);
-	}
-
-	sqlite3_reset(backend->st_write);
-	return (error == SQLITE_DONE) ? GIT_SUCCESS : git__throw(GIT_ERROR, "SQLite backend: Failed to write");
-}
-
-
-void sqlite_backend__free(git_odb_backend *_backend)
-{
-	sqlite_backend *backend;
-	assert(_backend);
-	backend = (sqlite_backend *)_backend;
-
-	sqlite3_finalize(backend->st_read);
-	sqlite3_finalize(backend->st_read_header);
-	sqlite3_finalize(backend->st_write);
-	sqlite3_close(backend->db);
-
-	free(backend);
-}
-
-static int create_table(sqlite3 *db)
-{
-	static const char *sql_creat =
-		"CREATE TABLE '" GIT2_TABLE_NAME "' ("
-		"'oid' CHARACTER(20) PRIMARY KEY NOT NULL,"
-		"'type' INTEGER NOT NULL,"
-		"'size' INTEGER NOT NULL,"
-		"'data' BLOB);";
-
-	if (sqlite3_exec(db, sql_creat, NULL, NULL, NULL) != SQLITE_OK)
-		return git__throw(GIT_ERROR, "SQLite backend: Failed to create table");
-
-	return GIT_SUCCESS;
-}
-
-static int init_db(sqlite3 *db)
-{
-	static const char *sql_check =
-		"SELECT name FROM sqlite_master WHERE type='table' AND name='" GIT2_TABLE_NAME "';";
-
-	sqlite3_stmt *st_check;
-	int error;
-
-	if (sqlite3_prepare_v2(db, sql_check, -1, &st_check, NULL) != SQLITE_OK)
-		return git__throw(GIT_ERROR, "SQLite backend: Failed to initialize database");
-
-	switch (sqlite3_step(st_check)) {
-	case SQLITE_DONE:
-		/* the table was not found */
-		error = create_table(db);
-		break;
-
-	case SQLITE_ROW:
-		/* the table was found */
-		error = GIT_SUCCESS;
-		break;
-
-	default:
-		error = GIT_ERROR;
-		break;
-	}
-
-	sqlite3_finalize(st_check);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "SQLite backend: Failed to initialize database");
-}
-
-static int init_statements(sqlite_backend *backend)
-{
-	static const char *sql_read =
-		"SELECT type, size, data FROM '" GIT2_TABLE_NAME "' WHERE oid = ?;";
-
-	static const char *sql_read_header =
-		"SELECT type, size FROM '" GIT2_TABLE_NAME "' WHERE oid = ?;";
-
-	static const char *sql_write =
-		"INSERT OR IGNORE INTO '" GIT2_TABLE_NAME "' VALUES (?, ?, ?, ?);";
-
-	if (sqlite3_prepare_v2(backend->db, sql_read, -1, &backend->st_read, NULL) != SQLITE_OK)
-		return git__throw(GIT_ERROR, "SQLite backend: Failed to initialize statements");
-
-	if (sqlite3_prepare_v2(backend->db, sql_read_header, -1, &backend->st_read_header, NULL) != SQLITE_OK)
-		return git__throw(GIT_ERROR, "SQLite backend: Failed to initialize statements");
-
-	if (sqlite3_prepare_v2(backend->db, sql_write, -1, &backend->st_write, NULL) != SQLITE_OK)
-		return git__throw(GIT_ERROR, "SQLite backend: Failed to initialize statements");
-
-	return GIT_SUCCESS;
-}
-
-int git_odb_backend_sqlite(git_odb_backend **backend_out, const char *sqlite_db)
-{
-	sqlite_backend *backend;
-	int error;
-
-	backend = git__calloc(1, sizeof(sqlite_backend));
-	if (backend == NULL)
-		return GIT_ENOMEM;
-
-	if (sqlite3_open(sqlite_db, &backend->db) != SQLITE_OK)
-		goto cleanup;
-
-	error = init_db(backend->db);
-	if (error < 0)
-		goto cleanup;
-
-	error = init_statements(backend);
-	if (error < 0)
-		goto cleanup;
-
-	backend->parent.read = &sqlite_backend__read;
-	backend->parent.read_prefix = &sqlite_backend__read_prefix;
-	backend->parent.read_header = &sqlite_backend__read_header;
-	backend->parent.write = &sqlite_backend__write;
-	backend->parent.exists = &sqlite_backend__exists;
-	backend->parent.free = &sqlite_backend__free;
-
-	*backend_out = (git_odb_backend *)backend;
-	return GIT_SUCCESS;
-
-cleanup:
-	sqlite_backend__free((git_odb_backend *)backend);
-	return git__throw(GIT_ERROR, "SQLite backend: Failed to get ODB backend");
-}
-
-#else
-
-int git_odb_backend_sqlite(git_odb_backend **GIT_UNUSED(backend_out), const char *GIT_UNUSED(sqlite_db))
-{
-	GIT_UNUSED_ARG(backend_out);
-	GIT_UNUSED_ARG(sqlite_db);
-	return git__throw(GIT_ENOTIMPLEMENTED, "SQLite backend: Failed to get ODB backend. Operation not yet implemented");
-}
-
-#endif /* HAVE_SQLITE3 */
diff --git a/src/blob.c b/src/blob.c
index 6ab58d6b2..699adec6b 100644
--- a/src/blob.c
+++ b/src/blob.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "git2/common.h"
@@ -29,6 +11,7 @@
 
 #include "common.h"
 #include "blob.h"
+#include "filter.h"
 
 const void *git_blob_rawcontent(git_blob *blob)
 {
@@ -36,16 +19,22 @@ const void *git_blob_rawcontent(git_blob *blob)
 	return blob->odb_object->raw.data;
 }
 
-int git_blob_rawsize(git_blob *blob)
+size_t git_blob_rawsize(git_blob *blob)
 {
 	assert(blob);
 	return blob->odb_object->raw.len;
 }
 
+int git_blob__getbuf(git_buf *buffer, git_blob *blob)
+{
+	return git_buf_set(
+		buffer, blob->odb_object->raw.data, blob->odb_object->raw.len);
+}
+
 void git_blob__free(git_blob *blob)
 {
-	git_odb_object_close(blob->odb_object);
-	free(blob);
+	git_odb_object_free(blob->odb_object);
+	git__free(blob);
 }
 
 int git_blob__parse(git_blob *blob, git_odb_object *odb_obj)
@@ -53,79 +42,247 @@ int git_blob__parse(git_blob *blob, git_odb_object *odb_obj)
 	assert(blob);
 	git_cached_obj_incref((git_cached_obj *)odb_obj);
 	blob->odb_object = odb_obj;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 int git_blob_create_frombuffer(git_oid *oid, git_repository *repo, const void *buffer, size_t len)
 {
 	int error;
+	git_odb *odb;
 	git_odb_stream *stream;
 
-	if ((error = git_odb_open_wstream(&stream, repo->db, len, GIT_OBJ_BLOB)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to create blob");
+	if ((error = git_repository_odb__weakptr(&odb, repo)) < 0 ||
+		(error = git_odb_open_wstream(&stream, odb, len, GIT_OBJ_BLOB)) < 0)
+		return error;
 
-	if ((error = stream->write(stream, buffer, len)) < GIT_SUCCESS) {
-		stream->free(stream);
+	if ((error = stream->write(stream, buffer, len)) == 0)
+		error = stream->finalize_write(oid, stream);
+
+	stream->free(stream);
+	return error;
+}
+
+static int write_file_stream(
+	git_oid *oid, git_odb *odb, const char *path, git_off_t file_size)
+{
+	int fd, error;
+	char buffer[4096];
+	git_odb_stream *stream = NULL;
+
+	if ((error = git_odb_open_wstream(
+			&stream, odb, (size_t)file_size, GIT_OBJ_BLOB)) < 0)
 		return error;
+
+	if ((fd = git_futils_open_ro(path)) < 0) {
+		stream->free(stream);
+		return -1;
+	}
+
+	while (!error && file_size > 0) {
+		ssize_t read_len = p_read(fd, buffer, sizeof(buffer));
+
+		if (read_len < 0) {
+			giterr_set(
+				GITERR_OS, "Failed to create blob. Can't read whole file");
+			error = -1;
+		}
+		else if (!(error = stream->write(stream, buffer, read_len)))
+			file_size -= read_len;
 	}
 
-	error = stream->finalize_write(oid, stream);
+	p_close(fd);
+
+	if (!error)
+		error = stream->finalize_write(oid, stream);
+
 	stream->free(stream);
+	return error;
+}
+
+static int write_file_filtered(
+	git_oid *oid,
+	git_odb *odb,
+	const char *full_path,
+	git_vector *filters)
+{
+	int error;
+	git_buf source = GIT_BUF_INIT;
+	git_buf dest = GIT_BUF_INIT;
+
+	if ((error = git_futils_readbuffer(&source, full_path)) < 0)
+		return error;
+
+	error = git_filters_apply(&dest, &source, filters);
+
+	/* Free the source as soon as possible. This can be big in memory,
+	 * and we don't want to ODB write to choke */
+	git_buf_free(&source);
 
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to create blob");
+	/* Write the file to disk if it was properly filtered */
+	if (!error)
+		error = git_odb_write(oid, odb, dest.ptr, dest.size, GIT_OBJ_BLOB);
 
-	return GIT_SUCCESS;
+	git_buf_free(&dest);
+	return error;
 }
 
-int git_blob_create_fromfile(git_oid *oid, git_repository *repo, const char *path)
+static int write_symlink(
+	git_oid *oid, git_odb *odb, const char *path, size_t link_size)
+{
+	char *link_data;
+	ssize_t read_len;
+	int error;
+
+	link_data = git__malloc(link_size);
+	GITERR_CHECK_ALLOC(link_data);
+
+	read_len = p_readlink(path, link_data, link_size);
+	if (read_len != (ssize_t)link_size) {
+		giterr_set(GITERR_OS, "Failed to create blob.  Can't read symlink '%s'", path);
+		git__free(link_data);
+		return -1;
+	}
+
+	error = git_odb_write(oid, odb, (void *)link_data, link_size, GIT_OBJ_BLOB);
+	git__free(link_data);
+	return error;
+}
+
+static int blob_create_internal(git_oid *oid, git_repository *repo, const char *content_path, const char *hint_path, bool try_load_filters)
 {
-	int error, fd;
-	char full_path[GIT_PATH_MAX];
-	char buffer[2048];
+	int error;
+	struct stat st;
+	git_odb *odb = NULL;
 	git_off_t size;
-	git_odb_stream *stream;
 
-	if (repo->path_workdir == NULL)
-		return git__throw(GIT_ENOTFOUND, "Failed to create blob. (No working directory found)");
+	assert(hint_path || !try_load_filters);
 
-	git__joinpath(full_path, repo->path_workdir, path);
+	if ((error = git_path_lstat(content_path, &st)) < 0 || (error = git_repository_odb__weakptr(&odb, repo)) < 0)
+		return error;
 
-	if ((fd = gitfo_open(full_path, O_RDONLY)) < 0)
-		return git__throw(GIT_ENOTFOUND, "Failed to create blob. Could not open '%s'", full_path);
+	size = st.st_size;
 
-	if ((size = gitfo_size(fd)) < 0 || !git__is_sizet(size)) {
-		gitfo_close(fd);
-		return git__throw(GIT_EOSERR, "Failed to create blob. '%s' appears to be corrupted", full_path);
-	}
+	if (S_ISLNK(st.st_mode)) {
+		error = write_symlink(oid, odb, content_path, (size_t)size);
+	} else {
+		git_vector write_filters = GIT_VECTOR_INIT;
+		int filter_count = 0;
+
+		if (try_load_filters) {
+			/* Load the filters for writing this file to the ODB */
+			filter_count = git_filters_load(
+				&write_filters, repo, hint_path, GIT_FILTER_TO_ODB);
+		}
+
+		if (filter_count < 0) {
+			/* Negative value means there was a critical error */
+			error = filter_count;
+		} else if (filter_count == 0) {
+			/* No filters need to be applied to the document: we can stream
+			 * directly from disk */
+			error = write_file_stream(oid, odb, content_path, size);
+		} else {
+			/* We need to apply one or more filters */
+			error = write_file_filtered(oid, odb, content_path, &write_filters);
+		}
 
-	if ((error = git_odb_open_wstream(&stream, repo->db, (size_t)size, GIT_OBJ_BLOB)) < GIT_SUCCESS) {
-		gitfo_close(fd);
-		return git__rethrow(error, "Failed to create blob");
+		git_filters_free(&write_filters);
+
+		/*
+		 * TODO: eventually support streaming filtered files, for files
+		 * which are bigger than a given threshold. This is not a priority
+		 * because applying a filter in streaming mode changes the final
+		 * size of the blob, and without knowing its final size, the blob
+		 * cannot be written in stream mode to the ODB.
+		 *
+		 * The plan is to do streaming writes to a tempfile on disk and then
+		 * opening streaming that file to the ODB, using
+		 * `write_file_stream`.
+		 *
+		 * CAREFULLY DESIGNED APIS YO
+		 */
 	}
 
-	while (size > 0) {
-		ssize_t read_len;
+	return error;
+}
 
-		read_len = read(fd, buffer, sizeof(buffer));
+int git_blob_create_fromfile(git_oid *oid, git_repository *repo, const char *path)
+{
+	git_buf full_path = GIT_BUF_INIT;
+	const char *workdir;
+	int error;
 
-		if (read_len < 0) {
-			gitfo_close(fd);
-			stream->free(stream);
-			return git__throw(GIT_EOSERR, "Failed to create blob. Can't read full file");
-		}
+	workdir = git_repository_workdir(repo);
+	assert(workdir); /* error to call this on bare repo */
 
-		stream->write(stream, buffer, read_len);
-		size -= read_len;
+	if (git_buf_joinpath(&full_path, workdir, path) < 0) {
+		git_buf_free(&full_path);
+		return -1;
 	}
 
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
-	gitfo_close(fd);
+	error = blob_create_internal(oid, repo, git_buf_cstr(&full_path), git_buf_cstr(&full_path), true);
+
+	git_buf_free(&full_path);
+	return error;
+}
+
+int git_blob_create_fromdisk(git_oid *oid, git_repository *repo, const char *path)
+{
+	int error;
+	git_buf full_path = GIT_BUF_INIT;
+
+	if ((error = git_path_prettify(&full_path, path, NULL)) < 0) {
+		git_buf_free(&full_path);
+		return error;
+	}
 
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to create blob");
+	error = blob_create_internal(oid, repo, git_buf_cstr(&full_path), git_buf_cstr(&full_path), true);
 
-	return GIT_SUCCESS;
+	git_buf_free(&full_path);
+	return error;
 }
 
+#define BUFFER_SIZE 4096
+
+int git_blob_create_fromchunks(
+	git_oid *oid,
+	git_repository *repo,
+	const char *hintpath,
+	int (*source_cb)(char *content, size_t max_length, void *payload),
+	void *payload)
+{
+	int error = -1, read_bytes;
+	char *content = NULL;
+	git_filebuf file = GIT_FILEBUF_INIT;
+
+	content = git__malloc(BUFFER_SIZE);
+	GITERR_CHECK_ALLOC(content);
+
+	if (git_filebuf_open(&file, hintpath == NULL ? "streamed" : hintpath, GIT_FILEBUF_TEMPORARY) < 0)
+		goto cleanup;
+
+	while (1) {
+		read_bytes = source_cb(content, BUFFER_SIZE, payload);
+
+		assert(read_bytes <= BUFFER_SIZE);
+
+		if (read_bytes <= 0)
+			break;
+
+		if (git_filebuf_write(&file, content, read_bytes) < 0)
+			goto cleanup;
+	}
+
+	if (read_bytes < 0)
+		goto cleanup;
+
+	if (git_filebuf_flush(&file) < 0)
+		goto cleanup;
+
+	error = blob_create_internal(oid, repo, file.path_lock, hintpath, hintpath != NULL);
+
+cleanup:
+	git_filebuf_cleanup(&file);
+	git__free(content);
+	return error;
+}
diff --git a/src/blob.h b/src/blob.h
index 4300d7e54..0305e9473 100644
--- a/src/blob.h
+++ b/src/blob.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_blob_h__
 #define INCLUDE_blob_h__
 
@@ -13,5 +19,6 @@ struct git_blob {
 
 void git_blob__free(git_blob *blob);
 int git_blob__parse(git_blob *blob, git_odb_object *obj);
+int git_blob__getbuf(git_buf *buffer, git_blob *blob);
 
 #endif
diff --git a/src/branch.c b/src/branch.c
new file mode 100644
index 000000000..5d5a24038
--- /dev/null
+++ b/src/branch.c
@@ -0,0 +1,208 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "commit.h"
+#include "branch.h"
+#include "tag.h"
+
+static int retrieve_branch_reference(
+	git_reference **branch_reference_out,
+	git_repository *repo,
+	const char *branch_name,
+	int is_remote)
+{
+	git_reference *branch;
+	int error = -1;
+	char *prefix;
+	git_buf ref_name = GIT_BUF_INIT;
+
+	*branch_reference_out = NULL;
+
+	prefix = is_remote ? GIT_REFS_REMOTES_DIR : GIT_REFS_HEADS_DIR;
+
+	if (git_buf_joinpath(&ref_name, prefix, branch_name) < 0)
+		goto cleanup;
+
+	if ((error = git_reference_lookup(&branch, repo, ref_name.ptr)) < 0) {
+		giterr_set(GITERR_REFERENCE,
+			"Cannot locate %s branch '%s'.", is_remote ? "remote-tracking" : "local", branch_name);
+		goto cleanup;
+	}
+
+	*branch_reference_out = branch;
+
+cleanup:
+	git_buf_free(&ref_name);
+	return error;
+}
+
+static int create_error_invalid(const char *msg)
+{
+	giterr_set(GITERR_INVALID, "Cannot create branch - %s", msg);
+	return -1;
+}
+
+int git_branch_create(
+		git_oid *oid_out,
+		git_repository *repo,
+		const char *branch_name,
+		const git_object *target,
+		int force)
+{
+	git_otype target_type = GIT_OBJ_BAD;
+	git_object *commit = NULL;
+	git_reference *branch = NULL;
+	git_buf canonical_branch_name = GIT_BUF_INIT;
+	int error = -1;
+
+	assert(repo && branch_name && target && oid_out);
+
+	if (git_object_owner(target) != repo)
+		return create_error_invalid("The given target does not belong to this repository");
+
+	target_type = git_object_type(target);
+
+	switch (target_type)
+	{
+	case GIT_OBJ_TAG:
+		if (git_tag_peel(&commit, (git_tag *)target) < 0)
+			goto cleanup;
+
+		if (git_object_type(commit) != GIT_OBJ_COMMIT) {
+			create_error_invalid("The given target does not resolve to a commit");
+			goto cleanup;
+		}
+		break;
+
+	case GIT_OBJ_COMMIT:
+		commit = (git_object *)target;
+		break;
+
+	default:
+		return create_error_invalid("Only git_tag and git_commit objects are valid targets.");
+	}
+
+	if (git_buf_joinpath(&canonical_branch_name, GIT_REFS_HEADS_DIR, branch_name) < 0)
+		goto cleanup;
+
+	if (git_reference_create_oid(&branch, repo, git_buf_cstr(&canonical_branch_name), git_object_id(commit), force) < 0)
+		goto cleanup;
+
+	git_oid_cpy(oid_out, git_reference_oid(branch));
+	error = 0;
+
+cleanup:
+	if (target_type == GIT_OBJ_TAG)
+		git_object_free(commit);
+
+	git_reference_free(branch);
+	git_buf_free(&canonical_branch_name);
+	return error;
+}
+
+int git_branch_delete(git_repository *repo, const char *branch_name, git_branch_t branch_type)
+{
+	git_reference *branch = NULL;
+	git_reference *head = NULL;
+	int error;
+
+	assert((branch_type == GIT_BRANCH_LOCAL) || (branch_type == GIT_BRANCH_REMOTE));
+
+	if ((error = retrieve_branch_reference(&branch, repo, branch_name, branch_type == GIT_BRANCH_REMOTE)) < 0)
+		return error;
+
+	if (git_reference_lookup(&head, repo, GIT_HEAD_FILE) < 0) {
+		giterr_set(GITERR_REFERENCE, "Cannot locate HEAD.");
+		goto on_error;
+	}
+
+	if ((git_reference_type(head) == GIT_REF_SYMBOLIC)
+		&& (strcmp(git_reference_target(head), git_reference_name(branch)) == 0)) {
+			giterr_set(GITERR_REFERENCE,
+					"Cannot delete branch '%s' as it is the current HEAD of the repository.", branch_name);
+			goto on_error;
+	}
+
+	if (git_reference_delete(branch) < 0)
+		goto on_error;
+
+	git_reference_free(head);
+	return 0;
+
+on_error:
+	git_reference_free(head);
+	git_reference_free(branch);
+	return -1;
+}
+
+typedef struct {
+	git_vector *branchlist;
+	unsigned int branch_type;
+} branch_filter_data;
+
+static int branch_list_cb(const char *branch_name, void *payload)
+{
+	branch_filter_data *filter = (branch_filter_data *)payload;
+
+	if ((filter->branch_type & GIT_BRANCH_LOCAL && git__prefixcmp(branch_name, GIT_REFS_HEADS_DIR) == 0)
+		|| (filter->branch_type & GIT_BRANCH_REMOTE && git__prefixcmp(branch_name, GIT_REFS_REMOTES_DIR) == 0))
+		return git_vector_insert(filter->branchlist, git__strdup(branch_name));
+
+	return 0;
+}
+
+int git_branch_list(git_strarray *branch_names, git_repository *repo, unsigned int list_flags)
+{
+	int error;
+	branch_filter_data filter;
+	git_vector branchlist;
+
+	assert(branch_names && repo);
+
+	if (git_vector_init(&branchlist, 8, NULL) < 0)
+		return -1;
+
+	filter.branchlist = &branchlist;
+	filter.branch_type = list_flags;
+
+	error = git_reference_foreach(repo, GIT_REF_LISTALL, &branch_list_cb, (void *)&filter);
+	if (error < 0) {
+		git_vector_free(&branchlist);
+		return -1;
+	}
+
+	branch_names->strings = (char **)branchlist.contents;
+	branch_names->count = branchlist.length;
+	return 0;
+}
+
+int git_branch_move(git_repository *repo, const char *old_branch_name, const char *new_branch_name, int force)
+{
+	git_reference *reference = NULL;
+	git_buf old_reference_name = GIT_BUF_INIT, new_reference_name = GIT_BUF_INIT;
+	int error = 0;
+
+	if ((error = git_buf_joinpath(&old_reference_name, GIT_REFS_HEADS_DIR, old_branch_name)) < 0)
+		goto cleanup;
+
+	/* We need to be able to return GIT_ENOTFOUND */
+	if ((error = git_reference_lookup(&reference, repo, git_buf_cstr(&old_reference_name))) < 0)
+		goto cleanup;
+
+	if ((error = git_buf_joinpath(&new_reference_name, GIT_REFS_HEADS_DIR, new_branch_name)) < 0)
+		goto cleanup;
+
+	error = git_reference_rename(reference, git_buf_cstr(&new_reference_name), force);
+
+cleanup:
+	git_reference_free(reference);
+	git_buf_free(&old_reference_name);
+	git_buf_free(&new_reference_name);
+
+	return error;
+}
diff --git a/src/branch.h b/src/branch.h
new file mode 100644
index 000000000..d0e5abc8b
--- /dev/null
+++ b/src/branch.h
@@ -0,0 +1,17 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_branch_h__
+#define INCLUDE_branch_h__
+
+#include "git2/branch.h"
+
+struct git_branch {
+	char *remote; /* TODO: Make this a git_remote */
+	char *merge;
+};
+
+#endif
diff --git a/src/bswap.h b/src/bswap.h
index b9211c3c8..995767a14 100644
--- a/src/bswap.h
+++ b/src/bswap.h
@@ -1,8 +1,8 @@
 /*
- * Let's make sure we always have a sane definition for ntohl()/htonl().
- * Some libraries define those as a function call, just to perform byte
- * shifting, bringing significant overhead to what should be a simple
- * operation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
@@ -14,8 +14,8 @@
 GIT_INLINE(uint32_t) default_swab32(uint32_t val)
 {
 	return (((val & 0xff000000) >> 24) |
-		((val & 0x00ff0000) >>  8) |
-		((val & 0x0000ff00) <<  8) |
+		((val & 0x00ff0000) >> 8) |
+		((val & 0x0000ff00) << 8) |
 		((val & 0x000000ff) << 24));
 }
 
diff --git a/src/buffer.c b/src/buffer.c
new file mode 100644
index 000000000..783a36eb8
--- /dev/null
+++ b/src/buffer.c
@@ -0,0 +1,461 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "buffer.h"
+#include "posix.h"
+#include <stdarg.h>
+#include <ctype.h>
+
+/* Used as default value for git_buf->ptr so that people can always
+ * assume ptr is non-NULL and zero terminated even for new git_bufs.
+ */
+char git_buf__initbuf[1];
+
+char git_buf__oom[1];
+
+#define ENSURE_SIZE(b, d) \
+	if ((d) > buf->asize && git_buf_grow(b, (d)) < 0)\
+		return -1;
+
+
+void git_buf_init(git_buf *buf, size_t initial_size)
+{
+	buf->asize = 0;
+	buf->size = 0;
+	buf->ptr = git_buf__initbuf;
+
+	if (initial_size)
+		git_buf_grow(buf, initial_size);
+}
+
+int git_buf_grow(git_buf *buf, size_t target_size)
+{
+	int error = git_buf_try_grow(buf, target_size);
+	if (error != 0)
+		buf->ptr = git_buf__oom;
+	return error;
+}
+
+int git_buf_try_grow(git_buf *buf, size_t target_size)
+{
+	char *new_ptr;
+	size_t new_size;
+
+	if (buf->ptr == git_buf__oom)
+		return -1;
+
+	if (target_size <= buf->asize)
+		return 0;
+
+	if (buf->asize == 0) {
+		new_size = target_size;
+		new_ptr = NULL;
+	} else {
+		new_size = buf->asize;
+		new_ptr = buf->ptr;
+	}
+
+	/* grow the buffer size by 1.5, until it's big enough
+	 * to fit our target size */
+	while (new_size < target_size)
+		new_size = (new_size << 1) - (new_size >> 1);
+
+	/* round allocation up to multiple of 8 */
+	new_size = (new_size + 7) & ~7;
+
+	new_ptr = git__realloc(new_ptr, new_size);
+	if (!new_ptr)
+		return -1;
+
+	buf->asize = new_size;
+	buf->ptr   = new_ptr;
+
+	/* truncate the existing buffer size if necessary */
+	if (buf->size >= buf->asize)
+		buf->size = buf->asize - 1;
+	buf->ptr[buf->size] = '\0';
+
+	return 0;
+}
+
+void git_buf_free(git_buf *buf)
+{
+	if (!buf) return;
+
+	if (buf->ptr != git_buf__initbuf && buf->ptr != git_buf__oom)
+		git__free(buf->ptr);
+
+	git_buf_init(buf, 0);
+}
+
+void git_buf_clear(git_buf *buf)
+{
+	buf->size = 0;
+	if (buf->asize > 0)
+		buf->ptr[0] = '\0';
+}
+
+int git_buf_set(git_buf *buf, const char *data, size_t len)
+{
+	if (len == 0 || data == NULL) {
+		git_buf_clear(buf);
+	} else {
+		if (data != buf->ptr) {
+			ENSURE_SIZE(buf, len + 1);
+			memmove(buf->ptr, data, len);
+		}
+		buf->size = len;
+		buf->ptr[buf->size] = '\0';
+	}
+	return 0;
+}
+
+int git_buf_sets(git_buf *buf, const char *string)
+{
+	return git_buf_set(buf, string, string ? strlen(string) : 0);
+}
+
+int git_buf_putc(git_buf *buf, char c)
+{
+	ENSURE_SIZE(buf, buf->size + 2);
+	buf->ptr[buf->size++] = c;
+	buf->ptr[buf->size] = '\0';
+	return 0;
+}
+
+int git_buf_put(git_buf *buf, const char *data, size_t len)
+{
+	ENSURE_SIZE(buf, buf->size + len + 1);
+	memmove(buf->ptr + buf->size, data, len);
+	buf->size += len;
+	buf->ptr[buf->size] = '\0';
+	return 0;
+}
+
+int git_buf_puts(git_buf *buf, const char *string)
+{
+	assert(string);
+	return git_buf_put(buf, string, strlen(string));
+}
+
+int git_buf_vprintf(git_buf *buf, const char *format, va_list ap)
+{
+	int len;
+
+	ENSURE_SIZE(buf, buf->size + (strlen(format) * 2));
+
+	while (1) {
+		va_list args;
+		va_copy(args, ap);
+
+		len = p_vsnprintf(
+			buf->ptr + buf->size,
+			buf->asize - buf->size,
+			format, args
+		);
+
+		if (len < 0) {
+			git__free(buf->ptr);
+			buf->ptr = git_buf__oom;
+			return -1;
+		}
+
+		if ((size_t)len + 1 <= buf->asize - buf->size) {
+			buf->size += len;
+			break;
+		}
+
+		ENSURE_SIZE(buf, buf->size + len + 1);
+	}
+
+	return 0;
+}
+
+int git_buf_printf(git_buf *buf, const char *format, ...)
+{
+	int r;
+	va_list ap;
+
+	va_start(ap, format);
+	r = git_buf_vprintf(buf, format, ap);
+	va_end(ap);
+
+	return r;
+}
+
+void git_buf_copy_cstr(char *data, size_t datasize, const git_buf *buf)
+{
+	size_t copylen;
+
+	assert(data && datasize && buf);
+
+	data[0] = '\0';
+
+	if (buf->size == 0 || buf->asize <= 0)
+		return;
+
+	copylen = buf->size;
+	if (copylen > datasize - 1)
+		copylen = datasize - 1;
+	memmove(data, buf->ptr, copylen);
+	data[copylen] = '\0';
+}
+
+void git_buf_consume(git_buf *buf, const char *end)
+{
+	if (end > buf->ptr && end <= buf->ptr + buf->size) {
+		size_t consumed = end - buf->ptr;
+		memmove(buf->ptr, end, buf->size - consumed);
+		buf->size -= consumed;
+		buf->ptr[buf->size] = '\0';
+	}
+}
+
+void git_buf_truncate(git_buf *buf, size_t len)
+{
+	if (len < buf->size) {
+		buf->size = len;
+		buf->ptr[buf->size] = '\0';
+	}
+}
+
+void git_buf_rtruncate_at_char(git_buf *buf, char separator)
+{
+	ssize_t idx = git_buf_rfind_next(buf, separator);
+	git_buf_truncate(buf, idx < 0 ? 0 : (size_t)idx);
+}
+
+void git_buf_swap(git_buf *buf_a, git_buf *buf_b)
+{
+	git_buf t = *buf_a;
+	*buf_a = *buf_b;
+	*buf_b = t;
+}
+
+char *git_buf_detach(git_buf *buf)
+{
+	char *data = buf->ptr;
+
+	if (buf->asize == 0 || buf->ptr == git_buf__oom)
+		return NULL;
+
+	git_buf_init(buf, 0);
+
+	return data;
+}
+
+void git_buf_attach(git_buf *buf, char *ptr, size_t asize)
+{
+	git_buf_free(buf);
+
+	if (ptr) {
+		buf->ptr = ptr;
+		buf->size = strlen(ptr);
+		if (asize)
+			buf->asize = (asize < buf->size) ? buf->size + 1 : asize;
+		else /* pass 0 to fall back on strlen + 1 */
+			buf->asize = buf->size + 1;
+	} else {
+		git_buf_grow(buf, asize);
+	}
+}
+
+int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...)
+{
+	va_list ap;
+	int i;
+	size_t total_size = 0, original_size = buf->size;
+	char *out, *original = buf->ptr;
+
+	if (buf->size > 0 && buf->ptr[buf->size - 1] != separator)
+		++total_size; /* space for initial separator */
+
+	/* Make two passes to avoid multiple reallocation */
+
+	va_start(ap, nbuf);
+	for (i = 0; i < nbuf; ++i) {
+		const char* segment;
+		size_t segment_len;
+
+		segment = va_arg(ap, const char *);
+		if (!segment)
+			continue;
+
+		segment_len = strlen(segment);
+		total_size += segment_len;
+		if (segment_len == 0 || segment[segment_len - 1] != separator)
+			++total_size; /* space for separator */
+	}
+	va_end(ap);
+
+	/* expand buffer if needed */
+	if (total_size == 0)
+		return 0;
+	if (git_buf_grow(buf, buf->size + total_size + 1) < 0)
+		return -1;
+
+	out = buf->ptr + buf->size;
+
+	/* append separator to existing buf if needed */
+	if (buf->size > 0 && out[-1] != separator)
+		*out++ = separator;
+
+	va_start(ap, nbuf);
+	for (i = 0; i < nbuf; ++i) {
+		const char* segment;
+		size_t segment_len;
+
+		segment = va_arg(ap, const char *);
+		if (!segment)
+			continue;
+
+		/* deal with join that references buffer's original content */
+		if (segment >= original && segment < original + original_size) {
+			size_t offset = (segment - original);
+			segment = buf->ptr + offset;
+			segment_len = original_size - offset;
+		} else {
+			segment_len = strlen(segment);
+		}
+
+		/* skip leading separators */
+		if (out > buf->ptr && out[-1] == separator)
+			while (segment_len > 0 && *segment == separator) {
+				segment++;
+				segment_len--;
+			}
+
+		/* copy over next buffer */
+		if (segment_len > 0) {
+			memmove(out, segment, segment_len);
+			out += segment_len;
+		}
+
+		/* append trailing separator (except for last item) */
+		if (i < nbuf - 1 && out > buf->ptr && out[-1] != separator)
+			*out++ = separator;
+	}
+	va_end(ap);
+
+	/* set size based on num characters actually written */
+	buf->size = out - buf->ptr;
+	buf->ptr[buf->size] = '\0';
+
+	return 0;
+}
+
+int git_buf_join(
+	git_buf *buf,
+	char separator,
+	const char *str_a,
+	const char *str_b)
+{
+	size_t strlen_a = str_a ? strlen(str_a) : 0;
+	size_t strlen_b = strlen(str_b);
+	int need_sep = 0;
+	ssize_t offset_a = -1;
+
+	/* not safe to have str_b point internally to the buffer */
+	assert(str_b < buf->ptr || str_b > buf->ptr + buf->size);
+
+	/* figure out if we need to insert a separator */
+	if (separator && strlen_a) {
+		while (*str_b == separator) { str_b++; strlen_b--; }
+		if (str_a[strlen_a - 1] != separator)
+			need_sep = 1;
+	}
+
+	/* str_a could be part of the buffer */
+	if (str_a >= buf->ptr && str_a < buf->ptr + buf->size)
+		offset_a = str_a - buf->ptr;
+
+	if (git_buf_grow(buf, strlen_a + strlen_b + need_sep + 1) < 0)
+		return -1;
+
+	/* fix up internal pointers */
+	if (offset_a >= 0)
+		str_a = buf->ptr + offset_a;
+
+	/* do the actual copying */
+	if (offset_a != 0)
+		memmove(buf->ptr, str_a, strlen_a);
+	if (need_sep)
+		buf->ptr[strlen_a] = separator;
+	memcpy(buf->ptr + strlen_a + need_sep, str_b, strlen_b);
+
+	buf->size = strlen_a + strlen_b + need_sep;
+	buf->ptr[buf->size] = '\0';
+
+	return 0;
+}
+
+void git_buf_rtrim(git_buf *buf)
+{
+	while (buf->size > 0) {
+		if (!git__isspace(buf->ptr[buf->size - 1]))
+			break;
+
+		buf->size--;
+	}
+
+	buf->ptr[buf->size] = '\0';
+}
+
+int git_buf_cmp(const git_buf *a, const git_buf *b)
+{
+	int result = memcmp(a->ptr, b->ptr, min(a->size, b->size));
+	return (result != 0) ? result :
+		(a->size < b->size) ? -1 : (a->size > b->size) ? 1 : 0;
+}
+
+int git_buf_common_prefix(git_buf *buf, const git_strarray *strings)
+{
+	size_t i;
+	const char *str, *pfx;
+
+	git_buf_clear(buf);
+
+	if (!strings || !strings->count)
+		return 0;
+
+	/* initialize common prefix to first string */
+	if (git_buf_sets(buf, strings->strings[0]) < 0)
+		return -1;
+
+	/* go through the rest of the strings, truncating to shared prefix */
+	for (i = 1; i < strings->count; ++i) {
+
+		for (str = strings->strings[i], pfx = buf->ptr;
+			 *str && *str == *pfx; str++, pfx++)
+			/* scanning */;
+
+		git_buf_truncate(buf, pfx - buf->ptr);
+
+		if (!buf->size)
+			break;
+	}
+
+	return 0;
+}
+
+bool git_buf_is_binary(const git_buf *buf)
+{
+	size_t i;
+	int printable = 0, nonprintable = 0;
+
+	for (i = 0; i < buf->size; i++) {
+		unsigned char c = buf->ptr[i];
+		if (c > 0x1F && c < 0x7F)
+			printable++;
+		else if (c == '\0')
+			return true;
+		else if (!git__isspace(c))
+			nonprintable++;
+	}
+
+	return ((printable >> 7) < nonprintable);
+}
+
diff --git a/src/buffer.h b/src/buffer.h
new file mode 100644
index 000000000..50c75f64e
--- /dev/null
+++ b/src/buffer.h
@@ -0,0 +1,135 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_buffer_h__
+#define INCLUDE_buffer_h__
+
+#include "common.h"
+#include <stdarg.h>
+
+typedef struct {
+	char *ptr;
+	size_t asize, size;
+} git_buf;
+
+extern char git_buf__initbuf[];
+extern char git_buf__oom[];
+
+#define GIT_BUF_INIT { git_buf__initbuf, 0, 0 }
+
+/**
+ * Initialize a git_buf structure.
+ *
+ * For the cases where GIT_BUF_INIT cannot be used to do static
+ * initialization.
+ */
+void git_buf_init(git_buf *buf, size_t initial_size);
+
+/**
+ * Grow the buffer to hold at least `target_size` bytes.
+ *
+ * If the allocation fails, this will return an error and the buffer
+ * will be marked as invalid for future operations.  The existing
+ * contents of the buffer will be preserved however.
+ * @return 0 on success or -1 on failure
+ */
+int git_buf_grow(git_buf *buf, size_t target_size);
+
+/**
+ * Attempt to grow the buffer to hold at least `target_size` bytes.
+ *
+ * This is just like `git_buf_grow` except that even if the allocation
+ * fails, the git_buf will still be left in a valid state.
+ */
+int git_buf_try_grow(git_buf *buf, size_t target_size);
+
+void git_buf_free(git_buf *buf);
+void git_buf_swap(git_buf *buf_a, git_buf *buf_b);
+char *git_buf_detach(git_buf *buf);
+void git_buf_attach(git_buf *buf, char *ptr, size_t asize);
+
+/**
+ * Test if there have been any reallocation failures with this git_buf.
+ *
+ * Any function that writes to a git_buf can fail due to memory allocation
+ * issues.  If one fails, the git_buf will be marked with an OOM error and
+ * further calls to modify the buffer will fail.  Check git_buf_oom() at the
+ * end of your sequence and it will be true if you ran out of memory at any
+ * point with that buffer.
+ *
+ * @return false if no error, true if allocation error
+ */
+GIT_INLINE(bool) git_buf_oom(const git_buf *buf)
+{
+	return (buf->ptr == git_buf__oom);
+}
+
+/*
+ * Functions below that return int value error codes will return 0 on
+ * success or -1 on failure (which generally means an allocation failed).
+ * Using a git_buf where the allocation has failed with result in -1 from
+ * all further calls using that buffer.  As a result, you can ignore the
+ * return code of these functions and call them in a series then just call
+ * git_buf_oom at the end.
+ */
+int git_buf_set(git_buf *buf, const char *data, size_t len);
+int git_buf_sets(git_buf *buf, const char *string);
+int git_buf_putc(git_buf *buf, char c);
+int git_buf_put(git_buf *buf, const char *data, size_t len);
+int git_buf_puts(git_buf *buf, const char *string);
+int git_buf_printf(git_buf *buf, const char *format, ...) GIT_FORMAT_PRINTF(2, 3);
+int git_buf_vprintf(git_buf *buf, const char *format, va_list ap);
+void git_buf_clear(git_buf *buf);
+void git_buf_consume(git_buf *buf, const char *end);
+void git_buf_truncate(git_buf *buf, size_t len);
+void git_buf_rtruncate_at_char(git_buf *path, char separator);
+
+int git_buf_join_n(git_buf *buf, char separator, int nbuf, ...);
+int git_buf_join(git_buf *buf, char separator, const char *str_a, const char *str_b);
+
+/**
+ * Join two strings as paths, inserting a slash between as needed.
+ * @return 0 on success, -1 on failure
+ */
+GIT_INLINE(int) git_buf_joinpath(git_buf *buf, const char *a, const char *b)
+{
+	return git_buf_join(buf, '/', a, b);
+}
+
+GIT_INLINE(const char *) git_buf_cstr(const git_buf *buf)
+{
+	return buf->ptr;
+}
+
+GIT_INLINE(size_t) git_buf_len(const git_buf *buf)
+{
+	return buf->size;
+}
+
+void git_buf_copy_cstr(char *data, size_t datasize, const git_buf *buf);
+
+#define git_buf_PUTS(buf, str) git_buf_put(buf, str, sizeof(str) - 1)
+
+GIT_INLINE(ssize_t) git_buf_rfind_next(git_buf *buf, char ch)
+{
+	ssize_t idx = (ssize_t)buf->size - 1;
+	while (idx >= 0 && buf->ptr[idx] == ch) idx--;
+	while (idx >= 0 && buf->ptr[idx] != ch) idx--;
+	return idx;
+}
+
+/* Remove whitespace from the end of the buffer */
+void git_buf_rtrim(git_buf *buf);
+
+int git_buf_cmp(const git_buf *a, const git_buf *b);
+
+/* Fill buf with the common prefix of a array of strings */
+int git_buf_common_prefix(git_buf *buf, const git_strarray *strings);
+
+/* Check if buffer looks like it contains binary data */
+bool git_buf_is_binary(const git_buf *buf);
+
+#endif
diff --git a/src/cache.c b/src/cache.c
index 433fc3d9c..31da3c36e 100644
--- a/src/cache.c
+++ b/src/cache.c
@@ -1,63 +1,34 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
 #include "repository.h"
 #include "commit.h"
 #include "thread-utils.h"
+#include "util.h"
 #include "cache.h"
 
 int git_cache_init(git_cache *cache, size_t size, git_cached_obj_freeptr free_ptr)
 {
-	size_t i;
-
 	if (size < 8)
 		size = 8;
+	size = git__size_t_powerof2(size);
 
-	/* round up size to closest power of 2 */
-	size--;
-	size |= size >> 1;
-	size |= size >> 2;
-	size |= size >> 4;
-	size |= size >> 8;
-	size |= size >> 16;
-
-	cache->size_mask = size;
+	cache->size_mask = size - 1;
 	cache->lru_count = 0;
 	cache->free_obj = free_ptr;
 
-	cache->nodes = git__malloc((size + 1) * sizeof(cache_node));
-	if (cache->nodes == NULL)
-		return GIT_ENOMEM;
+	git_mutex_init(&cache->lock);
 
-	for (i = 0; i < (size + 1); ++i) {
-		git_mutex_init(&cache->nodes[i].lock);
-		cache->nodes[i].ptr = NULL;
-	}
+	cache->nodes = git__malloc(size * sizeof(git_cached_obj *));
+	GITERR_CHECK_ALLOC(cache->nodes);
 
-	return GIT_SUCCESS;
+	memset(cache->nodes, 0x0, size * sizeof(git_cached_obj *));
+	return 0;
 }
 
 void git_cache_free(git_cache *cache)
@@ -65,65 +36,64 @@ void git_cache_free(git_cache *cache)
 	size_t i;
 
 	for (i = 0; i < (cache->size_mask + 1); ++i) {
-		if (cache->nodes[i].ptr)
-			git_cached_obj_decref(cache->nodes[i].ptr, cache->free_obj);
-
-		git_mutex_free(&cache->nodes[i].lock);
+		if (cache->nodes[i] != NULL)
+			git_cached_obj_decref(cache->nodes[i], cache->free_obj);
 	}
 
-	free(cache->nodes);
+	git__free(cache->nodes);
 }
 
 void *git_cache_get(git_cache *cache, const git_oid *oid)
 {
-	const uint32_t *hash;
-	cache_node *node = NULL;
-	void *result = NULL;
+	uint32_t hash;
+	git_cached_obj *node = NULL, *result = NULL;
 
-	hash = (const uint32_t *)oid->id;
-	node = &cache->nodes[hash[0] & cache->size_mask];
+	memcpy(&hash, oid->id, sizeof(hash));
 
-	git_mutex_lock(&node->lock);
+	git_mutex_lock(&cache->lock);
 	{
-		if (node->ptr && git_cached_obj_compare(node->ptr, oid) == 0) {
-			git_cached_obj_incref(node->ptr);
-			result = node->ptr;
+		node = cache->nodes[hash & cache->size_mask];
+
+		if (node != NULL && git_oid_cmp(&node->oid, oid) == 0) {
+			git_cached_obj_incref(node);
+			result = node;
 		}
 	}
-	git_mutex_unlock(&node->lock);
+	git_mutex_unlock(&cache->lock);
 
 	return result;
 }
 
-void *git_cache_try_store(git_cache *cache, void *entry)
+void *git_cache_try_store(git_cache *cache, void *_entry)
 {
-	const uint32_t *hash;
-	const git_oid *oid;
-	cache_node *node = NULL;
+	git_cached_obj *entry = _entry;
+	uint32_t hash;
 
-	oid = &((git_cached_obj*)entry)->oid;
-	hash = (const uint32_t *)oid->id;
-	node = &cache->nodes[hash[0] & cache->size_mask];
+	memcpy(&hash, &entry->oid, sizeof(hash));
 
 	/* increase the refcount on this object, because
 	 * the cache now owns it */
 	git_cached_obj_incref(entry);
-	git_mutex_lock(&node->lock);
-
-	if (node->ptr == NULL) {
-		node->ptr = entry;
-	} else if (git_cached_obj_compare(node->ptr, oid) == 0) {
-		git_cached_obj_decref(entry, cache->free_obj);
-		entry = node->ptr;
-	} else {
-		git_cached_obj_decref(node->ptr, cache->free_obj);
-		node->ptr = entry;
+
+	git_mutex_lock(&cache->lock);
+	{
+		git_cached_obj *node = cache->nodes[hash & cache->size_mask];
+
+		if (node == NULL) {
+			cache->nodes[hash & cache->size_mask] = entry;
+		} else if (git_oid_cmp(&node->oid, &entry->oid) == 0) {
+			git_cached_obj_decref(entry, cache->free_obj);
+			entry = node;
+		} else {
+			git_cached_obj_decref(node, cache->free_obj);
+			cache->nodes[hash & cache->size_mask] = entry;
+		}
 	}
+	git_mutex_unlock(&cache->lock);
 
 	/* increase the refcount again, because we are
 	 * returning it to the user */
 	git_cached_obj_incref(entry);
-	git_mutex_unlock(&node->lock);
 
 	return entry;
 }
diff --git a/src/cache.h b/src/cache.h
index 4794dea3a..6dc706897 100644
--- a/src/cache.h
+++ b/src/cache.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_cache_h__
 #define INCLUDE_cache_h__
 
@@ -17,42 +23,32 @@ typedef struct {
 } git_cached_obj;
 
 typedef struct {
-	git_cached_obj *ptr;
+	git_cached_obj **nodes;
 	git_mutex lock;
-} cache_node;
-
-typedef struct {
-	cache_node *nodes;
 
 	unsigned int lru_count;
 	size_t size_mask;
 	git_cached_obj_freeptr free_obj;
 } git_cache;
 
-
 int git_cache_init(git_cache *cache, size_t size, git_cached_obj_freeptr free_ptr);
 void git_cache_free(git_cache *cache);
 
 void *git_cache_try_store(git_cache *cache, void *entry);
 void *git_cache_get(git_cache *cache, const git_oid *oid);
 
-
-GIT_INLINE(int) git_cached_obj_compare(git_cached_obj *obj, const git_oid *oid)
-{
-	return git_oid_cmp(&obj->oid, oid);
-}
-
-GIT_INLINE(void) git_cached_obj_incref(git_cached_obj *obj)
+GIT_INLINE(void) git_cached_obj_incref(void *_obj)
 {
+	git_cached_obj *obj = _obj;
 	git_atomic_inc(&obj->refcount);
 }
 
-GIT_INLINE(void) git_cached_obj_decref(git_cached_obj *obj, git_cached_obj_freeptr free_obj)
+GIT_INLINE(void) git_cached_obj_decref(void *_obj, git_cached_obj_freeptr free_obj)
 {
+	git_cached_obj *obj = _obj;
+
 	if (git_atomic_dec(&obj->refcount) == 0)
 		free_obj(obj);
 }
 
-
-
 #endif
diff --git a/src/cc-compat.h b/src/cc-compat.h
index cf6cccf12..9f23dcae2 100644
--- a/src/cc-compat.h
+++ b/src/cc-compat.h
@@ -1,5 +1,8 @@
 /*
- * cc-compat.h - C compiler compat macros for internal use
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #ifndef INCLUDE_compat_h__
 #define INCLUDE_compat_h__
@@ -8,66 +11,59 @@
  * See if our compiler is known to support flexible array members.
  */
 #ifndef GIT_FLEX_ARRAY
-# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
-#  define GIT_FLEX_ARRAY /* empty */
-# elif defined(__GNUC__)
-#  if (__GNUC__ >= 3)
-#   define GIT_FLEX_ARRAY /* empty */
-#  else
-#   define GIT_FLEX_ARRAY 0 /* older GNU extension */
-#  endif
-# endif
+#	if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)
+#		define GIT_FLEX_ARRAY /* empty */
+#	elif defined(__GNUC__)
+#		if (__GNUC__ >= 3)
+#			define GIT_FLEX_ARRAY /* empty */
+#		else
+#			define GIT_FLEX_ARRAY 0 /* older GNU extension */
+#		endif
+#	endif
 
 /* Default to safer but a bit wasteful traditional style */
-# ifndef GIT_FLEX_ARRAY
-#  define GIT_FLEX_ARRAY 1
-# endif
+#	ifndef GIT_FLEX_ARRAY
+#		define GIT_FLEX_ARRAY 1
+#	endif
 #endif
 
 #ifdef __GNUC__
-# define GIT_TYPEOF(x) (__typeof__(x))
+#	define GIT_TYPEOF(x) (__typeof__(x))
 #else
-# define GIT_TYPEOF(x)
+#	define GIT_TYPEOF(x)
 #endif
 
-#ifdef __cplusplus
-# define GIT_UNUSED(x)
-#else
-# ifdef __GNUC__
-#  define GIT_UNUSED(x) x __attribute__ ((__unused__))
-# else
-#  define GIT_UNUSED(x) x
-# endif
-#endif
-
-#if defined(_MSC_VER)
-#define GIT_UNUSED_ARG(x) ((void)(x)); /* note trailing ; */
-#else
-#define GIT_UNUSED_ARG(x)
-#endif
-
-/*
- * Does our compiler/platform support the C99 <inttypes.h> and
- * <stdint.h> header files. (C99 requires that <inttypes.h>
- * includes <stdint.h>).
- */
-#if !defined(_MSC_VER)
-# define GIT_HAVE_INTTYPES_H 1
-#endif
+#define GIT_UNUSED(x) ((void)(x))
 
 /* Define the printf format specifer to use for size_t output */
 #if defined(_MSC_VER) || defined(__MINGW32__)
-# define PRIuZ "Iu"
+#	define PRIuZ "Iu"
 #else
-# define PRIuZ "zu"
+#	define PRIuZ "zu"
 #endif
 
 /* Micosoft Visual C/C++ */
 #if defined(_MSC_VER)
 /* disable "deprecated function" warnings */
-# pragma warning ( disable : 4996 )
+#	pragma warning ( disable : 4996 )
 /* disable "conditional expression is constant" level 4 warnings */
-# pragma warning ( disable : 4127 )
+#	pragma warning ( disable : 4127 )
+#endif
+
+#if defined (_MSC_VER)
+	typedef unsigned char bool;
+#	define true 1
+#	define false 0
+#else
+#	include <stdbool.h>
+#endif
+
+#ifndef va_copy
+#	ifdef __va_copy
+#		define va_copy(dst, src) __va_copy(dst, src)
+#	else
+#		define va_copy(dst, src) ((dst) = (src))
+#	endif
 #endif
 
 #endif /* INCLUDE_compat_h__ */
diff --git a/src/commit.c b/src/commit.c
index bfae0592e..2f40dc67d 100644
--- a/src/commit.c
+++ b/src/commit.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "git2/common.h"
@@ -32,25 +14,17 @@
 #include "odb.h"
 #include "commit.h"
 #include "signature.h"
+#include "message.h"
 
 #include <stdarg.h>
 
-#define COMMIT_BASIC_PARSE 0x0
-#define COMMIT_FULL_PARSE 0x1
-
-#define COMMIT_PRINT(commit) {\
-	char oid[41]; oid[40] = 0;\
-	git_oid_fmt(oid, &commit->object.id);\
-	printf("Oid: %s | In degree: %d | Time: %u\n", oid, commit->in_degree, commit->commit_time);\
-}
-
 static void clear_parents(git_commit *commit)
 {
 	unsigned int i;
 
 	for (i = 0; i < commit->parent_oids.length; ++i) {
 		git_oid *parent = git_vector_get(&commit->parent_oids, i);
-		free(parent);
+		git__free(parent);
 	}
 
 	git_vector_clear(&commit->parent_oids);
@@ -64,9 +38,9 @@ void git_commit__free(git_commit *commit)
 	git_signature_free(commit->author);
 	git_signature_free(commit->committer);
 
-	free(commit->message);
-	free(commit->message_short);
-	free(commit);
+	git__free(commit->message);
+	git__free(commit->message_encoding);
+	git__free(commit);
 }
 
 const git_oid *git_commit_id(git_commit *c)
@@ -74,97 +48,97 @@ const git_oid *git_commit_id(git_commit *c)
 	return git_object_id((git_object *)c);
 }
 
-
 int git_commit_create_v(
 		git_oid *oid,
 		git_repository *repo,
 		const char *update_ref,
 		const git_signature *author,
 		const git_signature *committer,
+		const char *message_encoding,
 		const char *message,
-		const git_oid *tree_oid,
+		const git_tree *tree,
 		int parent_count,
 		...)
 {
 	va_list ap;
-	int i, error;
-	const git_oid **oids;
+	int i, res;
+	const git_commit **parents;
 
-	oids = git__malloc(parent_count * sizeof(git_oid *));
+	parents = git__malloc(parent_count * sizeof(git_commit *));
+	GITERR_CHECK_ALLOC(parents);
 
 	va_start(ap, parent_count);
 	for (i = 0; i < parent_count; ++i)
-		oids[i] = va_arg(ap, const git_oid *);
+		parents[i] = va_arg(ap, const git_commit *);
 	va_end(ap);
 
-	error = git_commit_create(
-		oid, repo, update_ref, author, committer, message,
-		tree_oid, parent_count, oids);
-
-	free((void *)oids);
+	res = git_commit_create(
+		oid, repo, update_ref, author, committer,
+		message_encoding, message,
+		tree, parent_count, parents);
 
-	return error;
+	git__free((void *)parents);
+	return res;
 }
 
-int git_commit_create_ov(
-		git_oid *oid,
-		git_repository *repo,
-		const char *update_ref,
-		const git_signature *author,
-		const git_signature *committer,
-		const char *message,
-		const git_tree *tree,
-		int parent_count,
-		...)
+/* Update the reference named `ref_name` so it points to `oid` */
+static int update_reference(git_repository *repo, git_oid *oid, const char *ref_name)
 {
-	va_list ap;
-	int i, error;
-	const git_oid **oids;
+	git_reference *ref;
+	int res;
 
-	oids = git__malloc(parent_count * sizeof(git_oid *));
+	res = git_reference_lookup(&ref, repo, ref_name);
 
-	va_start(ap, parent_count);
-	for (i = 0; i < parent_count; ++i)
-		oids[i] = git_object_id(va_arg(ap, const git_object *));
-	va_end(ap);
+	/* If we haven't found the reference at all, we assume we need to create
+	 * a new reference and that's it */
+	if (res == GIT_ENOTFOUND) {
+		giterr_clear();
+		return git_reference_create_oid(NULL, repo, ref_name, oid, 1);
+	}
 
-	error = git_commit_create(
-		oid, repo, update_ref, author, committer, message,
-		git_object_id((git_object *)tree),
-		parent_count, oids);
+	if (res < 0)
+		return -1;
 
-	free((void *)oids);
+	/* If we have found a reference, but it's symbolic, we need to update
+	 * the direct reference it points to */
+	if (git_reference_type(ref) == GIT_REF_SYMBOLIC) {
+		git_reference *aux;
+		const char *sym_target;
 
-	return error;
-}
+		/* The target pointed at by this reference */
+		sym_target = git_reference_target(ref);
 
-int git_commit_create_o(
-		git_oid *oid,
-		git_repository *repo,
-		const char *update_ref,
-		const git_signature *author,
-		const git_signature *committer,
-		const char *message,
-		const git_tree *tree,
-		int parent_count,
-		const git_commit *parents[])
-{
-	int i, error;
-	const git_oid **oids;
+		/* resolve the reference to the target it points to */
+		res = git_reference_resolve(&aux, ref);
+
+		/*
+		 * if the symbolic reference pointed to an inexisting ref,
+		 * this is means we're creating a new branch, for example.
+		 * We need to create a new direct reference with that name
+		 */
+		if (res == GIT_ENOTFOUND) {
+			giterr_clear();
+			res = git_reference_create_oid(NULL, repo, sym_target, oid, 1);
+			git_reference_free(ref);
+			return res;
+		}
 
-	oids = git__malloc(parent_count * sizeof(git_oid *));
+		/* free the original symbolic reference now; not before because
+		 * we're using the `sym_target` pointer */
+		git_reference_free(ref);
 
-	for (i = 0; i < parent_count; ++i)
-		oids[i] = git_object_id((git_object *)parents[i]);
+		if (res < 0)
+			return -1;
 
-	error = git_commit_create(
-		oid, repo, update_ref, author, committer, message,
-		git_object_id((git_object *)tree),
-		parent_count, oids);
-	
-	free((void *)oids);
+		/* store the newly found direct reference in its place */
+		ref = aux;
+	}
 
-	return error;
+	/* ref is made to point to `oid`: ref is either the original reference,
+	 * or the target of the symbolic reference we've looked up */
+	res = git_reference_set_oid(ref, oid);
+	git_reference_free(ref);
+	return res;
 }
 
 int git_commit_create(
@@ -173,150 +147,137 @@ int git_commit_create(
 		const char *update_ref,
 		const git_signature *author,
 		const git_signature *committer,
+		const char *message_encoding,
 		const char *message,
-		const git_oid *tree_oid,
+		const git_tree *tree,
 		int parent_count,
-		const git_oid *parents[])
+		const git_commit *parents[])
 {
-	size_t final_size = 0;
-	int message_length, author_length, committer_length;
+	git_buf commit = GIT_BUF_INIT, cleaned_message = GIT_BUF_INIT;
+	int i;
+	git_odb *odb;
 
-	char *author_str, *committer_str;
+	assert(git_object_owner((const git_object *)tree) == repo);
 
-	int error, i;
-	git_odb_stream *stream;
+	git_oid__writebuf(&commit, "tree ", git_object_id((const git_object *)tree));
 
-	message_length = strlen(message);
-	author_length = git_signature__write(&author_str, "author", author);
-	committer_length = git_signature__write(&committer_str, "committer", committer);
-
-	if (author_length < 0 || committer_length < 0)
-		return git__throw(GIT_EINVALIDARGS, "Cannot create commit. Failed to parse signature");
+	for (i = 0; i < parent_count; ++i) {
+		assert(git_object_owner((const git_object *)parents[i]) == repo);
+		git_oid__writebuf(&commit, "parent ", git_object_id((const git_object *)parents[i]));
+	}
 
-	final_size += GIT_OID_LINE_LENGTH("tree");
-	final_size += GIT_OID_LINE_LENGTH("parent") * parent_count;
-	final_size += author_length;
-	final_size += committer_length;
-	final_size += 1 + message_length;
+	git_signature__writebuf(&commit, "author ", author);
+	git_signature__writebuf(&commit, "committer ", committer);
 
-	if ((error = git_odb_open_wstream(&stream, repo->db, final_size, GIT_OBJ_COMMIT)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to create commit");
+	if (message_encoding != NULL)
+		git_buf_printf(&commit, "encoding %s\n", message_encoding);
 
-	git__write_oid(stream, "tree", tree_oid);
+	git_buf_putc(&commit, '\n');
 
-	for (i = 0; i < parent_count; ++i)
-		git__write_oid(stream, "parent", parents[i]);
+	/* Remove comments by default */
+	if (git_message_prettify(&cleaned_message, message, 1) < 0)
+		goto on_error;
 
-	stream->write(stream, author_str, author_length);
-	free(author_str);
+	if (git_buf_puts(&commit, git_buf_cstr(&cleaned_message)) < 0)
+		goto on_error;
 
-	stream->write(stream, committer_str, committer_length);
-	free(committer_str);
+	git_buf_free(&cleaned_message);
 
+	if (git_repository_odb__weakptr(&odb, repo) < 0)
+		goto on_error;
 
-	stream->write(stream, "\n", 1);
-	stream->write(stream, message, message_length);
+	if (git_odb_write(oid, odb, commit.ptr, commit.size, GIT_OBJ_COMMIT) < 0)
+		goto on_error;
 
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
+	git_buf_free(&commit);
 
-	if (error == GIT_SUCCESS && update_ref != NULL) {
-		git_reference *head;
+	if (update_ref != NULL)
+		return update_reference(repo, oid, update_ref);
 
-		error = git_reference_lookup(&head, repo, update_ref);
-		if (error < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to create commit");
+	return 0;
 
-		error = git_reference_resolve(&head, head);
-		if (error < GIT_SUCCESS) {
-			if (error != GIT_ENOTFOUND)
-				return git__rethrow(error, "Failed to create commit");
-		/*
-		 * The target of the reference was not found. This can happen
-		 * just after a repository has been initialized (the master
-		 * branch doesn't exist yet, as it doesn't have anything to
-		 * point to) or after an orphan checkout, so if the target
-		 * branch doesn't exist yet, create it and return.
-		 */
-			return git_reference_create_oid_f(&head, repo, git_reference_target(head), oid);
-		}
-
-		error = git_reference_set_oid(head, oid);
-	}
-
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to create commit");
-
-	return GIT_SUCCESS;
+on_error:
+	git_buf_free(&commit);
+	git_buf_free(&cleaned_message);
+	giterr_set(GITERR_OBJECT, "Failed to create commit.");
+	return -1;
 }
 
-int commit_parse_buffer(git_commit *commit, const void *data, size_t len)
+int git_commit__parse_buffer(git_commit *commit, const void *data, size_t len)
 {
-	const char *buffer = (char *)data;
-	const char *buffer_end = (char *)data + len;
+	const char *buffer = data;
+	const char *buffer_end = (const char *)data + len;
 
 	git_oid parent_oid;
-	int error;
 
 	git_vector_init(&commit->parent_oids, 4, NULL);
 
-	if ((error = git__parse_oid(&commit->tree_oid, &buffer, buffer_end, "tree ")) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to parse buffer");
+	if (git_oid__parse(&commit->tree_oid, &buffer, buffer_end, "tree ") < 0)
+		goto bad_buffer;
 
 	/*
 	 * TODO: commit grafts!
 	 */
 
-	while (git__parse_oid(&parent_oid, &buffer, buffer_end, "parent ") == GIT_SUCCESS) {
+	while (git_oid__parse(&parent_oid, &buffer, buffer_end, "parent ") == 0) {
 		git_oid *new_oid;
 
 		new_oid = git__malloc(sizeof(git_oid));
+		GITERR_CHECK_ALLOC(new_oid);
+
 		git_oid_cpy(new_oid, &parent_oid);
 
-		if (git_vector_insert(&commit->parent_oids, new_oid) < GIT_SUCCESS)
-			return GIT_ENOMEM;
+		if (git_vector_insert(&commit->parent_oids, new_oid) < 0)
+			return -1;
 	}
 
 	commit->author = git__malloc(sizeof(git_signature));
-	if ((error = git_signature__parse(commit->author, &buffer, buffer_end, "author ")) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to parse buffer");
+	GITERR_CHECK_ALLOC(commit->author);
+
+	if (git_signature__parse(commit->author, &buffer, buffer_end, "author ", '\n') < 0)
+		return -1;
 
 	/* Always parse the committer; we need the commit time */
 	commit->committer = git__malloc(sizeof(git_signature));
-	if ((error = git_signature__parse(commit->committer, &buffer, buffer_end, "committer ")) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to parse buffer");
+	GITERR_CHECK_ALLOC(commit->committer);
 
-	/* parse commit message */
-	while (buffer <= buffer_end && *buffer == '\n')
-		buffer++;
+	if (git_signature__parse(commit->committer, &buffer, buffer_end, "committer ", '\n') < 0)
+		return -1;
 
-	if (buffer < buffer_end) {
-		const char *line_end;
-		size_t message_len;
+	if (git__prefixcmp(buffer, "encoding ") == 0) {
+		const char *encoding_end;
+		buffer += strlen("encoding ");
 
-		/* Long message */
-		message_len = buffer_end - buffer;
-		commit->message = git__malloc(message_len + 1);
-		memcpy(commit->message, buffer, message_len);
-		commit->message[message_len] = 0;
+		encoding_end = buffer;
+		while (encoding_end < buffer_end && *encoding_end != '\n')
+			encoding_end++;
 
-		/* Short message */
-		if((line_end = memchr(buffer, '\n', buffer_end - buffer)) == NULL)
-			line_end = buffer_end;
-		message_len = line_end - buffer;
+		commit->message_encoding = git__strndup(buffer, encoding_end - buffer);
+		GITERR_CHECK_ALLOC(commit->message_encoding);
 
-		commit->message_short = git__malloc(message_len + 1);
-		memcpy(commit->message_short, buffer, message_len);
-		commit->message_short[message_len] = 0;
+		buffer = encoding_end;
 	}
 
-	return GIT_SUCCESS;
+	/* parse commit message */
+	while (buffer < buffer_end - 1 && *buffer == '\n')
+		buffer++;
+
+	if (buffer <= buffer_end) {
+		commit->message = git__strndup(buffer, buffer_end - buffer);
+		GITERR_CHECK_ALLOC(commit->message);
+	}
+
+	return 0;
+
+bad_buffer:
+	giterr_set(GITERR_OBJECT, "Failed to parse bad commit object");
+	return -1;
 }
 
 int git_commit__parse(git_commit *commit, git_odb_object *obj)
 {
 	assert(commit);
-	return commit_parse_buffer(commit, obj->raw.data, obj->raw.len);
+	return git_commit__parse_buffer(commit, obj->raw.data, obj->raw.len);
 }
 
 #define GIT_COMMIT_GETTER(_rvalue, _name, _return) \
@@ -329,7 +290,7 @@ int git_commit__parse(git_commit *commit, git_odb_object *obj)
 GIT_COMMIT_GETTER(const git_signature *, author, commit->author)
 GIT_COMMIT_GETTER(const git_signature *, committer, commit->committer)
 GIT_COMMIT_GETTER(const char *, message, commit->message)
-GIT_COMMIT_GETTER(const char *, message_short, commit->message_short)
+GIT_COMMIT_GETTER(const char *, message_encoding, commit->message_encoding)
 GIT_COMMIT_GETTER(git_time_t, time, commit->committer->when.time)
 GIT_COMMIT_GETTER(int, time_offset, commit->committer->when.offset)
 GIT_COMMIT_GETTER(unsigned int, parentcount, commit->parent_oids.length)
@@ -348,8 +309,10 @@ int git_commit_parent(git_commit **parent, git_commit *commit, unsigned int n)
 	assert(commit);
 
 	parent_oid = git_vector_get(&commit->parent_oids, n);
-	if (parent_oid == NULL)
-		return git__throw(GIT_ENOTFOUND, "Parent %u does not exist", n);
+	if (parent_oid == NULL) {
+		giterr_set(GITERR_INVALID, "Parent %u does not exist", n);
+		return GIT_ENOTFOUND;
+	}
 
 	return git_commit_lookup(parent, commit->object.repo, parent_oid);
 }
diff --git a/src/commit.h b/src/commit.h
index 3d15c5044..d9f492862 100644
--- a/src/commit.h
+++ b/src/commit.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_commit_h__
 #define INCLUDE_commit_h__
 
@@ -17,11 +23,12 @@ struct git_commit {
 	git_signature *author;
 	git_signature *committer;
 
+	char *message_encoding;
 	char *message;
-	char *message_short;
 };
 
 void git_commit__free(git_commit *c);
 int git_commit__parse(git_commit *commit, git_odb_object *obj);
 
+int git_commit__parse_buffer(git_commit *commit, const void *data, size_t len);
 #endif
diff --git a/src/common.h b/src/common.h
index f4f11fd2f..e2a300291 100644
--- a/src/common.h
+++ b/src/common.h
@@ -1,19 +1,15 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_common_h__
 #define INCLUDE_common_h__
 
-/** Force 64 bit off_t size on POSIX. */
-#define _FILE_OFFSET_BITS 64
-
-#if defined(_WIN32) && !defined(__CYGWIN__)
-#define GIT_WIN32 1
-#endif
-
-#include "git2/thread-utils.h"
+#include "git2/common.h"
 #include "cc-compat.h"
 
-#ifdef GIT_HAVE_INTTYPES_H
-# include <inttypes.h>
-#endif
 #include <assert.h>
 #include <errno.h>
 #include <limits.h>
@@ -29,35 +25,47 @@
 # include <io.h>
 # include <direct.h>
 # include <windows.h>
-# include "msvc-compat.h"
-# include "mingw-compat.h"
+# include "win32/msvc-compat.h"
+# include "win32/mingw-compat.h"
 # ifdef GIT_THREADS
-#  include "win32/pthread.h"
+#	include "win32/pthread.h"
 #endif
 
 # define snprintf _snprintf
 
-typedef SSIZE_T ssize_t;
-
 #else
 # include <unistd.h>
-# include <arpa/inet.h>
 
 # ifdef GIT_THREADS
-#  include <pthread.h>
+#	include <pthread.h>
 # endif
 #endif
 
-#include "git2/common.h"
 #include "git2/types.h"
 #include "git2/errors.h"
 #include "thread-utils.h"
 #include "bswap.h"
 
-#define GIT_PATH_MAX 4096
-extern int git__throw(int error, const char *, ...) GIT_FORMAT_PRINTF(2, 3);
-extern int git__rethrow(int error, const char *, ...) GIT_FORMAT_PRINTF(2, 3);
+#include <regex.h>
+
+extern void git___throw(const char *, ...) GIT_FORMAT_PRINTF(1, 2);
+#define git__throw(error, ...) \
+	(git___throw(__VA_ARGS__), error)
+
+extern void git___rethrow(const char *, ...) GIT_FORMAT_PRINTF(1, 2);
+#define git__rethrow(error, ...) \
+	(git___rethrow(__VA_ARGS__), error)
+
+
+#define GITERR_CHECK_ALLOC(ptr) if (ptr == NULL) { return -1; }
+
+void giterr_set_oom(void);
+void giterr_set(int error_class, const char *string, ...);
+void giterr_clear(void);
+void giterr_set_str(int error_class, const char *string);
+void giterr_set_regex(const regex_t *regex, int error_code);
 
 #include "util.h"
 
+
 #endif /* INCLUDE_common_h__ */
diff --git a/src/compat/fnmatch.c b/src/compat/fnmatch.c
new file mode 100644
index 000000000..835d811bc
--- /dev/null
+++ b/src/compat/fnmatch.c
@@ -0,0 +1,180 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+/*
+ * Function fnmatch() as specified in POSIX 1003.2-1992, section B.6.
+ * Compares a filename or pathname to a pattern.
+ */
+
+#include <ctype.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "fnmatch.h"
+
+#define EOS		'\0'
+
+#define RANGE_MATCH		1
+#define RANGE_NOMATCH		0
+#define RANGE_ERROR		(-1)
+
+static int rangematch(const char *, char, int, char **);
+
+int
+p_fnmatch(const char *pattern, const char *string, int flags)
+{
+		const char *stringstart;
+		char *newp;
+		char c, test;
+
+		for (stringstart = string;;)
+				switch (c = *pattern++) {
+				case EOS:
+						if ((flags & FNM_LEADING_DIR) && *string == '/')
+								return (0);
+						return (*string == EOS ? 0 : FNM_NOMATCH);
+				case '?':
+						if (*string == EOS)
+								return (FNM_NOMATCH);
+						if (*string == '/' && (flags & FNM_PATHNAME))
+								return (FNM_NOMATCH);
+						if (*string == '.' && (flags & FNM_PERIOD) &&
+							(string == stringstart ||
+							((flags & FNM_PATHNAME) && *(string - 1) == '/')))
+								return (FNM_NOMATCH);
+						++string;
+						break;
+				case '*':
+						c = *pattern;
+						/* Collapse multiple stars. */
+						while (c == '*')
+								c = *++pattern;
+
+						if (*string == '.' && (flags & FNM_PERIOD) &&
+							(string == stringstart ||
+							((flags & FNM_PATHNAME) && *(string - 1) == '/')))
+								return (FNM_NOMATCH);
+
+						/* Optimize for pattern with * at end or before /. */
+						if (c == EOS) {
+								if (flags & FNM_PATHNAME)
+										return ((flags & FNM_LEADING_DIR) ||
+											strchr(string, '/') == NULL ?
+											0 : FNM_NOMATCH);
+								else
+										return (0);
+						} else if (c == '/' && (flags & FNM_PATHNAME)) {
+								if ((string = strchr(string, '/')) == NULL)
+										return (FNM_NOMATCH);
+								break;
+						}
+
+						/* General case, use recursion. */
+						while ((test = *string) != EOS) {
+								if (!p_fnmatch(pattern, string, flags & ~FNM_PERIOD))
+										return (0);
+								if (test == '/' && (flags & FNM_PATHNAME))
+										break;
+								++string;
+						}
+						return (FNM_NOMATCH);
+				case '[':
+						if (*string == EOS)
+								return (FNM_NOMATCH);
+						if (*string == '/' && (flags & FNM_PATHNAME))
+								return (FNM_NOMATCH);
+						if (*string == '.' && (flags & FNM_PERIOD) &&
+							(string == stringstart ||
+							((flags & FNM_PATHNAME) && *(string - 1) == '/')))
+								return (FNM_NOMATCH);
+
+						switch (rangematch(pattern, *string, flags, &newp)) {
+						case RANGE_ERROR:
+								/* not a good range, treat as normal text */
+								goto normal;
+						case RANGE_MATCH:
+								pattern = newp;
+								break;
+						case RANGE_NOMATCH:
+								return (FNM_NOMATCH);
+						}
+						++string;
+						break;
+				case '\\':
+						if (!(flags & FNM_NOESCAPE)) {
+								if ((c = *pattern++) == EOS) {
+										c = '\\';
+										--pattern;
+								}
+						}
+						/* FALLTHROUGH */
+				default:
+				normal:
+						if (c != *string && !((flags & FNM_CASEFOLD) &&
+									(tolower((unsigned char)c) ==
+									tolower((unsigned char)*string))))
+								return (FNM_NOMATCH);
+						++string;
+						break;
+				}
+		/* NOTREACHED */
+}
+
+static int
+rangematch(const char *pattern, char test, int flags, char **newp)
+{
+		int negate, ok;
+		char c, c2;
+
+		/*
+			* A bracket expression starting with an unquoted circumflex
+			* character produces unspecified results (IEEE 1003.2-1992,
+			* 3.13.2). This implementation treats it like '!', for
+			* consistency with the regular expression syntax.
+			* J.T. Conklin (conklin@ngai.kaleida.com)
+			*/
+		if ((negate = (*pattern == '!' || *pattern == '^')) != 0)
+				++pattern;
+
+		if (flags & FNM_CASEFOLD)
+				test = (char)tolower((unsigned char)test);
+
+		/*
+			* A right bracket shall lose its special meaning and represent
+			* itself in a bracket expression if it occurs first in the list.
+			* -- POSIX.2 2.8.3.2
+			*/
+		ok = 0;
+		c = *pattern++;
+		do {
+				if (c == '\\' && !(flags & FNM_NOESCAPE))
+						c = *pattern++;
+				if (c == EOS)
+						return (RANGE_ERROR);
+				if (c == '/' && (flags & FNM_PATHNAME))
+						return (RANGE_NOMATCH);
+				if ((flags & FNM_CASEFOLD))
+						c = (char)tolower((unsigned char)c);
+				if (*pattern == '-'
+					&& (c2 = *(pattern+1)) != EOS && c2 != ']') {
+						pattern += 2;
+						if (c2 == '\\' && !(flags & FNM_NOESCAPE))
+								c2 = *pattern++;
+						if (c2 == EOS)
+								return (RANGE_ERROR);
+						if (flags & FNM_CASEFOLD)
+								c2 = (char)tolower((unsigned char)c2);
+						if (c <= test && test <= c2)
+								ok = 1;
+				} else if (c == test)
+						ok = 1;
+		} while ((c = *pattern++) != ']');
+
+		*newp = (char *)pattern;
+		return (ok == negate ? RANGE_NOMATCH : RANGE_MATCH);
+}
+
diff --git a/src/compat/fnmatch.h b/src/compat/fnmatch.h
new file mode 100644
index 000000000..7faef09b3
--- /dev/null
+++ b/src/compat/fnmatch.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_fnmatch__compat_h__
+#define INCLUDE_fnmatch__compat_h__
+
+#include "common.h"
+
+#define FNM_NOMATCH		1		/* Match failed. */
+#define FNM_NOSYS		2		/* Function not supported (unused). */
+
+#define FNM_NOESCAPE		0x01		/* Disable backslash escaping. */
+#define FNM_PATHNAME		0x02		/* Slash must be matched by slash. */
+#define FNM_PERIOD		0x04		/* Period must be matched by period. */
+#define FNM_LEADING_DIR 0x08		/* Ignore /<tail> after Imatch. */
+#define FNM_CASEFOLD		0x10		/* Case insensitive search. */
+
+#define FNM_IGNORECASE	FNM_CASEFOLD
+#define FNM_FILE_NAME	FNM_PATHNAME
+
+extern int p_fnmatch(const char *pattern, const char *string, int flags);
+
+#endif /* _FNMATCH_H */
+
diff --git a/src/config.c b/src/config.c
index 29b9b799f..d18b85c30 100644
--- a/src/config.c
+++ b/src/config.c
@@ -1,34 +1,18 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
 #include "fileops.h"
-#include "hashtable.h"
 #include "config.h"
 #include "git2/config.h"
 #include "vector.h"
+#if GIT_WIN32
+# include <windows.h>
+#endif
 
 #include <ctype.h>
 
@@ -37,55 +21,7 @@ typedef struct {
 	int priority;
 } file_internal;
 
-int git_config_open_file(git_config **out, const char *path)
-{
-	git_config_file *file = NULL;
-	git_config *cfg = NULL;
-	int error = GIT_SUCCESS;
-
-	error = git_config_new(&cfg);
-	if (error < GIT_SUCCESS)
-		return error;
-
-	error = git_config_file__ondisk(&file, path);
-	if (error < GIT_SUCCESS) {
-		git_config_free(cfg);
-		return error;
-	}
-
-	error = git_config_add_file(cfg, file, 1);
-	if (error < GIT_SUCCESS) {
-		file->free(file);
-		git_config_free(cfg);
-		return error;
-	}
-
-	error = file->open(file);
-	if (error < GIT_SUCCESS) {
-		git_config_free(cfg);
-		return git__rethrow(error, "Failed to open config file");
-	}
-
-	*out = cfg;
-
-	return GIT_SUCCESS;
-}
-
-int git_config_open_global(git_config **out)
-{
-	char full_path[GIT_PATH_MAX];
-	const char *home;
-
-	home = getenv("HOME");
-	if (home == NULL)
-		return git__throw(GIT_EOSERR, "Failed to open global config file. Cannot find $HOME variable");
-
-	git__joinpath(full_path, home, GIT_CONFIG_FILENAME);
-
-	return git_config_open_file(out, full_path);
-}
-
-void git_config_free(git_config *cfg)
+static void config_free(git_config *cfg)
 {
 	unsigned int i;
 	git_config_file *file;
@@ -95,17 +31,25 @@ void git_config_free(git_config *cfg)
 		internal = git_vector_get(&cfg->files, i);
 		file = internal->file;
 		file->free(file);
-		free(internal);
+		git__free(internal);
 	}
 
 	git_vector_free(&cfg->files);
-	free(cfg);
+	git__free(cfg);
+}
+
+void git_config_free(git_config *cfg)
+{
+	if (cfg == NULL)
+		return;
+
+	GIT_REFCOUNT_DEC(cfg, config_free);
 }
 
 static int config_backend_cmp(const void *a, const void *b)
 {
-	const file_internal *bk_a = *(const file_internal **)(a);
-	const file_internal *bk_b = *(const file_internal **)(b);
+	const file_internal *bk_a = (const file_internal *)(a);
+	const file_internal *bk_b = (const file_internal *)(b);
 
 	return bk_b->priority - bk_a->priority;
 }
@@ -115,52 +59,86 @@ int git_config_new(git_config **out)
 	git_config *cfg;
 
 	cfg = git__malloc(sizeof(git_config));
-	if (cfg == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(cfg);
 
 	memset(cfg, 0x0, sizeof(git_config));
 
 	if (git_vector_init(&cfg->files, 3, config_backend_cmp) < 0) {
-		free(cfg);
-		return GIT_ENOMEM;
+		git__free(cfg);
+		return -1;
 	}
 
 	*out = cfg;
+	GIT_REFCOUNT_INC(cfg);
+	return 0;
+}
+
+int git_config_add_file_ondisk(git_config *cfg, const char *path, int priority)
+{
+	git_config_file *file = NULL;
+
+	if (git_config_file__ondisk(&file, path) < 0)
+		return -1;
+
+	if (git_config_add_file(cfg, file, priority) < 0) {
+		/*
+		 * free manually; the file is not owned by the config
+		 * instance yet and will not be freed on cleanup
+		 */
+		file->free(file);
+		return -1;
+	}
+
+	return 0;
+}
+
+int git_config_open_ondisk(git_config **cfg, const char *path)
+{
+	if (git_config_new(cfg) < 0)
+		return -1;
 
-	return GIT_SUCCESS;
+	if (git_config_add_file_ondisk(*cfg, path, 1) < 0) {
+		git_config_free(*cfg);
+		return -1;
+	}
+
+	return 0;
 }
 
 int git_config_add_file(git_config *cfg, git_config_file *file, int priority)
 {
 	file_internal *internal;
+	int result;
 
 	assert(cfg && file);
 
+	if ((result = file->open(file)) < 0)
+		return result;
+
 	internal = git__malloc(sizeof(file_internal));
-	if (internal == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(internal);
 
 	internal->file = file;
 	internal->priority = priority;
 
 	if (git_vector_insert(&cfg->files, internal) < 0) {
-		free(internal);
-		return GIT_ENOMEM;
+		git__free(internal);
+		return -1;
 	}
 
 	git_vector_sort(&cfg->files);
 	internal->file->cfg = cfg;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 /*
  * Loop over all the variables
  */
 
-int git_config_foreach(git_config *cfg, int (*fn)(const char *, void *), void *data)
+int git_config_foreach(git_config *cfg, int (*fn)(const char *, const char *, void *), void *data)
 {
-	int ret = GIT_SUCCESS;
+	int ret = 0;
 	unsigned int i;
 	file_internal *internal;
 	git_config_file *file;
@@ -174,25 +152,33 @@ int git_config_foreach(git_config *cfg, int (*fn)(const char *, void *), void *d
 	return ret;
 }
 
+int git_config_delete(git_config *cfg, const char *name)
+{
+	file_internal *internal;
+	git_config_file *file;
+
+	assert(cfg->files.length);
+
+	internal = git_vector_get(&cfg->files, 0);
+	file = internal->file;
+
+	return file->del(file, name);
+}
 
 /**************
  * Setters
  **************/
 
-/*
- * Internal function to actually set the string value of a variable
- */
-
-int git_config_set_long(git_config *cfg, const char *name, long int value)
+int git_config_set_int64(git_config *cfg, const char *name, int64_t value)
 {
 	char str_value[32]; /* All numbers should fit in here */
-	snprintf(str_value, sizeof(str_value), "%ld", value);
+	p_snprintf(str_value, sizeof(str_value), "%" PRId64, value);
 	return git_config_set_string(cfg, name, str_value);
 }
 
-int git_config_set_int(git_config *cfg, const char *name, int value)
+int git_config_set_int32(git_config *cfg, const char *name, int32_t value)
 {
-	return git_config_set_long(cfg, name, value);
+	return git_config_set_int64(cfg, name, (int64_t)value);
 }
 
 int git_config_set_bool(git_config *cfg, const char *name, int value)
@@ -205,8 +191,7 @@ int git_config_set_string(git_config *cfg, const char *name, const char *value)
 	file_internal *internal;
 	git_config_file *file;
 
-	if (cfg->files.length == 0)
-		return git__throw(GIT_EINVALIDARGS, "Cannot set variable value; no files open in the `git_config` instance");
+	assert(cfg->files.length);
 
 	internal = git_vector_get(&cfg->files, 0);
 	file = internal->file;
@@ -214,109 +199,309 @@ int git_config_set_string(git_config *cfg, const char *name, const char *value)
 	return file->set(file, name, value);
 }
 
-/***********
- * Getters
- ***********/
-
-int git_config_get_long(git_config *cfg, const char *name, long int *out)
+static int parse_int64(int64_t *out, const char *value)
 {
-	const char *value, *num_end;
-	int ret;
-	long int num;
-
-	ret = git_config_get_string(cfg, name, &value);
-	if (ret < GIT_SUCCESS)
-		return git__rethrow(ret, "Failed to get value for %s", name);
+	const char *num_end;
+	int64_t num;
 
-	ret = git__strtol32(&num, value, &num_end, 0);
-	if (ret < GIT_SUCCESS)
-		return git__rethrow(ret, "Failed to get value for %s", name);
+	if (git__strtol64(&num, value, &num_end, 0) < 0)
+		return -1;
 
 	switch (*num_end) {
-	case '\0':
-		break;
-	case 'k':
-	case 'K':
+	case 'g':
+	case 'G':
 		num *= 1024;
-		break;
+		/* fallthrough */
+
 	case 'm':
 	case 'M':
-		num *= 1024 * 1024;
-		break;
-	case 'g':
-	case 'G':
-		num *= 1024 * 1024 * 1024;
-		break;
+		num *= 1024;
+		/* fallthrough */
+
+	case 'k':
+	case 'K':
+		num *= 1024;
+
+		/* check that that there are no more characters after the
+		 * given modifier suffix */
+		if (num_end[1] != '\0')
+			return -1;
+
+		/* fallthrough */
+
+	case '\0':
+		*out = num;
+		return 0;
+
 	default:
-		return git__throw(GIT_EINVALIDTYPE, "Failed to get value for %s. Value is of invalid type", name);
+		return -1;
 	}
+}
 
-	*out = num;
+static int parse_int32(int32_t *out, const char *value)
+{
+	int64_t tmp;
+	int32_t truncate;
 
-	return GIT_SUCCESS;
+	if (parse_int64(&tmp, value) < 0)
+		return -1;
+
+	truncate = tmp & 0xFFFFFFFF;
+	if (truncate != tmp)
+		return -1;
+
+	*out = truncate;
+	return 0;
 }
 
-int git_config_get_int(git_config *cfg, const char *name, int *out)
+/***********
+ * Getters
+ ***********/
+int git_config_lookup_map_value(
+	git_cvar_map *maps, size_t map_n, const char *value, int *out)
 {
-	long int tmp;
+	size_t i;
+
+	if (!value)
+		return GIT_ENOTFOUND;
+
+	for (i = 0; i < map_n; ++i) {
+		git_cvar_map *m = maps + i;
+
+		switch (m->cvar_type) {
+		case GIT_CVAR_FALSE:
+		case GIT_CVAR_TRUE: {
+			int bool_val;
+
+			if (git__parse_bool(&bool_val, value) == 0 &&
+				bool_val == (int)m->cvar_type) {
+				*out = m->map_value;
+				return 0;
+			}
+			break;
+		}
+
+		case GIT_CVAR_INT32:
+			if (parse_int32(out, value) == 0)
+				return 0;
+			break;
+
+		case GIT_CVAR_STRING:
+			if (strcasecmp(value, m->str_match) == 0) {
+				*out = m->map_value;
+				return 0;
+			}
+			break;
+		}
+	}
+
+	return GIT_ENOTFOUND;
+}
+
+int git_config_get_mapped(
+	int *out,
+	git_config *cfg,
+	const char *name,
+	git_cvar_map *maps,
+	size_t map_n)
+{
+	const char *value;
 	int ret;
 
-	ret = git_config_get_long(cfg, name, &tmp);
+	ret = git_config_get_string(&value, cfg, name);
+	if (ret < 0)
+		return ret;
 
-	*out = (int) tmp;
+	if (!git_config_lookup_map_value(maps, map_n, value, out))
+		return 0;
 
-	return ret;
+	giterr_set(GITERR_CONFIG,
+		"Failed to map the '%s' config variable with a valid value", name);
+	return -1;
 }
 
-int git_config_get_bool(git_config *cfg, const char *name, int *out)
+int git_config_get_int64(int64_t *out, git_config *cfg, const char *name)
 {
 	const char *value;
-	int error = GIT_SUCCESS;
+	int ret;
 
-	error = git_config_get_string(cfg, name, &value);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to get value for %s", name);
+	ret = git_config_get_string(&value, cfg, name);
+	if (ret < 0)
+		return ret;
 
-	/* A missing value means true */
-	if (value == NULL) {
-		*out = 1;
-		return GIT_SUCCESS;
+	if (parse_int64(out, value) < 0) {
+		giterr_set(GITERR_CONFIG, "Failed to parse '%s' as an integer", value);
+		return -1;
 	}
 
-	if (!strcasecmp(value, "true") ||
-		!strcasecmp(value, "yes") ||
-		!strcasecmp(value, "on")) {
-		*out = 1;
-		return GIT_SUCCESS;
-	}
-	if (!strcasecmp(value, "false") ||
-		!strcasecmp(value, "no") ||
-		!strcasecmp(value, "off")) {
-		*out = 0;
-		return GIT_SUCCESS;
+	return 0;
+}
+
+int git_config_get_int32(int32_t *out, git_config *cfg, const char *name)
+{
+	const char *value;
+	int ret;
+
+	ret = git_config_get_string(&value, cfg, name);
+	if (ret < 0)
+		return ret;
+
+	if (parse_int32(out, value) < 0) {
+		giterr_set(GITERR_CONFIG, "Failed to parse '%s' as a 32-bit integer", value);
+		return -1;
 	}
 
-	/* Try to parse it as an integer */
-	error = git_config_get_int(cfg, name, out);
-	if (error == GIT_SUCCESS)
+	return 0;
+}
+
+int git_config_get_bool(int *out, git_config *cfg, const char *name)
+{
+	const char *value;
+	int ret;
+
+	ret = git_config_get_string(&value, cfg, name);
+	if (ret < 0)
+		return ret;
+
+	if (git__parse_bool(out, value) == 0)
+		return 0;
+
+	if (parse_int32(out, value) == 0) {
 		*out = !!(*out);
+		return 0;
+	}
 
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to get value for %s", name);
-	return error;
+	giterr_set(GITERR_CONFIG, "Failed to parse '%s' as a boolean value", value);
+	return -1;
+}
+
+int git_config_get_string(const char **out, git_config *cfg, const char *name)
+{
+	file_internal *internal;
+	unsigned int i;
+
+	assert(cfg->files.length);
+
+	*out = NULL;
+
+	git_vector_foreach(&cfg->files, i, internal) {
+		git_config_file *file = internal->file;
+		int ret = file->get(file, name, out);
+		if (ret != GIT_ENOTFOUND)
+			return ret;
+	}
+
+	return GIT_ENOTFOUND;
 }
 
-int git_config_get_string(git_config *cfg, const char *name, const char **out)
+int git_config_get_multivar(git_config *cfg, const char *name, const char *regexp,
+			    int (*fn)(const char *value, void *data), void *data)
 {
 	file_internal *internal;
 	git_config_file *file;
+	int ret = GIT_ENOTFOUND;
+	unsigned int i;
 
-	if (cfg->files.length == 0)
-		return git__throw(GIT_EINVALIDARGS, "Cannot get variable value; no files open in the `git_config` instance");
+	assert(cfg->files.length);
 
-	internal = git_vector_get(&cfg->files, 0);
-	file = internal->file;
+	/*
+	 * This loop runs the "wrong" way 'round because we need to
+	 * look at every value from the most general to most specific
+	 */
+	for (i = cfg->files.length; i > 0; --i) {
+		internal = git_vector_get(&cfg->files, i - 1);
+		file = internal->file;
+		ret = file->get_multivar(file, name, regexp, fn, data);
+		if (ret < 0 && ret != GIT_ENOTFOUND)
+			return ret;
+	}
+
+	return 0;
+}
+
+int git_config_set_multivar(git_config *cfg, const char *name, const char *regexp, const char *value)
+{
+	file_internal *internal;
+	git_config_file *file;
+	int ret = GIT_ENOTFOUND;
+	unsigned int i;
+
+	for (i = cfg->files.length; i > 0; --i) {
+		internal = git_vector_get(&cfg->files, i - 1);
+		file = internal->file;
+		ret = file->set_multivar(file, name, regexp, value);
+		if (ret < 0 && ret != GIT_ENOTFOUND)
+			return ret;
+	}
+
+	return 0;
+}
+
+int git_config_find_global_r(git_buf *path)
+{
+	return git_futils_find_global_file(path, GIT_CONFIG_FILENAME);
+}
 
-	return file->get(file, name, out);
+int git_config_find_global(char *global_config_path, size_t length)
+{
+	git_buf path  = GIT_BUF_INIT;
+	int     ret = git_config_find_global_r(&path);
+
+	if (ret < 0) {
+		git_buf_free(&path);
+		return ret;
+	}
+
+	if (path.size >= length) {
+		git_buf_free(&path);
+		giterr_set(GITERR_NOMEMORY,
+			"Path is to long to fit on the given buffer");
+		return -1;
+	}
+
+	git_buf_copy_cstr(global_config_path, length, &path);
+	git_buf_free(&path);
+	return 0;
+}
+
+int git_config_find_system_r(git_buf *path)
+{
+	return git_futils_find_system_file(path, GIT_CONFIG_FILENAME_SYSTEM);
+}
+
+int git_config_find_system(char *system_config_path, size_t length)
+{
+	git_buf path  = GIT_BUF_INIT;
+	int     ret = git_config_find_system_r(&path);
+
+	if (ret < 0) {
+		git_buf_free(&path);
+		return ret;
+	}
+
+	if (path.size >= length) {
+		git_buf_free(&path);
+		giterr_set(GITERR_NOMEMORY,
+			"Path is to long to fit on the given buffer");
+		return -1;
+	}
+
+	git_buf_copy_cstr(system_config_path, length, &path);
+	git_buf_free(&path);
+	return 0;
+}
+
+int git_config_open_global(git_config **out)
+{
+	int error;
+	git_buf path = GIT_BUF_INIT;
+
+	if ((error = git_config_find_global_r(&path)) < 0)
+		return error;
+
+	error = git_config_open_ondisk(out, git_buf_cstr(&path));
+	git_buf_free(&path);
+
+	return error;
 }
 
diff --git a/src/config.h b/src/config.h
index 8b521543c..82e98ce51 100644
--- a/src/config.h
+++ b/src/config.h
@@ -1,14 +1,33 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_config_h__
 #define INCLUDE_config_h__
 
 #include "git2.h"
 #include "git2/config.h"
 #include "vector.h"
+#include "repository.h"
 
 #define GIT_CONFIG_FILENAME ".gitconfig"
+#define GIT_CONFIG_FILENAME_INREPO "config"
+#define GIT_CONFIG_FILENAME_SYSTEM "gitconfig"
+#define GIT_CONFIG_FILE_MODE 0666
 
 struct git_config {
+	git_refcount rc;
 	git_vector files;
 };
 
+extern int git_config_find_global_r(git_buf *global_config_path);
+extern int git_config_find_system_r(git_buf *system_config_path);
+
+extern int git_config_parse_bool(int *out, const char *bool_string);
+
+extern int git_config_lookup_map_value(
+	git_cvar_map *maps, size_t map_n, const char *value, int *out);
+
 #endif
diff --git a/src/config_cache.c b/src/config_cache.c
new file mode 100644
index 000000000..ca9602e56
--- /dev/null
+++ b/src/config_cache.c
@@ -0,0 +1,94 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "fileops.h"
+#include "config.h"
+#include "git2/config.h"
+#include "vector.h"
+#include "filter.h"
+#include "repository.h"
+
+struct map_data {
+	const char *cvar_name;
+	git_cvar_map *maps;
+	size_t map_count;
+	int default_value;
+};
+
+/*
+ *	core.eol
+ *		Sets the line ending type to use in the working directory for
+ *	files that have the text property set. Alternatives are lf, crlf
+ *	and native, which uses the platform’s native line ending. The default
+ *	value is native. See gitattributes(5) for more information on
+ *	end-of-line conversion. 
+ */
+static git_cvar_map _cvar_map_eol[] = {
+	{GIT_CVAR_FALSE, NULL, GIT_EOL_UNSET},
+	{GIT_CVAR_STRING, "lf", GIT_EOL_LF},
+	{GIT_CVAR_STRING, "crlf", GIT_EOL_CRLF},
+	{GIT_CVAR_STRING, "native", GIT_EOL_NATIVE}
+};
+
+/*
+ *	core.autocrlf
+ *		Setting this variable to "true" is almost the same as setting 
+ *	the text attribute to "auto" on all files except that text files are
+ *	not guaranteed to be normalized: files that contain CRLF in the
+ *	repository will not be touched. Use this setting if you want to have
+ *	CRLF line endings in your working directory even though the repository
+ *	does not have normalized line endings. This variable can be set to input,
+ *	in which case no output conversion is performed.
+ */
+static git_cvar_map _cvar_map_autocrlf[] = {
+	{GIT_CVAR_FALSE, NULL, GIT_AUTO_CRLF_FALSE},
+	{GIT_CVAR_TRUE, NULL, GIT_AUTO_CRLF_TRUE},
+	{GIT_CVAR_STRING, "input", GIT_AUTO_CRLF_INPUT}
+};
+
+static struct map_data _cvar_maps[] = {
+	{"core.autocrlf", _cvar_map_autocrlf, ARRAY_SIZE(_cvar_map_autocrlf), GIT_AUTO_CRLF_DEFAULT},
+	{"core.eol", _cvar_map_eol, ARRAY_SIZE(_cvar_map_eol), GIT_EOL_DEFAULT}
+};
+
+int git_repository__cvar(int *out, git_repository *repo, git_cvar_cached cvar)
+{
+	*out = repo->cvar_cache[(int)cvar];
+
+	if (*out == GIT_CVAR_NOT_CACHED) {
+		struct map_data *data = &_cvar_maps[(int)cvar];
+		git_config *config;
+		int error;
+
+		error = git_repository_config__weakptr(&config, repo);
+		if (error < 0)
+			return error;
+
+		error = git_config_get_mapped(out,
+			config, data->cvar_name, data->maps, data->map_count);
+
+		if (error == GIT_ENOTFOUND)
+			*out = data->default_value;
+
+		else if (error < 0)
+			return error;
+
+		repo->cvar_cache[(int)cvar] = *out;
+	}
+
+	return 0;
+}
+
+void git_repository__cvar_cache_clear(git_repository *repo)
+{
+	int i;
+
+	for (i = 0; i < GIT_CVAR_CACHE_MAX; ++i)
+		repo->cvar_cache[i] = GIT_CVAR_NOT_CACHED;
+}
+
diff --git a/src/config_file.c b/src/config_file.c
index d76c6024d..1c748fad1 100644
--- a/src/config_file.c
+++ b/src/config_file.c
@@ -1,40 +1,28 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
 #include "config.h"
 #include "fileops.h"
+#include "filebuf.h"
+#include "buffer.h"
 #include "git2/config.h"
 #include "git2/types.h"
+#include "strmap.h"
 
 #include <ctype.h>
+#include <sys/types.h>
+#include <regex.h>
+
+GIT__USE_STRMAP;
 
 typedef struct cvar_t {
 	struct cvar_t *next;
-	char *section;
-	char *name;
+	char *key; /* TODO: we might be able to get rid of this */
 	char *value;
 } cvar_t;
 
@@ -84,10 +72,10 @@ typedef struct {
 typedef struct {
 	git_config_file parent;
 
-	cvar_t_list var_list;
+	git_strmap *values;
 
 	struct {
-		gitfo_buf buffer;
+		git_buf buffer;
 		char *read_ptr;
 		int line_number;
 		int eof;
@@ -98,290 +86,381 @@ typedef struct {
 
 static int config_parse(diskfile_backend *cfg_file);
 static int parse_variable(diskfile_backend *cfg, char **var_name, char **var_value);
+static int config_write(diskfile_backend *cfg, const char *key, const regex_t *preg, const char *value);
+
+static void set_parse_error(diskfile_backend *backend, int col, const char *error_str)
+{
+	giterr_set(GITERR_CONFIG, "Failed to parse config file: %s (in %s:%d, column %d)",
+		error_str, backend->file_path, backend->reader.line_number, col);
+}
 
 static void cvar_free(cvar_t *var)
 {
 	if (var == NULL)
 		return;
 
-	free(var->section);
-	free(var->name);
-	free(var->value);
-	free(var);
+	git__free(var->key);
+	git__free(var->value);
+	git__free(var);
 }
 
-static void cvar_list_free(cvar_t_list *list)
+/* Take something the user gave us and make it nice for our hash function */
+static int normalize_name(const char *in, char **out)
 {
-	cvar_t *cur;
+	char *name, *fdot, *ldot;
+
+	assert(in && out);
+
+	name = git__strdup(in);
+	GITERR_CHECK_ALLOC(name);
 
-	while (!CVAR_LIST_EMPTY(list)) {
-		cur = CVAR_LIST_HEAD(list);
-		CVAR_LIST_REMOVE_HEAD(list);
-		cvar_free(cur);
+	fdot = strchr(name, '.');
+	ldot = strrchr(name, '.');
+
+	if (fdot == NULL || ldot == NULL) {
+		git__free(name);
+		giterr_set(GITERR_CONFIG,
+			"Invalid variable name: '%s'", in);
+		return -1;
 	}
+
+	/* Downcase up to the first dot and after the last one */
+	git__strntolower(name, fdot - name);
+	git__strtolower(ldot);
+
+	*out = name;
+	return 0;
 }
 
-/*
- * Compare two strings according to the git section-subsection
- * rules. The order of the strings is important because local is
- * assumed to have the internal format (only the section name and with
- * case information) and input the normalized one (only dots, no case
- * information).
- */
-static int cvar_match_section(const char *local, const char *input)
+static void free_vars(git_strmap *values)
 {
-	char *first_dot, *last_dot;
-	char *local_sp = strchr(local, ' ');
-	int comparison_len;
+	cvar_t *var = NULL;
 
-	/*
-	 * If the local section name doesn't contain a space, then we can
-	 * just do a case-insensitive compare.
-	 */
-	if (local_sp == NULL)
-		return !strncasecmp(local, input, strlen(local));
+	if (values == NULL)
+		return;
 
-	/*
-	 * From here onwards, there is a space diving the section and the
-	 * subsection. Anything before the space in local is
-	 * case-insensitive.
-	 */
-	if (strncasecmp(local, input, local_sp - local))
-		return 0;
+	git_strmap_foreach_value(values, var,
+		while (var != NULL) {
+			cvar_t *next = CVAR_LIST_NEXT(var);
+			cvar_free(var);
+			var = next;
+		});
 
-	/*
-	 * We compare starting from the first character after the
-	 * quotation marks, which is two characters beyond the space. For
-	 * the input, we start one character beyond the dot. If the names
-	 * have different lengths, then we can fail early, as we know they
-	 * can't be the same.
-	 * The length is given by the length between the quotation marks.
-	 */
+	git_strmap_free(values);
+}
 
-	first_dot = strchr(input, '.');
-	last_dot = strrchr(input, '.');
-	comparison_len = strlen(local_sp + 2) - 1;
+static int config_open(git_config_file *cfg)
+{
+	int res;
+	diskfile_backend *b = (diskfile_backend *)cfg;
+
+	b->values = git_strmap_alloc();
+	GITERR_CHECK_ALLOC(b->values);
 
-	if (last_dot == first_dot || last_dot - first_dot - 1 != comparison_len)
+	git_buf_init(&b->reader.buffer, 0);
+	res = git_futils_readbuffer(&b->reader.buffer, b->file_path);
+
+	/* It's fine if the file doesn't exist */
+	if (res == GIT_ENOTFOUND)
 		return 0;
 
-	return !strncmp(local_sp + 2, first_dot + 1, comparison_len);
+	if (res < 0 || config_parse(b) <  0) {
+		free_vars(b->values);
+		b->values = NULL;
+		git_buf_free(&b->reader.buffer);
+		return -1;
+	}
+
+	git_buf_free(&b->reader.buffer);
+	return 0;
 }
 
-static int cvar_match_name(const cvar_t *var, const char *str)
+static void backend_free(git_config_file *_backend)
 {
-	const char *name_start;
+	diskfile_backend *backend = (diskfile_backend *)_backend;
 
-	if (!cvar_match_section(var->section, str)) {
-		return 0;
-	}
-	/* Early exit if the lengths are different */
-	name_start = strrchr(str, '.') + 1;
-	if (strlen(var->name) != strlen(name_start))
-		return 0;
+	if (backend == NULL)
+		return;
 
-	return !strcasecmp(var->name, name_start);
+	git__free(backend->file_path);
+	free_vars(backend->values);
+	git__free(backend);
 }
 
-static cvar_t *cvar_list_find(cvar_t_list *list, const char *name)
+static int file_foreach(git_config_file *backend, int (*fn)(const char *, const char *, void *), void *data)
 {
-	cvar_t *iter;
+	diskfile_backend *b = (diskfile_backend *)backend;
+	cvar_t *var;
+	const char *key;
 
-	CVAR_LIST_FOREACH (list, iter) {
-		if (cvar_match_name(iter, name))
-			return iter;
-	}
+	if (!b->values)
+		return 0;
 
-	return NULL;
+	git_strmap_foreach(b->values, key, var,
+		do {
+			if (fn(key, var->value, data) < 0)
+				break;
+
+			var = CVAR_LIST_NEXT(var);
+		} while (var != NULL);
+	);
+
+	return 0;
 }
 
-static int cvar_normalize_name(cvar_t *var, char **output)
+static int config_set(git_config_file *cfg, const char *name, const char *value)
 {
-	char *section_sp = strchr(var->section, ' ');
-	char *quote, *name;
-	int len, ret;
+	cvar_t *var = NULL, *old_var;
+	diskfile_backend *b = (diskfile_backend *)cfg;
+	char *key;
+	khiter_t pos;
+	int rval;
+
+	if (normalize_name(name, &key) < 0)
+		return -1;
 
 	/*
-	 * The final string is going to be at most one char longer than
-	 * the input
+	 * Try to find it in the existing values and update it if it
+	 * only has one value.
 	 */
-	len = strlen(var->section) + strlen(var->name) + 1;
-	name = git__malloc(len + 1);
-	if (name == NULL)
-		return GIT_ENOMEM;
+	pos = git_strmap_lookup_index(b->values, key);
+	if (git_strmap_valid_index(b->values, pos)) {
+		cvar_t *existing = git_strmap_value_at(b->values, pos);
+		char *tmp = NULL;
+
+		git__free(key);
+		if (existing->next != NULL) {
+			giterr_set(GITERR_CONFIG, "Multivar incompatible with simple set");
+			return -1;
+		}
 
-	/* If there aren't any spaces in the section, it's easy */
-	if (section_sp == NULL) {
-		ret = snprintf(name, len + 1, "%s.%s", var->section, var->name);
-		if (ret < 0)
-			return git__throw(GIT_EOSERR, "Failed to normalize name. OS err: %s", strerror(errno));
+		if (value) {
+			tmp = git__strdup(value);
+			GITERR_CHECK_ALLOC(tmp);
+		}
 
-		*output = name;
-		return GIT_SUCCESS;
-	}
+		git__free(existing->value);
+		existing->value = tmp;
 
-	/*
-	 * If there are spaces, we replace the space by a dot, move
-	 * section name so it overwrites the first quotation mark and
-	 * replace the last quotation mark by a dot. We then append the
-	 * variable name.
-	 */
-	strcpy(name, var->section);
-	section_sp = strchr(name, ' ');
-	*section_sp = '.';
-	/* Remove first quote */
-	quote = strchr(name, '"');
-	memmove(quote, quote+1, strlen(quote+1));
-	/* Remove second quote */
-	quote = strchr(name, '"');
-	*quote = '.';
-	strcpy(quote+1, var->name);
-
-	*output = name;
-	return GIT_SUCCESS;
-}
+		return config_write(b, existing->key, NULL, value);
+	}
 
-static int config_open(git_config_file *cfg)
-{
-	int error;
-	diskfile_backend *b = (diskfile_backend *)cfg;
+	var = git__malloc(sizeof(cvar_t));
+	GITERR_CHECK_ALLOC(var);
 
-	error = gitfo_read_file(&b->reader.buffer, b->file_path);
-	if(error < GIT_SUCCESS)
-		goto cleanup;
+	memset(var, 0x0, sizeof(cvar_t));
 
-	error = config_parse(b);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	var->key = key;
+	var->value = NULL;
 
-	gitfo_free_buf(&b->reader.buffer);
+	if (value) {
+		var->value = git__strdup(value);
+		GITERR_CHECK_ALLOC(var->value);
+	}
 
-	return error;
+	if (config_write(b, key, NULL, value) < 0) {
+		cvar_free(var);
+		return -1;
+	}
 
- cleanup:
-	cvar_list_free(&b->var_list);
-	gitfo_free_buf(&b->reader.buffer);
+	git_strmap_insert2(b->values, key, var, old_var, rval);
+	if (rval < 0)
+		return -1;
+	if (old_var != NULL)
+		cvar_free(old_var);
 
-	return git__rethrow(error, "Failed to open config");
+	return 0;
 }
 
-static void backend_free(git_config_file *_backend)
+/*
+ * Internal function that actually gets the value in string form
+ */
+static int config_get(git_config_file *cfg, const char *name, const char **out)
 {
-	diskfile_backend *backend = (diskfile_backend *)_backend;
+	diskfile_backend *b = (diskfile_backend *)cfg;
+	char *key;
+	khiter_t pos;
 
-	if (backend == NULL)
-		return;
+	if (normalize_name(name, &key) < 0)
+		return -1;
 
-	free(backend->file_path);
-	cvar_list_free(&backend->var_list);
+	pos = git_strmap_lookup_index(b->values, key);
+	git__free(key);
 
-	free(backend);
+	/* no error message; the config system will write one */
+	if (!git_strmap_valid_index(b->values, pos))
+		return GIT_ENOTFOUND;
+
+	*out = ((cvar_t *)git_strmap_value_at(b->values, pos))->value;
+
+	return 0;
 }
 
-static int file_foreach(git_config_file *backend, int (*fn)(const char *, void *), void *data)
+static int config_get_multivar(
+	git_config_file *cfg,
+	const char *name,
+	const char *regex_str,
+	int (*fn)(const char *, void *),
+	void *data)
 {
-	int ret = GIT_SUCCESS;
 	cvar_t *var;
-	diskfile_backend *b = (diskfile_backend *)backend;
+	diskfile_backend *b = (diskfile_backend *)cfg;
+	char *key;
+	khiter_t pos;
 
-	CVAR_LIST_FOREACH(&b->var_list, var) {
-		char *normalized = NULL;
+	if (normalize_name(name, &key) < 0)
+		return -1;
 
-		ret = cvar_normalize_name(var, &normalized);
-		if (ret < GIT_SUCCESS)
-			return ret;
+	pos = git_strmap_lookup_index(b->values, key);
+	git__free(key);
 
-		ret = fn(normalized, data);
-		free(normalized);
-		if (ret)
-			break;
+	if (!git_strmap_valid_index(b->values, pos))
+		return GIT_ENOTFOUND;
+
+	var = git_strmap_value_at(b->values, pos);
+
+	if (regex_str != NULL) {
+		regex_t regex;
+		int result;
+
+		/* regex matching; build the regex */
+		result = regcomp(&regex, regex_str, REG_EXTENDED);
+		if (result < 0) {
+			giterr_set_regex(&regex, result);
+			return -1;
+		}
+
+		/* and throw the callback only on the variables that
+		 * match the regex */
+		do {
+			if (regexec(&regex, var->value, 0, NULL, 0) == 0) {
+				/* early termination by the user is not an error;
+				 * just break and return successfully */
+				if (fn(var->value, data) < 0)
+					break;
+			}
+
+			var = var->next;
+		} while (var != NULL);
+		regfree(&regex);
+	} else {
+		/* no regex; go through all the variables */
+		do {
+			/* early termination by the user is not an error;
+			 * just break and return successfully */
+			if (fn(var->value, data) < 0)
+				break;
+
+			var = var->next;
+		} while (var != NULL);
 	}
 
-	return ret;
+	return 0;
 }
 
-static int config_set(git_config_file *cfg, const char *name, const char *value)
+static int config_set_multivar(
+	git_config_file *cfg, const char *name, const char *regexp, const char *value)
 {
-	cvar_t *var = NULL;
-	cvar_t *existing = NULL;
-	int error = GIT_SUCCESS;
-	const char *last_dot;
+	int replaced = 0;
+	cvar_t *var, *newvar;
 	diskfile_backend *b = (diskfile_backend *)cfg;
+	char *key;
+	regex_t preg;
+	int result;
+	khiter_t pos;
 
-	/*
-	 * If it already exists, we just need to update its value.
-	 */
-	existing = cvar_list_find(&b->var_list, name);
-	if (existing != NULL) {
-		char *tmp = value ? git__strdup(value) : NULL;
-		if (tmp == NULL && value != NULL)
-			return GIT_ENOMEM;
+	assert(regexp);
 
-		free(existing->value);
-		existing->value = tmp;
+	if (normalize_name(name, &key) < 0)
+		return -1;
 
-		return GIT_SUCCESS;
+	pos = git_strmap_lookup_index(b->values, key);
+	if (!git_strmap_valid_index(b->values, pos)) {
+		git__free(key);
+		return GIT_ENOTFOUND;
 	}
 
-	/*
-	 * Otherwise, create it and stick it at the end of the queue.
-	 */
+	var = git_strmap_value_at(b->values, pos);
 
-	last_dot = strrchr(name, '.');
-	if (last_dot == NULL) {
-		return git__throw(GIT_EINVALIDTYPE, "Variables without section aren't allowed");
+	result = regcomp(&preg, regexp, REG_EXTENDED);
+	if (result < 0) {
+		git__free(key);
+		giterr_set_regex(&preg, result);
+		return -1;
 	}
 
-	var = git__malloc(sizeof(cvar_t));
-	if (var == NULL)
-		return GIT_ENOMEM;
+	for (;;) {
+		if (regexec(&preg, var->value, 0, NULL, 0) == 0) {
+			char *tmp = git__strdup(value);
+			GITERR_CHECK_ALLOC(tmp);
 
-	memset(var, 0x0, sizeof(cvar_t));
+			git__free(var->value);
+			var->value = tmp;
+			replaced = 1;
+		}
 
-	var->section = git__strndup(name, last_dot - name);
-	if (var->section == NULL) {
-		error = GIT_ENOMEM;
-		goto out;
-	}
+		if (var->next == NULL)
+			break;
 
-	var->name = git__strdup(last_dot + 1);
-	if (var->name == NULL) {
-		error = GIT_ENOMEM;
-		goto out;
+		var = var->next;
 	}
 
-	var->value = value ? git__strdup(value) : NULL;
-	if (var->value == NULL && value != NULL) {
-		error = GIT_ENOMEM;
-		goto out;
+	/* If we've reached the end of the variables and we haven't found it yet, we need to append it */
+	if (!replaced) {
+		newvar = git__malloc(sizeof(cvar_t));
+		GITERR_CHECK_ALLOC(newvar);
+
+		memset(newvar, 0x0, sizeof(cvar_t));
+
+		newvar->key = git__strdup(var->key);
+		GITERR_CHECK_ALLOC(newvar->key);
+
+		newvar->value = git__strdup(value);
+		GITERR_CHECK_ALLOC(newvar->value);
+
+		var->next = newvar;
 	}
 
-	CVAR_LIST_APPEND(&b->var_list, var);
+	result = config_write(b, key, &preg, value);
 
- out:
-	if (error < GIT_SUCCESS)
-		cvar_free(var);
+	git__free(key);
+	regfree(&preg);
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to set config value");
+	return result;
 }
 
-/*
- * Internal function that actually gets the value in string form
- */
-static int config_get(git_config_file *cfg, const char *name, const char **out)
+static int config_delete(git_config_file *cfg, const char *name)
 {
 	cvar_t *var;
-	int error = GIT_SUCCESS;
 	diskfile_backend *b = (diskfile_backend *)cfg;
+	char *key;
+	int result;
+	khiter_t pos;
 
-	var = cvar_list_find(&b->var_list, name);
+	if (normalize_name(name, &key) < 0)
+		return -1;
 
-	if (var == NULL)
-		return git__throw(GIT_ENOTFOUND, "Variable '%s' not found", name);
+	pos = git_strmap_lookup_index(b->values, key);
+	git__free(key);
+
+	if (!git_strmap_valid_index(b->values, pos)) {
+		giterr_set(GITERR_CONFIG, "Could not find key '%s' to delete", name);
+		return GIT_ENOTFOUND;
+	}
+
+	var = git_strmap_value_at(b->values, pos);
+
+	if (var->next != NULL) {
+		giterr_set(GITERR_CONFIG, "Cannot delete multivar with a single delete");
+		return -1;
+	}
 
-	*out = var->value;
+	git_strmap_delete_at(b->values, pos);
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to get config value for %s", name);
+	result = config_write(b, var->key, NULL, NULL);
+
+	cvar_free(var);
+	return result;
 }
 
 int git_config_file__ondisk(git_config_file **out, const char *path)
@@ -389,26 +468,25 @@ int git_config_file__ondisk(git_config_file **out, const char *path)
 	diskfile_backend *backend;
 
 	backend = git__malloc(sizeof(diskfile_backend));
-	if (backend == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(backend);
 
 	memset(backend, 0x0, sizeof(diskfile_backend));
 
 	backend->file_path = git__strdup(path);
-	if (backend->file_path == NULL) {
-		free(backend);
-		return GIT_ENOMEM;
-	}
+	GITERR_CHECK_ALLOC(backend->file_path);
 
 	backend->parent.open = config_open;
 	backend->parent.get = config_get;
+	backend->parent.get_multivar = config_get_multivar;
 	backend->parent.set = config_set;
+	backend->parent.set_multivar = config_set_multivar;
+	backend->parent.del = config_delete;
 	backend->parent.foreach = file_foreach;
 	backend->parent.free = backend_free;
 
 	*out = (git_config_file *)backend;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static int cfg_getchar_raw(diskfile_backend *cfg)
@@ -449,7 +527,8 @@ static int cfg_getchar(diskfile_backend *cfg_file, int flags)
 	assert(cfg_file->reader.read_ptr);
 
 	do c = cfg_getchar_raw(cfg_file);
-	while (skip_whitespace && isspace(c));
+	while (skip_whitespace && git__isspace(c) &&
+	       !cfg_file->reader.eof);
 
 	if (skip_comments && (c == '#' || c == ';')) {
 		do c = cfg_getchar_raw(cfg_file);
@@ -486,21 +565,23 @@ static int cfg_peek(diskfile_backend *cfg, int flags)
 /*
  * Read and consume a line, returning it in newly-allocated memory.
  */
-static char *cfg_readline(diskfile_backend *cfg)
+static char *cfg_readline(diskfile_backend *cfg, bool skip_whitespace)
 {
 	char *line = NULL;
 	char *line_src, *line_end;
-	int line_len;
+	size_t line_len;
 
 	line_src = cfg->reader.read_ptr;
 
-	/* Skip empty empty lines */
-	while (isspace(*line_src))
-		++line_src;
+	if (skip_whitespace) {
+		/* Skip empty empty lines */
+		while (git__isspace(*line_src))
+			++line_src;
+	}
 
-    line_end = strchr(line_src, '\n');
+	line_end = strchr(line_src, '\n');
 
-    /* no newline at EOF */
+	/* no newline at EOF */
 	if (line_end == NULL)
 		line_end = strchr(line_src, 0);
 
@@ -512,10 +593,8 @@ static char *cfg_readline(diskfile_backend *cfg)
 
 	memcpy(line, line_src, line_len);
 
-	line[line_len] = '\0';
-
-	while (--line_len >= 0 && isspace(line[line_len]))
-		line[line_len] = '\0';
+	do line[line_len] = '\0';
+	while (line_len-- > 0 && git__isspace(line[line_len]));
 
 	if (*line_end == '\n')
 		line_end++;
@@ -532,7 +611,7 @@ static char *cfg_readline(diskfile_backend *cfg)
 /*
  * Consume a line, without storing it anywhere
  */
-void cfg_consume_line(diskfile_backend *cfg)
+static void cfg_consume_line(diskfile_backend *cfg)
 {
 	char *line_start, *line_end;
 
@@ -558,12 +637,11 @@ GIT_INLINE(int) config_keychar(int c)
 	return isalnum(c) || c == '-';
 }
 
-static int parse_section_header_ext(const char *line, const char *base_name, char **section_name)
+static int parse_section_header_ext(diskfile_backend *cfg, const char *line, const char *base_name, char **section_name)
 {
-	int buf_len, total_len, pos, rpos;
-	int c, ret;
-	char *subsection, *first_quote, *last_quote;
-	int error = GIT_SUCCESS;
+	int c, rpos;
+	char *first_quote, *last_quote;
+	git_buf buf = GIT_BUF_INIT;
 	int quote_marks;
 	/*
 	 * base_name is what came before the space. We should be at the
@@ -574,16 +652,14 @@ static int parse_section_header_ext(const char *line, const char *base_name, cha
 	first_quote = strchr(line, '"');
 	last_quote = strrchr(line, '"');
 
-	if (last_quote - first_quote == 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse ext header. There is no final quotation mark");
-
-	buf_len = last_quote - first_quote + 2;
+	if (last_quote - first_quote == 0) {
+		set_parse_error(cfg, 0, "Missing closing quotation mark in section header");
+		return -1;
+	}
 
-	subsection = git__malloc(buf_len + 2);
-	if (subsection == NULL)
-		return GIT_ENOMEM;
+	git_buf_grow(&buf, strlen(base_name) + last_quote - first_quote + 2);
+	git_buf_printf(&buf, "%s.", base_name);
 
-	pos = 0;
 	rpos = 0;
 	quote_marks = 0;
 
@@ -596,139 +672,123 @@ static int parse_section_header_ext(const char *line, const char *base_name, cha
 	 */
 	do {
 		if (quote_marks == 2) {
-			error = git__throw(GIT_EOBJCORRUPTED, "Falied to parse ext header. Text after closing quote");
-			goto out;
-
+			set_parse_error(cfg, rpos, "Unexpected text after closing quotes");
+			git_buf_free(&buf);
+			return -1;
 		}
 
 		switch (c) {
 		case '"':
 			++quote_marks;
-			break;
+			continue;
+
 		case '\\':
 			c = line[rpos++];
+
 			switch (c) {
 			case '"':
 			case '\\':
 				break;
+
 			default:
-				error = git__throw(GIT_EOBJCORRUPTED, "Failed to parse ext header. Unsupported escape char \\%c", c);
-				goto out;
+				set_parse_error(cfg, rpos, "Unsupported escape sequence");
+				git_buf_free(&buf);
+				return -1;
 			}
-			break;
+
 		default:
 			break;
 		}
 
-		subsection[pos++] = (char) c;
+		git_buf_putc(&buf, c);
 	} while ((c = line[rpos++]) != ']');
 
-	subsection[pos] = '\0';
-
-	total_len = strlen(base_name) + strlen(subsection) + 2;
-	*section_name = git__malloc(total_len);
-	if (*section_name == NULL) {
-		error = GIT_ENOMEM;
-		goto out;
-	}
-
-	ret = snprintf(*section_name, total_len, "%s %s", base_name, subsection);
-	if (ret >= total_len) {
-		/* If this fails, we've checked the length wrong */
-		error = git__throw(GIT_ERROR, "Failed to parse ext header. Wrong total length calculation");
-		goto out;
-	} else if (ret < 0) {
-		error = git__throw(GIT_EOSERR, "Failed to parse ext header. OS error: %s", strerror(errno));
-		goto out;
-	}
-
-	git__strntolower(*section_name, strchr(*section_name, ' ') - *section_name);
-
- out:
-	free(subsection);
-
-	return error;
+	*section_name = git_buf_detach(&buf);
+	return 0;
 }
 
 static int parse_section_header(diskfile_backend *cfg, char **section_out)
 {
 	char *name, *name_end;
 	int name_length, c, pos;
-	int error = GIT_SUCCESS;
+	int result;
 	char *line;
 
-	line = cfg_readline(cfg);
+	line = cfg_readline(cfg, true);
 	if (line == NULL)
-		return GIT_ENOMEM;
+		return -1;
 
 	/* find the end of the variable's name */
 	name_end = strchr(line, ']');
-	if (name_end == NULL)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse header. Can't find header name end");
+	if (name_end == NULL) {
+		git__free(line);
+		set_parse_error(cfg, 0, "Missing ']' in section header");
+		return -1;
+	}
 
 	name = (char *)git__malloc((size_t)(name_end - line) + 1);
-	if (name == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(name);
 
 	name_length = 0;
 	pos = 0;
 
 	/* Make sure we were given a section header */
 	c = line[pos++];
-	if (c != '[') {
-		error = git__throw(GIT_ERROR, "Failed to parse header. Didn't get section header. This is a bug");
-		goto error;
-	}
+	assert(c == '[');
 
 	c = line[pos++];
 
 	do {
-		if (cfg->reader.eof){
-			error = git__throw(GIT_EOBJCORRUPTED, "Failed to parse header. Config file ended unexpectedly");
-			goto error;
-		}
-
-		if (isspace(c)){
+		if (git__isspace(c)){
 			name[name_length] = '\0';
-			error = parse_section_header_ext(line, name, section_out);
-			free(line);
-			free(name);
-			return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to parse header");
+			result = parse_section_header_ext(cfg, line, name, section_out);
+			git__free(line);
+			git__free(name);
+			return result;
 		}
 
 		if (!config_keychar(c) && c != '.') {
-			error = git__throw(GIT_EOBJCORRUPTED, "Failed to parse header. Wrong format on header");
-			goto error;
+			set_parse_error(cfg, pos, "Unexpected character in header");
+			goto fail_parse;
 		}
 
 		name[name_length++] = (char) tolower(c);
 
 	} while ((c = line[pos++]) != ']');
 
+	if (line[pos - 1] != ']') {
+		set_parse_error(cfg, pos, "Unexpected end of file");
+		goto fail_parse;
+	}
+
+	git__free(line);
+
 	name[name_length] = 0;
-	free(line);
-	git__strtolower(name);
 	*section_out = name;
-	return GIT_SUCCESS;
 
-error:
-	free(line);
-	free(name);
-	return error;
+	return 0;
+
+fail_parse:
+	git__free(line);
+	git__free(name);
+	return -1;
 }
 
 static int skip_bom(diskfile_backend *cfg)
 {
-	static const char *utf8_bom = "\xef\xbb\xbf";
+	static const char utf8_bom[] = "\xef\xbb\xbf";
+
+	if (cfg->reader.buffer.size < sizeof(utf8_bom))
+		return 0;
 
 	if (memcmp(cfg->reader.read_ptr, utf8_bom, sizeof(utf8_bom)) == 0)
 		cfg->reader.read_ptr += sizeof(utf8_bom);
 
-	/*  TODO: the reference implementation does pretty stupid
+	/* TODO: the reference implementation does pretty stupid
 		shit with the BoM
 	*/
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 /*
@@ -770,9 +830,9 @@ static int skip_bom(diskfile_backend *cfg)
 	boolean_false = "no" | "0" | "false" | "off"
 */
 
-static void strip_comments(char *line)
+static int strip_comments(char *line, int in_quotes)
 {
-	int quote_count = 0;
+	int quote_count = in_quotes;
 	char *ptr;
 
 	for (ptr = line; *ptr; ++ptr) {
@@ -785,41 +845,49 @@ static void strip_comments(char *line)
 		}
 	}
 
-	if (isspace(ptr[-1])) {
-		/* TODO skip whitespace */
+	/* skip any space at the end */
+	if (git__isspace(ptr[-1])) {
+		ptr--;
 	}
+	ptr[0] = '\0';
+
+	return quote_count;
 }
 
 static int config_parse(diskfile_backend *cfg_file)
 {
-	int error = GIT_SUCCESS, c;
+	int c;
 	char *current_section = NULL;
 	char *var_name;
 	char *var_value;
-	cvar_t *var;
+	cvar_t *var, *existing;
+	git_buf buf = GIT_BUF_INIT;
+	int result = 0;
+	khiter_t pos;
 
 	/* Initialize the reading position */
-	cfg_file->reader.read_ptr = cfg_file->reader.buffer.data;
+	cfg_file->reader.read_ptr = cfg_file->reader.buffer.ptr;
 	cfg_file->reader.eof = 0;
 
 	/* If the file is empty, there's nothing for us to do */
 	if (*cfg_file->reader.read_ptr == '\0')
-		return GIT_SUCCESS;
+		return 0;
 
 	skip_bom(cfg_file);
 
-	while (error == GIT_SUCCESS && !cfg_file->reader.eof) {
+	while (result == 0 && !cfg_file->reader.eof) {
 
 		c = cfg_peek(cfg_file, SKIP_WHITESPACE);
 
 		switch (c) {
-		case '\0': /* We've arrived at the end of the file */
+		case '\n': /* EOF when peeking, set EOF in the reader to exit the loop */
+			cfg_file->reader.eof = 1;
 			break;
 
 		case '[': /* section header, new section begins */
-			free(current_section);
+			git__free(current_section);
 			current_section = NULL;
-			error = parse_section_header(cfg_file, &current_section);
+			result = parse_section_header(cfg_file, &current_section);
 			break;
 
 		case ';':
@@ -828,129 +896,376 @@ static int config_parse(diskfile_backend *cfg_file)
 			break;
 
 		default: /* assume variable declaration */
-			error = parse_variable(cfg_file, &var_name, &var_value);
-
-			if (error < GIT_SUCCESS)
+			result = parse_variable(cfg_file, &var_name, &var_value);
+			if (result < 0)
 				break;
 
-			var = malloc(sizeof(cvar_t));
-			if (var == NULL) {
-				error = GIT_ENOMEM;
-				break;
-			}
+			var = git__malloc(sizeof(cvar_t));
+			GITERR_CHECK_ALLOC(var);
 
 			memset(var, 0x0, sizeof(cvar_t));
 
-			var->section = git__strdup(current_section);
-			if (var->section == NULL) {
-				error = GIT_ENOMEM;
-				free(var);
-				break;
-			}
+			git__strtolower(var_name);
+			git_buf_printf(&buf, "%s.%s", current_section, var_name);
+			git__free(var_name);
+
+			if (git_buf_oom(&buf))
+				return -1;
 
-			var->name = var_name;
+			var->key = git_buf_detach(&buf);
 			var->value = var_value;
-			git__strtolower(var->name);
 
-			CVAR_LIST_APPEND(&cfg_file->var_list, var);
+			/* Add or append the new config option */
+			pos = git_strmap_lookup_index(cfg_file->values, var->key);
+			if (!git_strmap_valid_index(cfg_file->values, pos)) {
+				git_strmap_insert(cfg_file->values, var->key, var, result);
+				if (result < 0)
+					break;
+				result = 0;
+			} else {
+				existing = git_strmap_value_at(cfg_file->values, pos);
+				while (existing->next != NULL) {
+					existing = existing->next;
+				}
+				existing->next = var;
+			}
 
 			break;
 		}
 	}
 
-	free(current_section);
+	git__free(current_section);
+	return result;
+}
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to parse config");
+static int write_section(git_filebuf *file, const char *key)
+{
+	int result;
+	const char *dot;
+	git_buf buf = GIT_BUF_INIT;
+
+	/* All of this just for [section "subsection"] */
+	dot = strchr(key, '.');
+	git_buf_putc(&buf, '[');
+	if (dot == NULL) {
+		git_buf_puts(&buf, key);
+	} else {
+		git_buf_put(&buf, key, dot - key);
+		/* TODO: escape  */
+		git_buf_printf(&buf, " \"%s\"", dot + 1);
+	}
+	git_buf_puts(&buf, "]\n");
+
+	if (git_buf_oom(&buf))
+		return -1;
+
+	result = git_filebuf_write(file, git_buf_cstr(&buf), buf.size);
+	git_buf_free(&buf);
+
+	return result;
 }
 
-static int is_multiline_var(const char *str)
+/*
+ * This is pretty much the parsing, except we write out anything we don't have
+ */
+static int config_write(diskfile_backend *cfg, const char *key, const regex_t *preg, const char* value)
+{
+	int result, c;
+	int section_matches = 0, last_section_matched = 0, preg_replaced = 0, write_trailer = 0;
+	const char *pre_end = NULL, *post_start = NULL, *data_start;
+	char *current_section = NULL, *section, *name, *ldot;
+	git_filebuf file = GIT_FILEBUF_INIT;
+
+	/* We need to read in our own config file */
+	result = git_futils_readbuffer(&cfg->reader.buffer, cfg->file_path);
+
+	/* Initialise the reading position */
+	if (result == GIT_ENOTFOUND) {
+		cfg->reader.read_ptr = NULL;
+		cfg->reader.eof = 1;
+		data_start = NULL;
+		git_buf_clear(&cfg->reader.buffer);
+	} else if (result == 0) {
+		cfg->reader.read_ptr = cfg->reader.buffer.ptr;
+		cfg->reader.eof = 0;
+		data_start = cfg->reader.read_ptr;
+	} else {
+		return -1; /* OS error when reading the file */
+	}
+
+	/* Lock the file */
+	if (git_filebuf_open(&file, cfg->file_path, 0) < 0)
+		return -1;
+
+	skip_bom(cfg);
+	ldot = strrchr(key, '.');
+	name = ldot + 1;
+	section = git__strndup(key, ldot - key);
+
+	while (!cfg->reader.eof) {
+		c = cfg_peek(cfg, SKIP_WHITESPACE);
+
+		if (c == '\0') { /* We've arrived at the end of the file */
+			break;
+
+		} else if (c == '[') { /* section header, new section begins */
+			/*
+			 * We set both positions to the current one in case we
+			 * need to add a variable to the end of a section. In that
+			 * case, we want both variables to point just before the
+			 * new section. If we actually want to replace it, the
+			 * default case will take care of updating them.
+			 */
+			pre_end = post_start = cfg->reader.read_ptr;
+
+			git__free(current_section);
+			current_section = NULL;
+			if (parse_section_header(cfg, &current_section) < 0)
+				goto rewrite_fail;
+
+			/* Keep track of when it stops matching */
+			last_section_matched = section_matches;
+			section_matches = !strcmp(current_section, section);
+		}
+
+		else if (c == ';' || c == '#') {
+			cfg_consume_line(cfg);
+		}
+
+		else {
+			/*
+			 * If the section doesn't match, but the last section did,
+			 * it means we need to add a variable (so skip the line
+			 * otherwise). If both the section and name match, we need
+			 * to overwrite the variable (so skip the line
+			 * otherwise). pre_end needs to be updated each time so we
+			 * don't loose that information, but we only need to
+			 * update post_start if we're going to use it in this
+			 * iteration.
+			 */
+			if (!section_matches) {
+				if (!last_section_matched) {
+					cfg_consume_line(cfg);
+					continue;
+				}
+			} else {
+				int has_matched = 0;
+				char *var_name, *var_value;
+
+				pre_end = cfg->reader.read_ptr;
+				if (parse_variable(cfg, &var_name, &var_value) < 0)
+					goto rewrite_fail;
+
+				/* First try to match the name of the variable */
+				if (strcasecmp(name, var_name) == 0)
+					has_matched = 1;
+
+				/* If the name matches, and we have a regex to match the
+				 * value, try to match it */
+				if (has_matched && preg != NULL)
+					has_matched = (regexec(preg, var_value, 0, NULL, 0) == 0);
+
+				git__free(var_name);
+				git__free(var_value);
+
+				/* if there is no match, keep going */
+				if (!has_matched)
+					continue;
+
+				post_start = cfg->reader.read_ptr;
+			}
+
+			/* We've found the variable we wanted to change, so
+			 * write anything up to it */
+			git_filebuf_write(&file, data_start, pre_end - data_start);
+			preg_replaced = 1;
+
+			/* Then replace the variable. If the value is NULL, it
+			 * means we want to delete it, so don't write anything. */
+			if (value != NULL) {
+				git_filebuf_printf(&file, "\t%s = %s\n", name, value);
+			}
+
+			/* multiline variable? we need to keep reading lines to match */
+			if (preg != NULL) {
+				data_start = post_start;
+				continue;
+			}
+
+			write_trailer = 1;
+			break; /* break from the loop */
+		}
+	}
+
+	/*
+	 * Being here can mean that
+	 *
+	 * 1) our section is the last one in the file and we're
+	 * adding a variable
+	 *
+	 * 2) we didn't find a section for us so we need to create it
+	 * ourselves.
+	 *
+	 * 3) we're setting a multivar with a regex, which means we
+	 * continue to search for matching values
+	 *
+	 * In the last case, if we've already replaced a value, we
+	 * want to write the rest of the file. Otherwise we need to write
+	 * out the whole file and then the new variable.
+	 */
+	if (write_trailer) {
+		/* Write out rest of the file */
+		git_filebuf_write(&file, post_start, cfg->reader.buffer.size - (post_start - data_start));
+	} else {
+		if (preg_replaced) {
+			git_filebuf_printf(&file, "\n%s", data_start);
+		} else {
+			git_filebuf_write(&file, cfg->reader.buffer.ptr, cfg->reader.buffer.size);
+
+			/* And now if we just need to add a variable */
+			if (!section_matches && write_section(&file, section) < 0)
+				goto rewrite_fail;
+
+			/* Sanity check: if we are here, and value is NULL, that means that somebody
+			 * touched the config file after our intial read. We should probably assert()
+			 * this, but instead we'll handle it gracefully with an error. */
+			if (value == NULL) {
+				giterr_set(GITERR_CONFIG,
+					"Race condition when writing a config file (a cvar has been removed)");
+				goto rewrite_fail;
+			}
+
+			git_filebuf_printf(&file, "\t%s = %s\n", name, value);
+		}
+	}
+
+	git__free(section);
+	git__free(current_section);
+
+	result = git_filebuf_commit(&file, GIT_CONFIG_FILE_MODE);
+	git_buf_free(&cfg->reader.buffer);
+	return result;
+
+rewrite_fail:
+	git__free(section);
+	git__free(current_section);
+
+	git_filebuf_cleanup(&file);
+	git_buf_free(&cfg->reader.buffer);
+	return -1;
+}
+
+/* '\"' -> '"' etc */
+static char *fixup_line(const char *ptr, int quote_count)
 {
-	char *end = strrchr(str, '\0') - 1;
+	char *str = git__malloc(strlen(ptr) + 1);
+	char *out = str, *esc;
+	const char *escapes = "ntb\"\\";
+	const char *escaped = "\n\t\b\"\\";
 
-	while (isspace(*end))
-		--end;
+	if (str == NULL)
+		return NULL;
 
-	return *end == '\\';
+	while (*ptr != '\0') {
+		if (*ptr == '"') {
+			quote_count++;
+		} else if (*ptr != '\\') {
+			*out++ = *ptr;
+		} else {
+			/* backslash, check the next char */
+			ptr++;
+			/* if we're at the end, it's a multiline, so keep the backslash */
+			if (*ptr == '\0') {
+				*out++ = '\\';
+				goto out;
+			}
+			if ((esc = strchr(escapes, *ptr)) != NULL) {
+				*out++ = escaped[esc - escapes];
+			} else {
+				git__free(str);
+				giterr_set(GITERR_CONFIG, "Invalid escape at %s", ptr);
+				return NULL;
+			}
+		}
+		ptr++;
+	}
+
+out:
+	*out = '\0';
+
+	return str;
+}
+
+static int is_multiline_var(const char *str)
+{
+	const char *end = str + strlen(str);
+	return (end > str) && (end[-1] == '\\');
 }
 
-static int parse_multiline_variable(diskfile_backend *cfg, const char *first, char **out)
+static int parse_multiline_variable(diskfile_backend *cfg, git_buf *value, int in_quotes)
 {
-	char *line = NULL, *end;
-	int error = GIT_SUCCESS, len, ret;
-	char *buf;
+	char *line = NULL, *proc_line = NULL;
+	int quote_count;
 
 	/* Check that the next line exists */
-	line = cfg_readline(cfg);
+	line = cfg_readline(cfg, false);
 	if (line == NULL)
-		return GIT_ENOMEM;
+		return -1;
 
 	/* We've reached the end of the file, there is input missing */
 	if (line[0] == '\0') {
-		error = git__throw(GIT_EOBJCORRUPTED, "Failed to parse multiline var. File ended unexpectedly");
-		goto out;
+		set_parse_error(cfg, 0, "Unexpected end of file while parsing multine var");
+		git__free(line);
+		return -1;
 	}
 
-	strip_comments(line);
+	quote_count = strip_comments(line, !!in_quotes);
 
 	/* If it was just a comment, pretend it didn't exist */
 	if (line[0] == '\0') {
-		error = parse_multiline_variable(cfg, first, out);
-		goto out;
+		git__free(line);
+		return parse_multiline_variable(cfg, value, quote_count);
+		/* TODO: unbounded recursion. This **could** be exploitable */
 	}
 
-	/* Find the continuation character '\' and strip the whitespace */
-	end = strrchr(first, '\\');
-	while (isspace(end[-1]))
-		--end;
+	/* Drop the continuation character '\': to closely follow the UNIX
+	 * standard, this character **has** to be last one in the buf, with
+	 * no whitespace after it */
+	assert(is_multiline_var(value->ptr));
+	git_buf_truncate(value, git_buf_len(value) - 1);
 
-	*end = '\0'; /* Terminate the string here */
-
-	len = strlen(first) + strlen(line) + 2;
-	buf = git__malloc(len);
-	if (buf == NULL) {
-		error = GIT_ENOMEM;
-		goto out;
-	}
-
-	ret = snprintf(buf, len, "%s %s", first, line);
-	if (ret < 0) {
-		error = git__throw(GIT_EOSERR, "Failed to parse multiline var. Failed to put together two lines. OS err: %s", strerror(errno));
-		free(buf);
-		goto out;
+	proc_line = fixup_line(line, in_quotes);
+	if (proc_line == NULL) {
+		git__free(line);
+		return -1;
 	}
+	/* add this line to the multiline var */
+	git_buf_puts(value, proc_line);
+	git__free(line);
+	git__free(proc_line);
 
 	/*
-	 * If we need to continue reading the next line, pretend
-	 * everything we've read up to now was in one line and call
-	 * ourselves.
+	 * If we need to continue reading the next line, let's just
+	 * keep putting stuff in the buffer
 	 */
-	if (is_multiline_var(buf)) {
-		char *final_val;
-		error = parse_multiline_variable(cfg, buf, &final_val);
-		free(buf);
-		buf = final_val;
-	}
-
-	*out = buf;
+	if (is_multiline_var(value->ptr))
+		return parse_multiline_variable(cfg, value, quote_count);
 
- out:
-	free(line);
-	return error;
+	return 0;
 }
 
 static int parse_variable(diskfile_backend *cfg, char **var_name, char **var_value)
 {
-	char *tmp;
-	int error = GIT_SUCCESS;
 	const char *var_end = NULL;
 	const char *value_start = NULL;
 	char *line;
+	int quote_count;
 
-	line = cfg_readline(cfg);
+	line = cfg_readline(cfg, true);
 	if (line == NULL)
-		return GIT_ENOMEM;
+		return -1;
 
-	strip_comments(line);
+	quote_count = strip_comments(line, 0);
 
 	var_end = strchr(line, '=');
 
@@ -959,51 +1274,47 @@ static int parse_variable(diskfile_backend *cfg, char **var_name, char **var_val
 	else
 		value_start = var_end + 1;
 
-	if (isspace(var_end[-1])) {
+	if (git__isspace(var_end[-1])) {
 		do var_end--;
-		while (isspace(var_end[0]));
+		while (git__isspace(var_end[0]));
 	}
 
-	tmp = git__strndup(line, var_end - line + 1);
-	if (tmp == NULL) {
-		error = GIT_ENOMEM;
-		goto out;
-	}
+	*var_name = git__strndup(line, var_end - line + 1);
+	GITERR_CHECK_ALLOC(*var_name);
 
-	*var_name = tmp;
+	/* If there is no value, boolean true is assumed */
+	*var_value = NULL;
 
 	/*
 	 * Now, let's try to parse the value
 	 */
 	if (value_start != NULL) {
-
-		while (isspace(value_start[0]))
+		while (git__isspace(value_start[0]))
 			value_start++;
 
-		if (value_start[0] == '\0')
-			goto out;
-
 		if (is_multiline_var(value_start)) {
-			error = parse_multiline_variable(cfg, value_start, var_value);
-			if (error < GIT_SUCCESS)
-				free(*var_name);
-			goto out;
-		}
+			git_buf multi_value = GIT_BUF_INIT;
+			char *proc_line = fixup_line(value_start, 0);
+			GITERR_CHECK_ALLOC(proc_line);
+			git_buf_puts(&multi_value, proc_line);
+			git__free(proc_line);
+			if (parse_multiline_variable(cfg, &multi_value, quote_count) < 0 || git_buf_oom(&multi_value)) {
+				git__free(*var_name);
+				git__free(line);
+				git_buf_free(&multi_value);
+				return -1;
+			}
 
-		tmp = strdup(value_start);
-		if (tmp == NULL) {
-			free(*var_name);
-			error = GIT_ENOMEM;
-			goto out;
+			*var_value = git_buf_detach(&multi_value);
+
+		}
+		else if (value_start[0] != '\0') {
+			*var_value = fixup_line(value_start, 0);
+			GITERR_CHECK_ALLOC(*var_value);
 		}
 
-		*var_value = tmp;
-	} else {
-		/* If there is no value, boolean true is assumed */
-		*var_value = NULL;
 	}
 
- out:
-	free(line);
-	return error;
+	git__free(line);
+	return 0;
 }
diff --git a/src/config_file.h b/src/config_file.h
new file mode 100644
index 000000000..0080b5713
--- /dev/null
+++ b/src/config_file.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_config_file_h__
+#define INCLUDE_config_file_h__
+
+#include "git2/config.h"
+
+GIT_INLINE(int) git_config_file_open(git_config_file *cfg)
+{
+	return cfg->open(cfg);
+}
+
+GIT_INLINE(void) git_config_file_free(git_config_file *cfg)
+{
+	cfg->free(cfg);
+}
+
+GIT_INLINE(int) git_config_file_foreach(
+	git_config_file *cfg,
+	int (*fn)(const char *key, const char *value, void *data),
+	void *data)
+{
+	return cfg->foreach(cfg, fn, data);
+}
+
+#endif
+
diff --git a/src/crlf.c b/src/crlf.c
new file mode 100644
index 000000000..303a46d3b
--- /dev/null
+++ b/src/crlf.c
@@ -0,0 +1,228 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "fileops.h"
+#include "hash.h"
+#include "filter.h"
+#include "repository.h"
+
+#include "git2/attr.h"
+
+struct crlf_attrs {
+	int crlf_action;
+	int eol;
+};
+
+struct crlf_filter {
+	git_filter f;
+	struct crlf_attrs attrs;
+};
+
+static int check_crlf(const char *value)
+{
+	if (GIT_ATTR_TRUE(value))
+		return GIT_CRLF_TEXT;
+
+	if (GIT_ATTR_FALSE(value))
+		return GIT_CRLF_BINARY;
+
+	if (GIT_ATTR_UNSPECIFIED(value))
+		return GIT_CRLF_GUESS;
+
+	if (strcmp(value, "input") == 0)
+		return GIT_CRLF_INPUT;
+
+	if (strcmp(value, "auto") == 0)
+		return GIT_CRLF_AUTO;
+
+	return GIT_CRLF_GUESS;
+}
+
+static int check_eol(const char *value)
+{
+	if (GIT_ATTR_UNSPECIFIED(value))
+		return GIT_EOL_UNSET;
+
+	if (strcmp(value, "lf") == 0)
+		return GIT_EOL_LF;
+
+	if (strcmp(value, "crlf") == 0)
+		return GIT_EOL_CRLF;
+
+	return GIT_EOL_UNSET;
+}
+
+static int crlf_input_action(struct crlf_attrs *ca)
+{
+	if (ca->crlf_action == GIT_CRLF_BINARY)
+		return GIT_CRLF_BINARY;
+
+	if (ca->eol == GIT_EOL_LF)
+		return GIT_CRLF_INPUT;
+
+	if (ca->eol == GIT_EOL_CRLF)
+		return GIT_CRLF_CRLF;
+
+	return ca->crlf_action;
+}
+
+static int crlf_load_attributes(struct crlf_attrs *ca, git_repository *repo, const char *path)
+{
+#define NUM_CONV_ATTRS 3
+
+	static const char *attr_names[NUM_CONV_ATTRS] = {
+		"crlf", "eol", "text",
+	};
+
+	const char *attr_vals[NUM_CONV_ATTRS];
+	int error;
+
+	error = git_attr_get_many(attr_vals,
+		repo, 0, path, NUM_CONV_ATTRS, attr_names);
+
+	if (error == GIT_ENOTFOUND) {
+		ca->crlf_action = GIT_CRLF_GUESS;
+		ca->eol = GIT_EOL_UNSET;
+		return 0;
+	}
+
+	if (error == 0) {
+		ca->crlf_action = check_crlf(attr_vals[2]); /* text */
+		if (ca->crlf_action == GIT_CRLF_GUESS)
+			ca->crlf_action = check_crlf(attr_vals[0]); /* clrf */
+
+		ca->eol = check_eol(attr_vals[1]); /* eol */
+		return 0;
+	}
+
+	return -1;
+}
+
+static int drop_crlf(git_buf *dest, const git_buf *source)
+{
+	const char *scan = source->ptr, *next;
+	const char *scan_end = git_buf_cstr(source) + git_buf_len(source);
+
+	/* Main scan loop.  Find the next carriage return and copy the
+	 * whole chunk up to that point to the destination buffer.
+	 */
+	while ((next = memchr(scan, '\r', scan_end - scan)) != NULL) {
+		/* copy input up to \r */
+		if (next > scan)
+			git_buf_put(dest, scan, next - scan);
+
+		/* Do not drop \r unless it is followed by \n */
+		if (*(next + 1) != '\n')
+			git_buf_putc(dest, '\r');
+
+		scan = next + 1;
+	}
+
+	/* If there was no \r, then tell the library to skip this filter */
+	if (scan == source->ptr)
+		return -1;
+
+	/* Copy remaining input into dest */
+	git_buf_put(dest, scan, scan_end - scan);
+	return 0;
+}
+
+static int crlf_apply_to_odb(git_filter *self, git_buf *dest, const git_buf *source)
+{
+	struct crlf_filter *filter = (struct crlf_filter *)self;
+
+	assert(self && dest && source);
+
+	/* Empty file? Nothing to do */
+	if (git_buf_len(source) == 0)
+		return 0;
+
+	/* Heuristics to see if we can skip the conversion.
+	 * Straight from Core Git.
+	 */
+	if (filter->attrs.crlf_action == GIT_CRLF_AUTO ||
+		filter->attrs.crlf_action == GIT_CRLF_GUESS) {
+
+		git_text_stats stats;
+		git_text_gather_stats(&stats, source);
+
+		/*
+		 * We're currently not going to even try to convert stuff
+		 * that has bare CR characters. Does anybody do that crazy
+		 * stuff?
+		 */
+		if (stats.cr != stats.crlf)
+			return -1;
+
+		/*
+		 * And add some heuristics for binary vs text, of course...
+		 */
+		if (git_text_is_binary(&stats))
+			return -1;
+
+#if 0
+		if (crlf_action == CRLF_GUESS) {
+			/*
+			 * If the file in the index has any CR in it, do not convert.
+			 * This is the new safer autocrlf handling.
+			 */
+			if (has_cr_in_index(path))
+				return 0;
+		}
+#endif
+
+		if (!stats.cr)
+			return -1;
+	}
+
+	/* Actually drop the carriage returns */
+	return drop_crlf(dest, source);
+}
+
+int git_filter_add__crlf_to_odb(git_vector *filters, git_repository *repo, const char *path)
+{
+	struct crlf_attrs ca;
+	struct crlf_filter *filter;
+	int error;
+
+	/* Load gitattributes for the path */
+	if ((error = crlf_load_attributes(&ca, repo, path)) < 0)
+		return error;
+
+	/*
+	 * Use the core Git logic to see if we should perform CRLF for this file
+	 * based on its attributes & the value of `core.auto_crlf`
+	 */
+	ca.crlf_action = crlf_input_action(&ca);
+
+	if (ca.crlf_action == GIT_CRLF_BINARY)
+		return 0;
+
+	if (ca.crlf_action == GIT_CRLF_GUESS) {
+		int auto_crlf;
+
+		if ((error = git_repository__cvar(
+			&auto_crlf, repo, GIT_CVAR_AUTO_CRLF)) < 0)
+			return error;
+
+		if (auto_crlf == GIT_AUTO_CRLF_FALSE)
+			return 0;
+	}
+
+	/* If we're good, we create a new filter object and push it
+	 * into the filters array */
+	filter = git__malloc(sizeof(struct crlf_filter));
+	GITERR_CHECK_ALLOC(filter);
+
+	filter->f.apply = &crlf_apply_to_odb;
+	filter->f.do_free = NULL;
+	memcpy(&filter->attrs, &ca, sizeof(struct crlf_attrs));
+
+	return git_vector_insert(filters, filter);
+}
+
diff --git a/src/delta-apply.c b/src/delta-apply.c
index a6b711436..815ca8f16 100644
--- a/src/delta-apply.c
+++ b/src/delta-apply.c
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #include "common.h"
 #include "git2/odb.h"
 #include "delta-apply.h"
@@ -5,7 +11,7 @@
 /*
  * This file was heavily cribbed from BinaryDelta.java in JGit, which
  * itself was heavily cribbed from <code>patch-delta.c</code> in the
- * GIT project.   The original delta patching code was written by
+ * GIT project.	The original delta patching code was written by
  * Nicolas Pitre <nico@cam.org>.
  */
 
@@ -45,14 +51,19 @@ int git__delta_apply(
 	 * if not we would underflow while accessing data from the
 	 * base object, resulting in data corruption or segfault.
 	 */
-	if ((hdr_sz(&base_sz, &delta, delta_end) < 0) || (base_sz != base_len))
-		return git__throw(GIT_ERROR, "Failed to apply delta. Base size does not match given data");
+	if ((hdr_sz(&base_sz, &delta, delta_end) < 0) || (base_sz != base_len)) {
+		giterr_set(GITERR_INVALID, "Failed to apply delta. Base size does not match given data");
+		return -1;
+	}
 
-	if (hdr_sz(&res_sz, &delta, delta_end) < 0)
-		return git__throw(GIT_ERROR, "Failed to apply delta. Base size does not match given data");
+	if (hdr_sz(&res_sz, &delta, delta_end) < 0) {
+		giterr_set(GITERR_INVALID, "Failed to apply delta. Base size does not match given data");
+		return -1;
+	}
+
+	res_dp = git__malloc(res_sz + 1);
+	GITERR_CHECK_ALLOC(res_dp);
 
-	if ((res_dp = git__malloc(res_sz + 1)) == NULL)
-		return GIT_ENOMEM;
 	res_dp[res_sz] = '\0';
 	out->data = res_dp;
 	out->len = res_sz;
@@ -64,15 +75,15 @@ int git__delta_apply(
 			 */
 			size_t off = 0, len = 0;
 
-			if (cmd & 0x01) off  = *delta++;
-			if (cmd & 0x02) off |= *delta++ <<  8;
+			if (cmd & 0x01) off = *delta++;
+			if (cmd & 0x02) off |= *delta++ << 8;
 			if (cmd & 0x04) off |= *delta++ << 16;
 			if (cmd & 0x08) off |= *delta++ << 24;
 
-			if (cmd & 0x10) len  = *delta++;
-			if (cmd & 0x20) len |= *delta++ <<  8;
+			if (cmd & 0x10) len = *delta++;
+			if (cmd & 0x20) len |= *delta++ << 8;
 			if (cmd & 0x40) len |= *delta++ << 16;
-			if (!len)       len  = 0x10000;
+			if (!len)		len = 0x10000;
 
 			if (base_len < off + len || res_sz < len)
 				goto fail;
@@ -87,7 +98,7 @@ int git__delta_apply(
 			if (delta_end - delta < cmd || res_sz < cmd)
 				goto fail;
 			memcpy(res_dp, delta, cmd);
-			delta  += cmd;
+			delta += cmd;
 			res_dp += cmd;
 			res_sz -= cmd;
 
@@ -100,10 +111,11 @@ int git__delta_apply(
 
 	if (delta != delta_end || res_sz)
 		goto fail;
-	return GIT_SUCCESS;
+	return 0;
 
 fail:
-	free(out->data);
+	git__free(out->data);
 	out->data = NULL;
-	return git__throw(GIT_ERROR, "Failed to apply delta");
+	giterr_set(GITERR_INVALID, "Failed to apply delta");
+	return -1;
 }
diff --git a/src/delta-apply.h b/src/delta-apply.h
index 36c5cc60d..66fa76d43 100644
--- a/src/delta-apply.h
+++ b/src/delta-apply.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_delta_apply_h__
 #define INCLUDE_delta_apply_h__
 
@@ -14,7 +20,7 @@
  * @param delta the delta to execute copy/insert instructions from.
  * @param delta_len total number of bytes in the delta.
  * @return
- * - GIT_SUCCESS on a successful delta unpack.
+ * - 0 on a successful delta unpack.
  * - GIT_ERROR if the delta is corrupt or doesn't match the base.
  */
 extern int git__delta_apply(
diff --git a/src/diff.c b/src/diff.c
new file mode 100644
index 000000000..90baa9588
--- /dev/null
+++ b/src/diff.c
@@ -0,0 +1,784 @@
+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "git2/diff.h"
+#include "diff.h"
+#include "fileops.h"
+#include "config.h"
+#include "attr_file.h"
+
+static char *diff_prefix_from_pathspec(const git_strarray *pathspec)
+{
+	git_buf prefix = GIT_BUF_INIT;
+	const char *scan;
+
+	if (git_buf_common_prefix(&prefix, pathspec) < 0)
+		return NULL;
+
+	/* diff prefix will only be leading non-wildcards */
+	for (scan = prefix.ptr; *scan && !git__iswildcard(*scan); ++scan);
+	git_buf_truncate(&prefix, scan - prefix.ptr);
+
+	if (prefix.size > 0)
+		return git_buf_detach(&prefix);
+
+	git_buf_free(&prefix);
+	return NULL;
+}
+
+static bool diff_pathspec_is_interesting(const git_strarray *pathspec)
+{
+	const char *str;
+
+	if (pathspec == NULL || pathspec->count == 0)
+		return false;
+	if (pathspec->count > 1)
+		return true;
+
+	str = pathspec->strings[0];
+	if (!str || !str[0] || (!str[1] && (str[0] == '*' || str[0] == '.')))
+		return false;
+	return true;
+}
+
+static bool diff_path_matches_pathspec(git_diff_list *diff, const char *path)
+{
+	unsigned int i;
+	git_attr_fnmatch *match;
+
+	if (!diff->pathspec.length)
+		return true;
+
+	git_vector_foreach(&diff->pathspec, i, match) {
+		int result = p_fnmatch(match->pattern, path, 0);
+
+		/* if we didn't match, look for exact dirname prefix match */
+		if (result == FNM_NOMATCH &&
+			(match->flags & GIT_ATTR_FNMATCH_HASWILD) == 0 &&
+			strncmp(path, match->pattern, match->length) == 0 &&
+			path[match->length] == '/')
+			result = 0;
+
+		if (result == 0)
+			return (match->flags & GIT_ATTR_FNMATCH_NEGATIVE) ? false : true;
+	}
+
+	return false;
+}
+
+static git_diff_delta *diff_delta__alloc(
+	git_diff_list *diff,
+	git_delta_t status,
+	const char *path)
+{
+	git_diff_delta *delta = git__calloc(1, sizeof(git_diff_delta));
+	if (!delta)
+		return NULL;
+
+	delta->old_file.path = git_pool_strdup(&diff->pool, path);
+	if (delta->old_file.path == NULL) {
+		git__free(delta);
+		return NULL;
+	}
+
+	delta->new_file.path = delta->old_file.path;
+
+	if (diff->opts.flags & GIT_DIFF_REVERSE) {
+		switch (status) {
+		case GIT_DELTA_ADDED:   status = GIT_DELTA_DELETED; break;
+		case GIT_DELTA_DELETED: status = GIT_DELTA_ADDED; break;
+		default: break; /* leave other status values alone */
+		}
+	}
+	delta->status = status;
+
+	return delta;
+}
+
+static git_diff_delta *diff_delta__dup(
+	const git_diff_delta *d, git_pool *pool)
+{
+	git_diff_delta *delta = git__malloc(sizeof(git_diff_delta));
+	if (!delta)
+		return NULL;
+
+	memcpy(delta, d, sizeof(git_diff_delta));
+
+	delta->old_file.path = git_pool_strdup(pool, d->old_file.path);
+	if (delta->old_file.path == NULL)
+		goto fail;
+
+	if (d->new_file.path != d->old_file.path) {
+		delta->new_file.path = git_pool_strdup(pool, d->new_file.path);
+		if (delta->new_file.path == NULL)
+			goto fail;
+	} else {
+		delta->new_file.path = delta->old_file.path;
+	}
+
+	return delta;
+
+fail:
+	git__free(delta);
+	return NULL;
+}
+
+static git_diff_delta *diff_delta__merge_like_cgit(
+	const git_diff_delta *a, const git_diff_delta *b, git_pool *pool)
+{
+	git_diff_delta *dup = diff_delta__dup(a, pool);
+	if (!dup)
+		return NULL;
+
+	if (git_oid_cmp(&dup->new_file.oid, &b->new_file.oid) == 0)
+		return dup;
+
+	git_oid_cpy(&dup->new_file.oid, &b->new_file.oid);
+
+	dup->new_file.mode = b->new_file.mode;
+	dup->new_file.size = b->new_file.size;
+	dup->new_file.flags = b->new_file.flags;
+
+	/* Emulate C git for merging two diffs (a la 'git diff <sha>').
+	 *
+	 * When C git does a diff between the work dir and a tree, it actually
+	 * diffs with the index but uses the workdir contents.  This emulates
+	 * those choices so we can emulate the type of diff.
+	 */
+	if (git_oid_cmp(&dup->old_file.oid, &dup->new_file.oid) == 0) {
+		if (dup->status == GIT_DELTA_DELETED)
+			/* preserve pending delete info */;
+		else if (b->status == GIT_DELTA_UNTRACKED ||
+				 b->status == GIT_DELTA_IGNORED)
+			dup->status = b->status;
+		else
+			dup->status = GIT_DELTA_UNMODIFIED;
+	}
+	else if (dup->status == GIT_DELTA_UNMODIFIED ||
+			 b->status == GIT_DELTA_DELETED)
+		dup->status = b->status;
+
+	return dup;
+}
+
+static int diff_delta__from_one(
+	git_diff_list *diff,
+	git_delta_t   status,
+	const git_index_entry *entry)
+{
+	git_diff_delta *delta;
+
+	if (status == GIT_DELTA_IGNORED &&
+		(diff->opts.flags & GIT_DIFF_INCLUDE_IGNORED) == 0)
+		return 0;
+
+	if (status == GIT_DELTA_UNTRACKED &&
+		(diff->opts.flags & GIT_DIFF_INCLUDE_UNTRACKED) == 0)
+		return 0;
+
+	if (!diff_path_matches_pathspec(diff, entry->path))
+		return 0;
+
+	delta = diff_delta__alloc(diff, status, entry->path);
+	GITERR_CHECK_ALLOC(delta);
+
+	/* This fn is just for single-sided diffs */
+	assert(status != GIT_DELTA_MODIFIED);
+
+	if (delta->status == GIT_DELTA_DELETED) {
+		delta->old_file.mode = entry->mode;
+		delta->old_file.size = entry->file_size;
+		git_oid_cpy(&delta->old_file.oid, &entry->oid);
+	} else /* ADDED, IGNORED, UNTRACKED */ {
+		delta->new_file.mode = entry->mode;
+		delta->new_file.size = entry->file_size;
+		git_oid_cpy(&delta->new_file.oid, &entry->oid);
+	}
+
+	delta->old_file.flags |= GIT_DIFF_FILE_VALID_OID;
+	delta->new_file.flags |= GIT_DIFF_FILE_VALID_OID;
+
+	if (git_vector_insert(&diff->deltas, delta) < 0) {
+		git__free(delta);
+		return -1;
+	}
+
+	return 0;
+}
+
+static int diff_delta__from_two(
+	git_diff_list *diff,
+	git_delta_t   status,
+	const git_index_entry *old_entry,
+	const git_index_entry *new_entry,
+	git_oid *new_oid)
+{
+	git_diff_delta *delta;
+
+	if (status == GIT_DELTA_UNMODIFIED &&
+		(diff->opts.flags & GIT_DIFF_INCLUDE_UNMODIFIED) == 0)
+		return 0;
+
+	if ((diff->opts.flags & GIT_DIFF_REVERSE) != 0) {
+		const git_index_entry *temp = old_entry;
+		old_entry = new_entry;
+		new_entry = temp;
+	}
+
+	delta = diff_delta__alloc(diff, status, old_entry->path);
+	GITERR_CHECK_ALLOC(delta);
+
+	delta->old_file.mode = old_entry->mode;
+	git_oid_cpy(&delta->old_file.oid, &old_entry->oid);
+	delta->old_file.flags |= GIT_DIFF_FILE_VALID_OID;
+
+	delta->new_file.mode = new_entry->mode;
+	git_oid_cpy(&delta->new_file.oid, new_oid ? new_oid : &new_entry->oid);
+	if (new_oid || !git_oid_iszero(&new_entry->oid))
+		delta->new_file.flags |= GIT_DIFF_FILE_VALID_OID;
+
+	if (git_vector_insert(&diff->deltas, delta) < 0) {
+		git__free(delta);
+		return -1;
+	}
+
+	return 0;
+}
+
+static char *diff_strdup_prefix(git_pool *pool, const char *prefix)
+{
+	size_t len = strlen(prefix);
+
+	/* append '/' at end if needed */
+	if (len > 0 && prefix[len - 1] != '/')
+		return git_pool_strcat(pool, prefix, "/");
+	else
+		return git_pool_strndup(pool, prefix, len + 1);
+}
+
+static int diff_delta__cmp(const void *a, const void *b)
+{
+	const git_diff_delta *da = a, *db = b;
+	int val = strcmp(da->old_file.path, db->old_file.path);
+	return val ? val : ((int)da->status - (int)db->status);
+}
+
+static int config_bool(git_config *cfg, const char *name, int defvalue)
+{
+	int val = defvalue;
+
+	if (git_config_get_bool(&val, cfg, name) < 0)
+		giterr_clear();
+
+	return val;
+}
+
+static git_diff_list *git_diff_list_alloc(
+	git_repository *repo, const git_diff_options *opts)
+{
+	git_config *cfg;
+	size_t i;
+	git_diff_list *diff = git__calloc(1, sizeof(git_diff_list));
+	if (diff == NULL)
+		return NULL;
+
+	diff->repo = repo;
+
+	if (git_vector_init(&diff->deltas, 0, diff_delta__cmp) < 0 ||
+		git_pool_init(&diff->pool, 1, 0) < 0)
+		goto fail;
+
+	/* load config values that affect diff behavior */
+	if (git_repository_config__weakptr(&cfg, repo) < 0)
+		goto fail;
+	if (config_bool(cfg, "core.symlinks", 1))
+		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_HAS_SYMLINKS;
+	if (config_bool(cfg, "core.ignorestat", 0))
+		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_ASSUME_UNCHANGED;
+	if (config_bool(cfg, "core.filemode", 1))
+		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_TRUST_EXEC_BIT;
+	if (config_bool(cfg, "core.trustctime", 1))
+		diff->diffcaps = diff->diffcaps | GIT_DIFFCAPS_TRUST_CTIME;
+	/* Don't set GIT_DIFFCAPS_USE_DEV - compile time option in core git */
+
+	if (opts == NULL)
+		return diff;
+
+	memcpy(&diff->opts, opts, sizeof(git_diff_options));
+	memset(&diff->opts.pathspec, 0, sizeof(diff->opts.pathspec));
+
+	diff->opts.old_prefix = diff_strdup_prefix(&diff->pool,
+		opts->old_prefix ? opts->old_prefix : DIFF_OLD_PREFIX_DEFAULT);
+	diff->opts.new_prefix = diff_strdup_prefix(&diff->pool,
+		opts->new_prefix ? opts->new_prefix : DIFF_NEW_PREFIX_DEFAULT);
+
+	if (!diff->opts.old_prefix || !diff->opts.new_prefix)
+		goto fail;
+
+	if (diff->opts.flags & GIT_DIFF_REVERSE) {
+		char *swap = diff->opts.old_prefix;
+		diff->opts.old_prefix = diff->opts.new_prefix;
+		diff->opts.new_prefix = swap;
+	}
+
+	/* only copy pathspec if it is "interesting" so we can test
+	 * diff->pathspec.length > 0 to know if it is worth calling
+	 * fnmatch as we iterate.
+	 */
+	if (!diff_pathspec_is_interesting(&opts->pathspec))
+		return diff;
+
+	if (git_vector_init(
+		&diff->pathspec, (unsigned int)opts->pathspec.count, NULL) < 0)
+		goto fail;
+
+	for (i = 0; i < opts->pathspec.count; ++i) {
+		int ret;
+		const char *pattern = opts->pathspec.strings[i];
+		git_attr_fnmatch *match = git__calloc(1, sizeof(git_attr_fnmatch));
+		if (!match)
+			goto fail;
+		match->flags = GIT_ATTR_FNMATCH_ALLOWSPACE;
+		ret = git_attr_fnmatch__parse(match, &diff->pool, NULL, &pattern);
+		if (ret == GIT_ENOTFOUND) {
+			git__free(match);
+			continue;
+		} else if (ret < 0)
+			goto fail;
+
+		if (git_vector_insert(&diff->pathspec, match) < 0)
+			goto fail;
+	}
+
+	return diff;
+
+fail:
+	git_diff_list_free(diff);
+	return NULL;
+}
+
+void git_diff_list_free(git_diff_list *diff)
+{
+	git_diff_delta *delta;
+	git_attr_fnmatch *match;
+	unsigned int i;
+
+	if (!diff)
+		return;
+
+	git_vector_foreach(&diff->deltas, i, delta) {
+		git__free(delta);
+		diff->deltas.contents[i] = NULL;
+	}
+	git_vector_free(&diff->deltas);
+
+	git_vector_foreach(&diff->pathspec, i, match) {
+		git__free(match);
+		diff->pathspec.contents[i] = NULL;
+	}
+	git_vector_free(&diff->pathspec);
+
+	git_pool_clear(&diff->pool);
+	git__free(diff);
+}
+
+static int oid_for_workdir_item(
+	git_repository *repo,
+	const git_index_entry *item,
+	git_oid *oid)
+{
+	int result;
+	git_buf full_path = GIT_BUF_INIT;
+
+	if (git_buf_joinpath(&full_path, git_repository_workdir(repo), item->path) < 0)
+		return -1;
+
+	/* calculate OID for file if possible*/
+	if (S_ISLNK(item->mode))
+		result = git_odb__hashlink(oid, full_path.ptr);
+	else if (!git__is_sizet(item->file_size)) {
+		giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
+		result = -1;
+	} else {
+		int fd = git_futils_open_ro(full_path.ptr);
+		if (fd < 0)
+			result = fd;
+		else {
+			result = git_odb__hashfd(
+				oid, fd, (size_t)item->file_size, GIT_OBJ_BLOB);
+			p_close(fd);
+		}
+	}
+
+	git_buf_free(&full_path);
+
+	return result;
+}
+
+#define EXEC_BIT_MASK 0000111
+
+static int maybe_modified(
+	git_iterator *old_iter,
+	const git_index_entry *oitem,
+	git_iterator *new_iter,
+	const git_index_entry *nitem,
+	git_diff_list *diff)
+{
+	git_oid noid, *use_noid = NULL;
+	git_delta_t status = GIT_DELTA_MODIFIED;
+	unsigned int omode = oitem->mode;
+	unsigned int nmode = nitem->mode;
+
+	GIT_UNUSED(old_iter);
+
+	if (!diff_path_matches_pathspec(diff, oitem->path))
+		return 0;
+
+	/* on platforms with no symlinks, promote plain files to symlinks */
+	if (S_ISLNK(omode) && S_ISREG(nmode) &&
+		!(diff->diffcaps & GIT_DIFFCAPS_HAS_SYMLINKS))
+		nmode = GIT_MODE_TYPE(omode) | (nmode & GIT_MODE_PERMS_MASK);
+
+	/* on platforms with no execmode, clear exec bit from comparisons */
+	if (!(diff->diffcaps & GIT_DIFFCAPS_TRUST_EXEC_BIT)) {
+		omode = omode & ~EXEC_BIT_MASK;
+		nmode = nmode & ~EXEC_BIT_MASK;
+	}
+
+	/* support "assume unchanged" (badly, b/c we still stat everything) */
+	if ((diff->diffcaps & GIT_DIFFCAPS_ASSUME_UNCHANGED) != 0)
+		status = (oitem->flags_extended & GIT_IDXENTRY_INTENT_TO_ADD) ?
+			GIT_DELTA_MODIFIED : GIT_DELTA_UNMODIFIED;
+
+	/* support "skip worktree" index bit */
+	else if ((oitem->flags_extended & GIT_IDXENTRY_SKIP_WORKTREE) != 0)
+		status = GIT_DELTA_UNMODIFIED;
+
+	/* if basic type of file changed, then split into delete and add */
+	else if (GIT_MODE_TYPE(omode) != GIT_MODE_TYPE(nmode)) {
+		if (diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem) < 0 ||
+			diff_delta__from_one(diff, GIT_DELTA_ADDED, nitem) < 0)
+			return -1;
+		return 0;
+	}
+
+	/* if oids and modes match, then file is unmodified */
+	else if (git_oid_cmp(&oitem->oid, &nitem->oid) == 0 &&
+			 omode == nmode)
+		status = GIT_DELTA_UNMODIFIED;
+
+	/* if we have a workdir item with an unknown oid, check deeper */
+	else if (git_oid_iszero(&nitem->oid) && new_iter->type == GIT_ITERATOR_WORKDIR) {
+		/* TODO: add check against index file st_mtime to avoid racy-git */
+
+		/* if they files look exactly alike, then we'll assume the same */
+		if (oitem->file_size == nitem->file_size &&
+			(!(diff->diffcaps & GIT_DIFFCAPS_TRUST_CTIME) ||
+			 (oitem->ctime.seconds == nitem->ctime.seconds)) &&
+			oitem->mtime.seconds == nitem->mtime.seconds &&
+			(!(diff->diffcaps & GIT_DIFFCAPS_USE_DEV) ||
+			 (oitem->dev == nitem->dev)) &&
+			oitem->ino == nitem->ino &&
+			oitem->uid == nitem->uid &&
+			oitem->gid == nitem->gid)
+			status = GIT_DELTA_UNMODIFIED;
+
+		else if (S_ISGITLINK(nmode)) {
+			git_submodule *sub;
+
+			if ((diff->opts.flags & GIT_DIFF_IGNORE_SUBMODULES) != 0)
+				status = GIT_DELTA_UNMODIFIED;
+			else if (git_submodule_lookup(&sub, diff->repo, nitem->path) < 0)
+				return -1;
+			else if (sub->ignore == GIT_SUBMODULE_IGNORE_ALL)
+				status = GIT_DELTA_UNMODIFIED;
+			else {
+				/* TODO: support other GIT_SUBMODULE_IGNORE values */
+				status = GIT_DELTA_UNMODIFIED;
+			}
+		}
+
+		/* TODO: check git attributes so we will not have to read the file
+		 * in if it is marked binary.
+		 */
+
+		else if (oid_for_workdir_item(diff->repo, nitem, &noid) < 0)
+			return -1;
+
+		else if (git_oid_cmp(&oitem->oid, &noid) == 0 &&
+				 omode == nmode)
+			status = GIT_DELTA_UNMODIFIED;
+
+		/* store calculated oid so we don't have to recalc later */
+		use_noid = &noid;
+	}
+
+	return diff_delta__from_two(diff, status, oitem, nitem, use_noid);
+}
+
+static int diff_from_iterators(
+	git_repository *repo,
+	const git_diff_options *opts, /**< can be NULL for defaults */
+	git_iterator *old_iter,
+	git_iterator *new_iter,
+	git_diff_list **diff_ptr)
+{
+	const git_index_entry *oitem, *nitem;
+	git_buf ignore_prefix = GIT_BUF_INIT;
+	git_diff_list *diff = git_diff_list_alloc(repo, opts);
+	if (!diff)
+		goto fail;
+
+	diff->old_src = old_iter->type;
+	diff->new_src = new_iter->type;
+
+	if (git_iterator_current(old_iter, &oitem) < 0 ||
+		git_iterator_current(new_iter, &nitem) < 0)
+		goto fail;
+
+	/* run iterators building diffs */
+	while (oitem || nitem) {
+
+		/* create DELETED records for old items not matched in new */
+		if (oitem && (!nitem || strcmp(oitem->path, nitem->path) < 0)) {
+			if (diff_delta__from_one(diff, GIT_DELTA_DELETED, oitem) < 0 ||
+				git_iterator_advance(old_iter, &oitem) < 0)
+				goto fail;
+		}
+
+		/* create ADDED, TRACKED, or IGNORED records for new items not
+		 * matched in old (and/or descend into directories as needed)
+		 */
+		else if (nitem && (!oitem || strcmp(oitem->path, nitem->path) > 0)) {
+			git_delta_t delta_type = GIT_DELTA_UNTRACKED;
+
+			/* check if contained in ignored parent directory */
+			if (git_buf_len(&ignore_prefix) &&
+				git__prefixcmp(nitem->path, git_buf_cstr(&ignore_prefix)) == 0)
+				delta_type = GIT_DELTA_IGNORED;
+
+			if (S_ISDIR(nitem->mode)) {
+				/* recurse into directory only if there are tracked items in
+				 * it or if the user requested the contents of untracked
+				 * directories and it is not under an ignored directory.
+				 */
+				if ((oitem && git__prefixcmp(oitem->path, nitem->path) == 0) ||
+					(delta_type == GIT_DELTA_UNTRACKED &&
+					 (diff->opts.flags & GIT_DIFF_RECURSE_UNTRACKED_DIRS) != 0))
+				{
+					/* if this directory is ignored, remember it as the
+					 * "ignore_prefix" for processing contained items
+					 */
+					if (delta_type == GIT_DELTA_UNTRACKED &&
+						git_iterator_current_is_ignored(new_iter))
+						git_buf_sets(&ignore_prefix, nitem->path);
+
+					if (git_iterator_advance_into_directory(new_iter, &nitem) < 0)
+						goto fail;
+
+					continue;
+				}
+			}
+
+			/* In core git, the next two "else if" clauses are effectively
+			 * reversed -- i.e. when an untracked file contained in an
+			 * ignored directory is individually ignored, it shows up as an
+			 * ignored file in the diff list, even though other untracked
+			 * files in the same directory are skipped completely.
+			 *
+			 * To me, this is odd.  If the directory is ignored and the file
+			 * is untracked, we should skip it consistently, regardless of
+			 * whether it happens to match a pattern in the ignore file.
+			 *
+			 * To match the core git behavior, just reverse the following
+			 * two "else if" cases so that individual file ignores are
+			 * checked before container directory exclusions are used to
+			 * skip the file.
+			 */
+			else if (delta_type == GIT_DELTA_IGNORED) {
+				if (git_iterator_advance(new_iter, &nitem) < 0)
+					goto fail;
+				continue; /* ignored parent directory, so skip completely */
+			}
+
+			else if (git_iterator_current_is_ignored(new_iter))
+				delta_type = GIT_DELTA_IGNORED;
+
+			else if (new_iter->type != GIT_ITERATOR_WORKDIR)
+				delta_type = GIT_DELTA_ADDED;
+
+			if (diff_delta__from_one(diff, delta_type, nitem) < 0 ||
+				git_iterator_advance(new_iter, &nitem) < 0)
+				goto fail;
+		}
+
+		/* otherwise item paths match, so create MODIFIED record
+		 * (or ADDED and DELETED pair if type changed)
+		 */
+		else {
+			assert(oitem && nitem && strcmp(oitem->path, nitem->path) == 0);
+
+			if (maybe_modified(old_iter, oitem, new_iter, nitem, diff) < 0 ||
+				git_iterator_advance(old_iter, &oitem) < 0 ||
+				git_iterator_advance(new_iter, &nitem) < 0)
+				goto fail;
+		}
+	}
+
+	git_iterator_free(old_iter);
+	git_iterator_free(new_iter);
+	git_buf_free(&ignore_prefix);
+
+	*diff_ptr = diff;
+	return 0;
+
+fail:
+	git_iterator_free(old_iter);
+	git_iterator_free(new_iter);
+	git_buf_free(&ignore_prefix);
+
+	git_diff_list_free(diff);
+	*diff_ptr = NULL;
+	return -1;
+}
+
+
+int git_diff_tree_to_tree(
+	git_repository *repo,
+	const git_diff_options *opts, /**< can be NULL for defaults */
+	git_tree *old_tree,
+	git_tree *new_tree,
+	git_diff_list **diff)
+{
+	git_iterator *a = NULL, *b = NULL;
+	char *prefix = opts ? diff_prefix_from_pathspec(&opts->pathspec) : NULL;
+
+	assert(repo && old_tree && new_tree && diff);
+
+	if (git_iterator_for_tree_range(&a, repo, old_tree, prefix, prefix) < 0 ||
+		git_iterator_for_tree_range(&b, repo, new_tree, prefix, prefix) < 0)
+		return -1;
+
+	git__free(prefix);
+
+	return diff_from_iterators(repo, opts, a, b, diff);
+}
+
+int git_diff_index_to_tree(
+	git_repository *repo,
+	const git_diff_options *opts,
+	git_tree *old_tree,
+	git_diff_list **diff)
+{
+	git_iterator *a = NULL, *b = NULL;
+	char *prefix = opts ? diff_prefix_from_pathspec(&opts->pathspec) : NULL;
+
+	assert(repo && diff);
+
+	if (git_iterator_for_tree_range(&a, repo, old_tree, prefix, prefix) < 0 ||
+		git_iterator_for_index_range(&b, repo, prefix, prefix) < 0)
+		return -1;
+
+	git__free(prefix);
+
+	return diff_from_iterators(repo, opts, a, b, diff);
+}
+
+int git_diff_workdir_to_index(
+	git_repository *repo,
+	const git_diff_options *opts,
+	git_diff_list **diff)
+{
+	git_iterator *a = NULL, *b = NULL;
+	char *prefix = opts ? diff_prefix_from_pathspec(&opts->pathspec) : NULL;
+
+	assert(repo && diff);
+
+	if (git_iterator_for_index_range(&a, repo, prefix, prefix) < 0 ||
+		git_iterator_for_workdir_range(&b, repo, prefix, prefix) < 0)
+		return -1;
+
+	git__free(prefix);
+
+	return diff_from_iterators(repo, opts, a, b, diff);
+}
+
+
+int git_diff_workdir_to_tree(
+	git_repository *repo,
+	const git_diff_options *opts,
+	git_tree *old_tree,
+	git_diff_list **diff)
+{
+	git_iterator *a = NULL, *b = NULL;
+	char *prefix = opts ? diff_prefix_from_pathspec(&opts->pathspec) : NULL;
+
+	assert(repo && old_tree && diff);
+
+	if (git_iterator_for_tree_range(&a, repo, old_tree, prefix, prefix) < 0 ||
+		git_iterator_for_workdir_range(&b, repo, prefix, prefix) < 0)
+		return -1;
+
+	git__free(prefix);
+
+	return diff_from_iterators(repo, opts, a, b, diff);
+}
+
+int git_diff_merge(
+	git_diff_list *onto,
+	const git_diff_list *from)
+{
+	int error = 0;
+	git_pool onto_pool;
+	git_vector onto_new;
+	git_diff_delta *delta;
+	unsigned int i, j;
+
+	assert(onto && from);
+
+	if (!from->deltas.length)
+		return 0;
+
+	if (git_vector_init(&onto_new, onto->deltas.length, diff_delta__cmp) < 0 ||
+		git_pool_init(&onto_pool, 1, 0) < 0)
+		return -1;
+
+	for (i = 0, j = 0; i < onto->deltas.length || j < from->deltas.length; ) {
+		git_diff_delta *o = GIT_VECTOR_GET(&onto->deltas, i);
+		const git_diff_delta *f = GIT_VECTOR_GET(&from->deltas, j);
+		int cmp = !f ? -1 : !o ? 1 : strcmp(o->old_file.path, f->old_file.path);
+
+		if (cmp < 0) {
+			delta = diff_delta__dup(o, &onto_pool);
+			i++;
+		} else if (cmp > 0) {
+			delta = diff_delta__dup(f, &onto_pool);
+			j++;
+		} else {
+			delta = diff_delta__merge_like_cgit(o, f, &onto_pool);
+			i++;
+			j++;
+		}
+
+		if ((error = !delta ? -1 : git_vector_insert(&onto_new, delta)) < 0)
+			break;
+	}
+
+	if (!error) {
+		git_vector_swap(&onto->deltas, &onto_new);
+		git_pool_swap(&onto->pool, &onto_pool);
+		onto->new_src = from->new_src;
+	}
+
+	git_vector_foreach(&onto_new, i, delta)
+		git__free(delta);
+	git_vector_free(&onto_new);
+	git_pool_clear(&onto_pool);
+
+	return error;
+}
+
diff --git a/src/diff.h b/src/diff.h
new file mode 100644
index 000000000..ac2457956
--- /dev/null
+++ b/src/diff.h
@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_diff_h__
+#define INCLUDE_diff_h__
+
+#include <stdio.h>
+#include "vector.h"
+#include "buffer.h"
+#include "iterator.h"
+#include "repository.h"
+#include "pool.h"
+
+#define DIFF_OLD_PREFIX_DEFAULT "a/"
+#define DIFF_NEW_PREFIX_DEFAULT "b/"
+
+enum {
+	GIT_DIFFCAPS_HAS_SYMLINKS     = (1 << 0), /* symlinks on platform? */
+	GIT_DIFFCAPS_ASSUME_UNCHANGED = (1 << 1), /* use stat? */
+	GIT_DIFFCAPS_TRUST_EXEC_BIT   = (1 << 2), /* use st_mode exec bit? */
+	GIT_DIFFCAPS_TRUST_CTIME      = (1 << 3), /* use st_ctime? */
+	GIT_DIFFCAPS_USE_DEV          = (1 << 4), /* use st_dev? */
+};
+
+struct git_diff_list {
+	git_repository   *repo;
+	git_diff_options opts;
+	git_vector       pathspec;
+	git_vector       deltas;    /* vector of git_diff_file_delta */
+	git_pool pool;
+	git_iterator_type_t old_src;
+	git_iterator_type_t new_src;
+	uint32_t diffcaps;
+};
+
+#endif
+
diff --git a/src/diff_output.c b/src/diff_output.c
new file mode 100644
index 000000000..5ffa641c4
--- /dev/null
+++ b/src/diff_output.c
@@ -0,0 +1,786 @@
+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "git2/diff.h"
+#include "git2/attr.h"
+#include "git2/blob.h"
+#include "xdiff/xdiff.h"
+#include <ctype.h>
+#include "diff.h"
+#include "map.h"
+#include "fileops.h"
+#include "filter.h"
+
+typedef struct {
+	git_diff_list *diff;
+	void *cb_data;
+	git_diff_hunk_fn hunk_cb;
+	git_diff_data_fn line_cb;
+	unsigned int index;
+	git_diff_delta *delta;
+	git_diff_range range;
+} diff_output_info;
+
+static int read_next_int(const char **str, int *value)
+{
+	const char *scan = *str;
+	int v = 0, digits = 0;
+	/* find next digit */
+	for (scan = *str; *scan && !isdigit(*scan); scan++);
+	/* parse next number */
+	for (; isdigit(*scan); scan++, digits++)
+		v = (v * 10) + (*scan - '0');
+	*str = scan;
+	*value = v;
+	return (digits > 0) ? 0 : -1;
+}
+
+static int diff_output_cb(void *priv, mmbuffer_t *bufs, int len)
+{
+	diff_output_info *info = priv;
+
+	if (len == 1 && info->hunk_cb) {
+		git_diff_range range = { -1, 0, -1, 0 };
+		const char *scan = bufs[0].ptr;
+
+		/* expect something of the form "@@ -%d[,%d] +%d[,%d] @@" */
+		if (*scan != '@')
+			return -1;
+
+		if (read_next_int(&scan, &range.old_start) < 0)
+			return -1;
+		if (*scan == ',' && read_next_int(&scan, &range.old_lines) < 0)
+			return -1;
+
+		if (read_next_int(&scan, &range.new_start) < 0)
+			return -1;
+		if (*scan == ',' && read_next_int(&scan, &range.new_lines) < 0)
+			return -1;
+
+		if (range.old_start < 0 || range.new_start < 0)
+			return -1;
+
+		memcpy(&info->range, &range, sizeof(git_diff_range));
+
+		return info->hunk_cb(
+			info->cb_data, info->delta, &range, bufs[0].ptr, bufs[0].size);
+	}
+
+	if ((len == 2 || len == 3) && info->line_cb) {
+		int origin;
+
+		/* expect " "/"-"/"+", then data, then maybe newline */
+		origin =
+			(*bufs[0].ptr == '+') ? GIT_DIFF_LINE_ADDITION :
+			(*bufs[0].ptr == '-') ? GIT_DIFF_LINE_DELETION :
+			GIT_DIFF_LINE_CONTEXT;
+
+		if (info->line_cb(
+			info->cb_data, info->delta, &info->range, origin, bufs[1].ptr, bufs[1].size) < 0)
+			return -1;
+
+		/* deal with adding and removing newline at EOF */
+		if (len == 3) {
+			if (origin == GIT_DIFF_LINE_ADDITION)
+				origin = GIT_DIFF_LINE_ADD_EOFNL;
+			else
+				origin = GIT_DIFF_LINE_DEL_EOFNL;
+
+			return info->line_cb(
+				info->cb_data, info->delta, &info->range, origin, bufs[2].ptr, bufs[2].size);
+		}
+	}
+
+	return 0;
+}
+
+#define BINARY_DIFF_FLAGS (GIT_DIFF_FILE_BINARY|GIT_DIFF_FILE_NOT_BINARY)
+
+static int update_file_is_binary_by_attr(git_repository *repo, git_diff_file *file)
+{
+	const char *value;
+	if (git_attr_get(&value, repo, 0, file->path, "diff") < 0)
+		return -1;
+
+	if (GIT_ATTR_FALSE(value))
+		file->flags |= GIT_DIFF_FILE_BINARY;
+	else if (GIT_ATTR_TRUE(value))
+		file->flags |= GIT_DIFF_FILE_NOT_BINARY;
+	/* otherwise leave file->flags alone */
+
+	return 0;
+}
+
+static void update_delta_is_binary(git_diff_delta *delta)
+{
+	if ((delta->old_file.flags & GIT_DIFF_FILE_BINARY) != 0 ||
+		(delta->new_file.flags & GIT_DIFF_FILE_BINARY) != 0)
+		delta->binary = 1;
+	else if ((delta->old_file.flags & GIT_DIFF_FILE_NOT_BINARY) != 0 ||
+			 (delta->new_file.flags & GIT_DIFF_FILE_NOT_BINARY) != 0)
+		delta->binary = 0;
+	/* otherwise leave delta->binary value untouched */
+}
+
+static int file_is_binary_by_attr(
+	git_diff_list *diff,
+	git_diff_delta *delta)
+{
+	int error = 0, mirror_new;
+
+	delta->binary = -1;
+
+	/* make sure files are conceivably mmap-able */
+	if ((git_off_t)((size_t)delta->old_file.size) != delta->old_file.size ||
+		(git_off_t)((size_t)delta->new_file.size) != delta->new_file.size)
+	{
+		delta->old_file.flags |= GIT_DIFF_FILE_BINARY;
+		delta->new_file.flags |= GIT_DIFF_FILE_BINARY;
+		delta->binary = 1;
+		return 0;
+	}
+
+	/* check if user is forcing us to text diff these files */
+	if (diff->opts.flags & GIT_DIFF_FORCE_TEXT) {
+		delta->old_file.flags |= GIT_DIFF_FILE_NOT_BINARY;
+		delta->new_file.flags |= GIT_DIFF_FILE_NOT_BINARY;
+		delta->binary = 0;
+		return 0;
+	}
+
+	/* check diff attribute +, -, or 0 */
+	if (update_file_is_binary_by_attr(diff->repo, &delta->old_file) < 0)
+		return -1;
+
+	mirror_new = (delta->new_file.path == delta->old_file.path ||
+				  strcmp(delta->new_file.path, delta->old_file.path) == 0);
+	if (mirror_new)
+		delta->new_file.flags &= (delta->old_file.flags & BINARY_DIFF_FLAGS);
+	else
+		error = update_file_is_binary_by_attr(diff->repo, &delta->new_file);
+
+	update_delta_is_binary(delta);
+
+	return error;
+}
+
+static int file_is_binary_by_content(
+	git_diff_delta *delta,
+	git_map *old_data,
+	git_map *new_data)
+{
+	git_buf search;
+
+	if ((delta->old_file.flags & BINARY_DIFF_FLAGS) == 0) {
+		search.ptr  = old_data->data;
+		search.size = min(old_data->len, 4000);
+
+		if (git_buf_is_binary(&search))
+			delta->old_file.flags |= GIT_DIFF_FILE_BINARY;
+		else
+			delta->old_file.flags |= GIT_DIFF_FILE_NOT_BINARY;
+	}
+
+	if ((delta->new_file.flags & BINARY_DIFF_FLAGS) == 0) {
+		search.ptr  = new_data->data;
+		search.size = min(new_data->len, 4000);
+
+		if (git_buf_is_binary(&search))
+			delta->new_file.flags |= GIT_DIFF_FILE_BINARY;
+		else
+			delta->new_file.flags |= GIT_DIFF_FILE_NOT_BINARY;
+	}
+
+	update_delta_is_binary(delta);
+
+	/* TODO: if value != NULL, implement diff drivers */
+
+	return 0;
+}
+
+static void setup_xdiff_options(
+	git_diff_options *opts, xdemitconf_t *cfg, xpparam_t *param)
+{
+	memset(cfg, 0, sizeof(xdemitconf_t));
+	memset(param, 0, sizeof(xpparam_t));
+
+	cfg->ctxlen =
+		(!opts || !opts->context_lines) ? 3 : opts->context_lines;
+	cfg->interhunkctxlen =
+		(!opts || !opts->interhunk_lines) ? 3 : opts->interhunk_lines;
+
+	if (!opts)
+		return;
+
+	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE)
+		param->flags |= XDF_WHITESPACE_FLAGS;
+	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE_CHANGE)
+		param->flags |= XDF_IGNORE_WHITESPACE_CHANGE;
+	if (opts->flags & GIT_DIFF_IGNORE_WHITESPACE_EOL)
+		param->flags |= XDF_IGNORE_WHITESPACE_AT_EOL;
+}
+
+static int get_blob_content(
+	git_repository *repo,
+	const git_oid *oid,
+	git_map *map,
+	git_blob **blob)
+{
+	if (git_oid_iszero(oid))
+		return 0;
+
+	if (git_blob_lookup(blob, repo, oid) < 0)
+		return -1;
+
+	map->data = (void *)git_blob_rawcontent(*blob);
+	map->len  = git_blob_rawsize(*blob);
+	return 0;
+}
+
+static int get_workdir_content(
+	git_repository *repo,
+	git_diff_file *file,
+	git_map *map)
+{
+	int error = 0;
+	git_buf path = GIT_BUF_INIT;
+
+	if (git_buf_joinpath(&path, git_repository_workdir(repo), file->path) < 0)
+		return -1;
+
+	if (S_ISLNK(file->mode)) {
+		ssize_t read_len;
+
+		file->flags |= GIT_DIFF_FILE_FREE_DATA;
+		file->flags |= GIT_DIFF_FILE_BINARY;
+
+		map->data = git__malloc((size_t)file->size + 1);
+		GITERR_CHECK_ALLOC(map->data);
+
+		read_len = p_readlink(path.ptr, map->data, (size_t)file->size + 1);
+		if (read_len != (ssize_t)file->size) {
+			giterr_set(GITERR_OS, "Failed to read symlink '%s'", file->path);
+			error = -1;
+		} else
+			map->len = read_len;
+	}
+	else {
+		error = git_futils_mmap_ro_file(map, path.ptr);
+		file->flags |= GIT_DIFF_FILE_UNMAP_DATA;
+	}
+	git_buf_free(&path);
+	return error;
+}
+
+static void release_content(git_diff_file *file, git_map *map, git_blob *blob)
+{
+	if (blob != NULL)
+		git_blob_free(blob);
+
+	if (file->flags & GIT_DIFF_FILE_FREE_DATA) {
+		git__free(map->data);
+		map->data = NULL;
+		file->flags &= ~GIT_DIFF_FILE_FREE_DATA;
+	}
+	else if (file->flags & GIT_DIFF_FILE_UNMAP_DATA) {
+		git_futils_mmap_free(map);
+		map->data = NULL;
+		file->flags &= ~GIT_DIFF_FILE_UNMAP_DATA;
+	}
+}
+
+static void fill_map_from_mmfile(git_map *dst, mmfile_t *src) {
+	assert(dst && src);
+
+	dst->data = src->ptr;
+	dst->len = src->size;
+#ifdef GIT_WIN32
+	dst->fmh = NULL;
+#endif
+}
+
+int git_diff_foreach(
+	git_diff_list *diff,
+	void *data,
+	git_diff_file_fn file_cb,
+	git_diff_hunk_fn hunk_cb,
+	git_diff_data_fn line_cb)
+{
+	int error = 0;
+	diff_output_info info;
+	git_diff_delta *delta;
+	xpparam_t    xdiff_params;
+	xdemitconf_t xdiff_config;
+	xdemitcb_t   xdiff_callback;
+
+	info.diff    = diff;
+	info.cb_data = data;
+	info.hunk_cb = hunk_cb;
+	info.line_cb = line_cb;
+
+	setup_xdiff_options(&diff->opts, &xdiff_config, &xdiff_params);
+	memset(&xdiff_callback, 0, sizeof(xdiff_callback));
+	xdiff_callback.outf = diff_output_cb;
+	xdiff_callback.priv = &info;
+
+	git_vector_foreach(&diff->deltas, info.index, delta) {
+		git_blob *old_blob = NULL, *new_blob = NULL;
+		git_map old_data, new_data;
+		mmfile_t old_xdiff_data, new_xdiff_data;
+
+		if (delta->status == GIT_DELTA_UNMODIFIED &&
+			(diff->opts.flags & GIT_DIFF_INCLUDE_UNMODIFIED) == 0)
+			continue;
+
+		if (delta->status == GIT_DELTA_IGNORED &&
+			(diff->opts.flags & GIT_DIFF_INCLUDE_IGNORED) == 0)
+			continue;
+
+		if (delta->status == GIT_DELTA_UNTRACKED &&
+			(diff->opts.flags & GIT_DIFF_INCLUDE_UNTRACKED) == 0)
+			continue;
+
+		if ((error = file_is_binary_by_attr(diff, delta)) < 0)
+			goto cleanup;
+
+		old_data.data = "";
+		old_data.len = 0;
+		new_data.data = "";
+		new_data.len  = 0;
+
+		/* TODO: Partial blob reading to defer loading whole blob.
+		 * I.e. I want a blob with just the first 4kb loaded, then
+		 * later on I will read the rest of the blob if needed.
+		 */
+
+		/* map files */
+		if (delta->binary != 1 &&
+			(hunk_cb || line_cb) &&
+			(delta->status == GIT_DELTA_DELETED ||
+			 delta->status == GIT_DELTA_MODIFIED))
+		{
+			if (diff->old_src == GIT_ITERATOR_WORKDIR)
+				error = get_workdir_content(diff->repo, &delta->old_file, &old_data);
+			else
+				error = get_blob_content(
+					diff->repo, &delta->old_file.oid, &old_data, &old_blob);
+
+			if (error < 0)
+				goto cleanup;
+		}
+
+		if (delta->binary != 1 &&
+			(hunk_cb || line_cb || git_oid_iszero(&delta->new_file.oid)) &&
+			(delta->status == GIT_DELTA_ADDED ||
+			 delta->status == GIT_DELTA_MODIFIED))
+		{
+			if (diff->new_src == GIT_ITERATOR_WORKDIR)
+				error = get_workdir_content(diff->repo, &delta->new_file, &new_data);
+			else
+				error = get_blob_content(
+					diff->repo, &delta->new_file.oid, &new_data, &new_blob);
+
+			if (error < 0)
+				goto cleanup;
+
+			if ((delta->new_file.flags & GIT_DIFF_FILE_VALID_OID) == 0) {
+				error = git_odb_hash(
+					&delta->new_file.oid, new_data.data, new_data.len, GIT_OBJ_BLOB);
+
+				if (error < 0)
+					goto cleanup;
+
+				/* since we did not have the definitive oid, we may have
+				 * incorrect status and need to skip this item.
+				 */
+				if (git_oid_cmp(&delta->old_file.oid, &delta->new_file.oid) == 0) {
+					delta->status = GIT_DELTA_UNMODIFIED;
+					if ((diff->opts.flags & GIT_DIFF_INCLUDE_UNMODIFIED) == 0)
+						goto cleanup;
+				}
+			}
+		}
+
+		/* if we have not already decided whether file is binary,
+		 * check the first 4K for nul bytes to decide...
+		 */
+		if (delta->binary == -1) {
+			error = file_is_binary_by_content(
+				delta, &old_data, &new_data);
+			if (error < 0)
+				goto cleanup;
+		}
+
+		/* TODO: if ignore_whitespace is set, then we *must* do text
+		 * diffs to tell if a file has really been changed.
+		 */
+
+		if (file_cb != NULL) {
+			error = file_cb(data, delta, (float)info.index / diff->deltas.length);
+			if (error < 0)
+				goto cleanup;
+		}
+
+		/* don't do hunk and line diffs if file is binary */
+		if (delta->binary == 1)
+			goto cleanup;
+
+		/* nothing to do if we did not get data */
+		if (!old_data.len && !new_data.len)
+			goto cleanup;
+
+		assert(hunk_cb || line_cb);
+
+		info.delta = delta;
+		old_xdiff_data.ptr = old_data.data;
+		old_xdiff_data.size = old_data.len;
+		new_xdiff_data.ptr = new_data.data;
+		new_xdiff_data.size = new_data.len;
+
+		xdl_diff(&old_xdiff_data, &new_xdiff_data,
+			&xdiff_params, &xdiff_config, &xdiff_callback);
+
+cleanup:
+		release_content(&delta->old_file, &old_data, old_blob);
+		release_content(&delta->new_file, &new_data, new_blob);
+
+		if (error < 0)
+			break;
+	}
+
+	return error;
+}
+
+
+typedef struct {
+	git_diff_list *diff;
+	git_diff_data_fn print_cb;
+	void *cb_data;
+	git_buf *buf;
+} diff_print_info;
+
+static char pick_suffix(int mode)
+{
+	if (S_ISDIR(mode))
+		return '/';
+	else if (mode & 0100)
+		/* in git, modes are very regular, so we must have 0100755 mode */
+		return '*';
+	else
+		return ' ';
+}
+
+static int print_compact(void *data, git_diff_delta *delta, float progress)
+{
+	diff_print_info *pi = data;
+	char code, old_suffix, new_suffix;
+
+	GIT_UNUSED(progress);
+
+	switch (delta->status) {
+	case GIT_DELTA_ADDED: code = 'A'; break;
+	case GIT_DELTA_DELETED: code = 'D'; break;
+	case GIT_DELTA_MODIFIED: code = 'M'; break;
+	case GIT_DELTA_RENAMED: code = 'R'; break;
+	case GIT_DELTA_COPIED: code = 'C'; break;
+	case GIT_DELTA_IGNORED: code = 'I'; break;
+	case GIT_DELTA_UNTRACKED: code = '?'; break;
+	default: code = 0;
+	}
+
+	if (!code)
+		return 0;
+
+	old_suffix = pick_suffix(delta->old_file.mode);
+	new_suffix = pick_suffix(delta->new_file.mode);
+
+	git_buf_clear(pi->buf);
+
+	if (delta->old_file.path != delta->new_file.path &&
+		strcmp(delta->old_file.path,delta->new_file.path) != 0)
+		git_buf_printf(pi->buf, "%c\t%s%c -> %s%c\n", code,
+			delta->old_file.path, old_suffix, delta->new_file.path, new_suffix);
+	else if (delta->old_file.mode != delta->new_file.mode &&
+		delta->old_file.mode != 0 && delta->new_file.mode != 0)
+		git_buf_printf(pi->buf, "%c\t%s%c (%o -> %o)\n", code,
+			delta->old_file.path, new_suffix, delta->old_file.mode, delta->new_file.mode);
+	else if (old_suffix != ' ')
+		git_buf_printf(pi->buf, "%c\t%s%c\n", code, delta->old_file.path, old_suffix);
+	else
+		git_buf_printf(pi->buf, "%c\t%s\n", code, delta->old_file.path);
+
+	if (git_buf_oom(pi->buf))
+		return -1;
+
+	return pi->print_cb(pi->cb_data, delta, NULL, GIT_DIFF_LINE_FILE_HDR, git_buf_cstr(pi->buf), git_buf_len(pi->buf));
+}
+
+int git_diff_print_compact(
+	git_diff_list *diff,
+	void *cb_data,
+	git_diff_data_fn print_cb)
+{
+	int error;
+	git_buf buf = GIT_BUF_INIT;
+	diff_print_info pi;
+
+	pi.diff     = diff;
+	pi.print_cb = print_cb;
+	pi.cb_data  = cb_data;
+	pi.buf      = &buf;
+
+	error = git_diff_foreach(diff, &pi, print_compact, NULL, NULL);
+
+	git_buf_free(&buf);
+
+	return error;
+}
+
+
+static int print_oid_range(diff_print_info *pi, git_diff_delta *delta)
+{
+	char start_oid[8], end_oid[8];
+
+	/* TODO: Determine a good actual OID range to print */
+	git_oid_tostr(start_oid, sizeof(start_oid), &delta->old_file.oid);
+	git_oid_tostr(end_oid, sizeof(end_oid), &delta->new_file.oid);
+
+	/* TODO: Match git diff more closely */
+	if (delta->old_file.mode == delta->new_file.mode) {
+		git_buf_printf(pi->buf, "index %s..%s %o\n",
+			start_oid, end_oid, delta->old_file.mode);
+	} else {
+		if (delta->old_file.mode == 0) {
+			git_buf_printf(pi->buf, "new file mode %o\n", delta->new_file.mode);
+		} else if (delta->new_file.mode == 0) {
+			git_buf_printf(pi->buf, "deleted file mode %o\n", delta->old_file.mode);
+		} else {
+			git_buf_printf(pi->buf, "old mode %o\n", delta->old_file.mode);
+			git_buf_printf(pi->buf, "new mode %o\n", delta->new_file.mode);
+		}
+		git_buf_printf(pi->buf, "index %s..%s\n", start_oid, end_oid);
+	}
+
+	if (git_buf_oom(pi->buf))
+		return -1;
+
+	return 0;
+}
+
+static int print_patch_file(void *data, git_diff_delta *delta, float progress)
+{
+	diff_print_info *pi = data;
+	const char *oldpfx = pi->diff->opts.old_prefix;
+	const char *oldpath = delta->old_file.path;
+	const char *newpfx = pi->diff->opts.new_prefix;
+	const char *newpath = delta->new_file.path;
+	int result;
+
+	GIT_UNUSED(progress);
+
+	if (!oldpfx)
+		oldpfx = DIFF_OLD_PREFIX_DEFAULT;
+
+	if (!newpfx)
+		newpfx = DIFF_NEW_PREFIX_DEFAULT;
+
+	git_buf_clear(pi->buf);
+	git_buf_printf(pi->buf, "diff --git %s%s %s%s\n", oldpfx, delta->old_file.path, newpfx, delta->new_file.path);
+
+	if (print_oid_range(pi, delta) < 0)
+		return -1;
+
+	if (git_oid_iszero(&delta->old_file.oid)) {
+		oldpfx = "";
+		oldpath = "/dev/null";
+	}
+	if (git_oid_iszero(&delta->new_file.oid)) {
+		newpfx = "";
+		newpath = "/dev/null";
+	}
+
+	if (delta->binary != 1) {
+		git_buf_printf(pi->buf, "--- %s%s\n", oldpfx, oldpath);
+		git_buf_printf(pi->buf, "+++ %s%s\n", newpfx, newpath);
+	}
+
+	if (git_buf_oom(pi->buf))
+		return -1;
+
+    result = pi->print_cb(pi->cb_data, delta, NULL, GIT_DIFF_LINE_FILE_HDR, git_buf_cstr(pi->buf), git_buf_len(pi->buf));
+    if (result < 0)
+        return result;
+
+    if (delta->binary != 1)
+        return 0;
+
+	git_buf_clear(pi->buf);
+	git_buf_printf(
+		pi->buf, "Binary files %s%s and %s%s differ\n",
+		oldpfx, oldpath, newpfx, newpath);
+	if (git_buf_oom(pi->buf))
+		return -1;
+
+	return pi->print_cb(pi->cb_data, delta, NULL, GIT_DIFF_LINE_BINARY, git_buf_cstr(pi->buf), git_buf_len(pi->buf));
+}
+
+static int print_patch_hunk(
+	void *data,
+	git_diff_delta *d,
+	git_diff_range *r,
+	const char *header,
+	size_t header_len)
+{
+	diff_print_info *pi = data;
+
+	git_buf_clear(pi->buf);
+	if (git_buf_printf(pi->buf, "%.*s", (int)header_len, header) < 0)
+		return -1;
+
+	return pi->print_cb(pi->cb_data, d, r, GIT_DIFF_LINE_HUNK_HDR, git_buf_cstr(pi->buf), git_buf_len(pi->buf));
+}
+
+static int print_patch_line(
+	void *data,
+	git_diff_delta *delta,
+	git_diff_range *range,
+	char line_origin, /* GIT_DIFF_LINE value from above */
+	const char *content,
+	size_t content_len)
+{
+	diff_print_info *pi = data;
+
+	git_buf_clear(pi->buf);
+
+	if (line_origin == GIT_DIFF_LINE_ADDITION ||
+		line_origin == GIT_DIFF_LINE_DELETION ||
+		line_origin == GIT_DIFF_LINE_CONTEXT)
+		git_buf_printf(pi->buf, "%c%.*s", line_origin, (int)content_len, content);
+	else if (content_len > 0)
+		git_buf_printf(pi->buf, "%.*s", (int)content_len, content);
+
+	if (git_buf_oom(pi->buf))
+		return -1;
+
+	return pi->print_cb(pi->cb_data, delta, range, line_origin, git_buf_cstr(pi->buf), git_buf_len(pi->buf));
+}
+
+int git_diff_print_patch(
+	git_diff_list *diff,
+	void *cb_data,
+	git_diff_data_fn print_cb)
+{
+	int error;
+	git_buf buf = GIT_BUF_INIT;
+	diff_print_info pi;
+
+	pi.diff     = diff;
+	pi.print_cb = print_cb;
+	pi.cb_data  = cb_data;
+	pi.buf      = &buf;
+
+	error = git_diff_foreach(
+		diff, &pi, print_patch_file, print_patch_hunk, print_patch_line);
+
+	git_buf_free(&buf);
+
+	return error;
+}
+
+int git_diff_blobs(
+	git_blob *old_blob,
+	git_blob *new_blob,
+	git_diff_options *options,
+	void *cb_data,
+	git_diff_file_fn file_cb,
+	git_diff_hunk_fn hunk_cb,
+	git_diff_data_fn line_cb)
+{
+	diff_output_info info;
+	git_diff_delta delta;
+	mmfile_t old_data, new_data;
+	git_map old_map, new_map;
+	xpparam_t xdiff_params;
+	xdemitconf_t xdiff_config;
+	xdemitcb_t xdiff_callback;
+	git_blob *new, *old;
+
+	memset(&delta, 0, sizeof(delta));
+
+	new = new_blob;
+	old = old_blob;
+
+	if (options && (options->flags & GIT_DIFF_REVERSE)) {
+		git_blob *swap = old;
+		old = new;
+		new = swap;
+	}
+
+	if (old) {
+		old_data.ptr  = (char *)git_blob_rawcontent(old);
+		old_data.size = git_blob_rawsize(old);
+		git_oid_cpy(&delta.old_file.oid, git_object_id((const git_object *)old));
+	} else {
+		old_data.ptr  = "";
+		old_data.size = 0;
+	}
+
+	if (new) {
+		new_data.ptr  = (char *)git_blob_rawcontent(new);
+		new_data.size = git_blob_rawsize(new);
+		git_oid_cpy(&delta.new_file.oid, git_object_id((const git_object *)new));
+	} else {
+		new_data.ptr  = "";
+		new_data.size = 0;
+	}
+
+	/* populate a "fake" delta record */
+	delta.status = new ?
+		(old ? GIT_DELTA_MODIFIED : GIT_DELTA_ADDED) :
+		(old ? GIT_DELTA_DELETED : GIT_DELTA_UNTRACKED);
+
+	if (git_oid_cmp(&delta.new_file.oid, &delta.old_file.oid) == 0)
+		delta.status = GIT_DELTA_UNMODIFIED;
+
+	delta.old_file.size = old_data.size;
+	delta.new_file.size = new_data.size;
+
+	fill_map_from_mmfile(&old_map, &old_data);
+	fill_map_from_mmfile(&new_map, &new_data);
+
+	if (file_is_binary_by_content(&delta, &old_map, &new_map) < 0)
+		return -1;
+
+	if (file_cb != NULL) {
+		int error = file_cb(cb_data, &delta, 1);
+		if (error < 0)
+			return error;
+	}
+
+	/* don't do hunk and line diffs if the two blobs are identical */
+	if (delta.status == GIT_DELTA_UNMODIFIED)
+		return 0;
+
+	/* don't do hunk and line diffs if file is binary */
+	if (delta.binary == 1)
+		return 0;
+
+	info.diff    = NULL;
+	info.delta   = &delta;
+	info.cb_data = cb_data;
+	info.hunk_cb = hunk_cb;
+	info.line_cb = line_cb;
+
+	setup_xdiff_options(options, &xdiff_config, &xdiff_params);
+	memset(&xdiff_callback, 0, sizeof(xdiff_callback));
+	xdiff_callback.outf = diff_output_cb;
+	xdiff_callback.priv = &info;
+
+	xdl_diff(&old_data, &new_data, &xdiff_params, &xdiff_config, &xdiff_callback);
+
+	return 0;
+}
diff --git a/src/dir.h b/src/dir.h
deleted file mode 100644
index c01c3fae7..000000000
--- a/src/dir.h
+++ /dev/null
@@ -1,41 +0,0 @@
-#ifndef INCLUDE_dir_h__
-#define INCLUDE_dir_h__
-
-#include "common.h"
-
-#ifndef GIT_WIN32
-# include <dirent.h>
-#endif
-
-#ifdef GIT_WIN32
-
-struct git__dirent {
-	int  d_ino;
-	char d_name[261];
-};
-
-typedef struct {
-	HANDLE h;
-	WIN32_FIND_DATA f;
-	struct git__dirent entry;
-	char *dir;
-	int first;
-} git__DIR;
-
-extern git__DIR *git__opendir(const char *);
-extern struct git__dirent *git__readdir(git__DIR *);
-extern void git__rewinddir(git__DIR *);
-extern int git__closedir(git__DIR *);
-
-# ifndef GIT__WIN32_NO_WRAP_DIR
-#  define dirent git__dirent
-#  define DIR git__DIR
-#  define opendir   git__opendir
-#  define readdir   git__readdir
-#  define rewinddir git__rewinddir
-#  define closedir  git__closedir
-# endif
-
-#endif
-
-#endif /* INCLUDE_dir_h__ */
diff --git a/src/errors.c b/src/errors.c
index 7da02b4f7..d43d7d9b5 100644
--- a/src/errors.c
+++ b/src/errors.c
@@ -1,113 +1,119 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #include "common.h"
-#include "git2/thread-utils.h" /* for GIT_TLS */
-#include "thread-utils.h" /* for GIT_TLS */
-
+#include "global.h"
+#include "posix.h"
+#include "buffer.h"
 #include <stdarg.h>
 
-static GIT_TLS char g_last_error[1024];
-
-static struct {
-	int num;
-	const char *str;
-} error_codes[] = {
-	{GIT_ERROR, "Unspecified error"},
-	{GIT_ENOTOID, "Input was not a properly formatted Git object id."},
-	{GIT_ENOTFOUND, "Object does not exist in the scope searched."},
-	{GIT_ENOMEM, "Not enough space available."},
-	{GIT_EOSERR, "Consult the OS error information."},
-	{GIT_EOBJTYPE, "The specified object is of invalid type"},
-	{GIT_EOBJCORRUPTED, "The specified object has its data corrupted"},
-	{GIT_ENOTAREPO, "The specified repository is invalid"},
-	{GIT_EINVALIDTYPE, "The object or config variable type is invalid or doesn't match"},
-	{GIT_EMISSINGOBJDATA, "The object cannot be written that because it's missing internal data"},
-	{GIT_EPACKCORRUPTED, "The packfile for the ODB is corrupted"},
-	{GIT_EFLOCKFAIL, "Failed to adquire or release a file lock"},
-	{GIT_EZLIB, "The Z library failed to inflate/deflate an object's data"},
-	{GIT_EBUSY, "The queried object is currently busy"},
-	{GIT_EINVALIDPATH, "The path is invalid"},
-	{GIT_EBAREINDEX, "The index file is not backed up by an existing repository"},
-	{GIT_EINVALIDREFNAME, "The name of the reference is not valid"},
-	{GIT_EREFCORRUPTED, "The specified reference has its data corrupted"},
-	{GIT_ETOONESTEDSYMREF, "The specified symbolic reference is too deeply nested"},
-	{GIT_EPACKEDREFSCORRUPTED, "The pack-refs file is either corrupted of its format is not currently supported"},
-	{GIT_EINVALIDPATH, "The path is invalid" },
-	{GIT_EREVWALKOVER, "The revision walker is empty; there are no more commits left to iterate"},
-	{GIT_EINVALIDREFSTATE, "The state of the reference is not valid"},
-	{GIT_ENOTIMPLEMENTED, "This feature has not been implemented yet"},
-	{GIT_EEXISTS, "A reference with this name already exists"},
-	{GIT_EOVERFLOW, "The given integer literal is too large to be parsed"},
-	{GIT_ENOTNUM, "The given literal is not a valid number"},
-	{GIT_EAMBIGUOUSOIDPREFIX, "The given oid prefix is ambiguous"},
+/********************************************
+ * New error handling
+ ********************************************/
+
+static git_error g_git_oom_error = {
+	"Out of memory",
+	GITERR_NOMEMORY
 };
 
-const char *git_strerror(int num)
+static void set_error(int error_class, char *string)
 {
-	size_t i;
+	git_error *error = &GIT_GLOBAL->error_t;
 
-	if (num == GIT_EOSERR)
-		return strerror(errno);
-	for (i = 0; i < ARRAY_SIZE(error_codes); i++)
-		if (num == error_codes[i].num)
-			return error_codes[i].str;
+	git__free(error->message);
 
-	return "Unknown error";
+	error->message = string;
+	error->klass = error_class;
+
+	GIT_GLOBAL->last_error = error;
 }
 
-int git__rethrow(int error, const char *msg, ...)
+void giterr_set_oom(void)
 {
-	char new_error[1024];
-	char *old_error = NULL;
-
-	va_list va;
+	GIT_GLOBAL->last_error = &g_git_oom_error;
+}
 
-	va_start(va, msg);
-	vsnprintf(new_error, sizeof(new_error), msg, va);
-	va_end(va);
+void giterr_set(int error_class, const char *string, ...)
+{
+	git_buf buf = GIT_BUF_INIT;
+	va_list arglist;
+
+	int unix_error_code = 0;
+
+#ifdef GIT_WIN32
+	DWORD win32_error_code = 0;
+#endif
+
+	if (error_class == GITERR_OS) {
+		unix_error_code = errno;
+		errno = 0;
+
+#ifdef GIT_WIN32
+		win32_error_code = GetLastError();
+		SetLastError(0);
+#endif
+	}
+
+	va_start(arglist, string);
+	git_buf_vprintf(&buf, string, arglist);
+	va_end(arglist);
+
+	/* automatically suffix strerror(errno) for GITERR_OS errors */
+	if (error_class == GITERR_OS) {
+
+		if (unix_error_code != 0) {
+			git_buf_PUTS(&buf, ": ");
+			git_buf_puts(&buf, strerror(unix_error_code));
+		}
+
+#ifdef GIT_WIN32
+		else if (win32_error_code != 0) {
+			LPVOID lpMsgBuf = NULL;
+
+			FormatMessage(
+				FORMAT_MESSAGE_ALLOCATE_BUFFER | 
+				FORMAT_MESSAGE_FROM_SYSTEM |
+				FORMAT_MESSAGE_IGNORE_INSERTS,
+				NULL, win32_error_code, 0, (LPTSTR) &lpMsgBuf, 0, NULL);
+
+			if (lpMsgBuf) {
+				git_buf_PUTS(&buf, ": ");
+				git_buf_puts(&buf, lpMsgBuf);
+				LocalFree(lpMsgBuf);
+			}
+		}
+#endif
+	}
+
+	if (!git_buf_oom(&buf))
+		set_error(error_class, git_buf_detach(&buf));
+}
 
-	old_error = strdup(g_last_error);
-	snprintf(g_last_error, sizeof(g_last_error), "%s \n    - %s", new_error, old_error);
-	free(old_error);
+void giterr_set_str(int error_class, const char *string)
+{
+	char *message = git__strdup(string);
 
-	return error;
+	if (message)
+		set_error(error_class, message);
 }
 
-int git__throw(int error, const char *msg, ...)
+void giterr_set_regex(const regex_t *regex, int error_code)
 {
-	va_list va;
-
-	va_start(va, msg);
-	vsnprintf(g_last_error, sizeof(g_last_error), msg, va);
-	va_end(va);
+	char error_buf[1024];
+	regerror(error_code, regex, error_buf, sizeof(error_buf));
+	giterr_set_str(GITERR_REGEX, error_buf);
+}
 
-	return error;
+void giterr_clear(void)
+{
+	GIT_GLOBAL->last_error = NULL;
 }
 
-const char *git_lasterror(void)
+const git_error *giterr_last(void)
 {
-	return g_last_error;
+	return GIT_GLOBAL->last_error;
 }
 
diff --git a/src/fetch.c b/src/fetch.c
new file mode 100644
index 000000000..96b263faa
--- /dev/null
+++ b/src/fetch.c
@@ -0,0 +1,200 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "git2/remote.h"
+#include "git2/oid.h"
+#include "git2/refs.h"
+#include "git2/revwalk.h"
+#include "git2/indexer.h"
+
+#include "common.h"
+#include "transport.h"
+#include "remote.h"
+#include "refspec.h"
+#include "pack.h"
+#include "fetch.h"
+#include "netops.h"
+
+struct filter_payload {
+	git_remote *remote;
+	const git_refspec *spec;
+	git_odb *odb;
+	int found_head;
+};
+
+static int filter_ref__cb(git_remote_head *head, void *payload)
+{
+	struct filter_payload *p = payload;
+
+	if (!p->found_head && strcmp(head->name, GIT_HEAD_FILE) == 0) {
+		p->found_head = 1;
+	} else {
+		/* If it doesn't match the refpec, we don't want it */
+		if (!git_refspec_src_matches(p->spec, head->name))
+			return 0;
+
+		/* Don't even try to ask for the annotation target */
+		if (!git__suffixcmp(head->name, "^{}"))
+			return 0;
+	}
+
+	/* If we have the object, mark it so we don't ask for it */
+	if (git_odb_exists(p->odb, &head->oid))
+		head->local = 1;
+	else
+		p->remote->need_pack = 1;
+
+	return git_vector_insert(&p->remote->refs, head);
+}
+
+static int filter_wants(git_remote *remote)
+{
+	struct filter_payload p;
+
+	git_vector_clear(&remote->refs);
+
+	/*
+	 * The fetch refspec can be NULL, and what this means is that the
+	 * user didn't specify one. This is fine, as it means that we're
+	 * not interested in any particular branch but just the remote's
+	 * HEAD, which will be stored in FETCH_HEAD after the fetch.
+	 */
+	p.spec = git_remote_fetchspec(remote);
+	p.found_head = 0;
+	p.remote = remote;
+
+	if (git_repository_odb__weakptr(&p.odb, remote->repo) < 0)
+		return -1;
+
+	return remote->transport->ls(remote->transport, &filter_ref__cb, &p);
+}
+
+/*
+ * In this first version, we push all our refs in and start sending
+ * them out. When we get an ACK we hide that commit and continue
+ * traversing until we're done
+ */
+int git_fetch_negotiate(git_remote *remote)
+{
+	git_transport *t = remote->transport;
+
+	if (filter_wants(remote) < 0) {
+		giterr_set(GITERR_NET, "Failed to filter the reference list for wants");
+		return -1;
+	}
+
+	/* Don't try to negotiate when we don't want anything */
+	if (remote->refs.length == 0 || !remote->need_pack)
+		return 0;
+
+	/*
+	 * Now we have everything set up so we can start tell the server
+	 * what we want and what we have.
+	 */
+	return t->negotiate_fetch(t, remote->repo, &remote->refs);
+}
+
+int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
+{
+	if(!remote->need_pack)
+		return 0;
+
+	return remote->transport->download_pack(remote->transport, remote->repo, bytes, stats);
+}
+
+/* Receiving data from a socket and storing it is pretty much the same for git and HTTP */
+int git_fetch__download_pack(
+	const char *buffered,
+	size_t buffered_size,
+	git_transport *t,
+	git_repository *repo,
+	git_off_t *bytes,
+	git_indexer_stats *stats)
+{
+	int recvd;
+	char buff[1024];
+	gitno_buffer buf;
+	git_indexer_stream *idx;
+
+	gitno_buffer_setup(t, &buf, buff, sizeof(buff));
+
+	if (memcmp(buffered, "PACK", strlen("PACK"))) {
+		giterr_set(GITERR_NET, "The pack doesn't start with the signature");
+		return -1;
+	}
+
+	if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0)
+		return -1;
+
+	memset(stats, 0, sizeof(git_indexer_stats));
+	if (git_indexer_stream_add(idx, buffered, buffered_size, stats) < 0)
+		goto on_error;
+
+	*bytes = buffered_size;
+
+	do {
+		if (git_indexer_stream_add(idx, buf.data, buf.offset, stats) < 0)
+			goto on_error;
+
+		gitno_consume_n(&buf, buf.offset);
+		if ((recvd = gitno_recv(&buf)) < 0)
+			goto on_error;
+
+		*bytes += recvd;
+	} while(recvd > 0);
+
+	if (git_indexer_stream_finalize(idx, stats))
+		goto on_error;
+
+	git_indexer_stream_free(idx);
+	return 0;
+
+on_error:
+	git_indexer_stream_free(idx);
+	return -1;
+}
+
+int git_fetch_setup_walk(git_revwalk **out, git_repository *repo)
+{
+	git_revwalk *walk;
+	git_strarray refs;
+	unsigned int i;
+	git_reference *ref;
+
+	if (git_reference_list(&refs, repo, GIT_REF_LISTALL) < 0)
+		return -1;
+
+	if (git_revwalk_new(&walk, repo) < 0)
+		return -1;
+
+	git_revwalk_sorting(walk, GIT_SORT_TIME);
+
+	for (i = 0; i < refs.count; ++i) {
+		/* No tags */
+		if (!git__prefixcmp(refs.strings[i], GIT_REFS_TAGS_DIR))
+			continue;
+
+		if (git_reference_lookup(&ref, repo, refs.strings[i]) < 0)
+			goto on_error;
+
+		if (git_reference_type(ref) == GIT_REF_SYMBOLIC)
+			continue;
+		if (git_revwalk_push(walk, git_reference_oid(ref)) < 0)
+			goto on_error;
+
+		git_reference_free(ref);
+	}
+
+	git_strarray_free(&refs);
+	*out = walk;
+	return 0;
+
+on_error:
+	git_reference_free(ref);
+	git_strarray_free(&refs);
+	return -1;
+}
diff --git a/src/fetch.h b/src/fetch.h
new file mode 100644
index 000000000..a7f126520
--- /dev/null
+++ b/src/fetch.h
@@ -0,0 +1,19 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_fetch_h__
+#define INCLUDE_fetch_h__
+
+#include "netops.h"
+
+int git_fetch_negotiate(git_remote *remote);
+int git_fetch_download_pack(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats);
+
+int git_fetch__download_pack(const char *buffered, size_t buffered_size, git_transport *t,
+			     git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
+int git_fetch_setup_walk(git_revwalk **out, git_repository *repo);
+
+#endif
diff --git a/src/filebuf.c b/src/filebuf.c
index 63f2897cb..876f8e3e7 100644
--- a/src/filebuf.c
+++ b/src/filebuf.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #include <stdarg.h>
 
@@ -28,68 +10,117 @@
 #include "filebuf.h"
 #include "fileops.h"
 
+#define GIT_LOCK_FILE_MODE 0644
+
 static const size_t WRITE_BUFFER_SIZE = (4096 * 2);
 
+enum buferr_t {
+	BUFERR_OK = 0,
+	BUFERR_WRITE,
+	BUFERR_ZLIB,
+	BUFERR_MEM
+};
+
+#define ENSURE_BUF_OK(buf) if ((buf)->last_error != BUFERR_OK) { return -1; }
+
+static int verify_last_error(git_filebuf *file)
+{
+	switch (file->last_error) {
+	case BUFERR_WRITE:
+		giterr_set(GITERR_OS, "Failed to write out file");
+		return -1;
+
+	case BUFERR_MEM:
+		giterr_set_oom();
+		return -1;
+
+	case BUFERR_ZLIB:
+		giterr_set(GITERR_ZLIB,
+			"Buffer error when writing out ZLib data");
+		return -1;
+
+	default:
+		return 0;
+	}
+}
+
 static int lock_file(git_filebuf *file, int flags)
 {
-	if (gitfo_exists(file->path_lock) == 0) {
+	if (git_path_exists(file->path_lock) == true) {
 		if (flags & GIT_FILEBUF_FORCE)
-			gitfo_unlink(file->path_lock);
-		else
-			return git__throw(GIT_EOSERR, "Failed to lock file");
+			p_unlink(file->path_lock);
+		else {
+			giterr_set(GITERR_OS,
+				"Failed to lock file '%s' for writing", file->path_lock);
+			return -1;
+		}
 	}
 
 	/* create path to the file buffer is required */
 	if (flags & GIT_FILEBUF_FORCE) {
-		file->fd = gitfo_creat_force(file->path_lock, 0644);
+		/* XXX: Should dirmode here be configurable? Or is 0777 always fine? */
+		file->fd = git_futils_creat_locked_withpath(file->path_lock, 0777, GIT_LOCK_FILE_MODE);
 	} else {
-		file->fd = gitfo_creat(file->path_lock, 0644);
+		file->fd = git_futils_creat_locked(file->path_lock, GIT_LOCK_FILE_MODE);
 	}
 
 	if (file->fd < 0)
-		return git__throw(GIT_EOSERR, "Failed to create lock");
+		return -1;
 
-	/* TODO: do a flock() in the descriptor file_lock */
+	file->fd_is_open = true;
 
-	if ((flags & GIT_FILEBUF_APPEND) && gitfo_exists(file->path_original) == 0) {
+	if ((flags & GIT_FILEBUF_APPEND) && git_path_exists(file->path_original) == true) {
 		git_file source;
 		char buffer[2048];
 		size_t read_bytes;
 
-		source = gitfo_open(file->path_original, O_RDONLY);
-		if (source < 0)
-			return git__throw(GIT_EOSERR, "Failed to lock file. Could not open %s", file->path_original);
+		source = p_open(file->path_original, O_RDONLY);
+		if (source < 0) {
+			giterr_set(GITERR_OS,
+				"Failed to open file '%s' for reading",
+				file->path_original);
+			return -1;
+		}
 
-		while ((read_bytes = gitfo_read(source, buffer, 2048)) > 0) {
-			gitfo_write(file->fd, buffer, read_bytes);
+		while ((read_bytes = p_read(source, buffer, 2048)) > 0) {
+			p_write(file->fd, buffer, read_bytes);
 			if (file->digest)
 				git_hash_update(file->digest, buffer, read_bytes);
 		}
 
-		gitfo_close(source);
+		p_close(source);
 	}
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 void git_filebuf_cleanup(git_filebuf *file)
 {
-	if (file->fd >= 0)
-		gitfo_close(file->fd);
+	if (file->fd_is_open && file->fd >= 0)
+		p_close(file->fd);
 
-	if (file->path_lock && gitfo_exists(file->path_lock) == GIT_SUCCESS)
-		gitfo_unlink(file->path_lock);
+	if (file->fd_is_open && file->path_lock && git_path_exists(file->path_lock))
+		p_unlink(file->path_lock);
 
 	if (file->digest)
 		git_hash_free_ctx(file->digest);
 
-	free(file->buffer);
-	free(file->z_buf);
+	if (file->buffer)
+		git__free(file->buffer);
+
+	/* use the presence of z_buf to decide if we need to deflateEnd */
+	if (file->z_buf) {
+		git__free(file->z_buf);
+		deflateEnd(&file->zs);
+	}
 
-	deflateEnd(&file->zs);
+	if (file->path_original)
+		git__free(file->path_original);
+	if (file->path_lock)
+		git__free(file->path_lock);
 
-	free(file->path_original);
-	free(file->path_lock);
+	memset(file, 0x0, sizeof(git_filebuf));
+	file->fd = -1;
 }
 
 GIT_INLINE(int) flush_buffer(git_filebuf *file)
@@ -99,43 +130,53 @@ GIT_INLINE(int) flush_buffer(git_filebuf *file)
 	return result;
 }
 
-static int write_normal(git_filebuf *file, const void *source, size_t len)
+int git_filebuf_flush(git_filebuf *file)
 {
-	int result = 0;
+	return flush_buffer(file);
+}
 
+static int write_normal(git_filebuf *file, void *source, size_t len)
+{
 	if (len > 0) {
-		result = gitfo_write(file->fd, (void *)source, len);
+		if (p_write(file->fd, (void *)source, len) < 0) {
+			file->last_error = BUFERR_WRITE;
+			return -1;
+		}
+
 		if (file->digest)
 			git_hash_update(file->digest, source, len);
 	}
 
-	return result;
+	return 0;
 }
 
-static int write_deflate(git_filebuf *file, const void *source, size_t len)
+static int write_deflate(git_filebuf *file, void *source, size_t len)
 {
-	int result = Z_OK;
 	z_stream *zs = &file->zs;
 
 	if (len > 0 || file->flush_mode == Z_FINISH) {
-		zs->next_in = (void *)source;
-		zs->avail_in = len;
+		zs->next_in = source;
+		zs->avail_in = (uInt)len;
 
 		do {
-			int have;
+			size_t have;
 
 			zs->next_out = file->z_buf;
-			zs->avail_out = file->buf_size;
+			zs->avail_out = (uInt)file->buf_size;
 
-            result = deflate(zs, file->flush_mode);
-            assert(result != Z_STREAM_ERROR);
+			if (deflate(zs, file->flush_mode) == Z_STREAM_ERROR) {
+				file->last_error = BUFERR_ZLIB;
+				return -1;
+			}
 
-            have = file->buf_size - zs->avail_out;
+			have = file->buf_size - (size_t)zs->avail_out;
 
-			if (gitfo_write(file->fd, file->z_buf, have) < GIT_SUCCESS)
-				return git__throw(GIT_EOSERR, "Failed to write to file");
+			if (p_write(file->fd, file->z_buf, have) < 0) {
+				file->last_error = BUFERR_WRITE;
+				return -1;
+			}
 
-        } while (zs->avail_out == 0);
+		} while (zs->avail_out == 0);
 
 		assert(zs->avail_in == 0);
 
@@ -143,52 +184,54 @@ static int write_deflate(git_filebuf *file, const void *source, size_t len)
 			git_hash_update(file->digest, source, len);
 	}
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 {
-	int error;
+	int compression;
 	size_t path_len;
 
-	assert(file && path);
+	/* opening an already open buffer is a programming error;
+	 * assert that this never happens instead of returning
+	 * an error code */
+	assert(file && path && file->buffer == NULL);
 
 	memset(file, 0x0, sizeof(git_filebuf));
 
+	if (flags & GIT_FILEBUF_DO_NOT_BUFFER)
+		file->do_not_buffer = true;
+
 	file->buf_size = WRITE_BUFFER_SIZE;
 	file->buf_pos = 0;
 	file->fd = -1;
+	file->last_error = BUFERR_OK;
 
 	/* Allocate the main cache buffer */
-	file->buffer = git__malloc(file->buf_size);
-	if (file->buffer == NULL){
-		error = GIT_ENOMEM;
-		goto cleanup;
+	if (!file->do_not_buffer) {
+		file->buffer = git__malloc(file->buf_size);
+		GITERR_CHECK_ALLOC(file->buffer);
 	}
 
 	/* If we are hashing on-write, allocate a new hash context */
 	if (flags & GIT_FILEBUF_HASH_CONTENTS) {
-		if ((file->digest = git_hash_new_ctx()) == NULL) {
-			error = GIT_ENOMEM;
-			goto cleanup;
-		}
+		file->digest = git_hash_new_ctx();
+		GITERR_CHECK_ALLOC(file->digest);
 	}
 
-	/* If we are deflating on-write, */
-	if (flags & GIT_FILEBUF_DEFLATE_CONTENTS) {
+	compression = flags >> GIT_FILEBUF_DEFLATE_SHIFT;
 
+	/* If we are deflating on-write, */
+	if (compression != 0) {
 		/* Initialize the ZLib stream */
-		if (deflateInit(&file->zs, Z_BEST_SPEED) != Z_OK) {
-			error = git__throw(GIT_EZLIB, "Failed to initialize zlib");
+		if (deflateInit(&file->zs, compression) != Z_OK) {
+			giterr_set(GITERR_ZLIB, "Failed to initialize zlib");
 			goto cleanup;
 		}
 
 		/* Allocate the Zlib cache buffer */
 		file->z_buf = git__malloc(file->buf_size);
-		if (file->z_buf == NULL){
-			error = GIT_ENOMEM;
-			goto cleanup;
-		}
+		GITERR_CHECK_ALLOC(file->z_buf);
 
 		/* Never flush */
 		file->flush_mode = Z_NO_FLUSH;
@@ -199,102 +242,105 @@ int git_filebuf_open(git_filebuf *file, const char *path, int flags)
 
 	/* If we are writing to a temp file */
 	if (flags & GIT_FILEBUF_TEMPORARY) {
-		char tmp_path[GIT_PATH_MAX];
+		git_buf tmp_path = GIT_BUF_INIT;
 
 		/* Open the file as temporary for locking */
-		file->fd = gitfo_mktemp(tmp_path, path); 
+		file->fd = git_futils_mktmp(&tmp_path, path);
+
 		if (file->fd < 0) {
-			error = GIT_EOSERR;
+			git_buf_free(&tmp_path);
 			goto cleanup;
 		}
+		file->fd_is_open = true;
 
 		/* No original path */
 		file->path_original = NULL;
-		file->path_lock = git__strdup(tmp_path);
-
-		if (file->path_lock == NULL) {
-			error = GIT_ENOMEM;
-			goto cleanup;
-		}
+		file->path_lock = git_buf_detach(&tmp_path);
+		GITERR_CHECK_ALLOC(file->path_lock);
 	} else {
 		path_len = strlen(path);
 
 		/* Save the original path of the file */
 		file->path_original = git__strdup(path);
-		if (file->path_original == NULL) {
-			error = GIT_ENOMEM;
-			goto cleanup;
-		}
+		GITERR_CHECK_ALLOC(file->path_original);
 
 		/* create the locking path by appending ".lock" to the original */
 		file->path_lock = git__malloc(path_len + GIT_FILELOCK_EXTLENGTH);
-		if (file->path_lock == NULL) {
-			error = GIT_ENOMEM;
-			goto cleanup;
-		}
+		GITERR_CHECK_ALLOC(file->path_lock);
 
 		memcpy(file->path_lock, file->path_original, path_len);
 		memcpy(file->path_lock + path_len, GIT_FILELOCK_EXTENSION, GIT_FILELOCK_EXTLENGTH);
 
 		/* open the file for locking */
-		if ((error = lock_file(file, flags)) < GIT_SUCCESS)
+		if (lock_file(file, flags) < 0)
 			goto cleanup;
 	}
 
-	return GIT_SUCCESS;
+	return 0;
 
 cleanup:
 	git_filebuf_cleanup(file);
-	return git__rethrow(error, "Failed to open file buffer for '%s'", path);
+	return -1;
 }
 
 int git_filebuf_hash(git_oid *oid, git_filebuf *file)
 {
-	int error;
-
 	assert(oid && file && file->digest);
 
-	if ((error = flush_buffer(file)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to get hash for file");
+	flush_buffer(file);
+
+	if (verify_last_error(file) < 0)
+		return -1;
 
 	git_hash_final(oid, file->digest);
 	git_hash_free_ctx(file->digest);
 	file->digest = NULL;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-int git_filebuf_commit_at(git_filebuf *file, const char *path)
+int git_filebuf_commit_at(git_filebuf *file, const char *path, mode_t mode)
 {
-	free(file->path_original);
+	git__free(file->path_original);
 	file->path_original = git__strdup(path);
-	if (file->path_original == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(file->path_original);
 
-	return git_filebuf_commit(file);
+	return git_filebuf_commit(file, mode);
 }
 
-int git_filebuf_commit(git_filebuf *file)
+int git_filebuf_commit(git_filebuf *file, mode_t mode)
 {
-	int error;
-
 	/* temporary files cannot be committed */
 	assert(file && file->path_original);
 
 	file->flush_mode = Z_FINISH;
-	if ((error = flush_buffer(file)) < GIT_SUCCESS)
-		goto cleanup;
+	flush_buffer(file);
+
+	if (verify_last_error(file) < 0)
+		goto on_error;
 
-	gitfo_close(file->fd);
+	p_close(file->fd);
 	file->fd = -1;
+	file->fd_is_open = false;
 
-	error = gitfo_mv(file->path_lock, file->path_original);
+	if (p_chmod(file->path_lock, mode)) {
+		giterr_set(GITERR_OS, "Failed to set attributes for file at '%s'", file->path_lock);
+		goto on_error;
+	}
 
-cleanup:
+	p_unlink(file->path_original);
+
+	if (p_rename(file->path_lock, file->path_original) < 0) {
+		giterr_set(GITERR_OS, "Failed to rename lockfile to '%s'", file->path_original);
+		goto on_error;
+	}
+
+	git_filebuf_cleanup(file);
+	return 0;
+
+on_error:
 	git_filebuf_cleanup(file);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to commit locked file from buffer");
-	return GIT_SUCCESS;
+	return -1;
 }
 
 GIT_INLINE(void) add_to_cache(git_filebuf *file, const void *buf, size_t len)
@@ -305,80 +351,107 @@ GIT_INLINE(void) add_to_cache(git_filebuf *file, const void *buf, size_t len)
 
 int git_filebuf_write(git_filebuf *file, const void *buff, size_t len)
 {
-	int error;
 	const unsigned char *buf = buff;
 
+	ENSURE_BUF_OK(file);
+
+	if (file->do_not_buffer)
+		return file->write(file, (void *)buff, len);
+
 	for (;;) {
 		size_t space_left = file->buf_size - file->buf_pos;
 
 		/* cache if it's small */
 		if (space_left > len) {
 			add_to_cache(file, buf, len);
-			return GIT_SUCCESS;
+			return 0;
 		}
 
-		/* flush the cache if it doesn't fit */
-		if (file->buf_pos > 0) {
-			add_to_cache(file, buf, space_left);
-
-			if ((error = flush_buffer(file)) < GIT_SUCCESS)
-				return git__rethrow(error, "Failed to write to buffer");
-
-			len -= space_left;
-			buf += space_left;
-		}
+		add_to_cache(file, buf, space_left);
+		if (flush_buffer(file) < 0)
+			return -1;
 
-		/* write too-large chunks immediately */
-		if (len > file->buf_size) {
-			error = file->write(file, buf, len);
-			if (error < GIT_SUCCESS)
-				return git__rethrow(error, "Failed to write to buffer");
-		}
+		len -= space_left;
+		buf += space_left;
 	}
 }
 
 int git_filebuf_reserve(git_filebuf *file, void **buffer, size_t len)
 {
-	int error;
 	size_t space_left = file->buf_size - file->buf_pos;
 
 	*buffer = NULL;
 
-	if (len > file->buf_size)
-		return GIT_ENOMEM;
+	ENSURE_BUF_OK(file);
+
+	if (len > file->buf_size) {
+		file->last_error = BUFERR_MEM;
+		return -1;
+	}
 
 	if (space_left <= len) {
-		if ((error = flush_buffer(file)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to reserve buffer");
+		if (flush_buffer(file) < 0)
+			return -1;
 	}
 
 	*buffer = (file->buffer + file->buf_pos);
 	file->buf_pos += len;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 int git_filebuf_printf(git_filebuf *file, const char *format, ...)
 {
 	va_list arglist;
-	size_t space_left = file->buf_size - file->buf_pos;
-	int len, error;
+	size_t space_left;
+	int len, res;
+	char *tmp_buffer;
 
-	va_start(arglist, format);
+	ENSURE_BUF_OK(file);
 
-	len = vsnprintf((char *)file->buffer + file->buf_pos, space_left, format, arglist);
+	space_left = file->buf_size - file->buf_pos;
 
-	if (len < 0 || (size_t)len >= space_left) {
-		if ((error = flush_buffer(file)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to output to buffer");
+	do {
+		va_start(arglist, format);
+		len = p_vsnprintf((char *)file->buffer + file->buf_pos, space_left, format, arglist);
+		va_end(arglist);
+
+		if (len < 0) {
+			file->last_error = BUFERR_MEM;
+			return -1;
+		}
+
+		if ((size_t)len + 1 <= space_left) {
+			file->buf_pos += len;
+			return 0;
+		}
 
-		len = vsnprintf((char *)file->buffer + file->buf_pos, space_left, format, arglist);
-		if (len < 0 || (size_t)len > file->buf_size)
-			return GIT_ENOMEM;
+		if (flush_buffer(file) < 0)
+			return -1;
+
+		space_left = file->buf_size - file->buf_pos;
+
+	} while ((size_t)len + 1 <= space_left);
+
+	tmp_buffer = git__malloc(len + 1);
+	if (!tmp_buffer) {
+		file->last_error = BUFERR_MEM;
+		return -1;
 	}
 
-	file->buf_pos += len;
-	return GIT_SUCCESS;
+	va_start(arglist, format);
+	len = p_vsnprintf(tmp_buffer, len + 1, format, arglist);
+	va_end(arglist);
+
+	if (len < 0) {
+		git__free(tmp_buffer);
+		file->last_error = BUFERR_MEM;
+		return -1;
+	}
+
+	res = git_filebuf_write(file, tmp_buffer, len);
+	git__free(tmp_buffer);
 
+	return res;
 }
 
diff --git a/src/filebuf.h b/src/filebuf.h
index 37cb36784..377883147 100644
--- a/src/filebuf.h
+++ b/src/filebuf.h
@@ -1,9 +1,15 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_filebuf_h__
 #define INCLUDE_filebuf_h__
 
 #include "fileops.h"
 #include "hash.h"
-#include "git2/zlib.h"
+#include <zlib.h>
 
 #ifdef GIT_THREADS
 #	define GIT_FILEBUF_THREADS
@@ -13,7 +19,8 @@
 #define GIT_FILEBUF_APPEND				(1 << 2)
 #define GIT_FILEBUF_FORCE				(1 << 3)
 #define GIT_FILEBUF_TEMPORARY			(1 << 4)
-#define GIT_FILEBUF_DEFLATE_CONTENTS	(1 << 5)
+#define GIT_FILEBUF_DO_NOT_BUFFER		(1 << 5)
+#define GIT_FILEBUF_DEFLATE_SHIFT		(6)
 
 #define GIT_FILELOCK_EXTENSION ".lock\0"
 #define GIT_FILELOCK_EXTLENGTH 6
@@ -22,8 +29,7 @@ struct git_filebuf {
 	char *path_original;
 	char *path_lock;
 
-	int (*write)(struct git_filebuf *file,
-			const void *source, size_t len);
+	int (*write)(struct git_filebuf *file, void *source, size_t len);
 
 	git_hash_ctx *digest;
 
@@ -35,18 +41,46 @@ struct git_filebuf {
 
 	size_t buf_size, buf_pos;
 	git_file fd;
+	bool fd_is_open;
+	bool do_not_buffer;
+	int last_error;
 };
 
 typedef struct git_filebuf git_filebuf;
 
+#define GIT_FILEBUF_INIT {0}
+
+/*
+ * The git_filebuf object lifecycle is:
+ * - Allocate git_filebuf, preferably using GIT_FILEBUF_INIT.
+ *
+ * - Call git_filebuf_open() to initialize the filebuf for use.
+ *
+ * - Make as many calls to git_filebuf_write(), git_filebuf_printf(),
+ *   git_filebuf_reserve() as you like. The error codes for these
+ *   functions don't need to be checked. They are stored internally
+ *   by the file buffer.
+ *
+ * - While you are writing, you may call git_filebuf_hash() to get
+ *   the hash of all you have written so far. This function will
+ *   fail if any of the previous writes to the buffer failed.
+ *
+ * - To close the git_filebuf, you may call git_filebuf_commit() or
+ *   git_filebuf_commit_at() to save the file, or
+ *   git_filebuf_cleanup() to abandon the file.  All of these will
+ *   free the git_filebuf object. Likewise, all of these will fail
+ *   if any of the previous writes to the buffer failed, and set
+ *   an error code accordingly.
+ */
 int git_filebuf_write(git_filebuf *lock, const void *buff, size_t len);
 int git_filebuf_reserve(git_filebuf *file, void **buff, size_t len);
-int git_filebuf_printf(git_filebuf *file, const char *format, ...);
+int git_filebuf_printf(git_filebuf *file, const char *format, ...) GIT_FORMAT_PRINTF(2, 3);
 
 int git_filebuf_open(git_filebuf *lock, const char *path, int flags);
-int git_filebuf_commit(git_filebuf *lock);
-int git_filebuf_commit_at(git_filebuf *lock, const char *path);
+int git_filebuf_commit(git_filebuf *lock, mode_t mode);
+int git_filebuf_commit_at(git_filebuf *lock, const char *path, mode_t mode);
 void git_filebuf_cleanup(git_filebuf *lock);
 int git_filebuf_hash(git_oid *oid, git_filebuf *file);
+int git_filebuf_flush(git_filebuf *file);
 
 #endif
diff --git a/src/fileops.c b/src/fileops.c
index 73939349d..95a65893c 100644
--- a/src/fileops.c
+++ b/src/fileops.c
@@ -1,350 +1,283 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #include "common.h"
 #include "fileops.h"
 #include <ctype.h>
 
-int gitfo_mkdir_2file(const char *file_path)
+int git_futils_mkpath2file(const char *file_path, const mode_t mode)
 {
-	const int mode = 0755; /* or 0777 ? */
-	int error = GIT_SUCCESS;
-	char target_folder_path[GIT_PATH_MAX];
+	int result = 0;
+	git_buf target_folder = GIT_BUF_INIT;
 
-	error = git__dirname_r(target_folder_path, sizeof(target_folder_path), file_path);
-	if (error < GIT_SUCCESS)
-		return git__throw(GIT_EINVALIDPATH, "Failed to recursively build `%s` tree structure. Unable to parse parent folder name", file_path);
+	if (git_path_dirname_r(&target_folder, file_path) < 0)
+		return -1;
 
 	/* Does the containing folder exist? */
-	if (gitfo_isdir(target_folder_path)) {
-		git__joinpath(target_folder_path, target_folder_path, ""); /* Ensure there's a trailing slash */
-
+	if (git_path_isdir(target_folder.ptr) == false)
 		/* Let's create the tree structure */
-		error = gitfo_mkdir_recurs(target_folder_path, mode);
-		if (error < GIT_SUCCESS)
-			return error;	/* The callee already takes care of setting the correct error message. */
-	}
+		result = git_futils_mkdir_r(target_folder.ptr, NULL, mode);
 
-	return GIT_SUCCESS;
+	git_buf_free(&target_folder);
+	return result;
 }
 
-int gitfo_mktemp(char *path_out, const char *filename)
+int git_futils_mktmp(git_buf *path_out, const char *filename)
 {
 	int fd;
 
-	strcpy(path_out, filename);
-	strcat(path_out, "_git2_XXXXXX");
-
-#if defined(_MSC_VER)
-	/* FIXME: there may be race conditions when multi-threading
-	 * with the library */
-	if (_mktemp_s(path_out, GIT_PATH_MAX) != 0)
-		return git__throw(GIT_EOSERR, "Failed to make temporary file %s", path_out);
-
-	fd = gitfo_creat(path_out, 0744);
-#else
-	fd = mkstemp(path_out);
-#endif
+	git_buf_sets(path_out, filename);
+	git_buf_puts(path_out, "_git2_XXXXXX");
 
-	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to create temporary file %s", path_out);
-}
+	if (git_buf_oom(path_out))
+		return -1;
 
-int gitfo_open(const char *path, int flags)
-{
-	int fd = open(path, flags | O_BINARY);
-	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to open %s", path);
-}
+	if ((fd = p_mkstemp(path_out->ptr)) < 0) {
+		giterr_set(GITERR_OS,
+			"Failed to create temporary file '%s'", path_out->ptr);
+		return -1;
+	}
 
-int gitfo_creat(const char *path, int mode)
-{
-	int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, mode);
-	return fd >= 0 ? fd : git__throw(GIT_EOSERR, "Failed to create file. Could not open %s", path);
+	return fd;
 }
 
-int gitfo_creat_force(const char *path, int mode)
+int git_futils_creat_withpath(const char *path, const mode_t dirmode, const mode_t mode)
 {
-	if (gitfo_mkdir_2file(path) < GIT_SUCCESS)
-		return git__throw(GIT_EOSERR, "Failed to create file %s", path);
+	int fd;
 
-	return gitfo_creat(path, mode);
-}
+	if (git_futils_mkpath2file(path, dirmode) < 0)
+		return -1;
 
-int gitfo_read(git_file fd, void *buf, size_t cnt)
-{
-	char *b = buf;
-	while (cnt) {
-		ssize_t r = read(fd, b, cnt);
-		if (r < 0) {
-			if (errno == EINTR || errno == EAGAIN)
-				continue;
-			return git__throw(GIT_EOSERR, "Failed to read from file");
-		}
-		if (!r) {
-			errno = EPIPE;
-			return git__throw(GIT_EOSERR, "Failed to read from file");
-		}
-		cnt -= r;
-		b += r;
+	fd = p_creat(path, mode);
+	if (fd < 0) {
+		giterr_set(GITERR_OS, "Failed to create file '%s'", path);
+		return -1;
 	}
-	return GIT_SUCCESS;
-}
 
-int gitfo_write(git_file fd, void *buf, size_t cnt)
-{
-	char *b = buf;
-	while (cnt) {
-		ssize_t r = write(fd, b, cnt);
-		if (r < 0) {
-			if (errno == EINTR || errno == EAGAIN)
-				continue;
-			return git__throw(GIT_EOSERR, "Failed to write to file");
-		}
-		if (!r) {
-			errno = EPIPE;
-			return git__throw(GIT_EOSERR, "Failed to write to file");
-		}
-		cnt -= r;
-		b += r;
-	}
-	return GIT_SUCCESS;
+	return fd;
 }
 
-int gitfo_isdir(const char *path)
+int git_futils_creat_locked(const char *path, const mode_t mode)
 {
-	struct stat st;
-	int len, stat_error;
+	int fd;
 
-	if (!path)
-		return git__throw(GIT_ENOTFOUND, "No path given to gitfo_isdir");
+#ifdef GIT_WIN32
+	wchar_t* buf;
 
-	len = strlen(path);
+	buf = gitwin_to_utf16(path);
+	fd = _wopen(buf, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_EXCL, mode);
+	git__free(buf);
+#else
+	fd = open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY | O_EXCL, mode);
+#endif
 
-	/* win32: stat path for folders cannot end in a slash */
-	if (path[len - 1] == '/') {
-		char *path_fixed = NULL;
-		path_fixed = git__strdup(path);
-		path_fixed[len - 1] = 0;
-		stat_error = gitfo_stat(path_fixed, &st);
-		free(path_fixed);
-	} else {
-		stat_error = gitfo_stat(path, &st);
+	if (fd < 0) {
+		giterr_set(GITERR_OS, "Failed to create locked file '%s'", path);
+		return -1;
 	}
 
-	if (stat_error < GIT_SUCCESS)
-		return git__throw(GIT_ENOTFOUND, "%s does not exist", path);
+	return fd;
+}
 
-	if (!S_ISDIR(st.st_mode))
-		return git__throw(GIT_ENOTFOUND, "%s is a file", path);
+int git_futils_creat_locked_withpath(const char *path, const mode_t dirmode, const mode_t mode)
+{
+	if (git_futils_mkpath2file(path, dirmode) < 0)
+		return -1;
 
-	return GIT_SUCCESS;
+	return git_futils_creat_locked(path, mode);
 }
 
-int gitfo_exists(const char *path)
+int git_futils_open_ro(const char *path)
 {
-	assert(path);
-	return access(path, F_OK);
+	int fd = p_open(path, O_RDONLY);
+	if (fd < 0) {
+		if (errno == ENOENT)
+			fd = GIT_ENOTFOUND;
+		giterr_set(GITERR_OS, "Failed to open '%s'", path);
+	}
+	return fd;
 }
 
-git_off_t gitfo_size(git_file fd)
+git_off_t git_futils_filesize(git_file fd)
 {
 	struct stat sb;
-	if (gitfo_fstat(fd, &sb))
-		return git__throw(GIT_EOSERR, "Failed to get size of file. File missing or corrupted");
+
+	if (p_fstat(fd, &sb)) {
+		giterr_set(GITERR_OS, "Failed to stat file descriptor");
+		return -1;
+	}
+
 	return sb.st_size;
 }
 
-int gitfo_read_file(gitfo_buf *obj, const char *path)
+mode_t git_futils_canonical_mode(mode_t raw_mode)
+{
+	if (S_ISREG(raw_mode))
+		return S_IFREG | GIT_CANONICAL_PERMS(raw_mode);
+	else if (S_ISLNK(raw_mode))
+		return S_IFLNK;
+	else if (S_ISGITLINK(raw_mode))
+		return S_IFGITLINK;
+	else if (S_ISDIR(raw_mode))
+		return S_IFDIR;
+	else
+		return 0;
+}
+
+int git_futils_readbuffer_updated(git_buf *buf, const char *path, time_t *mtime, int *updated)
 {
 	git_file fd;
 	size_t len;
-	git_off_t size;
-	unsigned char *buff;
+	struct stat st;
 
-	assert(obj && path && *path);
+	assert(buf && path && *path);
 
-	if ((fd = gitfo_open(path, O_RDONLY)) < 0)
-		return git__throw(GIT_ERROR, "Failed to open %s for reading", path);
+	if (updated != NULL)
+		*updated = 0;
 
-	if (((size = gitfo_size(fd)) < 0) || !git__is_sizet(size+1)) {
-		gitfo_close(fd);
-		return git__throw(GIT_ERROR, "Failed to read file `%s`. An error occured while calculating its size", path);
-	}
-	len = (size_t) size;
+	if ((fd = git_futils_open_ro(path)) < 0)
+		return fd;
 
-	if ((buff = git__malloc(len + 1)) == NULL) {
-		gitfo_close(fd);
-		return GIT_ENOMEM;
+	if (p_fstat(fd, &st) < 0 || S_ISDIR(st.st_mode) || !git__is_sizet(st.st_size+1)) {
+		p_close(fd);
+		giterr_set(GITERR_OS, "Invalid regular file stat for '%s'", path);
+		return -1;
 	}
 
-	if (gitfo_read(fd, buff, len) < 0) {
-		gitfo_close(fd);
-		free(buff);
-		return git__throw(GIT_ERROR, "Failed to read file `%s`", path);
+	/*
+	 * If we were given a time, we only want to read the file if it
+	 * has been modified.
+	 */
+	if (mtime != NULL && *mtime >= st.st_mtime) {
+		p_close(fd);
+		return 0;
 	}
-	buff[len] = '\0';
 
-	gitfo_close(fd);
+	if (mtime != NULL)
+		*mtime = st.st_mtime;
 
-	obj->data = buff;
-	obj->len  = len;
+	len = (size_t) st.st_size;
 
-	return GIT_SUCCESS;
-}
-
-void gitfo_free_buf(gitfo_buf *obj)
-{
-	assert(obj);
-	free(obj->data);
-	obj->data = NULL;
-}
+	git_buf_clear(buf);
 
-int gitfo_mv(const char *from, const char *to)
-{
-	int error;
-
-#ifdef GIT_WIN32
-	/*
-	 * Win32 POSIX compilance my ass. If the destination
-	 * file exists, the `rename` call fails. This is as
-	 * close as it gets with the Win32 API.
-	 */
-	error = MoveFileEx(from, to, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) ? GIT_SUCCESS : GIT_EOSERR;
-#else
-	/* Don't even try this on Win32 */
-	if (!link(from, to)) {
-		gitfo_unlink(from);
-		return GIT_SUCCESS;
+	if (git_buf_grow(buf, len + 1) < 0) {
+		p_close(fd);
+		return -1;
 	}
 
-	if (!rename(from, to))
-		return GIT_SUCCESS;
+	buf->ptr[len] = '\0';
 
-	error = GIT_EOSERR;
-#endif
+	while (len > 0) {
+		ssize_t read_size = p_read(fd, buf->ptr, len);
 
-	if (error < GIT_SUCCESS)
-		return git__throw(error, "Failed to move file from `%s` to `%s`", from, to);
+		if (read_size < 0) {
+			p_close(fd);
+			giterr_set(GITERR_OS, "Failed to read descriptor for '%s'", path);
+			return -1;
+		}
 
-	return GIT_SUCCESS;
-}
+		len -= read_size;
+		buf->size += read_size;
+	}
 
-int gitfo_mv_force(const char *from, const char *to)
-{
-	if (gitfo_mkdir_2file(to) < GIT_SUCCESS)
-		return GIT_EOSERR;	/* The callee already takes care of setting the correct error message. */
+	p_close(fd);
 
-	return gitfo_mv(from, to);	/* The callee already takes care of setting the correct error message. */
-}
+	if (updated != NULL)
+		*updated = 1;
 
-int gitfo_map_ro(git_map *out, git_file fd, git_off_t begin, size_t len)
-{
-	if (git__mmap(out, len, GIT_PROT_READ, GIT_MAP_SHARED, fd, begin) < GIT_SUCCESS)
-		return GIT_EOSERR;
-	return GIT_SUCCESS;
+	return 0;
 }
 
-void gitfo_free_map(git_map *out)
+int git_futils_readbuffer(git_buf *buf, const char *path)
 {
-	git__munmap(out);
+	return git_futils_readbuffer_updated(buf, path, NULL, NULL);
 }
 
-int gitfo_dirent(
-	char *path,
-	size_t path_sz,
-	int (*fn)(void *, char *),
-	void *arg)
+int git_futils_mv_withpath(const char *from, const char *to, const mode_t dirmode)
 {
-	size_t wd_len = strlen(path);
-	DIR *dir;
-	struct dirent *de;
-
-	if (!wd_len || path_sz < wd_len + 2)
-		return git__throw(GIT_EINVALIDARGS, "Failed to process `%s` tree structure. Path is either empty or buffer size is too short", path);
-
-	while (path[wd_len - 1] == '/')
-		wd_len--;
-	path[wd_len++] = '/';
-	path[wd_len] = '\0';
-
-	dir = opendir(path);
-	if (!dir)
-		return git__throw(GIT_EOSERR, "Failed to process `%s` tree structure. An error occured while opening the directory", path);
-
-	while ((de = readdir(dir)) != NULL) {
-		size_t de_len;
-		int result;
-
-		/* always skip '.' and '..' */
-		if (de->d_name[0] == '.') {
-			if (de->d_name[1] == '\0')
-				continue;
-			if (de->d_name[1] == '.' && de->d_name[2] == '\0')
-				continue;
-		}
+	if (git_futils_mkpath2file(to, dirmode) < 0)
+		return -1;
 
-		de_len = strlen(de->d_name);
-		if (path_sz < wd_len + de_len + 1) {
-			closedir(dir);
-			return git__throw(GIT_ERROR, "Failed to process `%s` tree structure. Buffer size is too short", path);
-		}
-
-		strcpy(path + wd_len, de->d_name);
-		result = fn(arg, path);
-		if (result < GIT_SUCCESS) {
-			closedir(dir);
-			return result;	/* The callee is reponsible for setting the correct error message */
-		}
-		if (result > 0) {
-			closedir(dir);
-			return result;
-		}
+	if (p_rename(from, to) < 0) {
+		giterr_set(GITERR_OS, "Failed to rename '%s' to '%s'", from, to);
+		return -1;
 	}
 
-	closedir(dir);
-	return GIT_SUCCESS;
+	return 0;
 }
 
-
-int retrieve_path_root_offset(const char *path)
+int git_futils_mmap_ro(git_map *out, git_file fd, git_off_t begin, size_t len)
 {
-	int offset = 0;
-
-#ifdef GIT_WIN32
+	return p_mmap(out, len, GIT_PROT_READ, GIT_MAP_SHARED, fd, begin);
+}
 
-	/* Does the root of the path look like a windows drive ? */
-	if (isalpha(path[0]) && (path[1] == ':'))
-		offset += 2;
+int git_futils_mmap_ro_file(git_map *out, const char *path)
+{
+	git_file fd = git_futils_open_ro(path);
+	git_off_t len;
+	int result;
 
-#endif
+	if (fd < 0)
+		return fd;
 
-	if (*(path + offset) == '/')
-		return offset;
+	len = git_futils_filesize(fd);
+	if (!git__is_sizet(len)) {
+		giterr_set(GITERR_OS, "File `%s` too large to mmap", path);
+		return -1;
+	}
 
-	return -1;	/* Not a real error. Rather a signal than the path is not rooted */
+	result = git_futils_mmap_ro(out, fd, 0, (size_t)len);
+	p_close(fd);
+	return result;
 }
 
+void git_futils_mmap_free(git_map *out)
+{
+	p_munmap(out);
+}
 
-int gitfo_mkdir_recurs(const char *path, int mode)
+int git_futils_mkdir_r(const char *path, const char *base, const mode_t mode)
 {
-	int error, root_path_offset;
+	git_buf make_path = GIT_BUF_INIT;
+	size_t start = 0;
 	char *pp, *sp;
-    char *path_copy = git__strdup(path);
+	bool failed = false;
+
+	if (base != NULL) {
+		/*
+		 * when a base is being provided, it is supposed to already exist.
+		 * Therefore, no attempt is being made to recursively create this leading path
+		 * segment. It's just skipped. */
+		start = strlen(base);
+		if (git_buf_joinpath(&make_path, base, path) < 0)
+			return -1;
+	} else {
+		int root_path_offset;
 
-	if (path_copy == NULL)
-		return GIT_ENOMEM;
+		if (git_buf_puts(&make_path, path) < 0)
+			return -1;
 
-	error = GIT_SUCCESS;
-	pp = path_copy;
+		root_path_offset = git_path_root(make_path.ptr);
+		if (root_path_offset > 0) {
+			 /*
+			  * On Windows, will skip the drive name (eg. C: or D:)
+			  * or the leading part of a network path (eg. //computer_name ) */
+			start = root_path_offset;
+		}
+	}
 
-	root_path_offset = retrieve_path_root_offset(pp);
-	if (root_path_offset > 0)
-		pp += root_path_offset; /* On Windows, will skip the drive name (eg. C: or D:) */
+	pp = make_path.ptr + start;
 
-    while (error == GIT_SUCCESS && (sp = strchr(pp, '/')) != 0) {
-		if (sp != pp && gitfo_isdir(path_copy) < GIT_SUCCESS) {
+	while (!failed && (sp = strchr(pp, '/')) != NULL) {
+		if (sp != pp && git_path_isdir(make_path.ptr) == false) {
 			*sp = 0;
-			error = gitfo_mkdir(path_copy, mode);
 
 			/* Do not choke while trying to recreate an existing directory */
-			if (errno == EEXIST)
-				error = GIT_SUCCESS;
+			if (p_mkdir(make_path.ptr, mode) < 0 && errno != EEXIST)
+				failed = true;
 
 			*sp = '/';
 		}
@@ -352,201 +285,198 @@ int gitfo_mkdir_recurs(const char *path, int mode)
 		pp = sp + 1;
 	}
 
-	if (*(pp - 1) != '/' && error == GIT_SUCCESS)
-		error = gitfo_mkdir(path, mode);
+	if (*pp != '\0' && !failed) {
+		if (p_mkdir(make_path.ptr, mode) < 0 && errno != EEXIST)
+			failed = true;
+	}
 
-	free(path_copy);
+	git_buf_free(&make_path);
 
-	if (error < GIT_SUCCESS)
-		return git__throw(error, "Failed to recursively create `%s` tree structure", path);
+	if (failed) {
+		giterr_set(GITERR_OS,
+			"Failed to create directory structure at '%s'", path);
+		return -1;
+	}
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-static int retrieve_previous_path_component_start(const char *path)
+static int _rmdir_recurs_foreach(void *opaque, git_buf *path)
 {
-	int offset, len, root_offset, start = 0;
+	git_directory_removal_type removal_type = *(git_directory_removal_type *)opaque;
+
+	assert(removal_type == GIT_DIRREMOVAL_EMPTY_HIERARCHY
+		|| removal_type == GIT_DIRREMOVAL_FILES_AND_DIRS
+		|| removal_type == GIT_DIRREMOVAL_ONLY_EMPTY_DIRS);
 
-	root_offset = retrieve_path_root_offset(path);
-	if (root_offset > -1)
-		start += root_offset;
+	if (git_path_isdir(path->ptr) == true) {
+		if (git_path_direach(path, _rmdir_recurs_foreach, opaque) < 0)
+			return -1;
 
-	len = strlen(path);
-	offset = len - 1;
+		if (p_rmdir(path->ptr) < 0) {
+			if (removal_type == GIT_DIRREMOVAL_ONLY_EMPTY_DIRS && (errno == ENOTEMPTY || errno == EEXIST))
+				return 0;
+
+			giterr_set(GITERR_OS, "Could not remove directory '%s'", path->ptr);
+			return -1;
+		}
 
-	/* Skip leading slash */
-	if (path[start] == '/')
-		start++;
+		return 0;
+	}
 
-	/* Skip trailing slash */
-	if (path[offset] == '/')
-		offset--;
+	if (removal_type == GIT_DIRREMOVAL_FILES_AND_DIRS) {
+		if (p_unlink(path->ptr) < 0) {
+			giterr_set(GITERR_OS, "Could not remove directory.  File '%s' cannot be removed", path->ptr);
+			return -1;
+		}
 
-	if (offset < root_offset)
-		return git__throw(GIT_ERROR, "Failed to retrieve path component. Wrong offset");
+		return 0;
+	}
 
-	while (offset > start && path[offset-1] != '/') {
-		offset--;
+	if (removal_type == GIT_DIRREMOVAL_EMPTY_HIERARCHY) {
+		giterr_set(GITERR_OS, "Could not remove directory. File '%s' still present", path->ptr);
+		return -1;
 	}
 
-	return offset;
+	return 0;
 }
 
-int gitfo_prettify_dir_path(char *buffer_out, size_t size, const char *path)
+int git_futils_rmdir_r(const char *path, git_directory_removal_type removal_type)
 {
-	int len = 0, segment_len, only_dots, root_path_offset, error = GIT_SUCCESS;
-	char *current;
-	const char *buffer_out_start, *buffer_end;
-
-	current = (char *)path;
-	buffer_end = path + strlen(path);
-	buffer_out_start = buffer_out;
-
-	root_path_offset = retrieve_path_root_offset(path);
-	if (root_path_offset < 0) {
-		error = gitfo_getcwd(buffer_out, size);
-		if (error < GIT_SUCCESS)
-			return error;	/* The callee already takes care of setting the correct error message. */
-
-		len = strlen(buffer_out);
-		buffer_out += len;
-	}
+	int error;
+	git_buf p = GIT_BUF_INIT;
 
-	while (current < buffer_end) {
-		/* Prevent multiple slashes from being added to the output */
-		if (*current == '/' && len > 0 && buffer_out_start[len - 1] == '/') {
-			current++;
-			continue;
-		}
-		
-		only_dots = 1;
-		segment_len = 0;
-
-		/* Copy path segment to the output */
-		while (current < buffer_end && *current != '/')
-		{
-			only_dots &= (*current == '.');
-			*buffer_out++ = *current++;
-			segment_len++;
-			len++;
-		}
+	error = git_buf_sets(&p, path);
+	if (!error)
+		error = _rmdir_recurs_foreach(&removal_type, &p);
+	git_buf_free(&p);
+	return error;
+}
 
-		/* Skip current directory */
-		if (only_dots && segment_len == 1)
-		{
-			current++;
-			buffer_out -= segment_len;
-			len -= segment_len;
-			continue;
-		}
+#ifdef GIT_WIN32
+struct win32_path {
+	wchar_t path[MAX_PATH];
+	DWORD len;
+};
 
-		/* Handle the double-dot upward directory navigation */
-		if (only_dots && segment_len == 2)
-		{
-			current++;
-			buffer_out -= segment_len;
+static int win32_expand_path(struct win32_path *s_root, const wchar_t *templ)
+{
+	s_root->len = ExpandEnvironmentStringsW(templ, s_root->path, MAX_PATH);
+	return s_root->len ? 0 : -1;
+}
 
-			*buffer_out ='\0';
-			len = retrieve_previous_path_component_start(buffer_out_start);
+static int win32_find_file(git_buf *path, const struct win32_path *root, const char *filename)
+{
+	int error = 0;
+	size_t len;
+	wchar_t *file_utf16 = NULL;
+	char *file_utf8 = NULL;
 
-			/* Are we escaping out of the root dir? */
-			if (len < 0)
-				return git__throw(GIT_EINVALIDPATH, "Failed to normalize path `%s`. The path escapes out of the root directory", path);
+	if (!root || !filename || (len = strlen(filename)) == 0)
+		return GIT_ENOTFOUND;
 
-			buffer_out = (char *)buffer_out_start + len;
-			continue;
-		}
+	/* allocate space for wchar_t path to file */
+	file_utf16 = git__calloc(root->len + len + 2, sizeof(wchar_t));
+	GITERR_CHECK_ALLOC(file_utf16);
 
-		/* Guard against potential multiple dot path traversal (cf http://cwe.mitre.org/data/definitions/33.html) */
-		if (only_dots && segment_len > 0)
-			return git__throw(GIT_EINVALIDPATH, "Failed to normalize path `%s`. The path contains a segment with three `.` or more", path);
+	/* append root + '\\' + filename as wchar_t */
+	memcpy(file_utf16, root->path, root->len * sizeof(wchar_t));
 
-		*buffer_out++ = '/';
-		len++;
+	if (*filename == '/' || *filename == '\\')
+		filename++;
+
+	if (gitwin_append_utf16(file_utf16 + root->len - 1, filename, len + 1) !=
+		(int)len + 1) {
+		error = -1;
+		goto cleanup;
+	}
+
+	/* check access */
+	if (_waccess(file_utf16, F_OK) < 0) {
+		error = GIT_ENOTFOUND;
+		goto cleanup;
 	}
 
-	*buffer_out = '\0';
+	/* convert to utf8 */
+	if ((file_utf8 = gitwin_from_utf16(file_utf16)) == NULL)
+		error = -1;
+	else {
+		git_path_mkposix(file_utf8);
+		git_buf_attach(path, file_utf8, 0);
+	}
 
-	return GIT_SUCCESS;
+cleanup:
+	git__free(file_utf16);
+	return error;
 }
+#endif
 
-int gitfo_prettify_file_path(char *buffer_out, size_t size, const char *path)
+int git_futils_find_system_file(git_buf *path, const char *filename)
 {
-	int error, path_len, i;
-	const char* pattern = "/..";
-
-	path_len = strlen(path);
-
-	/* Let's make sure the filename isn't empty nor a dot */
-	if (path_len == 0 || (path_len == 1 && *path == '.'))
-		return git__throw(GIT_EINVALIDPATH, "Failed to normalize file path `%s`. The path is either empty or equals `.`", path);
+#ifdef GIT_WIN32
+	struct win32_path root;
 
-	/* Let's make sure the filename doesn't end with "/", "/." or "/.." */
-	for (i = 1; path_len > i && i < 4; i++) {
-		if (!strncmp(path + path_len - i, pattern, i))
-			return git__throw(GIT_EINVALIDPATH, "Failed to normalize file path `%s`. The path points to a folder", path);
+	if (win32_expand_path(&root, L"%PROGRAMFILES%\\Git\\etc\\") < 0 ||
+		root.path[0] == L'%') /* i.e. no expansion happened */
+	{
+		giterr_set(GITERR_OS, "Cannot locate the system's Program Files directory");
+		return -1;
 	}
 
-	error =  gitfo_prettify_dir_path(buffer_out, size, path);
-	if (error < GIT_SUCCESS)
-		return error;	/* The callee already takes care of setting the correct error message. */
+	if (win32_find_file(path, &root, filename) < 0) {
+		git_buf_clear(path);
+		return GIT_ENOTFOUND;
+	}
 
-	path_len = strlen(buffer_out);
-	if (path_len < 2)	/* TODO: Fixme. We should also take of detecting Windows rooted path (probably through usage of retrieve_path_root_offset) */
-		return git__throw(GIT_EINVALIDPATH, "Failed to normalize file path `%s`. The path points to a folder", path);
+	return 0;
 
-	/* Remove the trailing slash */
-	buffer_out[path_len - 1] = '\0';
+#else
+	if (git_buf_joinpath(path, "/etc", filename) < 0)
+		return -1;
 
-	return GIT_SUCCESS;
-}
+	if (git_path_exists(path->ptr) == true)
+		return 0;
 
-int gitfo_cmp_path(const char *name1, int len1, int isdir1,
-		const char *name2, int len2, int isdir2)
-{
-	int len = len1 < len2 ? len1 : len2;
-	int cmp;
-
-	cmp = memcmp(name1, name2, len);
-	if (cmp)
-		return cmp;
-	if (len1 < len2)
-		return ((!isdir1 && !isdir2) ? -1 :
-                        (isdir1 ? '/' - name2[len1] : name2[len1] - '/'));
-	if (len1 > len2)
-		return ((!isdir1 && !isdir2) ? 1 :
-                        (isdir2 ? name1[len2] - '/' : '/' - name1[len2]));
-	return 0;
+	git_buf_clear(path);
+	return GIT_ENOTFOUND;
+#endif
 }
 
-static void posixify_path(char *path)
+int git_futils_find_global_file(git_buf *path, const char *filename)
 {
-	while (*path) {
-		if (*path == '\\')
-			*path = '/';
+#ifdef GIT_WIN32
+	struct win32_path root;
 
-		path++;
+	if (win32_expand_path(&root, L"%USERPROFILE%\\") < 0 ||
+		root.path[0] == L'%') /* i.e. no expansion happened */
+	{
+		giterr_set(GITERR_OS, "Cannot locate the user's profile directory");
+		return -1;
 	}
-}
-
-int gitfo_getcwd(char *buffer_out, size_t size)
-{
-	char *cwd_buffer;
 
-	assert(buffer_out && size > 0);
+	if (win32_find_file(path, &root, filename) < 0) {
+		git_buf_clear(path);
+		return GIT_ENOTFOUND;
+	}
 
-#ifdef GIT_WIN32
-	cwd_buffer = _getcwd(buffer_out, size);
+	return 0;
 #else
-	cwd_buffer = getcwd(buffer_out, size);
-#endif
+	const char *home = getenv("HOME");
 
-	if (cwd_buffer == NULL)
-		return git__throw(GIT_EOSERR, "Failed to retrieve current working directory");
+	if (home == NULL) {
+		giterr_set(GITERR_OS, "Global file lookup failed. "
+			"Cannot locate the user's home directory");
+		return -1;
+	}
 
-	posixify_path(buffer_out);
+	if (git_buf_joinpath(path, home, filename) < 0)
+		return -1;
 
-	git__joinpath(buffer_out, buffer_out, "");	//Ensure the path ends with a trailing slash
+	if (git_path_exists(path->ptr) == false) {
+		git_buf_clear(path);
+		return GIT_ENOTFOUND;
+	}
 
-	return GIT_SUCCESS;
+	return 0;
+#endif
 }
diff --git a/src/fileops.h b/src/fileops.h
index d0381b675..b0c5779e5 100644
--- a/src/fileops.h
+++ b/src/fileops.h
@@ -1,94 +1,118 @@
 /*
- * fileops.h - OS agnostic disk io operations
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * This header describes the strictly internal part of the api
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #ifndef INCLUDE_fileops_h__
 #define INCLUDE_fileops_h__
 
 #include "common.h"
 #include "map.h"
-#include "dir.h"
-#include <fcntl.h>
-#include <time.h>
-
-#ifdef GIT_WIN32
-GIT_INLINE(int) link(const char *GIT_UNUSED(old), const char *GIT_UNUSED(new))
-{
-	GIT_UNUSED_ARG(old)
-	GIT_UNUSED_ARG(new)
-	errno = ENOSYS;
-	return -1;
-}
-
-GIT_INLINE(int) git__mkdir(const char *path, int GIT_UNUSED(mode))
-{
-	GIT_UNUSED_ARG(mode)
-	return mkdir(path);
-}
-
-extern int git__unlink(const char *path);
-extern int git__mkstemp(char *template);
-extern int git__fsync(int fd);
-
-# ifndef GIT__WIN32_NO_HIDE_FILEOPS
-#  define unlink(p) git__unlink(p)
-#  define mkstemp(t) git__mkstemp(t)
-#  define mkdir(p,m) git__mkdir(p, m)
-#  define fsync(fd) git__fsync(fd)
-# endif
-#endif  /* GIT_WIN32 */
-
-
-#if !defined(O_BINARY)
-#define O_BINARY 0
-#endif
-
-#define GITFO_BUF_INIT {NULL, 0}
-
-typedef int git_file;
-
-typedef struct {  /* file io buffer  */
-	void *data;  /* data bytes   */
-	size_t len;  /* data length  */
-} gitfo_buf;
-
-extern int gitfo_exists(const char *path);
-extern int gitfo_open(const char *path, int flags);
-extern int gitfo_creat(const char *path, int mode);
-extern int gitfo_creat_force(const char *path, int mode);
-extern int gitfo_mktemp(char *path_out, const char *filename);
-extern int gitfo_isdir(const char *path);
-extern int gitfo_mkdir_recurs(const char *path, int mode);
-extern int gitfo_mkdir_2file(const char *path);
-#define gitfo_close(fd) close(fd)
-
-extern int gitfo_read(git_file fd, void *buf, size_t cnt);
-extern int gitfo_write(git_file fd, void *buf, size_t cnt);
-#define gitfo_lseek(f,n,w) lseek(f, n, w)
-extern git_off_t gitfo_size(git_file fd);
-
-extern int gitfo_read_file(gitfo_buf *obj, const char *path);
-extern void gitfo_free_buf(gitfo_buf *obj);
-
-/* Move (rename) a file; this operation is atomic */
-extern int gitfo_mv(const char *from, const char *to);
-
-/* Move a file (forced); this will create the destination
- * path if it doesn't exist */
-extern int gitfo_mv_force(const char *from, const char *to);
-
-#define gitfo_stat(p,b) stat(p, b)
-#define gitfo_fstat(f,b) fstat(f, b)
-
-#define gitfo_unlink(p) unlink(p)
-#define gitfo_rmdir(p) rmdir(p)
-#define gitfo_chdir(p) chdir(p)
-#define gitfo_mkdir(p,m) mkdir(p, m)
-
-#define gitfo_mkstemp(t) mkstemp(t)
-#define gitfo_fsync(fd) fsync(fd)
-#define gitfo_chmod(p,m) chmod(p, m)
+#include "posix.h"
+#include "path.h"
+
+/**
+ * Filebuffer methods
+ *
+ * Read whole files into an in-memory buffer for processing
+ */
+extern int git_futils_readbuffer(git_buf *obj, const char *path);
+extern int git_futils_readbuffer_updated(git_buf *obj, const char *path, time_t *mtime, int *updated);
+
+/**
+ * File utils
+ *
+ * These are custom filesystem-related helper methods. They are
+ * rather high level, and wrap the underlying POSIX methods
+ *
+ * All these methods return 0 on success,
+ * or an error code on failure and an error message is set.
+ */
+
+/**
+ * Create and open a file, while also
+ * creating all the folders in its path
+ */
+extern int git_futils_creat_withpath(const char *path, const mode_t dirmode, const mode_t mode);
+
+/**
+ * Create an open a process-locked file
+ */
+extern int git_futils_creat_locked(const char *path, const mode_t mode);
+
+/**
+ * Create an open a process-locked file, while
+ * also creating all the folders in its path
+ */
+extern int git_futils_creat_locked_withpath(const char *path, const mode_t dirmode, const mode_t mode);
+
+/**
+ * Create a path recursively
+ *
+ * If a base parameter is being passed, it's expected to be valued with a path pointing to an already
+ * exisiting directory.
+ */
+extern int git_futils_mkdir_r(const char *path, const char *base, const mode_t mode);
+
+/**
+ * Create all the folders required to contain
+ * the full path of a file
+ */
+extern int git_futils_mkpath2file(const char *path, const mode_t mode);
+
+typedef enum {
+	GIT_DIRREMOVAL_EMPTY_HIERARCHY = 0,
+	GIT_DIRREMOVAL_FILES_AND_DIRS = 1,
+	GIT_DIRREMOVAL_ONLY_EMPTY_DIRS = 2,
+} git_directory_removal_type;
+
+/**
+ * Remove path and any files and directories beneath it.
+ *
+ * @param path Path to to top level directory to process.
+ *
+ * @param removal_type GIT_DIRREMOVAL_EMPTY_HIERARCHY to remove a hierarchy
+ * of empty directories (will fail if any file is found), GIT_DIRREMOVAL_FILES_AND_DIRS
+ * to remove a hierarchy of files and folders, GIT_DIRREMOVAL_ONLY_EMPTY_DIRS to only remove
+ * empty directories (no failure on file encounter).
+ *
+ * @return 0 on success; -1 on error.
+ */
+extern int git_futils_rmdir_r(const char *path, git_directory_removal_type removal_type);
+
+/**
+ * Create and open a temporary file with a `_git2_` suffix.
+ * Writes the filename into path_out.
+ * @return On success, an open file descriptor, else an error code < 0.
+ */
+extern int git_futils_mktmp(git_buf *path_out, const char *filename);
+
+/**
+ * Move a file on the filesystem, create the
+ * destination path if it doesn't exist
+ */
+extern int git_futils_mv_withpath(const char *from, const char *to, const mode_t dirmode);
+
+/**
+ * Open a file readonly and set error if needed.
+ */
+extern int git_futils_open_ro(const char *path);
+
+/**
+ * Get the filesize in bytes of a file
+ */
+extern git_off_t git_futils_filesize(git_file fd);
+
+#define GIT_MODE_PERMS_MASK			0777
+#define GIT_CANONICAL_PERMS(MODE)	(((MODE) & 0100) ? 0755 : 0644)
+#define GIT_MODE_TYPE(MODE)			((MODE) & ~GIT_MODE_PERMS_MASK)
+
+/**
+ * Convert a mode_t from the OS to a legal git mode_t value.
+ */
+extern mode_t git_futils_canonical_mode(mode_t raw_mode);
+
 
 /**
  * Read-only map all or part of a file into memory.
@@ -102,90 +126,57 @@ extern int gitfo_mv_force(const char *from, const char *to);
  * @param begin first byte to map, this should be page aligned.
  * @param end number of bytes to map.
  * @return
- * - GIT_SUCCESS on success;
- * - GIT_EOSERR on an unspecified OS related error.
+ * - 0 on success;
+ * - -1 on error.
  */
-extern int gitfo_map_ro(
+extern int git_futils_mmap_ro(
 	git_map *out,
 	git_file fd,
 	git_off_t begin,
 	size_t len);
 
 /**
+ * Read-only map an entire file.
+ *
+ * @param out buffer to populate with the mapping information.
+ * @param path path to file to be opened.
+ * @return
+ * - 0 on success;
+ * - GIT_ENOTFOUND if not found;
+ * - -1 on an unspecified OS related error.
+ */
+extern int git_futils_mmap_ro_file(
+	git_map *out,
+	const char *path);
+
+/**
  * Release the memory associated with a previous memory mapping.
  * @param map the mapping description previously configured.
  */
-extern void gitfo_free_map(git_map *map);
+extern void git_futils_mmap_free(git_map *map);
 
 /**
- * Walk each directory entry, except '.' and '..', calling fn(state).
- *
- * @param pathbuf buffer the function reads the initial directory
- * 		path from, and updates with each successive entry's name.
- * @param pathmax maximum allocation of pathbuf.
- * @param fn function to invoke with each entry.  The first arg is
- *		the input state and the second arg is pathbuf.  The function
- *		may modify the pathbuf, but only by appending new text.
- * @param state to pass to fn as the first arg.
- */
-extern int gitfo_dirent(
-	char *pathbuf,
-	size_t pathmax,
-	int (*fn)(void *, char *),
-	void *state);
-
-extern int gitfo_cmp_path(const char *name1, int len1, int isdir1,
-		const char *name2, int len2, int isdir2);
-
-extern int gitfo_getcwd(char *buffer_out, size_t size);
-
-/**
- * Clean up a provided absolute or relative directory path.
- * 
- * This prettification relies on basic operations such as coalescing 
- * multiple forward slashes into a single slash, removing '.' and 
- * './' current directory segments, and removing parent directory 
- * whenever '..' is encountered.
- *
- * If not empty, the returned path ends with a forward slash.
- *
- * For instance, this will turn "d1/s1///s2/..//../s3" into "d1/s3/".
+ * Find a "global" file (i.e. one in a user's home directory).
  *
- * This only performs a string based analysis of the path.
- * No checks are done to make sure the path actually makes sense from 
- * the file system perspective.
- *
- * @param buffer_out buffer to populate with the normalized path.
- * @param size buffer size.
- * @param path directory path to clean up.
+ * @param pathbuf buffer to write the full path into
+ * @param filename name of file to find in the home directory
  * @return
- * - GIT_SUCCESS on success;
- * - GIT_ERROR when the input path is invalid or escapes the current directory.
+ * - 0 if found;
+ * - GIT_ENOTFOUND if not found;
+ * - -1 on an unspecified OS related error.
  */
-int gitfo_prettify_dir_path(char *buffer_out, size_t size, const char *path);
+extern int git_futils_find_global_file(git_buf *path, const char *filename);
 
 /**
- * Clean up a provided absolute or relative file path.
- * 
- * This prettification relies on basic operations such as coalescing 
- * multiple forward slashes into a single slash, removing '.' and 
- * './' current directory segments, and removing parent directory 
- * whenever '..' is encountered.
- *
- * For instance, this will turn "d1/s1///s2/..//../s3" into "d1/s3".
- *
- * This only performs a string based analysis of the path.
- * No checks are done to make sure the path actually makes sense from 
- * the file system perspective.
+ * Find a "system" file (i.e. one shared for all users of the system).
  *
- * @param buffer_out buffer to populate with the normalized path.
- * @param size buffer size.
- * @param path file path to clean up.
+ * @param pathbuf buffer to write the full path into
+ * @param filename name of file to find in the home directory
  * @return
- * - GIT_SUCCESS on success;
- * - GIT_ERROR when the input path is invalid or escapes the current directory.
+ * - 0 if found;
+ * - GIT_ENOTFOUND if not found;
+ * - -1 on an unspecified OS related error.
  */
-int gitfo_prettify_file_path(char *buffer_out, size_t size, const char *path);
+extern int git_futils_find_system_file(git_buf *path, const char *filename);
 
-int retrieve_path_root_offset(const char *path);
 #endif /* INCLUDE_fileops_h__ */
diff --git a/src/filter.c b/src/filter.c
new file mode 100644
index 000000000..8fa3eb684
--- /dev/null
+++ b/src/filter.c
@@ -0,0 +1,165 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "fileops.h"
+#include "hash.h"
+#include "filter.h"
+#include "repository.h"
+#include "git2/config.h"
+
+/* Tweaked from Core Git. I wonder what we could use this for... */
+void git_text_gather_stats(git_text_stats *stats, const git_buf *text)
+{
+	size_t i;
+
+	memset(stats, 0, sizeof(*stats));
+
+	for (i = 0; i < git_buf_len(text); i++) {
+		unsigned char c = text->ptr[i];
+
+		if (c == '\r') {
+			stats->cr++;
+
+			if (i + 1 < git_buf_len(text) && text->ptr[i + 1] == '\n')
+				stats->crlf++;
+		}
+
+		else if (c == '\n')
+			stats->lf++;
+
+		else if (c == 0x85)
+			/* Unicode CR+LF */
+			stats->crlf++;
+
+		else if (c == 127)
+			/* DEL */
+			stats->nonprintable++;
+
+		else if (c <= 0x1F || (c >= 0x80 && c <= 0x9F)) {
+			switch (c) {
+				/* BS, HT, ESC and FF */
+			case '\b': case '\t': case '\033': case '\014':
+				stats->printable++;
+				break;
+			case 0:
+				stats->nul++;
+				/* fall through */
+			default:
+				stats->nonprintable++;
+			}
+		}
+
+		else
+			stats->printable++;
+	}
+
+	/* If file ends with EOF then don't count this EOF as non-printable. */
+	if (git_buf_len(text) >= 1 && text->ptr[text->size - 1] == '\032')
+		stats->nonprintable--;
+}
+
+/*
+ * Fresh from Core Git
+ */
+int git_text_is_binary(git_text_stats *stats)
+{
+	if (stats->nul)
+		return 1;
+
+	if ((stats->printable >> 7) < stats->nonprintable)
+		return 1;
+	/*
+	 * Other heuristics? Average line length might be relevant,
+	 * as might LF vs CR vs CRLF counts..
+	 *
+	 * NOTE! It might be normal to have a low ratio of CRLF to LF
+	 * (somebody starts with a LF-only file and edits it with an editor
+	 * that adds CRLF only to lines that are added..). But do  we
+	 * want to support CR-only? Probably not.
+	 */
+	return 0;
+}
+
+int git_filters_load(git_vector *filters, git_repository *repo, const char *path, int mode)
+{
+	int error;
+
+	if (mode == GIT_FILTER_TO_ODB) {
+		/* Load the CRLF cleanup filter when writing to the ODB */
+		error = git_filter_add__crlf_to_odb(filters, repo, path);
+		if (error < 0)
+			return error;
+	} else {
+		giterr_set(GITERR_INVALID, "Worktree filters are not implemented yet");
+		return -1;
+	}
+
+	return (int)filters->length;
+}
+
+void git_filters_free(git_vector *filters)
+{
+	size_t i;
+	git_filter *filter;
+
+	git_vector_foreach(filters, i, filter) {
+		if (filter->do_free != NULL)
+			filter->do_free(filter);
+		else
+			git__free(filter);
+	}
+
+	git_vector_free(filters);
+}
+
+int git_filters_apply(git_buf *dest, git_buf *source, git_vector *filters)
+{
+	unsigned int i, src;
+	git_buf *dbuffer[2];
+
+	dbuffer[0] = source;
+	dbuffer[1] = dest;
+
+	src = 0;
+
+	if (git_buf_len(source) == 0) {
+		git_buf_clear(dest);
+		return 0;
+	}
+
+	/* Pre-grow the destination buffer to more or less the size
+	 * we expect it to have */
+	if (git_buf_grow(dest, git_buf_len(source)) < 0)
+		return -1;
+
+	for (i = 0; i < filters->length; ++i) {
+		git_filter *filter = git_vector_get(filters, i);
+		unsigned int dst = 1 - src;
+
+		git_buf_clear(dbuffer[dst]);
+
+		/* Apply the filter from dbuffer[src] to the other buffer;
+		 * if the filtering is canceled by the user mid-filter,
+		 * we skip to the next filter without changing the source
+		 * of the double buffering (so that the text goes through
+		 * cleanly).
+		 */
+		if (filter->apply(filter, dbuffer[dst], dbuffer[src]) == 0)
+			src = dst;
+
+		if (git_buf_oom(dbuffer[dst]))
+			return -1;
+	}
+
+	/* Ensure that the output ends up in dbuffer[1] (i.e. the dest) */
+	if (src != 1)
+		git_buf_swap(dest, source);
+
+	return 0;
+}
+
diff --git a/src/filter.h b/src/filter.h
new file mode 100644
index 000000000..66e370aef
--- /dev/null
+++ b/src/filter.h
@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_filter_h__
+#define INCLUDE_filter_h__
+
+#include "common.h"
+#include "buffer.h"
+#include "git2/odb.h"
+#include "git2/repository.h"
+
+typedef struct git_filter {
+	int (*apply)(struct git_filter *self, git_buf *dest, const git_buf *source);
+	void (*do_free)(struct git_filter *self);
+} git_filter;
+
+typedef enum {
+	GIT_FILTER_TO_WORKTREE,
+	GIT_FILTER_TO_ODB
+} git_filter_mode;
+
+typedef enum {
+	GIT_CRLF_GUESS = -1,
+	GIT_CRLF_BINARY = 0,
+	GIT_CRLF_TEXT,
+	GIT_CRLF_INPUT,
+	GIT_CRLF_CRLF,
+	GIT_CRLF_AUTO,
+} git_crlf_t;
+
+typedef struct {
+	/* NUL, CR, LF and CRLF counts */
+	unsigned int nul, cr, lf, crlf;
+
+	/* These are just approximations! */
+	unsigned int printable, nonprintable;
+} git_text_stats;
+
+/*
+ * FILTER API
+ */
+
+/*
+ * For any given path in the working directory, fill the `filters`
+ * array with the relevant filters that need to be applied.
+ *
+ * Mode is either `GIT_FILTER_TO_WORKTREE` if you need to load the
+ * filters that will be used when checking out a file to the working
+ * directory, or `GIT_FILTER_TO_ODB` for the filters used when writing
+ * a file to the ODB.
+ *
+ * @param filters Vector where to store all the loaded filters
+ * @param repo Repository object that contains `path`
+ * @param path Relative path of the file to be filtered
+ * @param mode Filtering direction (WT->ODB or ODB->WT)
+ * @return the number of filters loaded for the file (0 if the file
+ *	doesn't need filtering), or a negative error code
+ */
+extern int git_filters_load(git_vector *filters, git_repository *repo, const char *path, int mode);
+
+/*
+ * Apply one or more filters to a file.
+ *
+ * The file must have been loaded as a `git_buf` object. Both the `source`
+ * and `dest` buffers are owned by the caller and must be freed once
+ * they are no longer needed.
+ *
+ * NOTE: Because of the double-buffering schema, the `source` buffer that contains
+ * the original file may be tampered once the filtering is complete. Regardless, 
+ * the `dest` buffer will always contain the final result of the filtering
+ *
+ * @param dest Buffer to store the result of the filtering
+ * @param source Buffer containing the document to filter
+ * @param filters A non-empty vector of filters as supplied by `git_filters_load`
+ * @return 0 on success, an error code otherwise
+ */
+extern int git_filters_apply(git_buf *dest, git_buf *source, git_vector *filters);
+
+/*
+ * Free the `filters` array generated by `git_filters_load`.
+ *
+ * Note that this frees both the array and its contents. The array will
+ * be clean/reusable after this call.
+ *
+ * @param filters A filters array as supplied by `git_filters_load`
+ */
+extern void git_filters_free(git_vector *filters);
+
+/*
+ * Available filters
+ */
+
+/* Strip CRLF, from Worktree to ODB */
+extern int git_filter_add__crlf_to_odb(git_vector *filters, git_repository *repo, const char *path);
+
+
+/*
+ * PLAINTEXT API
+ */
+
+/*
+ * Gather stats for a piece of text
+ *
+ * Fill the `stats` structure with information on the number of
+ * unreadable characters, carriage returns, etc, so it can be
+ * used in heuristics.
+ */
+extern void git_text_gather_stats(git_text_stats *stats, const git_buf *text);
+
+/*
+ * Process `git_text_stats` data generated by `git_text_stat` to see
+ * if it qualifies as a binary file
+ */
+extern int git_text_is_binary(git_text_stats *stats);
+
+#endif
diff --git a/src/global.c b/src/global.c
new file mode 100644
index 000000000..368c6c664
--- /dev/null
+++ b/src/global.c
@@ -0,0 +1,134 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "global.h"
+#include "git2/threads.h" 
+#include "thread-utils.h"
+
+/**
+ * Handle the global state with TLS
+ *
+ * If libgit2 is built with GIT_THREADS enabled,
+ * the `git_threads_init()` function must be called
+ * before calling any other function of the library.
+ *
+ * This function allocates a TLS index (using pthreads
+ * or the native Win32 API) to store the global state
+ * on a per-thread basis.
+ *
+ * Any internal method that requires global state will
+ * then call `git__global_state()` which returns a pointer
+ * to the global state structure; this pointer is lazily
+ * allocated on each thread.
+ *
+ * Before shutting down the library, the
+ * `git_threads_shutdown` method must be called to free
+ * the previously reserved TLS index.
+ *
+ * If libgit2 is built without threading support, the
+ * `git__global_statestate()` call returns a pointer to a single,
+ * statically allocated global state. The `git_thread_`
+ * functions are not available in that case.
+ */
+
+#if defined(GIT_THREADS) && defined(GIT_WIN32)
+
+static DWORD _tls_index;
+static int _tls_init = 0;
+
+void git_threads_init(void)
+{
+	if (_tls_init)
+		return;
+
+	_tls_index = TlsAlloc();
+	_tls_init = 1;
+}
+
+void git_threads_shutdown(void)
+{
+	TlsFree(_tls_index);
+	_tls_init = 0;
+}
+
+git_global_st *git__global_state(void)
+{
+	void *ptr;
+
+	if ((ptr = TlsGetValue(_tls_index)) != NULL)
+		return ptr;
+
+	ptr = git__malloc(sizeof(git_global_st));
+	if (!ptr)
+		return NULL;
+
+	memset(ptr, 0x0, sizeof(git_global_st));
+	TlsSetValue(_tls_index, ptr);
+	return ptr;
+}
+
+#elif defined(GIT_THREADS) && defined(_POSIX_THREADS)
+
+static pthread_key_t _tls_key;
+static int _tls_init = 0;
+
+static void cb__free_status(void *st)
+{
+	git__free(st);
+}
+
+void git_threads_init(void)
+{
+	if (_tls_init)
+		return;
+
+	pthread_key_create(&_tls_key, &cb__free_status);
+	_tls_init = 1;
+}
+
+void git_threads_shutdown(void)
+{
+	pthread_key_delete(_tls_key);
+	_tls_init = 0;
+}
+
+git_global_st *git__global_state(void)
+{
+	void *ptr;
+
+	if ((ptr = pthread_getspecific(_tls_key)) != NULL)
+		return ptr;
+
+	ptr = git__malloc(sizeof(git_global_st));
+	if (!ptr)
+		return NULL;
+
+	memset(ptr, 0x0, sizeof(git_global_st));
+	pthread_setspecific(_tls_key, ptr);
+	return ptr;
+}
+
+#else
+
+static git_global_st __state;
+
+void git_threads_init(void)
+{
+	/* noop */ 
+}
+
+void git_threads_shutdown(void)
+{
+	/* noop */
+}
+
+git_global_st *git__global_state(void)
+{
+	return &__state;
+}
+
+#endif /* GIT_THREADS */
diff --git a/src/global.h b/src/global.h
new file mode 100644
index 000000000..2b525ce07
--- /dev/null
+++ b/src/global.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_global_h__
+#define INCLUDE_global_h__
+
+#include "mwindow.h"
+
+typedef struct {
+	struct {
+		char last[1024];
+	} error;
+
+	git_error *last_error;
+	git_error error_t;
+
+	git_mwindow_ctl mem_ctl;
+} git_global_st;
+
+git_global_st *git__global_state(void);
+
+#define GIT_GLOBAL (git__global_state())
+
+#endif
diff --git a/src/hash.c b/src/hash.c
index 775e4b4c1..460756913 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
@@ -28,10 +10,8 @@
 
 #if defined(PPC_SHA1)
 # include "ppc/sha1.h"
-#elif defined(OPENSSL_SHA1)
-# include <openssl/sha.h>
 #else
-# include "block-sha1/sha1.h"
+# include "sha1.h"
 #endif
 
 struct git_hash_ctx {
@@ -52,7 +32,7 @@ git_hash_ctx *git_hash_new_ctx(void)
 
 void git_hash_free_ctx(git_hash_ctx *ctx)
 {
-	free(ctx);
+	git__free(ctx);
 }
 
 void git_hash_init(git_hash_ctx *ctx)
diff --git a/src/hash.h b/src/hash.h
index 2b769a4c9..33d7b20cd 100644
--- a/src/hash.h
+++ b/src/hash.h
@@ -1,5 +1,8 @@
 /*
- * hash.h
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #ifndef INCLUDE_hash_h__
 #define INCLUDE_hash_h__
diff --git a/src/hashtable.c b/src/hashtable.c
deleted file mode 100644
index b40737d67..000000000
--- a/src/hashtable.c
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
- *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- */
-
-#include "common.h"
-#include "repository.h"
-#include "commit.h"
-
-#define MAX_LOOPS 5
-static const double max_load_factor = 0.65;
-
-static int resize_to(git_hashtable *self, size_t new_size);
-static int set_size(git_hashtable *self, size_t new_size);
-static git_hashtable_node *node_with_hash(git_hashtable *self, const void *key, int hash_id);
-static void node_swap_with(git_hashtable_node *self, git_hashtable_node *other);
-static int node_insert(git_hashtable *self, git_hashtable_node *new_node);
-static int insert_nodes(git_hashtable *self, git_hashtable_node *old_nodes, size_t old_size);
-
-static int resize_to(git_hashtable *self, size_t new_size)
-{
-	git_hashtable_node *old_nodes = self->nodes;
-	size_t old_size = self->size;
-
-	self->is_resizing = 1;
-
-	do {
-		self->size = new_size;
-		self->size_mask = new_size - 1;
-		self->key_count = 0;
-		self->nodes = git__calloc(1, sizeof(git_hashtable_node) * self->size);
-
-		if (self->nodes == NULL)
-			return GIT_ENOMEM;
-
-		if (insert_nodes(self, old_nodes, old_size) == 0)
-			self->is_resizing = 0;
-		else {
-			new_size *= 2;
-			free(self->nodes);
-		}
-	} while(self->is_resizing);
-
-	free(old_nodes);
-	return GIT_SUCCESS;
-}
-
-static int set_size(git_hashtable *self, size_t new_size)
-{
-	self->nodes = realloc(self->nodes, new_size * sizeof(git_hashtable_node));
-	if (self->nodes == NULL)
-		return GIT_ENOMEM;
-
-	if (new_size > self->size) {
-		memset(&self->nodes[self->size], 0x0, (new_size - self->size) * sizeof(git_hashtable_node));
-	}
-
-	self->size = new_size;
-	self->size_mask = new_size - 1;
-	return GIT_SUCCESS;
-}
-
-static git_hashtable_node *node_with_hash(git_hashtable *self, const void *key, int hash_id)
-{
-	size_t pos = self->hash(key, hash_id) & self->size_mask;
-	return git_hashtable_node_at(self->nodes, pos);
-}
-
-static void node_swap_with(git_hashtable_node *self, git_hashtable_node *other)
-{
-	git_hashtable_node tmp = *self;
-	*self = *other;
-	*other = tmp;
-}
-
-static int node_insert(git_hashtable *self, git_hashtable_node *new_node)
-{	
-	int iteration, hash_id;
-
-	for (iteration = 0; iteration < MAX_LOOPS; iteration++) { 
-		for (hash_id = 0; hash_id < GIT_HASHTABLE_HASHES; ++hash_id) {
-			git_hashtable_node *node;
-			node = node_with_hash(self, new_node->key, hash_id);
-			node_swap_with(new_node, node);
-			if(new_node->key == 0x0){
-				self->key_count++;
-				return GIT_SUCCESS; 
-			}
-		}
-	}
-
-	if (self->is_resizing) 
-		return git__throw(GIT_EBUSY, "Failed to insert node. Hashtable is currently resizing");
-
-	resize_to(self, self->size * 2);
-	git_hashtable_insert(self, new_node->key, new_node->value);
-	return GIT_SUCCESS;
-}
-
-static int insert_nodes(git_hashtable *self, git_hashtable_node *old_nodes, size_t old_size)
-{
-	size_t i;
-
-	for (i = 0; i < old_size; ++i) {
-		git_hashtable_node *node = git_hashtable_node_at(old_nodes, i);
-		if (node->key && git_hashtable_insert(self, node->key, node->value) < GIT_SUCCESS)
-			return GIT_ENOMEM;
-	}
-
-	return GIT_SUCCESS;
-}
-
-git_hashtable *git_hashtable_alloc(size_t min_size, 
-		git_hash_ptr hash,
-		git_hash_keyeq_ptr key_eq)
-{
-	git_hashtable *table;
-
-	assert(hash && key_eq);
-
-	if ((table = git__malloc(sizeof(git_hashtable))) == NULL)
-		return NULL;
-
-	memset(table, 0x0, sizeof(git_hashtable));
-
-	if (min_size < 8)
-		min_size = 8;
-
-	/* round up size to closest power of 2 */
-	min_size--;
-	min_size |= min_size >> 1;
-	min_size |= min_size >> 2;
-	min_size |= min_size >> 4;
-	min_size |= min_size >> 8;
-	min_size |= min_size >> 16;
-
-	table->hash = hash;
-	table->key_equal = key_eq;
-
-	set_size(table, min_size + 1);
-
-	return table;
-}
-
-void git_hashtable_clear(git_hashtable *self)
-{
-	assert(self);
-
-	memset(self->nodes, 0x0, sizeof(git_hashtable_node) * self->size);
-	self->key_count = 0;
-}
-
-void git_hashtable_free(git_hashtable *self)
-{
-	assert(self);
-
-	free(self->nodes);
-	free(self);
-}
-
-
-int git_hashtable_insert2(git_hashtable *self, const void *key, void *value, void **old_value)
-{
-	int hash_id;
-	git_hashtable_node *node;
-
-	assert(self && self->nodes);
-
-	*old_value = NULL;
-
-	for (hash_id = 0; hash_id < GIT_HASHTABLE_HASHES; ++hash_id) {
-		node = node_with_hash(self, key, hash_id);
-
-		if (!node->key) {
-			node->key = key;
-			node->value = value;
-			self->key_count++;
-			return GIT_SUCCESS;
-		}
-
-		if (key == node->key || self->key_equal(key, node->key) == 0) {
-			*old_value = node->value;
-			node->key = key;
-			node->value = value;
-			return GIT_SUCCESS;
-		}
-	}
-
-	/* no space in table; must do cuckoo dance */
-	{
-		git_hashtable_node x;
-		x.key = key;
-		x.value = value;
-		return node_insert(self, &x);
-	}
-}
-
-void *git_hashtable_lookup(git_hashtable *self, const void *key)
-{
-	int hash_id;
-	git_hashtable_node *node;
-
-	assert(self && self->nodes);
-
-	for (hash_id = 0; hash_id < GIT_HASHTABLE_HASHES; ++hash_id) {
-		node = node_with_hash(self, key, hash_id);
-		if (node->key && self->key_equal(key, node->key) == 0)
-			return node->value;
-	}
-
-	return NULL;
-}
-
-int git_hashtable_remove(git_hashtable *self, const void *key)
-{
-	int hash_id;
-	git_hashtable_node *node;
-
-	assert(self && self->nodes);
-
-	for (hash_id = 0; hash_id < GIT_HASHTABLE_HASHES; ++hash_id) {
-		node = node_with_hash(self, key, hash_id);
-		if (node->key && self->key_equal(key, node->key) == 0) {
-			node->key = NULL;
-			node->value = NULL;
-			self->key_count--;
-			return GIT_SUCCESS;
-		}
-	}
-
-	return git__throw(GIT_ENOTFOUND, "Entry not found in hash table");
-}
-
-int git_hashtable_merge(git_hashtable *self, git_hashtable *other)
-{
-	if (resize_to(self, (self->size + other->size) * 2) < GIT_SUCCESS)
-		return GIT_ENOMEM;
-
-	return insert_nodes(self, other->nodes, other->key_count);
-}
-
diff --git a/src/hashtable.h b/src/hashtable.h
deleted file mode 100644
index c3475b6ed..000000000
--- a/src/hashtable.h
+++ /dev/null
@@ -1,73 +0,0 @@
-#ifndef INCLUDE_hashtable_h__
-#define INCLUDE_hashtable_h__
-
-#include "git2/common.h"
-#include "git2/oid.h"
-#include "git2/odb.h"
-
-#define GIT_HASHTABLE_HASHES 3
-
-typedef uint32_t (*git_hash_ptr)(const void *, int hash_id);
-typedef int (*git_hash_keyeq_ptr)(const void *key_a, const void *key_b);
-
-struct git_hashtable_node {
-	const void *key;
-	void *value;
-};
-
-struct git_hashtable {
-	struct git_hashtable_node *nodes;
-
-	size_t size_mask;
-	size_t size;
-	size_t key_count;
-
-	int is_resizing;
-
-	git_hash_ptr hash;
-	git_hash_keyeq_ptr key_equal;
-};
-
-typedef struct git_hashtable_node git_hashtable_node;
-typedef struct git_hashtable git_hashtable;
-
-git_hashtable *git_hashtable_alloc(size_t min_size, 
-		git_hash_ptr hash,
-		git_hash_keyeq_ptr key_eq);
-void *git_hashtable_lookup(git_hashtable *h, const void *key);
-int git_hashtable_remove(git_hashtable *table, const void *key);
-void git_hashtable_free(git_hashtable *h);
-void git_hashtable_clear(git_hashtable *h);
-int git_hashtable_merge(git_hashtable *self, git_hashtable *other);
-
-int git_hashtable_insert2(git_hashtable *h, const void *key, void *value, void **old_value);
-
-GIT_INLINE(int) git_hashtable_insert(git_hashtable *h, const void *key, void *value)
-{
-	void *_unused;
-	return git_hashtable_insert2(h, key, value, &_unused);
-}
-
-#define git_hashtable_node_at(nodes, pos) ((git_hashtable_node *)(&nodes[pos]))
-
-#define GIT_HASHTABLE_FOREACH(self, pkey, pvalue, code) {\
-	git_hashtable *_self = (self);\
-	git_hashtable_node *_nodes = _self->nodes;\
-	unsigned int _i, _size = _self->size;\
-	for (_i = 0; _i < _size; _i ++) {\
-		git_hashtable_node *_node = git_hashtable_node_at(_nodes, _i);\
-		if (_node->key)\
-		{\
-			pkey = _node->key;\
-			pvalue = _node->value;\
-			code;\
-		}\
-	}\
-}
-
-#define GIT_HASHTABLE_FOREACH_DELETE() {\
-	_node->key = NULL; _node->value = NULL; _self->key_count--;\
-}
-
-
-#endif
diff --git a/src/ignore.c b/src/ignore.c
new file mode 100644
index 000000000..fc6194bb5
--- /dev/null
+++ b/src/ignore.c
@@ -0,0 +1,203 @@
+#include "ignore.h"
+#include "path.h"
+
+#define GIT_IGNORE_INTERNAL		"[internal]exclude"
+#define GIT_IGNORE_FILE_INREPO	"info/exclude"
+#define GIT_IGNORE_FILE			".gitignore"
+
+static int parse_ignore_file(
+	git_repository *repo, const char *buffer, git_attr_file *ignores)
+{
+	int error = 0;
+	git_attr_fnmatch *match = NULL;
+	const char *scan = NULL;
+	char *context = NULL;
+
+	GIT_UNUSED(repo);
+
+	if (ignores->key && git__suffixcmp(ignores->key, "/" GIT_IGNORE_FILE) == 0) {
+		context = ignores->key + 2;
+		context[strlen(context) - strlen(GIT_IGNORE_FILE)] = '\0';
+	}
+
+	scan = buffer;
+
+	while (!error && *scan) {
+		if (!match) {
+			match = git__calloc(1, sizeof(*match));
+			GITERR_CHECK_ALLOC(match);
+		}
+
+		if (!(error = git_attr_fnmatch__parse(
+			match, ignores->pool, context, &scan)))
+		{
+			match->flags = match->flags | GIT_ATTR_FNMATCH_IGNORE;
+			scan = git__next_line(scan);
+			error = git_vector_insert(&ignores->rules, match);
+		}
+
+		if (error != 0) {
+			git__free(match->pattern);
+			match->pattern = NULL;
+
+			if (error == GIT_ENOTFOUND)
+				error = 0;
+		} else {
+			match = NULL; /* vector now "owns" the match */
+		}
+	}
+
+	git__free(match);
+	/* restore file path used for context */
+	if (context)
+		context[strlen(context)] = '.'; /* first char of GIT_IGNORE_FILE */
+
+	return error;
+}
+
+#define push_ignore_file(R,S,B,F) \
+	git_attr_cache__push_file((R),(B),(F),GIT_ATTR_FILE_FROM_FILE,parse_ignore_file,(S))
+
+static int push_one_ignore(void *ref, git_buf *path)
+{
+	git_ignores *ign = (git_ignores *)ref;
+	return push_ignore_file(ign->repo, &ign->ign_path, path->ptr, GIT_IGNORE_FILE);
+}
+
+int git_ignore__for_path(
+	git_repository *repo,
+	const char *path,
+	git_ignores *ignores)
+{
+	int error = 0;
+	const char *workdir = git_repository_workdir(repo);
+
+	assert(ignores);
+
+	ignores->repo = repo;
+	git_buf_init(&ignores->dir, 0);
+	ignores->ign_internal = NULL;
+
+	if ((error = git_vector_init(&ignores->ign_path, 8, NULL)) < 0 ||
+		(error = git_vector_init(&ignores->ign_global, 2, NULL)) < 0 ||
+		(error = git_attr_cache__init(repo)) < 0)
+		goto cleanup;
+
+	/* given a unrooted path in a non-bare repo, resolve it */
+	if (workdir && git_path_root(path) < 0)
+		error = git_path_find_dir(&ignores->dir, path, workdir);
+	else
+		error = git_buf_sets(&ignores->dir, path);
+	if (error < 0)
+		goto cleanup;
+
+	/* set up internals */
+	error = git_attr_cache__internal_file(
+		repo, GIT_IGNORE_INTERNAL, &ignores->ign_internal);
+	if (error < 0)
+		goto cleanup;
+
+	/* load .gitignore up the path */
+	if (workdir != NULL) {
+		error = git_path_walk_up(
+			&ignores->dir, workdir, push_one_ignore, ignores);
+		if (error < 0)
+			goto cleanup;
+	}
+
+	/* load .git/info/exclude */
+	error = push_ignore_file(repo, &ignores->ign_global,
+		git_repository_path(repo), GIT_IGNORE_FILE_INREPO);
+	if (error < 0)
+		goto cleanup;
+
+	/* load core.excludesfile */
+	if (git_repository_attr_cache(repo)->cfg_excl_file != NULL)
+		error = push_ignore_file(repo, &ignores->ign_global, NULL,
+			git_repository_attr_cache(repo)->cfg_excl_file);
+
+cleanup:
+	if (error < 0)
+		git_ignore__free(ignores);
+
+	return error;
+}
+
+int git_ignore__push_dir(git_ignores *ign, const char *dir)
+{
+	if (git_buf_joinpath(&ign->dir, ign->dir.ptr, dir) < 0)
+		return -1;
+	else
+		return push_ignore_file(
+			ign->repo, &ign->ign_path, ign->dir.ptr, GIT_IGNORE_FILE);
+}
+
+int git_ignore__pop_dir(git_ignores *ign)
+{
+	if (ign->ign_path.length > 0) {
+		git_attr_file *file = git_vector_last(&ign->ign_path);
+		if (git__suffixcmp(ign->dir.ptr, file->key + 2) == 0)
+			git_vector_pop(&ign->ign_path);
+		git_buf_rtruncate_at_char(&ign->dir, '/');
+	}
+	return 0;
+}
+
+void git_ignore__free(git_ignores *ignores)
+{
+	/* don't need to free ignores->ign_internal since it is in cache */
+	git_vector_free(&ignores->ign_path);
+	git_vector_free(&ignores->ign_global);
+	git_buf_free(&ignores->dir);
+}
+
+static bool ignore_lookup_in_rules(
+	git_vector *rules, git_attr_path *path, int *ignored)
+{
+	unsigned int j;
+	git_attr_fnmatch *match;
+
+	git_vector_rforeach(rules, j, match) {
+		if (git_attr_fnmatch__match(match, path)) {
+			*ignored = ((match->flags & GIT_ATTR_FNMATCH_NEGATIVE) == 0);
+			return true;
+		}
+	}
+
+	return false;
+}
+
+int git_ignore__lookup(
+	git_ignores *ignores, const char *pathname, int *ignored)
+{
+	unsigned int i;
+	git_attr_file *file;
+	git_attr_path path;
+
+	if (git_attr_path__init(
+		&path, pathname, git_repository_workdir(ignores->repo)) < 0)
+		return -1;
+
+	/* first process builtins - success means path was found */
+	if (ignore_lookup_in_rules(
+			&ignores->ign_internal->rules, &path, ignored))
+		goto cleanup;
+
+	/* next process files in the path */
+	git_vector_foreach(&ignores->ign_path, i, file) {
+		if (ignore_lookup_in_rules(&file->rules, &path, ignored))
+			goto cleanup;
+	}
+
+	/* last process global ignores */
+	git_vector_foreach(&ignores->ign_global, i, file) {
+		if (ignore_lookup_in_rules(&file->rules, &path, ignored))
+			goto cleanup;
+	}
+
+	*ignored = 0;
+
+cleanup:
+	git_attr_path__free(&path);
+	return 0;
+}
diff --git a/src/ignore.h b/src/ignore.h
new file mode 100644
index 000000000..809d2edbd
--- /dev/null
+++ b/src/ignore.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_ignore_h__
+#define INCLUDE_ignore_h__
+
+#include "repository.h"
+#include "vector.h"
+
+/* The git_ignores structure maintains three sets of ignores:
+ * - internal ignores
+ * - per directory ignores
+ * - global ignores (at lower priority than the others)
+ * As you traverse from one directory to another, you can push and pop
+ * directories onto git_ignores list efficiently.
+ */
+typedef struct {
+	git_repository *repo;
+	git_buf dir;
+	git_attr_file *ign_internal;
+	git_vector ign_path;
+	git_vector ign_global;
+} git_ignores;
+
+extern int git_ignore__for_path(git_repository *repo, const char *path, git_ignores *ign);
+
+extern int git_ignore__push_dir(git_ignores *ign, const char *dir);
+
+extern int git_ignore__pop_dir(git_ignores *ign);
+
+extern void git_ignore__free(git_ignores *ign);
+
+extern int git_ignore__lookup(git_ignores *ign, const char *path, int *ignored);
+
+#endif
diff --git a/src/index.c b/src/index.c
index c86d37a08..f1ae9a710 100644
--- a/src/index.c
+++ b/src/index.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include <stddef.h>
@@ -28,6 +10,8 @@
 #include "common.h"
 #include "repository.h"
 #include "index.h"
+#include "tree.h"
+#include "tree-cache.h"
 #include "hash.h"
 #include "git2/odb.h"
 #include "git2/blob.h"
@@ -48,6 +32,8 @@ static const unsigned int INDEX_HEADER_SIG = 0x44495243;
 static const char INDEX_EXT_TREECACHE_SIG[] = {'T', 'R', 'E', 'E'};
 static const char INDEX_EXT_UNMERGED_SIG[] = {'R', 'E', 'U', 'C'};
 
+#define INDEX_OWNER(idx) ((git_repository *)(GIT_REFCOUNT_OWNER(idx)))
+
 struct index_header {
 	uint32_t signature;
 	uint32_t version;
@@ -98,121 +84,100 @@ static size_t read_extension(git_index *index, const char *buffer, size_t buffer
 static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size);
 static int read_header(struct index_header *dest, const void *buffer);
 
-static int read_tree(git_index *index, const char *buffer, size_t buffer_size);
-static int read_tree_internal(git_index_tree **, const char **, const char *, git_index_tree *);
-
 static int parse_index(git_index *index, const char *buffer, size_t buffer_size);
 static int is_index_extended(git_index *index);
-static void sort_index(git_index *index);
 static int write_index(git_index *index, git_filebuf *file);
 
-int index_srch(const void *key, const void *array_member)
+static void index_entry_free(git_index_entry *entry);
+
+static int index_srch(const void *key, const void *array_member)
 {
-	const char *filename = (const char *)key;
-	const git_index_entry *entry = *(const git_index_entry **)(array_member);
+	const git_index_entry *entry = array_member;
 
-	return strcmp(filename, entry->path);
+	return strcmp(key, entry->path);
 }
 
-int index_cmp(const void *a, const void *b)
+static int index_cmp(const void *a, const void *b)
 {
-	const git_index_entry *entry_a = *(const git_index_entry **)(a);
-	const git_index_entry *entry_b = *(const git_index_entry **)(b);
+	const git_index_entry *entry_a = a;
+	const git_index_entry *entry_b = b;
 
 	return strcmp(entry_a->path, entry_b->path);
 }
 
-int unmerged_srch(const void *key, const void *array_member)
+static int unmerged_srch(const void *key, const void *array_member)
 {
-	const char *path = (const char *) key;
-	const git_index_entry_unmerged *entry = *(const git_index_entry_unmerged **) (array_member);
+	const git_index_entry_unmerged *entry = array_member;
 
-	return strcmp(path, entry->path);
+	return strcmp(key, entry->path);
 }
 
-int unmerged_cmp(const void *a, const void *b)
+static int unmerged_cmp(const void *a, const void *b)
 {
-	const git_index_entry_unmerged *info_a = *(const git_index_entry_unmerged **)(a);
-	const git_index_entry_unmerged *info_b = *(const git_index_entry_unmerged **)(b);
+	const git_index_entry_unmerged *info_a = a;
+	const git_index_entry_unmerged *info_b = b;
 
 	return strcmp(info_a->path, info_b->path);
 }
 
-static int index_initialize(git_index **index_out, git_repository *owner, const char *index_path)
+static unsigned int index_create_mode(unsigned int mode)
+{
+	if (S_ISLNK(mode))
+		return S_IFLNK;
+	if (S_ISDIR(mode) || (mode & S_IFMT) == (S_IFLNK | S_IFDIR))
+		return (S_IFLNK | S_IFDIR);
+	return S_IFREG | ((mode & 0100) ? 0755 : 0644);
+}
+
+int git_index_open(git_index **index_out, const char *index_path)
 {
 	git_index *index;
 
 	assert(index_out && index_path);
 
-	index = git__malloc(sizeof(git_index));
-	if (index == NULL)
-		return GIT_ENOMEM;
-
-	memset(index, 0x0, sizeof(git_index));
+	index = git__calloc(1, sizeof(git_index));
+	GITERR_CHECK_ALLOC(index);
 
 	index->index_file_path = git__strdup(index_path);
-	if (index->index_file_path == NULL) {
-		free(index);
-		return GIT_ENOMEM;
-	}
-
-	index->repository = owner;
+	GITERR_CHECK_ALLOC(index->index_file_path);
 
-	git_vector_init(&index->entries, 32, index_cmp);
+	if (git_vector_init(&index->entries, 32, index_cmp) < 0)
+		return -1;
 
 	/* Check if index file is stored on disk already */
-	if (gitfo_exists(index->index_file_path) == 0)
+	if (git_path_exists(index->index_file_path) == true)
 		index->on_disk = 1;
 
 	*index_out = index;
+	GIT_REFCOUNT_INC(index);
 	return git_index_read(index);
 }
 
-int git_index_open(git_index **index_out, const char *index_path)
-{
-	assert(index_out && index_path);
-	return index_initialize(index_out, NULL, index_path);
-}
-
-/*
- * Moved from `repository.c`
- */
-int git_repository_index(git_index **index_out, git_repository *repo)
+static void index_free(git_index *index)
 {
-	assert(index_out && repo);
-
-	if (repo->is_bare)
-		return git__throw(GIT_EBAREINDEX, "Failed to open index. Repository is bare");
-
-	return index_initialize(index_out, repo, repo->path_index);
-}
-
-void git_index_free(git_index *index)
-{
-	if (index == NULL)
-		return;
+	git_index_entry *e;
+	unsigned int i;
 
 	git_index_clear(index);
+	git_vector_foreach(&index->entries, i, e) {
+		index_entry_free(e);
+	}
 	git_vector_free(&index->entries);
+	git_vector_foreach(&index->unmerged, i, e) {
+		index_entry_free(e);
+	}
 	git_vector_free(&index->unmerged);
 
-	free(index->index_file_path);
-	free(index);
+	git__free(index->index_file_path);
+	git__free(index);
 }
 
-static void free_tree(git_index_tree *tree)
+void git_index_free(git_index *index)
 {
-	unsigned int i;
-
-	if (tree == NULL)
+	if (index == NULL)
 		return;
 
-	for (i = 0; i < tree->children_count; ++i)
-		free_tree(tree->children[i]);
-
-	free(tree->name);
-	free(tree->children);
-	free(tree);
+	GIT_REFCOUNT_DEC(index, index_free);
 }
 
 void git_index_clear(git_index *index)
@@ -224,90 +189,85 @@ void git_index_clear(git_index *index)
 	for (i = 0; i < index->entries.length; ++i) {
 		git_index_entry *e;
 		e = git_vector_get(&index->entries, i);
-		free((char *)e->path);
-		free(e);
+		git__free(e->path);
+		git__free(e);
 	}
 
 	for (i = 0; i < index->unmerged.length; ++i) {
 		git_index_entry_unmerged *e;
 		e = git_vector_get(&index->unmerged, i);
-		free((char *)e->path);
-		free(e);
+		git__free(e->path);
+		git__free(e);
 	}
 
 	git_vector_clear(&index->entries);
 	git_vector_clear(&index->unmerged);
 	index->last_modified = 0;
 
-	free_tree(index->tree);
+	git_tree_cache_free(index->tree);
 	index->tree = NULL;
 }
 
 int git_index_read(git_index *index)
 {
-	struct stat indexst;
-	int error = GIT_SUCCESS;
+	int error, updated;
+	git_buf buffer = GIT_BUF_INIT;
+	time_t mtime;
 
 	assert(index->index_file_path);
 
-	if (!index->on_disk || gitfo_exists(index->index_file_path) < 0) {
+	if (!index->on_disk || git_path_exists(index->index_file_path) == false) {
 		git_index_clear(index);
 		index->on_disk = 0;
-		return GIT_SUCCESS;
+		return 0;
 	}
 
-	if (gitfo_stat(index->index_file_path, &indexst) < 0)
-		return git__throw(GIT_EOSERR, "Failed to read index. %s does not exist or is corrupted", index->index_file_path);
-
-	if (!S_ISREG(indexst.st_mode))
-		return git__throw(GIT_ENOTFOUND, "Failed to read index. %s is not an index file", index->index_file_path);
-
-	if (indexst.st_mtime != index->last_modified) {
-
-		gitfo_buf buffer;
-
-		if ((error = gitfo_read_file(&buffer, index->index_file_path)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to read index");
+	/* We don't want to update the mtime if we fail to parse the index */
+	mtime = index->last_modified;
+	error = git_futils_readbuffer_updated(
+		&buffer, index->index_file_path, &mtime, &updated);
+	if (error < 0)
+		return error;
 
+	if (updated) {
 		git_index_clear(index);
-		error = parse_index(index, buffer.data, buffer.len);
+		error = parse_index(index, buffer.ptr, buffer.size);
 
-		if (error == GIT_SUCCESS)
-			index->last_modified = indexst.st_mtime;
+		if (!error)
+			index->last_modified = mtime;
 
-		gitfo_free_buf(&buffer);
+		git_buf_free(&buffer);
 	}
 
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to read index");
 	return error;
 }
 
 int git_index_write(git_index *index)
 {
-	git_filebuf file;
+	git_filebuf file = GIT_FILEBUF_INIT;
 	struct stat indexst;
 	int error;
 
-	sort_index(index);
+	git_vector_sort(&index->entries);
 
-	if ((error = git_filebuf_open(&file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write index");
+	if ((error = git_filebuf_open(
+			 &file, index->index_file_path, GIT_FILEBUF_HASH_CONTENTS)) < 0)
+		return error;
 
-	if ((error = write_index(index, &file)) < GIT_SUCCESS) {
+	if ((error = write_index(index, &file)) < 0) {
 		git_filebuf_cleanup(&file);
-		return git__rethrow(error, "Failed to write index");
+		return error;
 	}
 
-	if ((error = git_filebuf_commit(&file)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write index");
+	if ((error = git_filebuf_commit(&file, GIT_INDEX_FILE_MODE)) < 0)
+		return error;
 
-	if (gitfo_stat(index->index_file_path, &indexst) == 0) {
+	if (p_stat(index->index_file_path, &indexst) == 0) {
 		index->last_modified = indexst.st_mtime;
 		index->on_disk = 1;
 	}
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 unsigned int git_index_entrycount(git_index *index)
@@ -322,39 +282,110 @@ unsigned int git_index_entrycount_unmerged(git_index *index)
 	return index->unmerged.length;
 }
 
-git_index_entry *git_index_get(git_index *index, int n)
+git_index_entry *git_index_get(git_index *index, unsigned int n)
 {
-	assert(index);
-	sort_index(index);
-	return git_vector_get(&index->entries, (unsigned int)n);
+	git_vector_sort(&index->entries);
+	return git_vector_get(&index->entries, n);
 }
 
-static void sort_index(git_index *index)
+void git_index__init_entry_from_stat(struct stat *st, git_index_entry *entry)
 {
-	git_vector_sort(&index->entries);
+	entry->ctime.seconds = (git_time_t)st->st_ctime;
+	entry->mtime.seconds = (git_time_t)st->st_mtime;
+	/* entry->mtime.nanoseconds = st->st_mtimensec; */
+	/* entry->ctime.nanoseconds = st->st_ctimensec; */
+	entry->dev  = st->st_rdev;
+	entry->ino  = st->st_ino;
+	entry->mode = index_create_mode(st->st_mode);
+	entry->uid  = st->st_uid;
+	entry->gid  = st->st_gid;
+	entry->file_size = st->st_size;
 }
 
-static int index_insert(git_index *index, const git_index_entry *source_entry, int replace)
+static int index_entry_init(git_index_entry **entry_out, git_index *index, const char *rel_path, int stage)
 {
-	git_index_entry *entry;
-	size_t path_length;
-	int position;
+	git_index_entry *entry = NULL;
+	struct stat st;
+	git_oid oid;
+	const char *workdir;
+	git_buf full_path = GIT_BUF_INIT;
+	int error;
+
+	assert(stage >= 0 && stage <= 3);
+
+	if (INDEX_OWNER(index) == NULL ||
+		(workdir = git_repository_workdir(INDEX_OWNER(index))) == NULL)
+	{
+		giterr_set(GITERR_INDEX,
+			"Could not initialize index entry. Repository is bare");
+		return -1;
+	}
+
+	if ((error = git_buf_joinpath(&full_path, workdir, rel_path)) < 0)
+		return error;
+
+	if ((error = git_path_lstat(full_path.ptr, &st)) < 0) {
+		git_buf_free(&full_path);
+		return error;
+	}
+
+	git_buf_free(&full_path); /* done with full path */
+
+	/* There is no need to validate the rel_path here, since it will be
+	 * immediately validated by the call to git_blob_create_fromfile.
+	 */
+
+	/* write the blob to disk and get the oid */
+	if ((error = git_blob_create_fromfile(&oid, INDEX_OWNER(index), rel_path)) < 0)
+		return error;
+
+	entry = git__calloc(1, sizeof(git_index_entry));
+	GITERR_CHECK_ALLOC(entry);
+
+	git_index__init_entry_from_stat(&st, entry);
 
-	assert(index && source_entry);
+	entry->oid = oid;
+	entry->flags |= (stage << GIT_IDXENTRY_STAGESHIFT);
+	entry->path = git__strdup(rel_path);
+	GITERR_CHECK_ALLOC(entry->path);
+
+	*entry_out = entry;
+	return 0;
+}
 
-	if (source_entry->path == NULL)
-		return git__throw(GIT_EMISSINGOBJDATA, "Failed to insert into index. Entry has no path");
+static git_index_entry *index_entry_dup(const git_index_entry *source_entry)
+{
+	git_index_entry *entry;
 
 	entry = git__malloc(sizeof(git_index_entry));
-	if (entry == NULL)
-		return GIT_ENOMEM;
+	if (!entry)
+		return NULL;
 
 	memcpy(entry, source_entry, sizeof(git_index_entry));
 
 	/* duplicate the path string so we own it */
 	entry->path = git__strdup(entry->path);
-	if (entry->path == NULL)
-		return GIT_ENOMEM;
+	if (!entry->path)
+		return NULL;
+
+	return entry;
+}
+
+static void index_entry_free(git_index_entry *entry)
+{
+	if (!entry)
+		return;
+	git__free(entry->path);
+	git__free(entry);
+}
+
+static int index_insert(git_index *index, git_index_entry *entry, int replace)
+{
+	size_t path_length;
+	int position;
+	git_index_entry **entry_array;
+
+	assert(index && entry && entry->path != NULL);
 
 	/* make sure that the path length flag is correct */
 	path_length = strlen(entry->path);
@@ -366,120 +397,127 @@ static int index_insert(git_index *index, const git_index_entry *source_entry, i
 	else
 		entry->flags |= GIT_IDXENTRY_NAMEMASK;;
 
+	/*
+	 * replacing is not requested: just insert entry at the end;
+	 * the index is no longer sorted
+	 */
+	if (!replace)
+		return git_vector_insert(&index->entries, entry);
 
 	/* look if an entry with this path already exists */
-	position = git_index_find(index, source_entry->path);
+	position = git_index_find(index, entry->path);
 
-	/* if no entry exists and replace is not set,
-	 * add the entry at the end;
-	 * the index is no longer sorted */
-	if (!replace || position == GIT_ENOTFOUND) {
-		if (git_vector_insert(&index->entries, entry) < GIT_SUCCESS)
-			return GIT_ENOMEM;
+	/*
+	 * if no entry exists add the entry at the end;
+	 * the index is no longer sorted
+	 */
+	if (position == GIT_ENOTFOUND)
+		return git_vector_insert(&index->entries, entry);
 
-	/* if a previous entry exists and replace is set,
-	 * replace it */
-	} else {
-		git_index_entry **entry_array = (git_index_entry **)index->entries.contents;
+	/* exists, replace it */
+	entry_array = (git_index_entry **) index->entries.contents;
+	git__free(entry_array[position]->path);
+	git__free(entry_array[position]);
+	entry_array[position] = entry;
+
+	return 0;
+}
 
-		free((char *)entry_array[position]->path);
-		free(entry_array[position]);
+static int index_add(git_index *index, const char *path, int stage, int replace)
+{
+	git_index_entry *entry = NULL;
+	int ret;
 
-		entry_array[position] = entry;
+	if ((ret = index_entry_init(&entry, index, path, stage)) < 0 ||
+		(ret = index_insert(index, entry, replace)) < 0)
+	{
+		index_entry_free(entry);
+		return ret;
 	}
 
-	return GIT_SUCCESS;
+	git_tree_cache_invalidate_path(index->tree, entry->path);
+	return 0;
 }
 
-static int index_init_entry(git_index_entry *entry, git_index *index, const char *rel_path, int stage)
+int git_index_add(git_index *index, const char *path, int stage)
 {
-	char full_path[GIT_PATH_MAX];
-	struct stat st;
-	int error;
-
-	if (index->repository == NULL)
-		return git__throw(GIT_EBAREINDEX, "Failed to initialize entry. Repository is bare");
-
-	git__joinpath(full_path, index->repository->path_workdir, rel_path);
+	return index_add(index, path, stage, 1);
+}
 
-	if (gitfo_exists(full_path) < 0)
-		return git__throw(GIT_ENOTFOUND, "Failed to initialize entry. %s does not exist", full_path);
+int git_index_append(git_index *index, const char *path, int stage)
+{
+	return index_add(index, path, stage, 0);
+}
 
-	if (gitfo_stat(full_path, &st) < 0)
-		return git__throw(GIT_EOSERR, "Failed to initialize entry. %s appears to be corrupted", full_path);
+static int index_add2(
+	git_index *index, const git_index_entry *source_entry, int replace)
+{
+	git_index_entry *entry = NULL;
+	int ret;
 
-	if (stage < 0 || stage > 3)
-		return git__throw(GIT_ERROR, "Failed to initialize entry. Invalid stage %i", stage);
+	entry = index_entry_dup(source_entry);
+	if (entry == NULL)
+		return -1;
 
-	memset(entry, 0x0, sizeof(git_index_entry));
+	if ((ret = index_insert(index, entry, replace)) < 0) {
+		index_entry_free(entry);
+		return ret;
+	}
 
-	entry->ctime.seconds = (git_time_t)st.st_ctime;
-	entry->mtime.seconds = (git_time_t)st.st_mtime;
-	/* entry.mtime.nanoseconds = st.st_mtimensec; */
-	/* entry.ctime.nanoseconds = st.st_ctimensec; */
-	entry->dev= st.st_rdev;
-	entry->ino = st.st_ino;
-	entry->mode = st.st_mode;
-	entry->uid = st.st_uid;
-	entry->gid = st.st_gid;
-	entry->file_size = st.st_size;
+	git_tree_cache_invalidate_path(index->tree, entry->path);
+	return 0;
+}
 
-	/* write the blob to disk and get the oid */
-	if ((error = git_blob_create_fromfile(&entry->oid, index->repository, rel_path)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to initialize index entry");
+int git_index_add2(git_index *index, const git_index_entry *source_entry)
+{
+	return index_add2(index, source_entry, 1);
+}
 
-	entry->flags |= (stage << GIT_IDXENTRY_STAGESHIFT);
-	entry->path = (char *)rel_path; /* do not duplicate; index_insert already does this */
-	return GIT_SUCCESS;
+int git_index_append2(git_index *index, const git_index_entry *source_entry)
+{
+	return index_add2(index, source_entry, 1);
 }
 
-int git_index_add(git_index *index, const char *path, int stage)
+int git_index_remove(git_index *index, int position)
 {
 	int error;
-	git_index_entry entry;
+	git_index_entry *entry;
 
-	if ((error = index_init_entry(&entry, index, path, stage)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to add to index");
+	git_vector_sort(&index->entries);
 
-	return index_insert(index, &entry, 1);
-}
+	entry = git_vector_get(&index->entries, position);
+	if (entry != NULL)
+		git_tree_cache_invalidate_path(index->tree, entry->path);
 
-int git_index_append(git_index *index, const char *path, int stage)
-{
-	int error;
-	git_index_entry entry;
+	error = git_vector_remove(&index->entries, (unsigned int)position);
 
-	if ((error = index_init_entry(&entry, index, path, stage)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to append to index");
+	if (!error)
+		index_entry_free(entry);
 
-	return index_insert(index, &entry, 0);
+	return error;
 }
 
-int git_index_add2(git_index *index, const git_index_entry *source_entry)
+int git_index_find(git_index *index, const char *path)
 {
-	return index_insert(index, source_entry, 1);
+	return git_vector_bsearch2(&index->entries, index_srch, path);
 }
 
-int git_index_append2(git_index *index, const git_index_entry *source_entry)
+unsigned int git_index__prefix_position(git_index *index, const char *path)
 {
-	return index_insert(index, source_entry, 0);
-}
+	unsigned int pos;
 
+	git_vector_bsearch3(&pos, &index->entries, index_srch, path);
 
-int git_index_remove(git_index *index, int position)
-{
-	assert(index);
-	sort_index(index);
-	return git_vector_remove(&index->entries, (unsigned int)position);
+	return pos;
 }
 
-int git_index_find(git_index *index, const char *path)
+void git_index_uniq(git_index *index)
 {
-	sort_index(index);
-	return git_vector_bsearch2(&index->entries, index_srch, path);
+	git_vector_uniq(&index->entries);
 }
 
-const git_index_entry_unmerged *git_index_get_unmerged(git_index *index, const char *path)
+const git_index_entry_unmerged *git_index_get_unmerged_bypath(
+	git_index *index, const char *path)
 {
 	int pos;
 	assert(index && path);
@@ -487,129 +525,23 @@ const git_index_entry_unmerged *git_index_get_unmerged(git_index *index, const c
 	if (!index->unmerged.length)
 		return NULL;
 
-	if ((pos = git_vector_bsearch2(&index->unmerged, unmerged_srch, path)) < GIT_SUCCESS)
+	if ((pos = git_vector_bsearch2(&index->unmerged, unmerged_srch, path)) < 0)
 		return NULL;
 
 	return git_vector_get(&index->unmerged, pos);
 }
 
-
-static int read_tree_internal(git_index_tree **out,
-		const char **buffer_in, const char *buffer_end, git_index_tree *parent)
+const git_index_entry_unmerged *git_index_get_unmerged_byindex(
+	git_index *index, unsigned int n)
 {
-	git_index_tree *tree;
-	const char *name_start, *buffer;
-	long count;
-	int error = GIT_SUCCESS;
-
-	if ((tree = git__malloc(sizeof(git_index_tree))) == NULL)
-		return GIT_ENOMEM;
-
-	memset(tree, 0x0, sizeof(git_index_tree));
-	tree->parent = parent;
-
-	buffer = name_start = *buffer_in;
-
-	if ((buffer = memchr(buffer, '\0', buffer_end - buffer)) == NULL) {
-		error = GIT_EOBJCORRUPTED;
-		goto cleanup;
-	}
-
-	/* NUL-terminated tree name */
-	tree->name = git__strdup(name_start);
-	if (tree->name == NULL) {
-		error = GIT_ENOMEM;
-		goto cleanup;
-	}
-
-	if (++buffer >= buffer_end) {
-		error = GIT_EOBJCORRUPTED;
-		goto cleanup;
-	}
-
-	/* Blank-terminated ASCII decimal number of entries in this tree */
-	if (git__strtol32(&count, buffer, &buffer, 10) < GIT_SUCCESS || count < -1) {
-		error = GIT_EOBJCORRUPTED;
-		goto cleanup;
-	}
-
-	/* Invalidated TREE. Free the tree but report success */
-	if (count == -1) {
-		/* FIXME: return buffer_end or the end position for
-		 * this single tree entry */
-		*buffer_in = buffer_end; 
-		*out = NULL;
-		free_tree(tree); /* Needs to be done manually */
-		return GIT_SUCCESS;
-	}
-
-	tree->entries = (size_t)count;
-
-	if (*buffer != ' ' || ++buffer >= buffer_end) {
-		error = GIT_EOBJCORRUPTED;
-		goto cleanup;
-	}
-
-	 /* Number of children of the tree, newline-terminated */
-	if (git__strtol32(&count, buffer, &buffer, 10) < GIT_SUCCESS ||
-		count < 0) {
-		error = GIT_EOBJCORRUPTED;
-		goto cleanup;
-	}
-
-	tree->children_count = (size_t)count;
-
-	if (*buffer != '\n' || ++buffer >= buffer_end) {
-		error = GIT_EOBJCORRUPTED;
-		goto cleanup;
-	}
-
-	/* 160-bit SHA-1 for this tree and it's children */
-	if (buffer + GIT_OID_RAWSZ > buffer_end) {
-		error = GIT_EOBJCORRUPTED;
-		goto cleanup;
-	}
-
-	git_oid_mkraw(&tree->oid, (const unsigned char *)buffer);
-	buffer += GIT_OID_RAWSZ;
-
-	/* Parse children: */
-	if (tree->children_count > 0) {
-		unsigned int i;
-		int err;
-
-		tree->children = git__malloc(tree->children_count * sizeof(git_index_tree *));
-		if (tree->children == NULL)
-			goto cleanup;
-
-		for (i = 0; i < tree->children_count; ++i) {
-			err = read_tree_internal(&tree->children[i], &buffer, buffer_end, tree);
-
-			if (err < GIT_SUCCESS)
-				goto cleanup;
-		}
-	}
-
-	*buffer_in = buffer;
-	*out = tree;
-	return GIT_SUCCESS;
-
- cleanup:
-	free_tree(tree);
-	return error;
+	assert(index);
+	return git_vector_get(&index->unmerged, n);
 }
 
-static int read_tree(git_index *index, const char *buffer, size_t buffer_size)
+static int index_error_invalid(const char *message)
 {
-	const char *buffer_end = buffer + buffer_size;
-	int error;
-
-	error = read_tree_internal(&index->tree, &buffer, buffer_end, NULL);
-
-	if (buffer < buffer_end)
-		return GIT_EOBJCORRUPTED;
-
-	return error;
+	giterr_set(GITERR_INDEX, "Invalid data in index - %s", message);
+	return -1;
 }
 
 static int read_unmerged(git_index *index, const char *buffer, size_t size)
@@ -618,53 +550,62 @@ static int read_unmerged(git_index *index, const char *buffer, size_t size)
 	size_t len;
 	int i;
 
-	git_vector_init(&index->unmerged, 16, unmerged_cmp);
+	if (git_vector_init(&index->unmerged, 16, unmerged_cmp) < 0)
+		return -1;
 
 	while (size) {
 		git_index_entry_unmerged *lost;
 
 		len = strlen(buffer) + 1;
 		if (size <= len)
-			return git__throw(GIT_ERROR, "Failed to read unmerged entries");
+			return index_error_invalid("reading unmerged entries");
 
-		if ((lost = git__malloc(sizeof(git_index_entry_unmerged))) == NULL)
-			return GIT_ENOMEM;
+		lost = git__malloc(sizeof(git_index_entry_unmerged));
+		GITERR_CHECK_ALLOC(lost);
 
-		if (git_vector_insert(&index->unmerged, lost) < GIT_SUCCESS)
-			return git__throw(GIT_ERROR, "Failed to read unmerged entries");
+		if (git_vector_insert(&index->unmerged, lost) < 0)
+			return -1;
 
+		/* read NUL-terminated pathname for entry */
 		lost->path = git__strdup(buffer);
-		if (!lost->path)
-			return GIT_ENOMEM;
+		GITERR_CHECK_ALLOC(lost->path);
 
 		size -= len;
 		buffer += len;
 
+		/* read 3 ASCII octal numbers for stage entries */
 		for (i = 0; i < 3; i++) {
-			if (git__strtol32((long int *) &lost->mode[i], buffer, &endptr, 8) < GIT_SUCCESS ||
-				!endptr || endptr == buffer || *endptr)
-				return GIT_ERROR;
+			int tmp;
+
+			if (git__strtol32(&tmp, buffer, &endptr, 8) < 0 ||
+				!endptr || endptr == buffer || *endptr ||
+				(unsigned)tmp > UINT_MAX)
+				return index_error_invalid("reading unmerged entry stage");
 
-			len = (endptr + 1) - (char *) buffer;
+			lost->mode[i] = tmp;
+
+			len = (endptr + 1) - buffer;
 			if (size <= len)
-				return git__throw(GIT_ERROR, "Failed to read unmerged entries");
+				return index_error_invalid("reading unmerged entry stage");
 
 			size -= len;
 			buffer += len;
 		}
 
+		/* read up to 3 OIDs for stage entries */
 		for (i = 0; i < 3; i++) {
 			if (!lost->mode[i])
 				continue;
 			if (size < 20)
-				return git__throw(GIT_ERROR, "Failed to read unmerged entries");
-			git_oid_mkraw(&lost->oid[i], (unsigned char *) buffer);
+				return index_error_invalid("reading unmerged entry oid");
+
+			git_oid_fromraw(&lost->oid[i], (const unsigned char *) buffer);
 			size -= 20;
 			buffer += 20;
 		}
 	}
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffer_size)
@@ -672,15 +613,13 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe
 	size_t path_length, entry_size;
 	uint16_t flags_raw;
 	const char *path_ptr;
-	const struct entry_short *source;
+	const struct entry_short *source = buffer;
 
 	if (INDEX_FOOTER_SIZE + minimal_entry_size > buffer_size)
 		return 0;
 
 	memset(dest, 0x0, sizeof(git_index_entry));
 
-	source = (const struct entry_short *)(buffer);
-
 	dest->ctime.seconds = (git_time_t)ntohl(source->ctime.seconds);
 	dest->ctime.nanoseconds = ntohl(source->ctime.nanoseconds);
 	dest->mtime.seconds = (git_time_t)ntohl(source->mtime.seconds);
@@ -695,7 +634,7 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe
 	dest->flags = ntohs(source->flags);
 
 	if (dest->flags & GIT_IDXENTRY_EXTENDED) {
-		struct entry_long *source_l = (struct entry_long *)source;
+		const struct entry_long *source_l = (const struct entry_long *)source;
 		path_ptr = source_l->path;
 
 		flags_raw = ntohs(source_l->flags_extended);
@@ -712,7 +651,7 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe
 
 		path_end = memchr(path_ptr, '\0', buffer_size);
 		if (path_end == NULL)
-				return 0;
+			return 0;
 
 		path_length = path_end - path_ptr;
 	}
@@ -733,20 +672,19 @@ static size_t read_entry(git_index_entry *dest, const void *buffer, size_t buffe
 
 static int read_header(struct index_header *dest, const void *buffer)
 {
-	const struct index_header *source;
-	source = (const struct index_header *)(buffer);
+	const struct index_header *source = buffer;
 
 	dest->signature = ntohl(source->signature);
 	if (dest->signature != INDEX_HEADER_SIG)
-		return GIT_EOBJCORRUPTED;
+		return index_error_invalid("incorrect header signature");
 
 	dest->version = ntohl(source->version);
 	if (dest->version != INDEX_VERSION_NUMBER_EXT &&
 		dest->version != INDEX_VERSION_NUMBER)
-		return GIT_EOBJCORRUPTED;
+		return index_error_invalid("incorrect header version");
 
 	dest->entry_count = ntohl(source->entry_count);
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static size_t read_extension(git_index *index, const char *buffer, size_t buffer_size)
@@ -769,10 +707,10 @@ static size_t read_extension(git_index *index, const char *buffer, size_t buffer
 	if (dest.signature[0] >= 'A' && dest.signature[0] <= 'Z') {
 		/* tree cache */
 		if (memcmp(dest.signature, INDEX_EXT_TREECACHE_SIG, 4) == 0) {
-			if (read_tree(index, buffer + 8, dest.extension_size) < GIT_SUCCESS)
+			if (git_tree_cache_read(&index->tree, buffer + 8, dest.extension_size) < 0)
 				return 0;
 		} else if (memcmp(dest.signature, INDEX_EXT_UNMERGED_SIG, 4) == 0) {
-			if (read_unmerged(index, buffer + 8, dest.extension_size) < GIT_SUCCESS)
+			if (read_unmerged(index, buffer + 8, dest.extension_size) < 0)
 				return 0;
 		}
 		/* else, unsupported extension. We cannot parse this, but we can skip
@@ -794,21 +732,21 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 
 #define seek_forward(_increase) { \
 	if (_increase >= buffer_size) \
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to seek forward. Buffer size exceeded"); \
+		return index_error_invalid("ran out of data while parsing"); \
 	buffer += _increase; \
 	buffer_size -= _increase;\
 }
 
 	if (buffer_size < INDEX_HEADER_SIZE + INDEX_FOOTER_SIZE)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Buffer too small");
+		return index_error_invalid("insufficient buffer space");
 
 	/* Precalculate the SHA1 of the files's contents -- we'll match it to
 	 * the provided SHA1 in the footer */
-	git_hash_buf(&checksum_calculated, (const void *)buffer, buffer_size - INDEX_FOOTER_SIZE);
+	git_hash_buf(&checksum_calculated, buffer, buffer_size - INDEX_FOOTER_SIZE);
 
 	/* Parse header */
-	if (read_header(&header, buffer) < GIT_SUCCESS)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Header is corrupted");
+	if (read_header(&header, buffer) < 0)
+		return -1;
 
 	seek_forward(INDEX_HEADER_SIZE);
 
@@ -820,23 +758,22 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 		git_index_entry *entry;
 
 		entry = git__malloc(sizeof(git_index_entry));
-		if (entry == NULL)
-			return GIT_ENOMEM;
+		GITERR_CHECK_ALLOC(entry);
 
 		entry_size = read_entry(entry, buffer, buffer_size);
 
 		/* 0 bytes read means an object corruption */
 		if (entry_size == 0)
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Entry size is zero");
+			return index_error_invalid("invalid entry");
 
-		if (git_vector_insert(&index->entries, entry) < GIT_SUCCESS)
-			return GIT_ENOMEM;
+		if (git_vector_insert(&index->entries, entry) < 0)
+			return -1;
 
 		seek_forward(entry_size);
 	}
 
 	if (i != header.entry_count)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Header entries changed while parsing");
+		return index_error_invalid("header entries changed while parsing");
 
 	/* There's still space for some extensions! */
 	while (buffer_size > INDEX_FOOTER_SIZE) {
@@ -846,48 +783,49 @@ static int parse_index(git_index *index, const char *buffer, size_t buffer_size)
 
 		/* see if we have read any bytes from the extension */
 		if (extension_size == 0)
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Extension size is zero");
+			return index_error_invalid("extension size is zero");
 
 		seek_forward(extension_size);
 	}
 
 	if (buffer_size != INDEX_FOOTER_SIZE)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Buffer size does not match index footer size");
+		return index_error_invalid("buffer size does not match index footer size");
 
 	/* 160-bit SHA-1 over the content of the index file before this checksum. */
-	git_oid_mkraw(&checksum_expected, (const unsigned char *)buffer);
+	git_oid_fromraw(&checksum_expected, (const unsigned char *)buffer);
 
 	if (git_oid_cmp(&checksum_calculated, &checksum_expected) != 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse index. Calculated checksum does not match expected checksum");
+		return index_error_invalid("calculated checksum does not match expected");
 
 #undef seek_forward
 
 	/* force sorting in the vector: the entries are
 	 * assured to be sorted on the index */
 	index->entries.sorted = 1;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static int is_index_extended(git_index *index)
 {
 	unsigned int i, extended;
+	git_index_entry *entry;
 
 	extended = 0;
 
-	for (i = 0; i < index->entries.length; ++i) {
-		git_index_entry *entry;
-		entry = git_vector_get(&index->entries, i);
+	git_vector_foreach(&index->entries, i, entry) {
 		entry->flags &= ~GIT_IDXENTRY_EXTENDED;
 		if (entry->flags_extended & GIT_IDXENTRY_EXTENDED_FLAGS) {
 			extended++;
 			entry->flags |= GIT_IDXENTRY_EXTENDED;
 		}
 	}
+
 	return extended;
 }
 
 static int write_disk_entry(git_filebuf *file, git_index_entry *entry)
 {
+	void *mem = NULL;
 	struct entry_short *ondisk;
 	size_t path_len, disk_size;
 	char *path;
@@ -899,21 +837,33 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry)
 	else
 		disk_size = short_entry_size(path_len);
 
-	if (git_filebuf_reserve(file, (void **)&ondisk, disk_size) < GIT_SUCCESS)
-		return GIT_ENOMEM;
+	if (git_filebuf_reserve(file, &mem, disk_size) < 0)
+		return -1;
+
+	ondisk = (struct entry_short *)mem;
 
 	memset(ondisk, 0x0, disk_size);
 
-	ondisk->ctime.seconds = htonl((unsigned long)entry->ctime.seconds);
-	ondisk->mtime.seconds = htonl((unsigned long)entry->mtime.seconds);
+	/**
+	 * Yes, we have to truncate.
+	 *
+	 * The on-disk format for Index entries clearly defines
+	 * the time and size fields to be 4 bytes each -- so even if
+	 * we store these values with 8 bytes on-memory, they must
+	 * be truncated to 4 bytes before writing to disk.
+	 *
+	 * In 2038 I will be either too dead or too rich to care about this
+	 */
+	ondisk->ctime.seconds = htonl((uint32_t)entry->ctime.seconds);
+	ondisk->mtime.seconds = htonl((uint32_t)entry->mtime.seconds);
 	ondisk->ctime.nanoseconds = htonl(entry->ctime.nanoseconds);
 	ondisk->mtime.nanoseconds = htonl(entry->mtime.nanoseconds);
-	ondisk->dev  = htonl(entry->dev);
-	ondisk->ino  = htonl(entry->ino);
+	ondisk->dev = htonl(entry->dev);
+	ondisk->ino = htonl(entry->ino);
 	ondisk->mode = htonl(entry->mode);
-	ondisk->uid  = htonl(entry->uid);
-	ondisk->gid  = htonl(entry->gid);
-	ondisk->file_size = htonl((unsigned long)entry->file_size);
+	ondisk->uid = htonl(entry->uid);
+	ondisk->gid = htonl(entry->gid);
+	ondisk->file_size = htonl((uint32_t)entry->file_size);
 
 	git_oid_cpy(&ondisk->oid, &entry->oid);
 
@@ -930,7 +880,7 @@ static int write_disk_entry(git_filebuf *file, git_index_entry *entry)
 
 	memcpy(path, entry->path, path_len);
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static int write_entries(git_index *index, git_filebuf *file)
@@ -940,16 +890,15 @@ static int write_entries(git_index *index, git_filebuf *file)
 	for (i = 0; i < index->entries.length; ++i) {
 		git_index_entry *entry;
 		entry = git_vector_get(&index->entries, i);
-		if (write_disk_entry(file, entry) < GIT_SUCCESS)
-			return GIT_ENOMEM;
+		if (write_disk_entry(file, entry) < 0)
+			return -1;
 	}
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static int write_index(git_index *index, git_filebuf *file)
 {
-	int error = GIT_SUCCESS;
 	git_oid hash_final;
 
 	struct index_header header;
@@ -964,11 +913,11 @@ static int write_index(git_index *index, git_filebuf *file)
 	header.version = htonl(is_extended ? INDEX_VERSION_NUMBER_EXT : INDEX_VERSION_NUMBER);
 	header.entry_count = htonl(index->entries.length);
 
-	git_filebuf_write(file, &header, sizeof(struct index_header));
+	if (git_filebuf_write(file, &header, sizeof(struct index_header)) < 0)
+		return -1;
 
-	error = write_entries(index, file);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write index");
+	if (write_entries(index, file) < 0)
+		return -1;
 
 	/* TODO: write extensions (tree cache) */
 
@@ -976,12 +925,45 @@ static int write_index(git_index *index, git_filebuf *file)
 	git_filebuf_hash(&hash_final, file);
 
 	/* write it at the end of the file */
-	git_filebuf_write(file, hash_final.id, GIT_OID_RAWSZ);
-
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to write index");
+	return git_filebuf_write(file, hash_final.id, GIT_OID_RAWSZ);
 }
 
 int git_index_entry_stage(const git_index_entry *entry)
 {
 	return (entry->flags & GIT_IDXENTRY_STAGEMASK) >> GIT_IDXENTRY_STAGESHIFT;
 }
+
+static int read_tree_cb(const char *root, git_tree_entry *tentry, void *data)
+{
+	git_index *index = data;
+	git_index_entry *entry = NULL;
+	git_buf path = GIT_BUF_INIT;
+
+	if (git_tree_entry__is_tree(tentry))
+		return 0;
+
+	if (git_buf_joinpath(&path, root, tentry->filename) < 0)
+		return -1;
+
+	entry = git__calloc(1, sizeof(git_index_entry));
+	GITERR_CHECK_ALLOC(entry);
+
+	entry->mode = tentry->attr;
+	entry->oid = tentry->oid;
+	entry->path = git_buf_detach(&path);
+	git_buf_free(&path);
+
+	if (index_insert(index, entry, 0) < 0) {
+		index_entry_free(entry);
+		return -1;
+	}
+
+	return 0;
+}
+
+int git_index_read_tree(git_index *index, git_tree *tree)
+{
+	git_index_clear(index);
+
+	return git_tree_walk(tree, read_tree_cb, GIT_TREEWALK_POST, index);
+}
diff --git a/src/index.h b/src/index.h
index f2402fd71..8515f4fcb 100644
--- a/src/index.h
+++ b/src/index.h
@@ -1,36 +1,38 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_index_h__
 #define INCLUDE_index_h__
 
 #include "fileops.h"
 #include "filebuf.h"
 #include "vector.h"
+#include "tree-cache.h"
 #include "git2/odb.h"
 #include "git2/index.h"
 
-struct git_index_tree {
-	char *name;
-
-	struct git_index_tree *parent;
-	struct git_index_tree **children;
-	size_t children_count;
-
-	size_t entries;
-	git_oid oid;
-};
-
-typedef struct git_index_tree git_index_tree;
+#define GIT_INDEX_FILE "index"
+#define GIT_INDEX_FILE_MODE 0666
 
 struct git_index {
-	git_repository *repository;
+	git_refcount rc;
+
 	char *index_file_path;
 
 	time_t last_modified;
 	git_vector entries;
 
 	unsigned int on_disk:1;
-	git_index_tree *tree;
+	git_tree_cache *tree;
 
 	git_vector unmerged;
 };
 
+extern void git_index__init_entry_from_stat(struct stat *st, git_index_entry *entry);
+
+extern unsigned int git_index__prefix_position(git_index *index, const char *path);
+
 #endif
diff --git a/src/indexer.c b/src/indexer.c
new file mode 100644
index 000000000..f0e0a6381
--- /dev/null
+++ b/src/indexer.c
@@ -0,0 +1,898 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include <zlib.h>
+
+#include "git2/indexer.h"
+#include "git2/object.h"
+#include "git2/oid.h"
+
+#include "common.h"
+#include "pack.h"
+#include "mwindow.h"
+#include "posix.h"
+#include "pack.h"
+#include "filebuf.h"
+#include "sha1.h"
+
+#define UINT31_MAX (0x7FFFFFFF)
+
+struct entry {
+	git_oid oid;
+	uint32_t crc;
+	uint32_t offset;
+	uint64_t offset_long;
+};
+
+struct git_indexer {
+	struct git_pack_file *pack;
+	size_t nr_objects;
+	git_vector objects;
+	git_filebuf file;
+	unsigned int fanout[256];
+	git_oid hash;
+};
+
+struct git_indexer_stream {
+	unsigned int parsed_header :1,
+		opened_pack;
+	struct git_pack_file *pack;
+	git_filebuf pack_file;
+	git_filebuf index_file;
+	git_off_t off;
+	size_t nr_objects;
+	git_vector objects;
+	git_vector deltas;
+	unsigned int fanout[256];
+	git_oid hash;
+};
+
+struct delta_info {
+	git_off_t delta_off;
+};
+
+const git_oid *git_indexer_hash(git_indexer *idx)
+{
+	return &idx->hash;
+}
+
+const git_oid *git_indexer_stream_hash(git_indexer_stream *idx)
+{
+	return &idx->hash;
+}
+
+static int open_pack(struct git_pack_file **out, const char *filename)
+{
+	size_t namelen;
+	struct git_pack_file *pack;
+	struct stat st;
+	int fd;
+
+	namelen = strlen(filename);
+	pack = git__calloc(1, sizeof(struct git_pack_file) + namelen + 1);
+	GITERR_CHECK_ALLOC(pack);
+
+	memcpy(pack->pack_name, filename, namelen + 1);
+
+	if (p_stat(filename, &st) < 0) {
+		giterr_set(GITERR_OS, "Failed to stat packfile.");
+		goto cleanup;
+	}
+
+	if ((fd = p_open(pack->pack_name, O_RDONLY)) < 0) {
+		giterr_set(GITERR_OS, "Failed to open packfile.");
+		goto cleanup;
+	}
+
+	pack->mwf.fd = fd;
+	pack->mwf.size = (git_off_t)st.st_size;
+
+	*out = pack;
+	return 0;
+
+cleanup:
+	git__free(pack);
+	return -1;
+}
+
+static int parse_header(struct git_pack_header *hdr, struct git_pack_file *pack)
+{
+	int error;
+
+	/* Verify we recognize this pack file format. */
+	if ((error = p_read(pack->mwf.fd, hdr, sizeof(*hdr))) < 0) {
+		giterr_set(GITERR_OS, "Failed to read in pack header");
+		return error;
+	}
+
+	if (hdr->hdr_signature != ntohl(PACK_SIGNATURE)) {
+		giterr_set(GITERR_INDEXER, "Wrong pack signature");
+		return -1;
+	}
+
+	if (!pack_version_ok(hdr->hdr_version)) {
+		giterr_set(GITERR_INDEXER, "Wrong pack version");
+		return -1;
+	}
+
+	return 0;
+}
+
+static int objects_cmp(const void *a, const void *b)
+{
+	const struct entry *entrya = a;
+	const struct entry *entryb = b;
+
+	return git_oid_cmp(&entrya->oid, &entryb->oid);
+}
+
+static int cache_cmp(const void *a, const void *b)
+{
+	const struct git_pack_entry *ea = a;
+	const struct git_pack_entry *eb = b;
+
+	return git_oid_cmp(&ea->sha1, &eb->sha1);
+}
+
+int git_indexer_stream_new(git_indexer_stream **out, const char *prefix)
+{
+	git_indexer_stream *idx;
+	git_buf path = GIT_BUF_INIT;
+	static const char suff[] = "/objects/pack/pack-received";
+	int error;
+
+	idx = git__calloc(1, sizeof(git_indexer_stream));
+	GITERR_CHECK_ALLOC(idx);
+
+	error = git_buf_joinpath(&path, prefix, suff);
+	if (error < 0)
+		goto cleanup;
+
+	error = git_filebuf_open(&idx->pack_file, path.ptr,
+				 GIT_FILEBUF_TEMPORARY | GIT_FILEBUF_DO_NOT_BUFFER);
+	git_buf_free(&path);
+	if (error < 0)
+		goto cleanup;
+
+	*out = idx;
+	return 0;
+
+cleanup:
+	git_buf_free(&path);
+	git_filebuf_cleanup(&idx->pack_file);
+	git__free(idx);
+	return -1;
+}
+
+/* Try to store the delta so we can try to resolve it later */
+static int store_delta(git_indexer_stream *idx)
+{
+	git_otype type;
+	git_mwindow *w = NULL;
+	git_mwindow_file *mwf = &idx->pack->mwf;
+	git_off_t entry_start = idx->off;
+	struct delta_info *delta;
+	size_t entry_size;
+	git_rawobj obj;
+	int error;
+
+	/*
+	 * ref-delta objects can refer to object that we haven't
+	 * found yet, so give it another opportunity
+	 */
+	if (git_packfile_unpack_header(&entry_size, &type, mwf, &w, &idx->off) < 0)
+		return -1;
+
+	git_mwindow_close(&w);
+
+	/* If it's not a delta, mark it as failure, we can't do anything with it */
+	if (type != GIT_OBJ_REF_DELTA && type != GIT_OBJ_OFS_DELTA)
+		return -1;
+
+	if (type == GIT_OBJ_REF_DELTA) {
+		idx->off += GIT_OID_RAWSZ;
+	} else {
+		git_off_t base_off;
+
+		base_off = get_delta_base(idx->pack, &w, &idx->off, type, entry_start);
+		git_mwindow_close(&w);
+		if (base_off < 0)
+			return (int)base_off;
+	}
+
+	error = packfile_unpack_compressed(&obj, idx->pack, &w, &idx->off, entry_size, type);
+	if (error == GIT_EBUFS) {
+		idx->off = entry_start;
+		return GIT_EBUFS;
+	} else if (error < 0){
+		return -1;
+	}
+
+	delta = git__calloc(1, sizeof(struct delta_info));
+	GITERR_CHECK_ALLOC(delta);
+	delta->delta_off = entry_start;
+
+	git__free(obj.data);
+
+	if (git_vector_insert(&idx->deltas, delta) < 0)
+		return -1;
+
+	return 0;
+}
+
+static int hash_and_save(git_indexer_stream *idx, git_rawobj *obj, git_off_t entry_start)
+{
+	int i;
+	git_oid oid;
+	void *packed;
+	size_t entry_size;
+	unsigned int left;
+	struct entry *entry;
+	git_mwindow *w = NULL;
+	git_mwindow_file *mwf = &idx->pack->mwf;
+	struct git_pack_entry *pentry;
+
+	entry = git__calloc(1, sizeof(*entry));
+	GITERR_CHECK_ALLOC(entry);
+
+	if (entry_start > UINT31_MAX) {
+		entry->offset = UINT32_MAX;
+		entry->offset_long = entry_start;
+	} else {
+		entry->offset = (uint32_t)entry_start;
+	}
+
+	/* FIXME: Parse the object instead of hashing it */
+	if (git_odb__hashobj(&oid, obj) < 0) {
+		giterr_set(GITERR_INDEXER, "Failed to hash object");
+		return -1;
+	}
+
+	pentry = git__malloc(sizeof(struct git_pack_entry));
+	GITERR_CHECK_ALLOC(pentry);
+
+	git_oid_cpy(&pentry->sha1, &oid);
+	pentry->offset = entry_start;
+	if (git_vector_insert(&idx->pack->cache, pentry) < 0)
+		goto on_error;
+
+	git_oid_cpy(&entry->oid, &oid);
+	entry->crc = crc32(0L, Z_NULL, 0);
+
+	entry_size = (size_t)(idx->off - entry_start);
+	packed = git_mwindow_open(mwf, &w, entry_start, entry_size, &left);
+	if (packed == NULL)
+		goto on_error;
+
+	entry->crc = htonl(crc32(entry->crc, packed, (uInt)entry_size));
+	git_mwindow_close(&w);
+
+	/* Add the object to the list */
+	if (git_vector_insert(&idx->objects, entry) < 0)
+		goto on_error;
+
+	for (i = oid.id[0]; i < 256; ++i) {
+		idx->fanout[i]++;
+	}
+
+	return 0;
+
+on_error:
+	git__free(entry);
+	git__free(pentry);
+	git__free(obj->data);
+	return -1;
+}
+
+int git_indexer_stream_add(git_indexer_stream *idx, const void *data, size_t size, git_indexer_stats *stats)
+{
+	int error;
+	struct git_pack_header hdr;
+	size_t processed = stats->processed;
+	git_mwindow_file *mwf = &idx->pack->mwf;
+
+	assert(idx && data && stats);
+
+	if (git_filebuf_write(&idx->pack_file, data, size) < 0)
+		return -1;
+
+	/* Make sure we set the new size of the pack */
+	if (idx->opened_pack) {
+		idx->pack->mwf.size += size;
+		//printf("\nadding %zu for %zu\n", size, idx->pack->mwf.size);
+	} else {
+		if (open_pack(&idx->pack, idx->pack_file.path_lock) < 0)
+			return -1;
+		idx->opened_pack = 1;
+		mwf = &idx->pack->mwf;
+		if (git_mwindow_file_register(&idx->pack->mwf) < 0)
+			return -1;
+
+		return 0;
+	}
+
+	if (!idx->parsed_header) {
+		if ((unsigned)idx->pack->mwf.size < sizeof(hdr))
+			return 0;
+
+		if (parse_header(&hdr, idx->pack) < 0)
+			return -1;
+
+		idx->parsed_header = 1;
+		idx->nr_objects = ntohl(hdr.hdr_entries);
+		idx->off = sizeof(struct git_pack_header);
+
+		/* for now, limit to 2^32 objects */
+		assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
+
+		if (git_vector_init(&idx->pack->cache, (unsigned int)idx->nr_objects, cache_cmp) < 0)
+			return -1;
+
+		idx->pack->has_cache = 1;
+		if (git_vector_init(&idx->objects, (unsigned int)idx->nr_objects, objects_cmp) < 0)
+			return -1;
+
+		if (git_vector_init(&idx->deltas, (unsigned int)(idx->nr_objects / 2), NULL) < 0)
+			return -1;
+
+		stats->total = (unsigned int)idx->nr_objects;
+		stats->processed = 0;
+	}
+
+	/* Now that we have data in the pack, let's try to parse it */
+
+	/* As the file grows any windows we try to use will be out of date */
+	git_mwindow_free_all(mwf);
+	while (processed < idx->nr_objects) {
+		git_rawobj obj;
+		git_off_t entry_start = idx->off;
+
+		if (idx->pack->mwf.size <= idx->off + 20)
+			return 0;
+
+		error = git_packfile_unpack(&obj, idx->pack, &idx->off);
+		if (error == GIT_EBUFS) {
+			idx->off = entry_start;
+			return 0;
+		}
+
+		if (error < 0) {
+			idx->off = entry_start;
+			error = store_delta(idx);
+			if (error == GIT_EBUFS)
+				return 0;
+			if (error < 0)
+				return error;
+
+			continue;
+		}
+
+		if (hash_and_save(idx, &obj, entry_start) < 0)
+			goto on_error;
+
+		git__free(obj.data);
+
+		stats->processed = (unsigned int)++processed;
+	}
+
+	return 0;
+
+on_error:
+	git_mwindow_free_all(mwf);
+	return -1;
+}
+
+static int index_path_stream(git_buf *path, git_indexer_stream *idx, const char *suffix)
+{
+	const char prefix[] = "pack-";
+	size_t slash = (size_t)path->size;
+
+	/* search backwards for '/' */
+	while (slash > 0 && path->ptr[slash - 1] != '/')
+		slash--;
+
+	if (git_buf_grow(path, slash + 1 + strlen(prefix) +
+					 GIT_OID_HEXSZ + strlen(suffix) + 1) < 0)
+		return -1;
+
+	git_buf_truncate(path, slash);
+	git_buf_puts(path, prefix);
+	git_oid_fmt(path->ptr + git_buf_len(path), &idx->hash);
+	path->size += GIT_OID_HEXSZ;
+	git_buf_puts(path, suffix);
+
+	return git_buf_oom(path) ? -1 : 0;
+}
+
+static int resolve_deltas(git_indexer_stream *idx, git_indexer_stats *stats)
+{
+	unsigned int i;
+	struct delta_info *delta;
+
+	git_vector_foreach(&idx->deltas, i, delta) {
+		git_rawobj obj;
+
+		idx->off = delta->delta_off;
+		if (git_packfile_unpack(&obj, idx->pack, &idx->off) < 0)
+			return -1;
+
+		if (hash_and_save(idx, &obj, delta->delta_off) < 0)
+			return -1;
+
+		git__free(obj.data);
+		stats->processed++;
+	}
+
+	return 0;
+}
+
+int git_indexer_stream_finalize(git_indexer_stream *idx, git_indexer_stats *stats)
+{
+	git_mwindow *w = NULL;
+	unsigned int i, long_offsets = 0, left;
+	struct git_pack_idx_header hdr;
+	git_buf filename = GIT_BUF_INIT;
+	struct entry *entry;
+	void *packfile_hash;
+	git_oid file_hash;
+	SHA_CTX ctx;
+
+	/* Test for this before resolve_deltas(), as it plays with idx->off */
+	if (idx->off < idx->pack->mwf.size - GIT_OID_RAWSZ) {
+		giterr_set(GITERR_INDEXER, "Indexing error: junk at the end of the pack");
+		return -1;
+	}
+
+	if (idx->deltas.length > 0)
+		if (resolve_deltas(idx, stats) < 0)
+			return -1;
+
+	if (stats->processed != stats->total) {
+		giterr_set(GITERR_INDEXER, "Indexing error: early EOF");
+		return -1;
+	}
+
+	git_vector_sort(&idx->objects);
+
+	git_buf_sets(&filename, idx->pack->pack_name);
+	git_buf_truncate(&filename, filename.size - strlen("pack"));
+	git_buf_puts(&filename, "idx");
+	if (git_buf_oom(&filename))
+		return -1;
+
+	if (git_filebuf_open(&idx->index_file, filename.ptr, GIT_FILEBUF_HASH_CONTENTS) < 0)
+		goto on_error;
+
+	/* Write out the header */
+	hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
+	hdr.idx_version = htonl(2);
+	git_filebuf_write(&idx->index_file, &hdr, sizeof(hdr));
+
+	/* Write out the fanout table */
+	for (i = 0; i < 256; ++i) {
+		uint32_t n = htonl(idx->fanout[i]);
+		git_filebuf_write(&idx->index_file, &n, sizeof(n));
+	}
+
+	/* Write out the object names (SHA-1 hashes) */
+	SHA1_Init(&ctx);
+	git_vector_foreach(&idx->objects, i, entry) {
+		git_filebuf_write(&idx->index_file, &entry->oid, sizeof(git_oid));
+		SHA1_Update(&ctx, &entry->oid, GIT_OID_RAWSZ);
+	}
+	SHA1_Final(idx->hash.id, &ctx);
+
+	/* Write out the CRC32 values */
+	git_vector_foreach(&idx->objects, i, entry) {
+		git_filebuf_write(&idx->index_file, &entry->crc, sizeof(uint32_t));
+	}
+
+	/* Write out the offsets */
+	git_vector_foreach(&idx->objects, i, entry) {
+		uint32_t n;
+
+		if (entry->offset == UINT32_MAX)
+			n = htonl(0x80000000 | long_offsets++);
+		else
+			n = htonl(entry->offset);
+
+		git_filebuf_write(&idx->index_file, &n, sizeof(uint32_t));
+	}
+
+	/* Write out the long offsets */
+	git_vector_foreach(&idx->objects, i, entry) {
+		uint32_t split[2];
+
+		if (entry->offset != UINT32_MAX)
+			continue;
+
+		split[0] = htonl(entry->offset_long >> 32);
+		split[1] = htonl(entry->offset_long & 0xffffffff);
+
+		git_filebuf_write(&idx->index_file, &split, sizeof(uint32_t) * 2);
+	}
+
+	/* Write out the packfile trailer */
+	packfile_hash = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
+	if (packfile_hash == NULL) {
+		git_mwindow_close(&w);
+		goto on_error;
+	}
+
+	memcpy(&file_hash, packfile_hash, GIT_OID_RAWSZ);
+	git_mwindow_close(&w);
+
+	git_filebuf_write(&idx->index_file, &file_hash, sizeof(git_oid));
+
+	/* Write out the packfile trailer to the idx file as well */
+	if (git_filebuf_hash(&file_hash, &idx->index_file) < 0)
+		goto on_error;
+
+	git_filebuf_write(&idx->index_file, &file_hash, sizeof(git_oid));
+
+	/* Figure out what the final name should be */
+	if (index_path_stream(&filename, idx, ".idx") < 0)
+		goto on_error;
+
+	/* Commit file */
+	if (git_filebuf_commit_at(&idx->index_file, filename.ptr, GIT_PACK_FILE_MODE) < 0)
+		goto on_error;
+
+	git_mwindow_free_all(&idx->pack->mwf);
+	p_close(idx->pack->mwf.fd);
+
+	if (index_path_stream(&filename, idx, ".pack") < 0)
+		goto on_error;
+	/* And don't forget to rename the packfile to its new place. */
+	if (git_filebuf_commit_at(&idx->pack_file, filename.ptr, GIT_PACK_FILE_MODE) < 0)
+		return -1;
+
+	git_buf_free(&filename);
+	return 0;
+
+on_error:
+	git_mwindow_free_all(&idx->pack->mwf);
+	p_close(idx->pack->mwf.fd);
+	git_filebuf_cleanup(&idx->index_file);
+	git_buf_free(&filename);
+	return -1;
+}
+
+void git_indexer_stream_free(git_indexer_stream *idx)
+{
+	unsigned int i;
+	struct entry *e;
+	struct git_pack_entry *pe;
+	struct delta_info *delta;
+
+	if (idx == NULL)
+		return;
+
+	git_vector_foreach(&idx->objects, i, e)
+		git__free(e);
+	git_vector_free(&idx->objects);
+	git_vector_foreach(&idx->pack->cache, i, pe)
+		git__free(pe);
+	git_vector_free(&idx->pack->cache);
+	git_vector_foreach(&idx->deltas, i, delta)
+		git__free(delta);
+	git_vector_free(&idx->deltas);
+	git__free(idx->pack);
+	git__free(idx);
+}
+
+int git_indexer_new(git_indexer **out, const char *packname)
+{
+	git_indexer *idx;
+	struct git_pack_header hdr;
+	int error;
+
+	assert(out && packname);
+
+	if (git_path_root(packname) < 0) {
+		giterr_set(GITERR_INDEXER, "Path is not absolute");
+		return -1;
+	}
+
+	idx = git__calloc(1, sizeof(git_indexer));
+	GITERR_CHECK_ALLOC(idx);
+
+	open_pack(&idx->pack, packname);
+
+	if ((error = parse_header(&hdr, idx->pack)) < 0)
+		goto cleanup;
+
+	idx->nr_objects = ntohl(hdr.hdr_entries);
+
+	/* for now, limit to 2^32 objects */
+	assert(idx->nr_objects == (size_t)((unsigned int)idx->nr_objects));
+
+	error = git_vector_init(&idx->pack->cache, (unsigned int)idx->nr_objects, cache_cmp);
+	if (error < 0)
+		goto cleanup;
+
+	idx->pack->has_cache = 1;
+	error = git_vector_init(&idx->objects, (unsigned int)idx->nr_objects, objects_cmp);
+	if (error < 0)
+		goto cleanup;
+
+	*out = idx;
+
+	return 0;
+
+cleanup:
+	git_indexer_free(idx);
+
+	return -1;
+}
+
+static int index_path(git_buf *path, git_indexer *idx)
+{
+	const char prefix[] = "pack-", suffix[] = ".idx";
+	size_t slash = (size_t)path->size;
+
+	/* search backwards for '/' */
+	while (slash > 0 && path->ptr[slash - 1] != '/')
+		slash--;
+
+	if (git_buf_grow(path, slash + 1 + strlen(prefix) +
+					 GIT_OID_HEXSZ + strlen(suffix) + 1) < 0)
+		return -1;
+
+	git_buf_truncate(path, slash);
+	git_buf_puts(path, prefix);
+	git_oid_fmt(path->ptr + git_buf_len(path), &idx->hash);
+	path->size += GIT_OID_HEXSZ;
+	git_buf_puts(path, suffix);
+
+	return git_buf_oom(path) ? -1 : 0;
+}
+
+int git_indexer_write(git_indexer *idx)
+{
+	git_mwindow *w = NULL;
+	int error;
+	unsigned int i, long_offsets = 0, left;
+	struct git_pack_idx_header hdr;
+	git_buf filename = GIT_BUF_INIT;
+	struct entry *entry;
+	void *packfile_hash;
+	git_oid file_hash;
+	SHA_CTX ctx;
+
+	git_vector_sort(&idx->objects);
+
+	git_buf_sets(&filename, idx->pack->pack_name);
+	git_buf_truncate(&filename, filename.size - strlen("pack"));
+	git_buf_puts(&filename, "idx");
+	if (git_buf_oom(&filename))
+		return -1;
+
+	error = git_filebuf_open(&idx->file, filename.ptr, GIT_FILEBUF_HASH_CONTENTS);
+	if (error < 0)
+		goto cleanup;
+
+	/* Write out the header */
+	hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
+	hdr.idx_version = htonl(2);
+	error = git_filebuf_write(&idx->file, &hdr, sizeof(hdr));
+	if (error < 0)
+		goto cleanup;
+
+	/* Write out the fanout table */
+	for (i = 0; i < 256; ++i) {
+		uint32_t n = htonl(idx->fanout[i]);
+		error = git_filebuf_write(&idx->file, &n, sizeof(n));
+		if (error < 0)
+			goto cleanup;
+	}
+
+	/* Write out the object names (SHA-1 hashes) */
+	SHA1_Init(&ctx);
+	git_vector_foreach(&idx->objects, i, entry) {
+		error = git_filebuf_write(&idx->file, &entry->oid, sizeof(git_oid));
+		SHA1_Update(&ctx, &entry->oid, GIT_OID_RAWSZ);
+		if (error < 0)
+			goto cleanup;
+	}
+	SHA1_Final(idx->hash.id, &ctx);
+
+	/* Write out the CRC32 values */
+	git_vector_foreach(&idx->objects, i, entry) {
+		error = git_filebuf_write(&idx->file, &entry->crc, sizeof(uint32_t));
+		if (error < 0)
+			goto cleanup;
+	}
+
+	/* Write out the offsets */
+	git_vector_foreach(&idx->objects, i, entry) {
+		uint32_t n;
+
+		if (entry->offset == UINT32_MAX)
+			n = htonl(0x80000000 | long_offsets++);
+		else
+			n = htonl(entry->offset);
+
+		error = git_filebuf_write(&idx->file, &n, sizeof(uint32_t));
+		if (error < 0)
+			goto cleanup;
+	}
+
+	/* Write out the long offsets */
+	git_vector_foreach(&idx->objects, i, entry) {
+		uint32_t split[2];
+
+		if (entry->offset != UINT32_MAX)
+			continue;
+
+		split[0] = htonl(entry->offset_long >> 32);
+		split[1] = htonl(entry->offset_long & 0xffffffff);
+
+		error = git_filebuf_write(&idx->file, &split, sizeof(uint32_t) * 2);
+		if (error < 0)
+			goto cleanup;
+	}
+
+	/* Write out the packfile trailer */
+
+	packfile_hash = git_mwindow_open(&idx->pack->mwf, &w, idx->pack->mwf.size - GIT_OID_RAWSZ, GIT_OID_RAWSZ, &left);
+	git_mwindow_close(&w);
+	if (packfile_hash == NULL) {
+		error = -1;
+		goto cleanup;
+	}
+
+	memcpy(&file_hash, packfile_hash, GIT_OID_RAWSZ);
+
+	git_mwindow_close(&w);
+
+	error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid));
+	if (error < 0)
+		goto cleanup;
+
+	/* Write out the index sha */
+	error = git_filebuf_hash(&file_hash, &idx->file);
+	if (error < 0)
+		goto cleanup;
+
+	error = git_filebuf_write(&idx->file, &file_hash, sizeof(git_oid));
+	if (error < 0)
+		goto cleanup;
+
+	/* Figure out what the final name should be */
+	error = index_path(&filename, idx);
+	if (error < 0)
+		goto cleanup;
+
+	/* Commit file */
+	error = git_filebuf_commit_at(&idx->file, filename.ptr, GIT_PACK_FILE_MODE);
+
+cleanup:
+	git_mwindow_free_all(&idx->pack->mwf);
+	if (error < 0)
+		git_filebuf_cleanup(&idx->file);
+	git_buf_free(&filename);
+
+	return error;
+}
+
+int git_indexer_run(git_indexer *idx, git_indexer_stats *stats)
+{
+	git_mwindow_file *mwf;
+	git_off_t off = sizeof(struct git_pack_header);
+	int error;
+	struct entry *entry;
+	unsigned int left, processed;
+
+	assert(idx && stats);
+
+	mwf = &idx->pack->mwf;
+	error = git_mwindow_file_register(mwf);
+	if (error < 0)
+		return error;
+
+	stats->total = (unsigned int)idx->nr_objects;
+	stats->processed = processed = 0;
+
+	while (processed < idx->nr_objects) {
+		git_rawobj obj;
+		git_oid oid;
+		struct git_pack_entry *pentry;
+		git_mwindow *w = NULL;
+		int i;
+		git_off_t entry_start = off;
+		void *packed;
+		size_t entry_size;
+		char fmt[GIT_OID_HEXSZ] = {0};
+
+		entry = git__calloc(1, sizeof(*entry));
+		GITERR_CHECK_ALLOC(entry);
+
+		if (off > UINT31_MAX) {
+			entry->offset = UINT32_MAX;
+			entry->offset_long = off;
+		} else {
+			entry->offset = (uint32_t)off;
+		}
+
+		error = git_packfile_unpack(&obj, idx->pack, &off);
+		if (error < 0)
+			goto cleanup;
+
+		/* FIXME: Parse the object instead of hashing it */
+		error = git_odb__hashobj(&oid, &obj);
+		if (error < 0) {
+			giterr_set(GITERR_INDEXER, "Failed to hash object");
+			goto cleanup;
+		}
+
+		pentry = git__malloc(sizeof(struct git_pack_entry));
+		if (pentry == NULL) {
+			error = -1;
+			goto cleanup;
+		}
+
+		git_oid_cpy(&pentry->sha1, &oid);
+		pentry->offset = entry_start;
+		git_oid_fmt(fmt, &oid);
+		error = git_vector_insert(&idx->pack->cache, pentry);
+		if (error < 0)
+			goto cleanup;
+
+		git_oid_cpy(&entry->oid, &oid);
+		entry->crc = crc32(0L, Z_NULL, 0);
+
+		entry_size = (size_t)(off - entry_start);
+		packed = git_mwindow_open(mwf, &w, entry_start, entry_size, &left);
+		if (packed == NULL) {
+			error = -1;
+			goto cleanup;
+		}
+		entry->crc = htonl(crc32(entry->crc, packed, (uInt)entry_size));
+		git_mwindow_close(&w);
+
+		/* Add the object to the list */
+		error = git_vector_insert(&idx->objects, entry);
+		if (error < 0)
+			goto cleanup;
+
+		for (i = oid.id[0]; i < 256; ++i) {
+			idx->fanout[i]++;
+		}
+
+		git__free(obj.data);
+
+		stats->processed = ++processed;
+	}
+
+cleanup:
+	git_mwindow_free_all(mwf);
+
+	return error;
+
+}
+
+void git_indexer_free(git_indexer *idx)
+{
+	unsigned int i;
+	struct entry *e;
+	struct git_pack_entry *pe;
+
+	if (idx == NULL)
+		return;
+
+	p_close(idx->pack->mwf.fd);
+	git_vector_foreach(&idx->objects, i, e)
+		git__free(e);
+	git_vector_free(&idx->objects);
+	git_vector_foreach(&idx->pack->cache, i, pe)
+		git__free(pe);
+	git_vector_free(&idx->pack->cache);
+	git__free(idx->pack);
+	git__free(idx);
+}
+
diff --git a/src/iterator.c b/src/iterator.c
new file mode 100644
index 000000000..819b0e22a
--- /dev/null
+++ b/src/iterator.c
@@ -0,0 +1,748 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "iterator.h"
+#include "tree.h"
+#include "ignore.h"
+#include "buffer.h"
+#include "git2/submodule.h"
+
+#define ITERATOR_BASE_INIT(P,NAME_LC,NAME_UC) do { \
+	(P) = git__calloc(1, sizeof(NAME_LC ## _iterator)); \
+	GITERR_CHECK_ALLOC(P); \
+	(P)->base.type    = GIT_ITERATOR_ ## NAME_UC; \
+	(P)->base.start   = start ? git__strdup(start) : NULL; \
+	(P)->base.end     = end ? git__strdup(end) : NULL; \
+	(P)->base.current = NAME_LC ## _iterator__current; \
+	(P)->base.at_end  = NAME_LC ## _iterator__at_end; \
+	(P)->base.advance = NAME_LC ## _iterator__advance; \
+	(P)->base.seek    = NAME_LC ## _iterator__seek; \
+	(P)->base.reset   = NAME_LC ## _iterator__reset; \
+	(P)->base.free    = NAME_LC ## _iterator__free; \
+	if ((start && !(P)->base.start) || (end && !(P)->base.end)) \
+		return -1; \
+	} while (0)
+
+
+static int empty_iterator__no_item(
+	git_iterator *iter, const git_index_entry **entry)
+{
+	GIT_UNUSED(iter);
+	*entry = NULL;
+	return 0;
+}
+
+static int empty_iterator__at_end(git_iterator *iter)
+{
+	GIT_UNUSED(iter);
+	return 1;
+}
+
+static int empty_iterator__noop(git_iterator *iter)
+{
+	GIT_UNUSED(iter);
+	return 0;
+}
+
+static int empty_iterator__seek(git_iterator *iter, const char *prefix)
+{
+	GIT_UNUSED(iter);
+	GIT_UNUSED(prefix);
+	return -1;
+}
+
+static void empty_iterator__free(git_iterator *iter)
+{
+	GIT_UNUSED(iter);
+}
+
+int git_iterator_for_nothing(git_iterator **iter)
+{
+	git_iterator *i = git__calloc(1, sizeof(git_iterator));
+	GITERR_CHECK_ALLOC(i);
+
+	i->type    = GIT_ITERATOR_EMPTY;
+	i->current = empty_iterator__no_item;
+	i->at_end  = empty_iterator__at_end;
+	i->advance = empty_iterator__no_item;
+	i->seek    = empty_iterator__seek;
+	i->reset   = empty_iterator__noop;
+	i->free    = empty_iterator__free;
+
+	*iter = i;
+
+	return 0;
+}
+
+
+typedef struct tree_iterator_frame tree_iterator_frame;
+struct tree_iterator_frame {
+	tree_iterator_frame *next;
+	git_tree *tree;
+	char *start;
+	unsigned int index;
+};
+
+typedef struct {
+	git_iterator base;
+	git_repository *repo;
+	tree_iterator_frame *stack;
+	git_index_entry entry;
+	git_buf path;
+	bool path_has_filename;
+} tree_iterator;
+
+static const git_tree_entry *tree_iterator__tree_entry(tree_iterator *ti)
+{
+	return (ti->stack == NULL) ? NULL :
+		git_tree_entry_byindex(ti->stack->tree, ti->stack->index);
+}
+
+static char *tree_iterator__current_filename(
+	tree_iterator *ti, const git_tree_entry *te)
+{
+	if (!ti->path_has_filename) {
+		if (git_buf_joinpath(&ti->path, ti->path.ptr, te->filename) < 0)
+			return NULL;
+		ti->path_has_filename = true;
+	}
+
+	return ti->path.ptr;
+}
+
+static void tree_iterator__pop_frame(tree_iterator *ti)
+{
+	tree_iterator_frame *tf = ti->stack;
+	ti->stack = tf->next;
+	if (ti->stack != NULL) /* don't free the initial tree */
+		git_tree_free(tf->tree);
+	git__free(tf);
+}
+
+static int tree_iterator__to_end(tree_iterator *ti)
+{
+	while (ti->stack && ti->stack->next)
+		tree_iterator__pop_frame(ti);
+
+	if (ti->stack)
+		ti->stack->index = git_tree_entrycount(ti->stack->tree);
+
+	return 0;
+}
+
+static int tree_iterator__current(
+	git_iterator *self, const git_index_entry **entry)
+{
+	tree_iterator *ti = (tree_iterator *)self;
+	const git_tree_entry *te = tree_iterator__tree_entry(ti);
+
+	if (entry)
+		*entry = NULL;
+
+	if (te == NULL)
+		return 0;
+
+	ti->entry.mode = te->attr;
+	git_oid_cpy(&ti->entry.oid, &te->oid);
+
+	ti->entry.path = tree_iterator__current_filename(ti, te);
+	if (ti->entry.path == NULL)
+		return -1;
+
+	if (ti->base.end && git__prefixcmp(ti->entry.path, ti->base.end) > 0)
+		return tree_iterator__to_end(ti);
+
+	if (entry)
+		*entry = &ti->entry;
+
+	return 0;
+}
+
+static int tree_iterator__at_end(git_iterator *self)
+{
+	return (tree_iterator__tree_entry((tree_iterator *)self) == NULL);
+}
+
+static tree_iterator_frame *tree_iterator__alloc_frame(
+	git_tree *tree, char *start)
+{
+	tree_iterator_frame *tf = git__calloc(1, sizeof(tree_iterator_frame));
+	if (!tf)
+		return NULL;
+
+	tf->tree = tree;
+
+	if (start && *start) {
+		tf->start = start;
+		tf->index = git_tree__prefix_position(tree, start);
+	}
+
+	return tf;
+}
+
+static int tree_iterator__expand_tree(tree_iterator *ti)
+{
+	int error;
+	git_tree *subtree;
+	const git_tree_entry *te = tree_iterator__tree_entry(ti);
+	tree_iterator_frame *tf;
+	char *relpath;
+
+	while (te != NULL && git_tree_entry__is_tree(te)) {
+		if (git_buf_joinpath(&ti->path, ti->path.ptr, te->filename) < 0)
+			return -1;
+
+		/* check that we have not passed the range end */
+		if (ti->base.end != NULL &&
+			git__prefixcmp(ti->path.ptr, ti->base.end) > 0)
+			return tree_iterator__to_end(ti);
+
+		if ((error = git_tree_lookup(&subtree, ti->repo, &te->oid)) < 0)
+			return error;
+
+		relpath = NULL;
+
+		/* apply range start to new frame if relevant */
+		if (ti->stack->start &&
+			git__prefixcmp(ti->stack->start, te->filename) == 0)
+		{
+			size_t namelen = strlen(te->filename);
+			if (ti->stack->start[namelen] == '/')
+				relpath = ti->stack->start + namelen + 1;
+		}
+
+		if ((tf = tree_iterator__alloc_frame(subtree, relpath)) == NULL)
+			return -1;
+
+		tf->next  = ti->stack;
+		ti->stack = tf;
+
+		te = tree_iterator__tree_entry(ti);
+	}
+
+	return 0;
+}
+
+static int tree_iterator__advance(
+	git_iterator *self, const git_index_entry **entry)
+{
+	int error = 0;
+	tree_iterator *ti = (tree_iterator *)self;
+	const git_tree_entry *te = NULL;
+
+	if (entry != NULL)
+		*entry = NULL;
+
+	if (ti->path_has_filename) {
+		git_buf_rtruncate_at_char(&ti->path, '/');
+		ti->path_has_filename = false;
+	}
+
+	while (ti->stack != NULL) {
+		te = git_tree_entry_byindex(ti->stack->tree, ++ti->stack->index);
+		if (te != NULL)
+			break;
+
+		tree_iterator__pop_frame(ti);
+
+		git_buf_rtruncate_at_char(&ti->path, '/');
+	}
+
+	if (te && git_tree_entry__is_tree(te))
+		error = tree_iterator__expand_tree(ti);
+
+	if (!error)
+		error = tree_iterator__current(self, entry);
+
+	return error;
+}
+
+static int tree_iterator__seek(git_iterator *self, const char *prefix)
+{
+	GIT_UNUSED(self);
+	GIT_UNUSED(prefix);
+	/* pop stack until matches prefix */
+	/* seek item in current frame matching prefix */
+	/* push stack which matches prefix */
+	return -1;
+}
+
+static void tree_iterator__free(git_iterator *self)
+{
+	tree_iterator *ti = (tree_iterator *)self;
+	while (ti->stack != NULL)
+		tree_iterator__pop_frame(ti);
+	git_buf_free(&ti->path);
+}
+
+static int tree_iterator__reset(git_iterator *self)
+{
+	tree_iterator *ti = (tree_iterator *)self;
+
+	while (ti->stack && ti->stack->next)
+		tree_iterator__pop_frame(ti);
+
+	if (ti->stack)
+		ti->stack->index =
+			git_tree__prefix_position(ti->stack->tree, ti->base.start);
+
+	git_buf_clear(&ti->path);
+
+	return tree_iterator__expand_tree(ti);
+}
+
+int git_iterator_for_tree_range(
+	git_iterator **iter,
+	git_repository *repo,
+	git_tree *tree,
+	const char *start,
+	const char *end)
+{
+	int error;
+	tree_iterator *ti;
+
+	if (tree == NULL)
+		return git_iterator_for_nothing(iter);
+
+	ITERATOR_BASE_INIT(ti, tree, TREE);
+
+	ti->repo  = repo;
+	ti->stack = tree_iterator__alloc_frame(tree, ti->base.start);
+
+	if ((error = tree_iterator__expand_tree(ti)) < 0)
+		git_iterator_free((git_iterator *)ti);
+	else
+		*iter = (git_iterator *)ti;
+
+	return error;
+}
+
+
+typedef struct {
+	git_iterator base;
+	git_index *index;
+	unsigned int current;
+} index_iterator;
+
+static int index_iterator__current(
+	git_iterator *self, const git_index_entry **entry)
+{
+	index_iterator *ii = (index_iterator *)self;
+	git_index_entry *ie = git_index_get(ii->index, ii->current);
+
+	if (ie != NULL &&
+		ii->base.end != NULL &&
+		git__prefixcmp(ie->path, ii->base.end) > 0)
+	{
+		ii->current = git_index_entrycount(ii->index);
+		ie = NULL;
+	}
+
+	if (entry)
+		*entry = ie;
+
+	return 0;
+}
+
+static int index_iterator__at_end(git_iterator *self)
+{
+	index_iterator *ii = (index_iterator *)self;
+	return (ii->current >= git_index_entrycount(ii->index));
+}
+
+static int index_iterator__advance(
+	git_iterator *self, const git_index_entry **entry)
+{
+	index_iterator *ii = (index_iterator *)self;
+
+	if (ii->current < git_index_entrycount(ii->index))
+		ii->current++;
+
+	return index_iterator__current(self, entry);
+}
+
+static int index_iterator__seek(git_iterator *self, const char *prefix)
+{
+	GIT_UNUSED(self);
+	GIT_UNUSED(prefix);
+	/* find last item before prefix */
+	return -1;
+}
+
+static int index_iterator__reset(git_iterator *self)
+{
+	index_iterator *ii = (index_iterator *)self;
+	ii->current = 0;
+	return 0;
+}
+
+static void index_iterator__free(git_iterator *self)
+{
+	index_iterator *ii = (index_iterator *)self;
+	git_index_free(ii->index);
+	ii->index = NULL;
+}
+
+int git_iterator_for_index_range(
+	git_iterator **iter,
+	git_repository *repo,
+	const char *start,
+	const char *end)
+{
+	int error;
+	index_iterator *ii;
+
+	ITERATOR_BASE_INIT(ii, index, INDEX);
+
+	if ((error = git_repository_index(&ii->index, repo)) < 0)
+		git__free(ii);
+	else {
+		ii->current = start ? git_index__prefix_position(ii->index, start) : 0;
+		*iter = (git_iterator *)ii;
+	}
+
+	return error;
+}
+
+
+typedef struct workdir_iterator_frame workdir_iterator_frame;
+struct workdir_iterator_frame {
+	workdir_iterator_frame *next;
+	git_vector entries;
+	unsigned int index;
+	char *start;
+};
+
+typedef struct {
+	git_iterator base;
+	git_repository *repo;
+	size_t root_len;
+	workdir_iterator_frame *stack;
+	git_ignores ignores;
+	git_index_entry entry;
+	git_buf path;
+	int is_ignored;
+} workdir_iterator;
+
+static workdir_iterator_frame *workdir_iterator__alloc_frame(void)
+{
+	workdir_iterator_frame *wf = git__calloc(1, sizeof(workdir_iterator_frame));
+	if (wf == NULL)
+		return NULL;
+	if (git_vector_init(&wf->entries, 0, git_path_with_stat_cmp) != 0) {
+		git__free(wf);
+		return NULL;
+	}
+	return wf;
+}
+
+static void workdir_iterator__free_frame(workdir_iterator_frame *wf)
+{
+	unsigned int i;
+	git_path_with_stat *path;
+
+	git_vector_foreach(&wf->entries, i, path)
+		git__free(path);
+	git_vector_free(&wf->entries);
+	git__free(wf);
+}
+
+static int workdir_iterator__update_entry(workdir_iterator *wi);
+
+static int workdir_iterator__entry_cmp(const void *prefix, const void *item)
+{
+	const git_path_with_stat *ps = item;
+	return git__prefixcmp((const char *)prefix, ps->path);
+}
+
+static int workdir_iterator__expand_dir(workdir_iterator *wi)
+{
+	int error;
+	workdir_iterator_frame *wf = workdir_iterator__alloc_frame();
+	GITERR_CHECK_ALLOC(wf);
+
+	error = git_path_dirload_with_stat(wi->path.ptr, wi->root_len, &wf->entries);
+	if (error < 0 || wf->entries.length == 0) {
+		workdir_iterator__free_frame(wf);
+		return GIT_ENOTFOUND;
+	}
+
+	git_vector_sort(&wf->entries);
+
+	if (!wi->stack)
+		wf->start = wi->base.start;
+	else if (wi->stack->start &&
+		git__prefixcmp(wi->stack->start, wi->path.ptr + wi->root_len) == 0)
+		wf->start = wi->stack->start;
+
+	if (wf->start)
+		git_vector_bsearch3(
+			&wf->index, &wf->entries, workdir_iterator__entry_cmp, wf->start);
+
+	wf->next  = wi->stack;
+	wi->stack = wf;
+
+	/* only push new ignores if this is not top level directory */
+	if (wi->stack->next != NULL) {
+		ssize_t slash_pos = git_buf_rfind_next(&wi->path, '/');
+		(void)git_ignore__push_dir(&wi->ignores, &wi->path.ptr[slash_pos + 1]);
+	}
+
+	return workdir_iterator__update_entry(wi);
+}
+
+static int workdir_iterator__current(
+	git_iterator *self, const git_index_entry **entry)
+{
+	workdir_iterator *wi = (workdir_iterator *)self;
+	*entry = (wi->entry.path == NULL) ? NULL : &wi->entry;
+	return 0;
+}
+
+static int workdir_iterator__at_end(git_iterator *self)
+{
+	return (((workdir_iterator *)self)->entry.path == NULL);
+}
+
+static int workdir_iterator__advance(
+	git_iterator *self, const git_index_entry **entry)
+{
+	int error;
+	workdir_iterator *wi = (workdir_iterator *)self;
+	workdir_iterator_frame *wf;
+	git_path_with_stat *next;
+
+	if (entry != NULL)
+		*entry = NULL;
+
+	if (wi->entry.path == NULL)
+		return 0;
+
+	while ((wf = wi->stack) != NULL) {
+		next = git_vector_get(&wf->entries, ++wf->index);
+		if (next != NULL) {
+			if (strcmp(next->path, DOT_GIT "/") == 0)
+				continue;
+			/* else found a good entry */
+			break;
+		}
+
+		/* pop workdir directory stack */
+		wi->stack = wf->next;
+		workdir_iterator__free_frame(wf);
+		git_ignore__pop_dir(&wi->ignores);
+
+		if (wi->stack == NULL) {
+			memset(&wi->entry, 0, sizeof(wi->entry));
+			return 0;
+		}
+	}
+
+	error = workdir_iterator__update_entry(wi);
+
+	if (!error && entry != NULL)
+		error = workdir_iterator__current(self, entry);
+
+	return error;
+}
+
+static int workdir_iterator__seek(git_iterator *self, const char *prefix)
+{
+	GIT_UNUSED(self);
+	GIT_UNUSED(prefix);
+	/* pop stack until matching prefix */
+	/* find prefix item in current frame */
+	/* push subdirectories as deep as possible while matching */
+	return 0;
+}
+
+static int workdir_iterator__reset(git_iterator *self)
+{
+	workdir_iterator *wi = (workdir_iterator *)self;
+	while (wi->stack != NULL && wi->stack->next != NULL) {
+		workdir_iterator_frame *wf = wi->stack;
+		wi->stack = wf->next;
+		workdir_iterator__free_frame(wf);
+		git_ignore__pop_dir(&wi->ignores);
+	}
+	if (wi->stack)
+		wi->stack->index = 0;
+	return 0;
+}
+
+static void workdir_iterator__free(git_iterator *self)
+{
+	workdir_iterator *wi = (workdir_iterator *)self;
+
+	while (wi->stack != NULL) {
+		workdir_iterator_frame *wf = wi->stack;
+		wi->stack = wf->next;
+		workdir_iterator__free_frame(wf);
+	}
+
+	git_ignore__free(&wi->ignores);
+	git_buf_free(&wi->path);
+}
+
+static int workdir_iterator__update_entry(workdir_iterator *wi)
+{
+	git_path_with_stat *ps = git_vector_get(&wi->stack->entries, wi->stack->index);
+
+	git_buf_truncate(&wi->path, wi->root_len);
+	memset(&wi->entry, 0, sizeof(wi->entry));
+
+	if (!ps)
+		return 0;
+
+	if (git_buf_put(&wi->path, ps->path, ps->path_len) < 0)
+		return -1;
+
+	if (wi->base.end &&
+		git__prefixcmp(wi->path.ptr + wi->root_len, wi->base.end) > 0)
+		return 0;
+
+	wi->entry.path = ps->path;
+
+	/* skip over .git directory */
+	if (strcmp(ps->path, DOT_GIT "/") == 0)
+		return workdir_iterator__advance((git_iterator *)wi, NULL);
+
+	/* if there is an error processing the entry, treat as ignored */
+	wi->is_ignored = 1;
+
+	git_index__init_entry_from_stat(&ps->st, &wi->entry);
+
+	/* need different mode here to keep directories during iteration */
+	wi->entry.mode = git_futils_canonical_mode(ps->st.st_mode);
+
+	/* if this is a file type we don't handle, treat as ignored */
+	if (wi->entry.mode == 0)
+		return 0;
+
+	/* okay, we are far enough along to look up real ignore rule */
+	if (git_ignore__lookup(&wi->ignores, wi->entry.path, &wi->is_ignored) < 0)
+		return 0; /* if error, ignore it and ignore file */
+
+	/* detect submodules */
+	if (S_ISDIR(wi->entry.mode)) {
+		bool is_submodule = git_path_contains(&wi->path, DOT_GIT);
+
+		/* if there is no .git, still check submodules data */
+		if (!is_submodule) {
+			int res = git_submodule_lookup(NULL, wi->repo, wi->entry.path);
+			is_submodule = (res == 0);
+			if (res == GIT_ENOTFOUND)
+				giterr_clear();
+		}
+
+		/* if submodule, mark as GITLINK and remove trailing slash */
+		if (is_submodule) {
+			size_t len = strlen(wi->entry.path);
+			assert(wi->entry.path[len - 1] == '/');
+			wi->entry.path[len - 1] = '\0';
+			wi->entry.mode = S_IFGITLINK;
+		}
+	}
+
+	return 0;
+}
+
+int git_iterator_for_workdir_range(
+	git_iterator **iter,
+	git_repository *repo,
+	const char *start,
+	const char *end)
+{
+	int error;
+	workdir_iterator *wi;
+
+	assert(iter && repo);
+
+	if (git_repository_is_bare(repo)) {
+		giterr_set(GITERR_INVALID,
+			"Cannot scan working directory for bare repo");
+		return -1;
+	}
+
+	ITERATOR_BASE_INIT(wi, workdir, WORKDIR);
+
+	wi->repo = repo;
+
+	if (git_buf_sets(&wi->path, git_repository_workdir(repo)) < 0 ||
+		git_path_to_dir(&wi->path) < 0 ||
+		git_ignore__for_path(repo, "", &wi->ignores) < 0)
+	{
+		git__free(wi);
+		return -1;
+	}
+
+	wi->root_len = wi->path.size;
+
+	if ((error = workdir_iterator__expand_dir(wi)) < 0) {
+		if (error == GIT_ENOTFOUND)
+			error = 0;
+		else {
+			git_iterator_free((git_iterator *)wi);
+			wi = NULL;
+		}
+	}
+
+	*iter = (git_iterator *)wi;
+
+	return error;
+}
+
+
+int git_iterator_current_tree_entry(
+	git_iterator *iter, const git_tree_entry **tree_entry)
+{
+	*tree_entry = (iter->type != GIT_ITERATOR_TREE) ? NULL :
+		tree_iterator__tree_entry((tree_iterator *)iter);
+	return 0;
+}
+
+int git_iterator_current_is_ignored(git_iterator *iter)
+{
+	return (iter->type != GIT_ITERATOR_WORKDIR) ? 0 :
+		((workdir_iterator *)iter)->is_ignored;
+}
+
+int git_iterator_advance_into_directory(
+	git_iterator *iter, const git_index_entry **entry)
+{
+	workdir_iterator *wi = (workdir_iterator *)iter;
+
+	if (iter->type == GIT_ITERATOR_WORKDIR &&
+		wi->entry.path &&
+		S_ISDIR(wi->entry.mode) &&
+		!S_ISGITLINK(wi->entry.mode))
+	{
+		if (workdir_iterator__expand_dir(wi) < 0)
+			/* if error loading or if empty, skip the directory. */
+			return workdir_iterator__advance(iter, entry);
+	}
+
+	return entry ? git_iterator_current(iter, entry) : 0;
+}
+
+int git_iterator_cmp(
+	git_iterator *iter, const char *path_prefix)
+{
+	const git_index_entry *entry;
+
+	/* a "done" iterator is after every prefix */
+	if (git_iterator_current(iter, &entry) < 0 ||
+		entry == NULL)
+		return 1;
+
+	/* a NULL prefix is after any valid iterator */
+	if (!path_prefix)
+		return -1;
+
+	return git__prefixcmp(entry->path, path_prefix);
+}
+
diff --git a/src/iterator.h b/src/iterator.h
new file mode 100644
index 000000000..b916a9080
--- /dev/null
+++ b/src/iterator.h
@@ -0,0 +1,151 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_iterator_h__
+#define INCLUDE_iterator_h__
+
+#include "common.h"
+#include "git2/index.h"
+
+typedef struct git_iterator git_iterator;
+
+typedef enum {
+	GIT_ITERATOR_EMPTY = 0,
+	GIT_ITERATOR_TREE = 1,
+	GIT_ITERATOR_INDEX = 2,
+	GIT_ITERATOR_WORKDIR = 3
+} git_iterator_type_t;
+
+struct git_iterator {
+	git_iterator_type_t type;
+	char *start;
+	char *end;
+	int (*current)(git_iterator *, const git_index_entry **);
+	int (*at_end)(git_iterator *);
+	int (*advance)(git_iterator *, const git_index_entry **);
+	int (*seek)(git_iterator *, const char *prefix);
+	int (*reset)(git_iterator *);
+	void (*free)(git_iterator *);
+};
+
+extern int git_iterator_for_nothing(git_iterator **iter);
+
+extern int git_iterator_for_tree_range(
+	git_iterator **iter, git_repository *repo, git_tree *tree,
+	const char *start, const char *end);
+
+GIT_INLINE(int) git_iterator_for_tree(
+	git_iterator **iter, git_repository *repo, git_tree *tree)
+{
+	return git_iterator_for_tree_range(iter, repo, tree, NULL, NULL);
+}
+
+extern int git_iterator_for_index_range(
+	git_iterator **iter, git_repository *repo,
+	const char *start, const char *end);
+
+GIT_INLINE(int) git_iterator_for_index(
+	git_iterator **iter, git_repository *repo)
+{
+	return git_iterator_for_index_range(iter, repo, NULL, NULL);
+}
+
+extern int git_iterator_for_workdir_range(
+	git_iterator **iter, git_repository *repo,
+	const char *start, const char *end);
+
+GIT_INLINE(int) git_iterator_for_workdir(
+	git_iterator **iter, git_repository *repo)
+{
+	return git_iterator_for_workdir_range(iter, repo, NULL, NULL);
+}
+
+
+/* Entry is not guaranteed to be fully populated.  For a tree iterator,
+ * we will only populate the mode, oid and path, for example.  For a workdir
+ * iterator, we will not populate the oid.
+ *
+ * You do not need to free the entry.  It is still "owned" by the iterator.
+ * Once you call `git_iterator_advance`, then content of the old entry is
+ * no longer guaranteed to be valid.
+ */
+GIT_INLINE(int) git_iterator_current(
+	git_iterator *iter, const git_index_entry **entry)
+{
+	return iter->current(iter, entry);
+}
+
+GIT_INLINE(int) git_iterator_at_end(git_iterator *iter)
+{
+	return iter->at_end(iter);
+}
+
+GIT_INLINE(int) git_iterator_advance(
+	git_iterator *iter, const git_index_entry **entry)
+{
+	return iter->advance(iter, entry);
+}
+
+GIT_INLINE(int) git_iterator_seek(
+	git_iterator *iter, const char *prefix)
+{
+	return iter->seek(iter, prefix);
+}
+
+GIT_INLINE(int) git_iterator_reset(git_iterator *iter)
+{
+	return iter->reset(iter);
+}
+
+GIT_INLINE(void) git_iterator_free(git_iterator *iter)
+{
+	if (iter == NULL)
+		return;
+
+	iter->free(iter);
+
+	git__free(iter->start);
+	git__free(iter->end);
+
+	memset(iter, 0, sizeof(*iter));
+
+	git__free(iter);
+}
+
+GIT_INLINE(git_iterator_type_t) git_iterator_type(git_iterator *iter)
+{
+	return iter->type;
+}
+
+extern int git_iterator_current_tree_entry(
+	git_iterator *iter, const git_tree_entry **tree_entry);
+
+extern int git_iterator_current_is_ignored(git_iterator *iter);
+
+/**
+ * Iterate into a workdir directory.
+ *
+ * Workdir iterators do not automatically descend into directories (so that
+ * when comparing two iterator entries you can detect a newly created
+ * directory in the workdir).  As a result, you may get S_ISDIR items from
+ * a workdir iterator.  If you wish to iterate over the contents of the
+ * directories you encounter, then call this function when you encounter
+ * a directory.
+ *
+ * If there are no files in the directory, this will end up acting like a
+ * regular advance and will skip past the directory, so you should be
+ * prepared for that case.
+ *
+ * On non-workdir iterators or if not pointing at a directory, this is a
+ * no-op and will not advance the iterator.
+ */
+extern int git_iterator_advance_into_directory(
+	git_iterator *iter, const git_index_entry **entry);
+
+extern int git_iterator_cmp(
+	git_iterator *iter, const char *path_prefix);
+
+#endif
diff --git a/src/khash.h b/src/khash.h
new file mode 100644
index 000000000..bd67fe1f7
--- /dev/null
+++ b/src/khash.h
@@ -0,0 +1,608 @@
+/* The MIT License
+
+   Copyright (c) 2008, 2009, 2011 by Attractive Chaos <attractor@live.co.uk>
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/*
+  An example:
+
+#include "khash.h"
+KHASH_MAP_INIT_INT(32, char)
+int main() {
+	int ret, is_missing;
+	khiter_t k;
+	khash_t(32) *h = kh_init(32);
+	k = kh_put(32, h, 5, &ret);
+	kh_value(h, k) = 10;
+	k = kh_get(32, h, 10);
+	is_missing = (k == kh_end(h));
+	k = kh_get(32, h, 5);
+	kh_del(32, h, k);
+	for (k = kh_begin(h); k != kh_end(h); ++k)
+		if (kh_exist(h, k)) kh_value(h, k) = 1;
+	kh_destroy(32, h);
+	return 0;
+}
+*/
+
+/*
+  2011-12-29 (0.2.7):
+
+    * Minor code clean up; no actual effect.
+
+  2011-09-16 (0.2.6):
+
+	* The capacity is a power of 2. This seems to dramatically improve the
+	  speed for simple keys. Thank Zilong Tan for the suggestion. Reference:
+
+	   - http://code.google.com/p/ulib/
+	   - http://nothings.org/computer/judy/
+
+	* Allow to optionally use linear probing which usually has better
+	  performance for random input. Double hashing is still the default as it
+	  is more robust to certain non-random input.
+
+	* Added Wang's integer hash function (not used by default). This hash
+	  function is more robust to certain non-random input.
+
+  2011-02-14 (0.2.5):
+
+    * Allow to declare global functions.
+
+  2009-09-26 (0.2.4):
+
+    * Improve portability
+
+  2008-09-19 (0.2.3):
+
+	* Corrected the example
+	* Improved interfaces
+
+  2008-09-11 (0.2.2):
+
+	* Improved speed a little in kh_put()
+
+  2008-09-10 (0.2.1):
+
+	* Added kh_clear()
+	* Fixed a compiling error
+
+  2008-09-02 (0.2.0):
+
+	* Changed to token concatenation which increases flexibility.
+
+  2008-08-31 (0.1.2):
+
+	* Fixed a bug in kh_get(), which has not been tested previously.
+
+  2008-08-31 (0.1.1):
+
+	* Added destructor
+*/
+
+
+#ifndef __AC_KHASH_H
+#define __AC_KHASH_H
+
+/*!
+  @header
+
+  Generic hash table library.
+ */
+
+#define AC_VERSION_KHASH_H "0.2.6"
+
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+/* compipler specific configuration */
+
+#if UINT_MAX == 0xffffffffu
+typedef unsigned int khint32_t;
+#elif ULONG_MAX == 0xffffffffu
+typedef unsigned long khint32_t;
+#endif
+
+#if ULONG_MAX == ULLONG_MAX
+typedef unsigned long khint64_t;
+#else
+typedef unsigned long long khint64_t;
+#endif
+
+#ifdef _MSC_VER
+#define inline __inline
+#endif
+
+typedef khint32_t khint_t;
+typedef khint_t khiter_t;
+
+#define __ac_isempty(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&2)
+#define __ac_isdel(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&1)
+#define __ac_iseither(flag, i) ((flag[i>>4]>>((i&0xfU)<<1))&3)
+#define __ac_set_isdel_false(flag, i) (flag[i>>4]&=~(1ul<<((i&0xfU)<<1)))
+#define __ac_set_isempty_false(flag, i) (flag[i>>4]&=~(2ul<<((i&0xfU)<<1)))
+#define __ac_set_isboth_false(flag, i) (flag[i>>4]&=~(3ul<<((i&0xfU)<<1)))
+#define __ac_set_isdel_true(flag, i) (flag[i>>4]|=1ul<<((i&0xfU)<<1))
+
+#ifdef KHASH_LINEAR
+#define __ac_inc(k, m) 1
+#else
+#define __ac_inc(k, m) (((k)>>3 ^ (k)<<3) | 1) & (m)
+#endif
+
+#define __ac_fsize(m) ((m) < 16? 1 : (m)>>4)
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#ifndef kcalloc
+#define kcalloc(N,Z) calloc(N,Z)
+#endif
+#ifndef kmalloc
+#define kmalloc(Z) malloc(Z)
+#endif
+#ifndef krealloc
+#define krealloc(P,Z) realloc(P,Z)
+#endif
+#ifndef kfree
+#define kfree(P) free(P)
+#endif
+
+static const double __ac_HASH_UPPER = 0.77;
+
+#define __KHASH_TYPE(name, khkey_t, khval_t) \
+	typedef struct { \
+		khint_t n_buckets, size, n_occupied, upper_bound; \
+		khint32_t *flags; \
+		khkey_t *keys; \
+		khval_t *vals; \
+	} kh_##name##_t;
+
+#define __KHASH_PROTOTYPES(name, khkey_t, khval_t)	 					\
+	extern kh_##name##_t *kh_init_##name(void);							\
+	extern void kh_destroy_##name(kh_##name##_t *h);					\
+	extern void kh_clear_##name(kh_##name##_t *h);						\
+	extern khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key); 	\
+	extern int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets); \
+	extern khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret); \
+	extern void kh_del_##name(kh_##name##_t *h, khint_t x);
+
+#define __KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	SCOPE kh_##name##_t *kh_init_##name(void) {							\
+		return (kh_##name##_t*)kcalloc(1, sizeof(kh_##name##_t));		\
+	}																	\
+	SCOPE void kh_destroy_##name(kh_##name##_t *h)						\
+	{																	\
+		if (h) {														\
+			kfree((void *)h->keys); kfree(h->flags);					\
+			kfree((void *)h->vals);										\
+			kfree(h);													\
+		}																\
+	}																	\
+	SCOPE void kh_clear_##name(kh_##name##_t *h)						\
+	{																	\
+		if (h && h->flags) {											\
+			memset(h->flags, 0xaa, __ac_fsize(h->n_buckets) * sizeof(khint32_t)); \
+			h->size = h->n_occupied = 0;								\
+		}																\
+	}																	\
+	SCOPE khint_t kh_get_##name(const kh_##name##_t *h, khkey_t key) 	\
+	{																	\
+		if (h->n_buckets) {												\
+			khint_t inc, k, i, last, mask;								\
+			mask = h->n_buckets - 1;									\
+			k = __hash_func(key); i = k & mask;							\
+			inc = __ac_inc(k, mask); last = i; /* inc==1 for linear probing */ \
+			while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+				i = (i + inc) & mask; 									\
+				if (i == last) return h->n_buckets;						\
+			}															\
+			return __ac_iseither(h->flags, i)? h->n_buckets : i;		\
+		} else return 0;												\
+	}																	\
+	SCOPE int kh_resize_##name(kh_##name##_t *h, khint_t new_n_buckets) \
+	{ /* This function uses 0.25*n_bucktes bytes of working space instead of [sizeof(key_t+val_t)+.25]*n_buckets. */ \
+		khint32_t *new_flags = 0;										\
+		khint_t j = 1;													\
+		{																\
+			kroundup32(new_n_buckets); 									\
+			if (new_n_buckets < 4) new_n_buckets = 4;					\
+			if (h->size >= (khint_t)(new_n_buckets * __ac_HASH_UPPER + 0.5)) j = 0;	/* requested size is too small */ \
+			else { /* hash table size to be changed (shrink or expand); rehash */ \
+				new_flags = (khint32_t*)kmalloc(__ac_fsize(new_n_buckets) * sizeof(khint32_t));	\
+				if (!new_flags) return -1;								\
+				memset(new_flags, 0xaa, __ac_fsize(new_n_buckets) * sizeof(khint32_t)); \
+				if (h->n_buckets < new_n_buckets) {	/* expand */		\
+					khkey_t *new_keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+					if (!new_keys) return -1;							\
+					h->keys = new_keys;									\
+					if (kh_is_map) {									\
+						khval_t *new_vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+						if (!new_vals) return -1;						\
+						h->vals = new_vals;								\
+					}													\
+				} /* otherwise shrink */								\
+			}															\
+		}																\
+		if (j) { /* rehashing is needed */								\
+			for (j = 0; j != h->n_buckets; ++j) {						\
+				if (__ac_iseither(h->flags, j) == 0) {					\
+					khkey_t key = h->keys[j];							\
+					khval_t val;										\
+					khint_t new_mask;									\
+					new_mask = new_n_buckets - 1; 						\
+					if (kh_is_map) val = h->vals[j];					\
+					__ac_set_isdel_true(h->flags, j);					\
+					while (1) { /* kick-out process; sort of like in Cuckoo hashing */ \
+						khint_t inc, k, i;								\
+						k = __hash_func(key);							\
+						i = k & new_mask;								\
+						inc = __ac_inc(k, new_mask);					\
+						while (!__ac_isempty(new_flags, i)) i = (i + inc) & new_mask; \
+						__ac_set_isempty_false(new_flags, i);			\
+						if (i < h->n_buckets && __ac_iseither(h->flags, i) == 0) { /* kick out the existing element */ \
+							{ khkey_t tmp = h->keys[i]; h->keys[i] = key; key = tmp; } \
+							if (kh_is_map) { khval_t tmp = h->vals[i]; h->vals[i] = val; val = tmp; } \
+							__ac_set_isdel_true(h->flags, i); /* mark it as deleted in the old hash table */ \
+						} else { /* write the element and jump out of the loop */ \
+							h->keys[i] = key;							\
+							if (kh_is_map) h->vals[i] = val;			\
+							break;										\
+						}												\
+					}													\
+				}														\
+			}															\
+			if (h->n_buckets > new_n_buckets) { /* shrink the hash table */ \
+				h->keys = (khkey_t*)krealloc((void *)h->keys, new_n_buckets * sizeof(khkey_t)); \
+				if (kh_is_map) h->vals = (khval_t*)krealloc((void *)h->vals, new_n_buckets * sizeof(khval_t)); \
+			}															\
+			kfree(h->flags); /* free the working space */				\
+			h->flags = new_flags;										\
+			h->n_buckets = new_n_buckets;								\
+			h->n_occupied = h->size;									\
+			h->upper_bound = (khint_t)(h->n_buckets * __ac_HASH_UPPER + 0.5); \
+		}																\
+		return 0;														\
+	}																	\
+	SCOPE khint_t kh_put_##name(kh_##name##_t *h, khkey_t key, int *ret) \
+	{																	\
+		khint_t x;														\
+		if (h->n_occupied >= h->upper_bound) { /* update the hash table */ \
+			if (h->n_buckets > (h->size<<1)) {							\
+				if (kh_resize_##name(h, h->n_buckets - 1) < 0) { /* clear "deleted" elements */ \
+					*ret = -1; return h->n_buckets;						\
+				}														\
+			} else if (kh_resize_##name(h, h->n_buckets + 1) < 0) { /* expand the hash table */ \
+				*ret = -1; return h->n_buckets;							\
+			}															\
+		} /* TODO: to implement automatically shrinking; resize() already support shrinking */ \
+		{																\
+			khint_t inc, k, i, site, last, mask = h->n_buckets - 1;		\
+			x = site = h->n_buckets; k = __hash_func(key); i = k & mask; \
+			if (__ac_isempty(h->flags, i)) x = i; /* for speed up */	\
+			else {														\
+				inc = __ac_inc(k, mask); last = i;						\
+				while (!__ac_isempty(h->flags, i) && (__ac_isdel(h->flags, i) || !__hash_equal(h->keys[i], key))) { \
+					if (__ac_isdel(h->flags, i)) site = i;				\
+					i = (i + inc) & mask; 								\
+					if (i == last) { x = site; break; }					\
+				}														\
+				if (x == h->n_buckets) {								\
+					if (__ac_isempty(h->flags, i) && site != h->n_buckets) x = site; \
+					else x = i;											\
+				}														\
+			}															\
+		}																\
+		if (__ac_isempty(h->flags, x)) { /* not present at all */		\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size; ++h->n_occupied;									\
+			*ret = 1;													\
+		} else if (__ac_isdel(h->flags, x)) { /* deleted */				\
+			h->keys[x] = key;											\
+			__ac_set_isboth_false(h->flags, x);							\
+			++h->size;													\
+			*ret = 2;													\
+		} else *ret = 0; /* Don't touch h->keys[x] if present and not deleted */ \
+		return x;														\
+	}																	\
+	SCOPE void kh_del_##name(kh_##name##_t *h, khint_t x)				\
+	{																	\
+		if (x != h->n_buckets && !__ac_iseither(h->flags, x)) {			\
+			__ac_set_isdel_true(h->flags, x);							\
+			--h->size;													\
+		}																\
+	}
+
+#define KHASH_DECLARE(name, khkey_t, khval_t)		 					\
+	__KHASH_TYPE(name, khkey_t, khval_t) 								\
+	__KHASH_PROTOTYPES(name, khkey_t, khval_t)
+
+#define KHASH_INIT2(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	__KHASH_TYPE(name, khkey_t, khval_t) 								\
+	__KHASH_IMPL(name, SCOPE, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+#define KHASH_INIT(name, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal) \
+	KHASH_INIT2(name, static inline, khkey_t, khval_t, kh_is_map, __hash_func, __hash_equal)
+
+/* --- BEGIN OF HASH FUNCTIONS --- */
+
+/*! @function
+  @abstract     Integer hash function
+  @param  key   The integer [khint32_t]
+  @return       The hash value [khint_t]
+ */
+#define kh_int_hash_func(key) (khint32_t)(key)
+/*! @function
+  @abstract     Integer comparison function
+ */
+#define kh_int_hash_equal(a, b) ((a) == (b))
+/*! @function
+  @abstract     64-bit integer hash function
+  @param  key   The integer [khint64_t]
+  @return       The hash value [khint_t]
+ */
+#define kh_int64_hash_func(key) (khint32_t)((key)>>33^(key)^(key)<<11)
+/*! @function
+  @abstract     64-bit integer comparison function
+ */
+#define kh_int64_hash_equal(a, b) ((a) == (b))
+/*! @function
+  @abstract     const char* hash function
+  @param  s     Pointer to a null terminated string
+  @return       The hash value
+ */
+static inline khint_t __ac_X31_hash_string(const char *s)
+{
+	khint_t h = (khint_t)*s;
+	if (h) for (++s ; *s; ++s) h = (h << 5) - h + (khint_t)*s;
+	return h;
+}
+/*! @function
+  @abstract     Another interface to const char* hash function
+  @param  key   Pointer to a null terminated string [const char*]
+  @return       The hash value [khint_t]
+ */
+#define kh_str_hash_func(key) __ac_X31_hash_string(key)
+/*! @function
+  @abstract     Const char* comparison function
+ */
+#define kh_str_hash_equal(a, b) (strcmp(a, b) == 0)
+
+static inline khint_t __ac_Wang_hash(khint_t key)
+{
+    key += ~(key << 15);
+    key ^=  (key >> 10);
+    key +=  (key << 3);
+    key ^=  (key >> 6);
+    key += ~(key << 11);
+    key ^=  (key >> 16);
+    return key;
+}
+#define kh_int_hash_func2(k) __ac_Wang_hash((khint_t)key)
+
+/* --- END OF HASH FUNCTIONS --- */
+
+/* Other convenient macros... */
+
+/*!
+  @abstract Type of the hash table.
+  @param  name  Name of the hash table [symbol]
+ */
+#define khash_t(name) kh_##name##_t
+
+/*! @function
+  @abstract     Initiate a hash table.
+  @param  name  Name of the hash table [symbol]
+  @return       Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_init(name) kh_init_##name()
+
+/*! @function
+  @abstract     Destroy a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_destroy(name, h) kh_destroy_##name(h)
+
+/*! @function
+  @abstract     Reset a hash table without deallocating memory.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+ */
+#define kh_clear(name, h) kh_clear_##name(h)
+
+/*! @function
+  @abstract     Resize a hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  s     New size [khint_t]
+ */
+#define kh_resize(name, h, s) kh_resize_##name(h, s)
+
+/*! @function
+  @abstract     Insert a key to the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Key [type of keys]
+  @param  r     Extra return code: 0 if the key is present in the hash table;
+                1 if the bucket is empty (never used); 2 if the element in
+				the bucket has been deleted [int*]
+  @return       Iterator to the inserted element [khint_t]
+ */
+#define kh_put(name, h, k, r) kh_put_##name(h, k, r)
+
+/*! @function
+  @abstract     Retrieve a key from the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Key [type of keys]
+  @return       Iterator to the found element, or kh_end(h) is the element is absent [khint_t]
+ */
+#define kh_get(name, h, k) kh_get_##name(h, k)
+
+/*! @function
+  @abstract     Remove a key from the hash table.
+  @param  name  Name of the hash table [symbol]
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  k     Iterator to the element to be deleted [khint_t]
+ */
+#define kh_del(name, h, k) kh_del_##name(h, k)
+
+/*! @function
+  @abstract     Test whether a bucket contains data.
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       1 if containing data; 0 otherwise [int]
+ */
+#define kh_exist(h, x) (!__ac_iseither((h)->flags, (x)))
+
+/*! @function
+  @abstract     Get key given an iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       Key [type of keys]
+ */
+#define kh_key(h, x) ((h)->keys[x])
+
+/*! @function
+  @abstract     Get value given an iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  x     Iterator to the bucket [khint_t]
+  @return       Value [type of values]
+  @discussion   For hash sets, calling this results in segfault.
+ */
+#define kh_val(h, x) ((h)->vals[x])
+
+/*! @function
+  @abstract     Alias of kh_val()
+ */
+#define kh_value(h, x) ((h)->vals[x])
+
+/*! @function
+  @abstract     Get the start iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       The start iterator [khint_t]
+ */
+#define kh_begin(h) (khint_t)(0)
+
+/*! @function
+  @abstract     Get the end iterator
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       The end iterator [khint_t]
+ */
+#define kh_end(h) ((h)->n_buckets)
+
+/*! @function
+  @abstract     Get the number of elements in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       Number of elements in the hash table [khint_t]
+ */
+#define kh_size(h) ((h)->size)
+
+/*! @function
+  @abstract     Get the number of buckets in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @return       Number of buckets in the hash table [khint_t]
+ */
+#define kh_n_buckets(h) ((h)->n_buckets)
+
+/*! @function
+  @abstract     Iterate over the entries in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  kvar  Variable to which key will be assigned
+  @param  vvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach(h, kvar, vvar, code) { khint_t __i;		\
+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\
+		if (!kh_exist(h,__i)) continue;						\
+		(kvar) = kh_key(h,__i);								\
+		(vvar) = kh_val(h,__i);								\
+		code;												\
+	} }
+
+/*! @function
+  @abstract     Iterate over the values in the hash table
+  @param  h     Pointer to the hash table [khash_t(name)*]
+  @param  vvar  Variable to which value will be assigned
+  @param  code  Block of code to execute
+ */
+#define kh_foreach_value(h, vvar, code) { khint_t __i;		\
+	for (__i = kh_begin(h); __i != kh_end(h); ++__i) {		\
+		if (!kh_exist(h,__i)) continue;						\
+		(vvar) = kh_val(h,__i);								\
+		code;												\
+	} }
+
+/* More conenient interfaces */
+
+/*! @function
+  @abstract     Instantiate a hash set containing integer keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT(name)										\
+	KHASH_INIT(name, khint32_t, char, 0, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT(name, khval_t)								\
+	KHASH_INIT(name, khint32_t, khval_t, 1, kh_int_hash_func, kh_int_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing 64-bit integer keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_INT64(name)										\
+	KHASH_INIT(name, khint64_t, char, 0, kh_int64_hash_func, kh_int64_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing 64-bit integer keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_INT64(name, khval_t)								\
+	KHASH_INIT(name, khint64_t, khval_t, 1, kh_int64_hash_func, kh_int64_hash_equal)
+
+typedef const char *kh_cstr_t;
+/*! @function
+  @abstract     Instantiate a hash map containing const char* keys
+  @param  name  Name of the hash table [symbol]
+ */
+#define KHASH_SET_INIT_STR(name)										\
+	KHASH_INIT(name, kh_cstr_t, char, 0, kh_str_hash_func, kh_str_hash_equal)
+
+/*! @function
+  @abstract     Instantiate a hash map containing const char* keys
+  @param  name  Name of the hash table [symbol]
+  @param  khval_t  Type of values [type]
+ */
+#define KHASH_MAP_INIT_STR(name, khval_t)								\
+	KHASH_INIT(name, kh_cstr_t, khval_t, 1, kh_str_hash_func, kh_str_hash_equal)
+
+#endif /* __AC_KHASH_H */
diff --git a/src/map.h b/src/map.h
index be569abc8..96d879547 100644
--- a/src/map.h
+++ b/src/map.h
@@ -1,31 +1,42 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_map_h__
 #define INCLUDE_map_h__
 
 #include "common.h"
 
 
-/* git__mmap() prot values */
-#define GIT_PROT_NONE  0x0
-#define GIT_PROT_READ  0x1
+/* p_mmap() prot values */
+#define GIT_PROT_NONE 0x0
+#define GIT_PROT_READ 0x1
 #define GIT_PROT_WRITE 0x2
-#define GIT_PROT_EXEC  0x4
+#define GIT_PROT_EXEC 0x4
 
 /* git__mmmap() flags values */
-#define GIT_MAP_FILE    0
-#define GIT_MAP_SHARED  1
+#define GIT_MAP_FILE	0
+#define GIT_MAP_SHARED 1
 #define GIT_MAP_PRIVATE 2
-#define GIT_MAP_TYPE    0xf
-#define GIT_MAP_FIXED   0x10
+#define GIT_MAP_TYPE	0xf
+#define GIT_MAP_FIXED	0x10
 
-typedef struct {  /* memory mapped buffer   */
-	void *data;  /* data bytes          */
-	size_t len;  /* data length         */
+typedef struct { /* memory mapped buffer	*/
+	void *data; /* data bytes			*/
+	size_t len; /* data length			*/
 #ifdef GIT_WIN32
-	HANDLE fmh;  /* file mapping handle */
+	HANDLE fmh; /* file mapping handle */
 #endif
 } git_map;
 
-extern int git__mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset);
-extern int git__munmap(git_map *map);
+#define GIT_MMAP_VALIDATE(out, len, prot, flags) do { \
+	assert(out != NULL && len > 0); \
+	assert((prot & GIT_PROT_WRITE) || (prot & GIT_PROT_READ)); \
+	assert((flags & GIT_MAP_FIXED) == 0); } while (0)
+
+extern int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset);
+extern int p_munmap(git_map *map);
 
 #endif /* INCLUDE_map_h__ */
diff --git a/src/message.c b/src/message.c
new file mode 100644
index 000000000..aa0220fd0
--- /dev/null
+++ b/src/message.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "message.h"
+#include <ctype.h>
+
+static size_t line_length_without_trailing_spaces(const char *line, size_t len)
+{
+	while (len) {
+		unsigned char c = line[len - 1];
+		if (!git__isspace(c))
+			break;
+		len--;
+	}
+
+	return len;
+}
+
+/* Greatly inspired from git.git "stripspace" */
+/* see https://github.com/git/git/blob/497215d8811ac7b8955693ceaad0899ecd894ed2/builtin/stripspace.c#L4-67 */
+int git_message_prettify(git_buf *message_out, const char *message, int strip_comments)
+{
+	const size_t message_len = strlen(message);
+
+	int consecutive_empty_lines = 0;
+	size_t i, line_length, rtrimmed_line_length;
+	char *next_newline;
+
+	for (i = 0; i < strlen(message); i += line_length) {
+		next_newline = memchr(message + i, '\n', message_len - i);
+
+		if (next_newline != NULL) {
+			line_length = next_newline - (message + i) + 1;
+		} else {
+			line_length = message_len - i;
+		}
+
+		if (strip_comments && line_length && message[i] == '#')
+			continue;
+
+		rtrimmed_line_length = line_length_without_trailing_spaces(message + i, line_length);
+
+		if (!rtrimmed_line_length) {
+			consecutive_empty_lines++;
+			continue;
+		}
+
+		if (consecutive_empty_lines > 0 && message_out->size > 0)
+			git_buf_putc(message_out, '\n');
+
+		consecutive_empty_lines = 0;
+		git_buf_put(message_out, message + i, rtrimmed_line_length);
+		git_buf_putc(message_out, '\n');
+	}
+
+	return git_buf_oom(message_out) ? -1 : 0;
+}
diff --git a/src/message.h b/src/message.h
new file mode 100644
index 000000000..ddfa13e18
--- /dev/null
+++ b/src/message.h
@@ -0,0 +1,14 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_message_h__
+#define INCLUDE_message_h__
+
+#include "buffer.h"
+
+int git_message_prettify(git_buf *message_out, const char *message, int strip_comments);
+
+#endif /* INCLUDE_message_h__ */
diff --git a/src/mingw-compat.h b/src/mingw-compat.h
deleted file mode 100644
index b7919c2e8..000000000
--- a/src/mingw-compat.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef INCLUDE_mingw_compat__
-#define INCLUDE_mingw_compat__
-
-#if defined(__MINGW32__)
-
-/* use a 64-bit file offset type */
-# define lseek _lseeki64
-# define stat _stati64
-# define fstat _fstati64
-
-#endif
-
-#endif /* INCLUDE_mingw_compat__ */
diff --git a/src/msvc-compat.h b/src/msvc-compat.h
deleted file mode 100644
index 1ec85f91b..000000000
--- a/src/msvc-compat.h
+++ /dev/null
@@ -1,46 +0,0 @@
-#ifndef INCLUDE_msvc_compat__
-#define INCLUDE_msvc_compat__
-
-#if defined(_MSC_VER)
-
-/* access() mode parameter #defines   */
-# define F_OK 0  /* existence  check */
-# define W_OK 2  /* write mode check */
-# define R_OK 4  /* read  mode check */
-
-# define lseek _lseeki64
-# define stat _stat64
-# define fstat _fstat64
-
-/* stat: file mode type testing macros */
-# define S_ISDIR(m)   (((m) & _S_IFMT) == _S_IFDIR)
-# define S_ISREG(m)   (((m) & _S_IFMT) == _S_IFREG)
-# define S_ISFIFO(m)  (((m) & _S_IFMT) == _S_IFIFO)
-
-/* case-insensitive string comparison */
-# define strcasecmp   _stricmp
-# define strncasecmp  _strnicmp
-
-#if (_MSC_VER >= 1600)
-#	include <stdint.h>
-#else
-/* add some missing <stdint.h> typedef's */
-typedef signed char int8_t;
-typedef unsigned char uint8_t;
-
-typedef short int16_t;
-typedef unsigned short uint16_t;
-
-typedef long int32_t;
-typedef unsigned long uint32_t;
-
-typedef long long int64_t;
-typedef unsigned long long uint64_t;
-
-typedef long long intmax_t;
-typedef unsigned long long uintmax_t;
-#endif
-
-#endif
-
-#endif /* INCLUDE_msvc_compat__ */
diff --git a/src/mwindow.c b/src/mwindow.c
new file mode 100644
index 000000000..57adabd48
--- /dev/null
+++ b/src/mwindow.c
@@ -0,0 +1,270 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "mwindow.h"
+#include "vector.h"
+#include "fileops.h"
+#include "map.h"
+#include "global.h"
+
+#define DEFAULT_WINDOW_SIZE \
+	(sizeof(void*) >= 8 \
+		? 1 * 1024 * 1024 * 1024 \
+		: 32 * 1024 * 1024)
+
+#define DEFAULT_MAPPED_LIMIT \
+	((1024 * 1024) * (sizeof(void*) >= 8 ? 8192ULL : 256UL))
+
+/*
+ * These are the global options for mmmap limits.
+ * TODO: allow the user to change these
+ */
+static struct {
+	size_t window_size;
+	size_t mapped_limit;
+} _mw_options = {
+	DEFAULT_WINDOW_SIZE,
+	DEFAULT_MAPPED_LIMIT,
+};
+
+/*
+ * Free all the windows in a sequence, typically because we're done
+ * with the file
+ */
+void git_mwindow_free_all(git_mwindow_file *mwf)
+{
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
+	unsigned int i;
+	/*
+	 * Remove these windows from the global list
+	 */
+	for (i = 0; i < ctl->windowfiles.length; ++i){
+		if (git_vector_get(&ctl->windowfiles, i) == mwf) {
+			git_vector_remove(&ctl->windowfiles, i);
+			break;
+		}
+	}
+
+	if (ctl->windowfiles.length == 0) {
+		git_vector_free(&ctl->windowfiles);
+		ctl->windowfiles.contents = NULL;
+	}
+
+	while (mwf->windows) {
+		git_mwindow *w = mwf->windows;
+		assert(w->inuse_cnt == 0);
+
+		ctl->mapped -= w->window_map.len;
+		ctl->open_windows--;
+
+		git_futils_mmap_free(&w->window_map);
+
+		mwf->windows = w->next;
+		git__free(w);
+	}
+}
+
+/*
+ * Check if a window 'win' contains the address 'offset'
+ */
+int git_mwindow_contains(git_mwindow *win, git_off_t offset)
+{
+	git_off_t win_off = win->offset;
+	return win_off <= offset
+		&& offset <= (git_off_t)(win_off + win->window_map.len);
+}
+
+/*
+ * Find the least-recently-used window in a file
+ */
+void git_mwindow_scan_lru(
+	git_mwindow_file *mwf,
+	git_mwindow **lru_w,
+	git_mwindow **lru_l)
+{
+	git_mwindow *w, *w_l;
+
+	for (w_l = NULL, w = mwf->windows; w; w = w->next) {
+		if (!w->inuse_cnt) {
+			/*
+			 * If the current one is more recent than the last one,
+			 * store it in the output parameter. If lru_w is NULL,
+			 * it's the first loop, so store it as well.
+			 */
+			if (!*lru_w || w->last_used < (*lru_w)->last_used) {
+				*lru_w = w;
+				*lru_l = w_l;
+			}
+		}
+		w_l = w;
+	}
+}
+
+/*
+ * Close the least recently used window. You should check to see if
+ * the file descriptors need closing from time to time.
+ */
+static int git_mwindow_close_lru(git_mwindow_file *mwf)
+{
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
+	unsigned int i;
+	git_mwindow *lru_w = NULL, *lru_l = NULL, **list = &mwf->windows;
+
+	/* FIXME: Does this give us any advantage? */
+	if(mwf->windows)
+		git_mwindow_scan_lru(mwf, &lru_w, &lru_l);
+
+	for (i = 0; i < ctl->windowfiles.length; ++i) {
+		git_mwindow *last = lru_w;
+		git_mwindow_file *cur = git_vector_get(&ctl->windowfiles, i);
+		git_mwindow_scan_lru(cur, &lru_w, &lru_l);
+		if (lru_w != last)
+			list = &cur->windows;
+	}
+
+	if (!lru_w) {
+		giterr_set(GITERR_OS, "Failed to close memory window. Couldn't find LRU");
+		return -1;
+	}
+
+	ctl->mapped -= lru_w->window_map.len;
+	git_futils_mmap_free(&lru_w->window_map);
+
+	if (lru_l)
+		lru_l->next = lru_w->next;
+	else
+		*list = lru_w->next;
+
+	git__free(lru_w);
+	ctl->open_windows--;
+
+	return 0;
+}
+
+static git_mwindow *new_window(
+	git_mwindow_file *mwf,
+	git_file fd,
+	git_off_t size,
+	git_off_t offset)
+{
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
+	size_t walign = _mw_options.window_size / 2;
+	git_off_t len;
+	git_mwindow *w;
+
+	w = git__malloc(sizeof(*w));
+	if (w == NULL)
+		return NULL;
+
+	memset(w, 0x0, sizeof(*w));
+	w->offset = (offset / walign) * walign;
+
+	len = size - w->offset;
+	if (len > (git_off_t)_mw_options.window_size)
+		len = (git_off_t)_mw_options.window_size;
+
+	ctl->mapped += (size_t)len;
+
+	while (_mw_options.mapped_limit < ctl->mapped &&
+			git_mwindow_close_lru(mwf) == 0) /* nop */;
+
+	/*
+	 * We treat _mw_options.mapped_limit as a soft limit. If we can't find a
+	 * window to close and are above the limit, we still mmap the new
+	 * window.
+	 */
+
+	if (git_futils_mmap_ro(&w->window_map, fd, w->offset, (size_t)len) < 0) {
+		git__free(w);
+		return NULL;
+	}
+
+	ctl->mmap_calls++;
+	ctl->open_windows++;
+
+	if (ctl->mapped > ctl->peak_mapped)
+		ctl->peak_mapped = ctl->mapped;
+
+	if (ctl->open_windows > ctl->peak_open_windows)
+		ctl->peak_open_windows = ctl->open_windows;
+
+	return w;
+}
+
+/*
+ * Open a new window, closing the least recenty used until we have
+ * enough space. Don't forget to add it to your list
+ */
+unsigned char *git_mwindow_open(
+	git_mwindow_file *mwf,
+	git_mwindow **cursor,
+	git_off_t offset,
+	size_t extra,
+	unsigned int *left)
+{
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
+	git_mwindow *w = *cursor;
+
+	if (!w || !(git_mwindow_contains(w, offset) && git_mwindow_contains(w, offset + extra))) {
+		if (w) {
+			w->inuse_cnt--;
+		}
+
+		for (w = mwf->windows; w; w = w->next) {
+			if (git_mwindow_contains(w, offset) &&
+				git_mwindow_contains(w, offset + extra))
+				break;
+		}
+
+		/*
+		 * If there isn't a suitable window, we need to create a new
+		 * one.
+		 */
+		if (!w) {
+			w = new_window(mwf, mwf->fd, mwf->size, offset);
+			if (w == NULL)
+				return NULL;
+			w->next = mwf->windows;
+			mwf->windows = w;
+		}
+	}
+
+	/* If we changed w, store it in the cursor */
+	if (w != *cursor) {
+		w->last_used = ctl->used_ctr++;
+		w->inuse_cnt++;
+		*cursor = w;
+	}
+
+	offset -= w->offset;
+
+	if (left)
+		*left = (unsigned int)(w->window_map.len - offset);
+
+	return (unsigned char *) w->window_map.data + offset;
+}
+
+int git_mwindow_file_register(git_mwindow_file *mwf)
+{
+	git_mwindow_ctl *ctl = &GIT_GLOBAL->mem_ctl;
+
+	if (ctl->windowfiles.length == 0 &&
+		git_vector_init(&ctl->windowfiles, 8, NULL) < 0)
+		return -1;
+
+	return git_vector_insert(&ctl->windowfiles, mwf);
+}
+
+void git_mwindow_close(git_mwindow **window)
+{
+	git_mwindow *w = *window;
+	if (w) {
+		w->inuse_cnt--;
+		*window = NULL;
+	}
+}
diff --git a/src/mwindow.h b/src/mwindow.h
new file mode 100644
index 000000000..058027251
--- /dev/null
+++ b/src/mwindow.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_mwindow__
+#define INCLUDE_mwindow__
+
+#include "map.h"
+#include "vector.h"
+
+typedef struct git_mwindow {
+	struct git_mwindow *next;
+	git_map window_map;
+	git_off_t offset;
+	size_t last_used;
+	size_t inuse_cnt;
+} git_mwindow;
+
+typedef struct git_mwindow_file {
+	git_mwindow *windows;
+	int fd;
+	git_off_t size;
+} git_mwindow_file;
+
+typedef struct git_mwindow_ctl {
+	size_t mapped;
+	unsigned int open_windows;
+	unsigned int mmap_calls;
+	unsigned int peak_open_windows;
+	size_t peak_mapped;
+	size_t used_ctr;
+	git_vector windowfiles;
+} git_mwindow_ctl;
+
+int git_mwindow_contains(git_mwindow *win, git_off_t offset);
+void git_mwindow_free_all(git_mwindow_file *mwf);
+unsigned char *git_mwindow_open(git_mwindow_file *mwf, git_mwindow **cursor, git_off_t offset, size_t extra, unsigned int *left);
+void git_mwindow_scan_lru(git_mwindow_file *mwf, git_mwindow **lru_w, git_mwindow **lru_l);
+int git_mwindow_file_register(git_mwindow_file *mwf);
+void git_mwindow_close(git_mwindow **w_cursor);
+
+#endif
diff --git a/src/netops.c b/src/netops.c
new file mode 100644
index 000000000..e6f3e5627
--- /dev/null
+++ b/src/netops.c
@@ -0,0 +1,517 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef _WIN32
+#	include <sys/types.h>
+#	include <sys/socket.h>
+#	include <sys/select.h>
+#	include <sys/time.h>
+#	include <netdb.h>
+#       include <arpa/inet.h>
+#else
+#	include <winsock2.h>
+#	include <ws2tcpip.h>
+#	ifdef _MSC_VER
+#		pragma comment(lib, "ws2_32.lib")
+#	endif
+#endif
+
+#ifdef GIT_SSL
+# include <openssl/ssl.h>
+# include <openssl/x509v3.h>
+#endif
+
+#include <ctype.h>
+#include "git2/errors.h"
+
+#include "common.h"
+#include "netops.h"
+#include "posix.h"
+#include "buffer.h"
+#include "transport.h"
+
+#ifdef GIT_WIN32
+static void net_set_error(const char *str)
+{
+	int size, error = WSAGetLastError();
+	LPSTR err_str = NULL;
+
+	size = FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM,
+			     0, error, 0, (LPSTR)&err_str, 0, 0);
+
+	giterr_set(GITERR_NET, "%s: %s", str, err_str);
+	LocalFree(err_str);
+}
+#else
+static void net_set_error(const char *str)
+{
+	giterr_set(GITERR_NET, "%s: %s", str, strerror(errno));
+}
+#endif
+
+#ifdef GIT_SSL
+static int ssl_set_error(gitno_ssl *ssl, int error)
+{
+	int err;
+	err = SSL_get_error(ssl->ssl, error);
+	giterr_set(GITERR_NET, "SSL error: %s", ERR_error_string(err, NULL));
+	return -1;
+}
+#endif
+
+void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigned int len)
+{
+	memset(buf, 0x0, sizeof(gitno_buffer));
+	memset(data, 0x0, len);
+	buf->data = data;
+	buf->len = len;
+	buf->offset = 0;
+	buf->fd = t->socket;
+#ifdef GIT_SSL
+	if (t->encrypt)
+		buf->ssl = &t->ssl;
+#endif
+}
+
+#ifdef GIT_SSL
+static int ssl_recv(gitno_ssl *ssl, void *data, size_t len)
+{
+	int ret;
+
+	do {
+		ret = SSL_read(ssl->ssl, data, len);
+	} while (SSL_get_error(ssl->ssl, ret) == SSL_ERROR_WANT_READ);
+
+	if (ret < 0)
+		return ssl_set_error(ssl, ret);
+
+	return ret;
+}
+#endif
+
+int gitno_recv(gitno_buffer *buf)
+{
+	int ret;
+
+#ifdef GIT_SSL
+	if (buf->ssl != NULL) {
+		if ((ret = ssl_recv(buf->ssl, buf->data + buf->offset, buf->len - buf->offset)) < 0)
+			return -1;
+	} else {
+		ret = p_recv(buf->fd, buf->data + buf->offset, buf->len - buf->offset, 0);
+		if (ret < 0) {
+			net_set_error("Error receiving socket data");
+			return -1;
+		}
+	}
+#else
+	ret = p_recv(buf->fd, buf->data + buf->offset, buf->len - buf->offset, 0);
+	if (ret < 0) {
+		net_set_error("Error receiving socket data");
+		return -1;
+	}
+#endif
+
+	buf->offset += ret;
+	return ret;
+}
+
+/* Consume up to ptr and move the rest of the buffer to the beginning */
+void gitno_consume(gitno_buffer *buf, const char *ptr)
+{
+	size_t consumed;
+
+	assert(ptr - buf->data >= 0);
+	assert(ptr - buf->data <= (int) buf->len);
+
+	consumed = ptr - buf->data;
+
+	memmove(buf->data, ptr, buf->offset - consumed);
+	memset(buf->data + buf->offset, 0x0, buf->len - buf->offset);
+	buf->offset -= consumed;
+}
+
+/* Consume const bytes and move the rest of the buffer to the beginning */
+void gitno_consume_n(gitno_buffer *buf, size_t cons)
+{
+	memmove(buf->data, buf->data + cons, buf->len - buf->offset);
+	memset(buf->data + cons, 0x0, buf->len - buf->offset);
+	buf->offset -= cons;
+}
+
+int gitno_ssl_teardown(git_transport *t)
+{
+#ifdef GIT_SSL
+	int ret;
+#endif
+
+	if (!t->encrypt)
+		return 0;
+
+#ifdef GIT_SSL
+
+	do {
+		ret = SSL_shutdown(t->ssl.ssl);
+	} while (ret == 0);
+	if (ret < 0)
+		return ssl_set_error(&t->ssl, ret);
+
+	SSL_free(t->ssl.ssl);
+	SSL_CTX_free(t->ssl.ctx);
+#endif
+	return 0;
+}
+
+
+#ifdef GIT_SSL
+/* Match host names according to RFC 2818 rules */
+static int match_host(const char *pattern, const char *host)
+{
+	for (;;) {
+		char c = tolower(*pattern++);
+
+		if (c == '\0')
+			return *host ? -1 : 0;
+
+		if (c == '*') {
+			c = *pattern;
+			/* '*' at the end matches everything left */
+			if (c == '\0')
+				return 0;
+
+	/*
+	 * We've found a pattern, so move towards the next matching
+	 * char. The '.' is handled specially because wildcards aren't
+	 * allowed to cross subdomains.
+	 */
+
+			while(*host) {
+				char h = tolower(*host);
+				if (c == h)
+					return match_host(pattern, host++);
+				if (h == '.')
+					return match_host(pattern, host);
+				host++;
+			}
+			return -1;
+		}
+
+		if (c != tolower(*host++))
+			return -1;
+	}
+
+	return -1;
+}
+
+static int check_host_name(const char *name, const char *host)
+{
+	if (!strcasecmp(name, host))
+		return 0;
+
+	if (match_host(name, host) < 0)
+		return -1;
+
+	return 0;
+}
+
+static int verify_server_cert(git_transport *t, const char *host)
+{
+	X509 *cert;
+	X509_NAME *peer_name;
+	ASN1_STRING *str;
+	unsigned char *peer_cn = NULL;
+	int matched = -1, type = GEN_DNS;
+	GENERAL_NAMES *alts;
+	struct in6_addr addr6;
+	struct in_addr addr4;
+	void *addr;
+	int i = -1,j;
+
+
+	/* Try to parse the host as an IP address to see if it is */
+	if (inet_pton(AF_INET, host, &addr4)) {
+		type = GEN_IPADD;
+		addr = &addr4;
+	} else {
+		if(inet_pton(AF_INET6, host, &addr6)) {
+			type = GEN_IPADD;
+			addr = &addr6;
+		}
+	}
+
+
+	cert = SSL_get_peer_certificate(t->ssl.ssl);
+
+	/* Check the alternative names */
+	alts = X509_get_ext_d2i(cert, NID_subject_alt_name, NULL, NULL);
+	if (alts) {
+		int num;
+
+		num = sk_GENERAL_NAME_num(alts);
+		for (i = 0; i < num && matched != 1; i++) {
+			const GENERAL_NAME *gn = sk_GENERAL_NAME_value(alts, i);
+			const char *name = (char *) ASN1_STRING_data(gn->d.ia5);
+			size_t namelen = (size_t) ASN1_STRING_length(gn->d.ia5);
+
+			/* Skip any names of a type we're not looking for */
+			if (gn->type != type)
+				continue;
+
+			if (type == GEN_DNS) {
+				/* If it contains embedded NULs, don't even try */
+				if (memchr(name, '\0', namelen))
+					continue;
+
+				if (check_host_name(name, host) < 0)
+					matched = 0;
+				else
+					matched = 1;
+			} else if (type == GEN_IPADD) {
+				/* Here name isn't so much a name but a binary representation of the IP */
+				matched = !!memcmp(name, addr, namelen);
+			}
+		}
+	}
+	GENERAL_NAMES_free(alts);
+
+	if (matched == 0)
+		goto on_error;
+
+	if (matched == 1)
+		return 0;
+
+	/* If no alternative names are available, check the common name */
+	peer_name = X509_get_subject_name(cert);
+	if (peer_name == NULL)
+		goto on_error;
+
+	if (peer_name) {
+		/* Get the index of the last CN entry */
+		while ((j = X509_NAME_get_index_by_NID(peer_name, NID_commonName, i)) >= 0)
+			i = j;
+	}
+
+	if (i < 0)
+		goto on_error;
+
+	str = X509_NAME_ENTRY_get_data(X509_NAME_get_entry(peer_name, i));
+	if (str == NULL)
+		goto on_error;
+
+	/* Work around a bug in OpenSSL whereby ASN1_STRING_to_UTF8 fails if it's already in utf-8 */
+	if (ASN1_STRING_type(str) == V_ASN1_UTF8STRING) {
+		int size = ASN1_STRING_length(str);
+
+		if (size > 0) {
+			peer_cn = OPENSSL_malloc(size + 1);
+			GITERR_CHECK_ALLOC(peer_cn);
+			memcpy(peer_cn, ASN1_STRING_data(str), size);
+			peer_cn[size] = '\0';
+		}
+	} else {
+		int size = ASN1_STRING_to_UTF8(&peer_cn, str);
+		GITERR_CHECK_ALLOC(peer_cn);
+		if (memchr(peer_cn, '\0', size))
+			goto cert_fail;
+	}
+
+	if (check_host_name((char *)peer_cn, host) < 0)
+		goto cert_fail;
+
+	OPENSSL_free(peer_cn);
+
+	return 0;
+
+on_error:
+	OPENSSL_free(peer_cn);
+	return ssl_set_error(&t->ssl, 0);
+
+cert_fail:
+	OPENSSL_free(peer_cn);
+	giterr_set(GITERR_SSL, "Certificate host name check failed");
+	return -1;
+}
+
+static int ssl_setup(git_transport *t, const char *host)
+{
+	int ret;
+
+	SSL_library_init();
+	SSL_load_error_strings();
+	t->ssl.ctx = SSL_CTX_new(SSLv23_method());
+	if (t->ssl.ctx == NULL)
+		return ssl_set_error(&t->ssl, 0);
+
+	SSL_CTX_set_mode(t->ssl.ctx, SSL_MODE_AUTO_RETRY);
+	SSL_CTX_set_verify(t->ssl.ctx, SSL_VERIFY_PEER, NULL);
+	if (!SSL_CTX_set_default_verify_paths(t->ssl.ctx))
+		return ssl_set_error(&t->ssl, 0);
+
+	t->ssl.ssl = SSL_new(t->ssl.ctx);
+	if (t->ssl.ssl == NULL)
+		return ssl_set_error(&t->ssl, 0);
+
+	if((ret = SSL_set_fd(t->ssl.ssl, t->socket)) == 0)
+		return ssl_set_error(&t->ssl, ret);
+
+	if ((ret = SSL_connect(t->ssl.ssl)) <= 0)
+		return ssl_set_error(&t->ssl, ret);
+
+	if (t->check_cert && verify_server_cert(t, host) < 0)
+		return -1;
+
+	return 0;
+}
+#else
+static int ssl_setup(git_transport *t, const char *host)
+{
+	GIT_UNUSED(t);
+	GIT_UNUSED(host);
+	return 0;
+}
+#endif
+
+int gitno_connect(git_transport *t, const char *host, const char *port)
+{
+	struct addrinfo *info = NULL, *p;
+	struct addrinfo hints;
+	int ret;
+	GIT_SOCKET s = INVALID_SOCKET;
+
+	memset(&hints, 0x0, sizeof(struct addrinfo));
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_socktype = SOCK_STREAM;
+
+	if ((ret = getaddrinfo(host, port, &hints, &info)) < 0) {
+		giterr_set(GITERR_NET, "Failed to resolve address for %s: %s", host, gai_strerror(ret));
+		return -1;
+	}
+
+	for (p = info; p != NULL; p = p->ai_next) {
+		s = socket(p->ai_family, p->ai_socktype, p->ai_protocol);
+		if (s == INVALID_SOCKET) {
+			net_set_error("error creating socket");
+			break;
+		}
+
+		if (connect(s, p->ai_addr, (socklen_t)p->ai_addrlen) == 0)
+			break;
+
+		/* If we can't connect, try the next one */
+		gitno_close(s);
+		s = INVALID_SOCKET;
+	}
+
+	/* Oops, we couldn't connect to any address */
+	if (s == INVALID_SOCKET && p == NULL) {
+		giterr_set(GITERR_OS, "Failed to connect to %s", host);
+		return -1;
+	}
+
+	t->socket = s;
+	freeaddrinfo(info);
+
+	if (t->encrypt && ssl_setup(t, host) < 0)
+		return -1;
+
+	return 0;
+}
+
+#ifdef GIT_SSL
+static int send_ssl(gitno_ssl *ssl, const char *msg, size_t len)
+{
+	int ret;
+	size_t off = 0;
+
+	while (off < len) {
+		ret = SSL_write(ssl->ssl, msg + off, len - off);
+		if (ret <= 0)
+			return ssl_set_error(ssl, ret);
+
+		off += ret;
+	}
+
+	return off;
+}
+#endif
+
+int gitno_send(git_transport *t, const char *msg, size_t len, int flags)
+{
+	int ret;
+	size_t off = 0;
+
+#ifdef GIT_SSL
+	if (t->encrypt)
+		return send_ssl(&t->ssl, msg, len);
+#endif
+
+	while (off < len) {
+		errno = 0;
+		ret = p_send(t->socket, msg + off, len - off, flags);
+		if (ret < 0) {
+			net_set_error("Error sending data");
+			return -1;
+		}
+
+		off += ret;
+	}
+
+	return (int)off;
+}
+
+
+#ifdef GIT_WIN32
+int gitno_close(GIT_SOCKET s)
+{
+	return closesocket(s) == SOCKET_ERROR ? -1 : 0;
+}
+#else
+int gitno_close(GIT_SOCKET s)
+{
+	return close(s);
+}
+#endif
+
+int gitno_select_in(gitno_buffer *buf, long int sec, long int usec)
+{
+	fd_set fds;
+	struct timeval tv;
+
+	tv.tv_sec = sec;
+	tv.tv_usec = usec;
+
+	FD_ZERO(&fds);
+	FD_SET(buf->fd, &fds);
+
+	/* The select(2) interface is silly */
+	return select((int)buf->fd + 1, &fds, NULL, NULL, &tv);
+}
+
+int gitno_extract_host_and_port(char **host, char **port, const char *url, const char *default_port)
+{
+	char *colon, *slash, *delim;
+
+	colon = strchr(url, ':');
+	slash = strchr(url, '/');
+
+	if (slash == NULL) {
+		giterr_set(GITERR_NET, "Malformed URL: missing /");
+		return -1;
+	}
+
+	if (colon == NULL) {
+		*port = git__strdup(default_port);
+	} else {
+		*port = git__strndup(colon + 1, slash - colon - 1);
+	}
+	GITERR_CHECK_ALLOC(*port);
+
+	delim = colon == NULL ? slash : colon;
+	*host = git__strndup(url, delim - url);
+	GITERR_CHECK_ALLOC(*host);
+
+	return 0;
+}
diff --git a/src/netops.h b/src/netops.h
new file mode 100644
index 000000000..4976f87f8
--- /dev/null
+++ b/src/netops.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_netops_h__
+#define INCLUDE_netops_h__
+
+#include "posix.h"
+#include "transport.h"
+#include "common.h"
+
+typedef struct gitno_buffer {
+	char *data;
+	size_t len;
+	size_t offset;
+	GIT_SOCKET fd;
+#ifdef GIT_SSL
+	struct gitno_ssl *ssl;
+#endif
+} gitno_buffer;
+
+void gitno_buffer_setup(git_transport *t, gitno_buffer *buf, char *data, unsigned int len);
+int gitno_recv(gitno_buffer *buf);
+
+void gitno_consume(gitno_buffer *buf, const char *ptr);
+void gitno_consume_n(gitno_buffer *buf, size_t cons);
+
+int gitno_connect(git_transport *t, const char *host, const char *port);
+int gitno_send(git_transport *t, const char *msg, size_t len, int flags);
+int gitno_close(GIT_SOCKET s);
+int gitno_ssl_teardown(git_transport *t);
+int gitno_send_chunk_size(int s, size_t len);
+int gitno_select_in(gitno_buffer *buf, long int sec, long int usec);
+
+int gitno_extract_host_and_port(char **host, char **port, const char *url, const char *default_port);
+
+#endif
diff --git a/src/notes.c b/src/notes.c
new file mode 100644
index 000000000..84ad94087
--- /dev/null
+++ b/src/notes.c
@@ -0,0 +1,548 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "notes.h"
+
+#include "git2.h"
+#include "refs.h"
+#include "config.h"
+#include "iterator.h"
+
+static int find_subtree(git_tree **subtree, const git_oid *root,
+			git_repository *repo, const char *target, int *fanout)
+{
+	int error;
+	unsigned int i;
+	git_tree *tree;
+	const git_tree_entry *entry;
+
+	*subtree = NULL;
+
+	error = git_tree_lookup(&tree, repo, root);
+	if (error < 0)
+		return error;
+
+	for (i=0; i<git_tree_entrycount(tree); i++) {
+		entry = git_tree_entry_byindex(tree, i);
+
+		if (!git__ishex(git_tree_entry_name(entry)))
+			continue;
+
+		/*
+		 * A notes tree follows a strict byte-based progressive fanout
+		 * (i.e. using 2/38, 2/2/36, etc. fanouts, not e.g. 4/36 fanout)
+		 */
+
+		if (S_ISDIR(git_tree_entry_attributes(entry))
+			&& strlen(git_tree_entry_name(entry)) == 2
+			&& !strncmp(git_tree_entry_name(entry), target + *fanout, 2)) {
+
+			/* found matching subtree - unpack and resume lookup */
+
+			git_oid subtree_sha;
+			git_oid_cpy(&subtree_sha, git_tree_entry_id(entry));
+			git_tree_free(tree);
+
+			*fanout += 2;
+
+			return find_subtree(subtree, &subtree_sha, repo,
+					    target, fanout);
+		}
+	}
+
+	*subtree = tree;
+	return 0;
+}
+
+static int find_blob(git_oid *blob, git_tree *tree, const char *target)
+{
+	unsigned int i;
+	const git_tree_entry *entry;
+
+	for (i=0; i<git_tree_entrycount(tree); i++) {
+		entry = git_tree_entry_byindex(tree, i);
+
+		if (!strcmp(git_tree_entry_name(entry), target)) {
+			/* found matching note object - return */
+
+			git_oid_cpy(blob, git_tree_entry_id(entry));
+			return 0;
+		}
+	}
+	return GIT_ENOTFOUND;
+}
+
+static int note_write(git_oid *out, git_repository *repo,
+		      git_signature *author, git_signature *committer,
+		      const char *notes_ref, const char *note,
+		      const git_oid *tree_sha, const char *target,
+		      int nparents, git_commit **parents)
+{
+	int error, fanout = 0;
+	git_oid oid;
+	git_tree *tree = NULL;
+	git_tree_entry *entry;
+	git_treebuilder *tb;
+
+	/* check for existing notes tree */
+
+	if (tree_sha) {
+		error = find_subtree(&tree, tree_sha, repo, target, &fanout);
+		if (error < 0)
+			return error;
+
+		error = find_blob(&oid, tree, target + fanout);
+		if (error != GIT_ENOTFOUND) {
+			git_tree_free(tree);
+			if (!error) {
+				giterr_set(GITERR_REPOSITORY, "Note for '%s' exists already", target);
+				error = GIT_EEXISTS;
+			}
+			return error;
+		}
+	}
+
+	/* no matching tree entry - add note object to target tree */
+
+	error = git_treebuilder_create(&tb, tree);
+	git_tree_free(tree);
+
+	if (error < 0)
+		return error;
+
+	if (!tree_sha)
+		/* no notes tree yet - create fanout */
+		fanout += 2;
+
+	/* create note object */
+	error = git_blob_create_frombuffer(&oid, repo, note, strlen(note));
+	if (error < 0) {
+		git_treebuilder_free(tb);
+		return error;
+	}
+
+	error = git_treebuilder_insert(&entry, tb, target + fanout, &oid, 0100644);
+	if (error < 0) {
+		/* libgit2 doesn't support object removal (gc) yet */
+		/* we leave an orphaned blob object behind - TODO */
+
+		git_treebuilder_free(tb);
+		return error;
+	}
+
+	if (out)
+		git_oid_cpy(out, git_tree_entry_id(entry));
+
+	error = git_treebuilder_write(&oid, repo, tb);
+	git_treebuilder_free(tb);
+
+	if (error < 0)
+		return 0;
+
+	if (!tree_sha) {
+		/* create fanout subtree */
+
+		char subtree[3];
+		strncpy(subtree, target, 2);
+		subtree[2] = '\0';
+
+		error = git_treebuilder_create(&tb, NULL);
+		if (error < 0)
+			return error;
+
+		error = git_treebuilder_insert(NULL, tb, subtree, &oid, 0040000);
+		if (error < 0) {
+			git_treebuilder_free(tb);
+			return error;
+		}
+
+		error = git_treebuilder_write(&oid, repo, tb);
+
+		git_treebuilder_free(tb);
+
+		if (error < 0)
+			return error;
+	}
+
+	/* create new notes commit */
+
+	error = git_tree_lookup(&tree, repo, &oid);
+	if (error < 0)
+		return error;
+
+	error = git_commit_create(&oid, repo, notes_ref, author, committer,
+				  NULL, GIT_NOTES_DEFAULT_MSG_ADD,
+				  tree, nparents, (const git_commit **) parents);
+
+	git_tree_free(tree);
+
+	return error;
+}
+
+static int note_lookup(git_note **out, git_repository *repo,
+		       const git_oid *tree_sha, const char *target)
+{
+	int error, fanout = 0;
+	git_oid oid;
+	git_blob *blob;
+	git_tree *tree;
+	git_note *note;
+
+	error = find_subtree(&tree, tree_sha, repo, target, &fanout);
+	if (error < 0)
+		return error;
+
+	error = find_blob(&oid, tree, target + fanout);
+
+	git_tree_free(tree);
+	if (error < 0)
+		return error;
+
+	error = git_blob_lookup(&blob, repo, &oid);
+	if (error < 0)
+		return error;
+
+	note = git__malloc(sizeof(git_note));
+	GITERR_CHECK_ALLOC(note);
+
+	git_oid_cpy(&note->oid, &oid);
+	note->message = git__strdup(git_blob_rawcontent(blob));
+	GITERR_CHECK_ALLOC(note->message);
+
+	*out = note;
+
+	git_blob_free(blob);
+	return error;
+}
+
+static int note_remove(git_repository *repo,
+		       git_signature *author, git_signature *committer,
+		       const char *notes_ref, const git_oid *tree_sha,
+		       const char *target, int nparents, git_commit **parents)
+{
+	int error, fanout = 0;
+	git_oid oid;
+	git_tree *tree;
+	git_treebuilder *tb;
+
+	error = find_subtree(&tree, tree_sha, repo, target, &fanout);
+	if (error < 0)
+		return error;
+
+	error = find_blob(&oid, tree, target + fanout);
+	if (!error)
+		error = git_treebuilder_create(&tb, tree);
+
+	git_tree_free(tree);
+	if (error < 0)
+		return error;
+
+	error = git_treebuilder_remove(tb, target + fanout);
+	if (!error)
+		error = git_treebuilder_write(&oid, repo, tb);
+
+	git_treebuilder_free(tb);
+	if (error < 0)
+		return error;
+
+	/* create new notes commit */
+
+	error = git_tree_lookup(&tree, repo, &oid);
+	if (error < 0)
+		return error;
+
+	error = git_commit_create(&oid, repo, notes_ref, author, committer,
+				  NULL, GIT_NOTES_DEFAULT_MSG_RM,
+				  tree, nparents, (const git_commit **) parents);
+
+	git_tree_free(tree);
+
+	return error;
+}
+
+static int note_get_default_ref(const char **out, git_repository *repo)
+{
+	int ret;
+	git_config *cfg;
+
+	*out = NULL;
+
+	if (git_repository_config__weakptr(&cfg, repo) < 0)
+		return -1;
+
+	ret = git_config_get_string(out, cfg, "core.notesRef");
+	if (ret == GIT_ENOTFOUND) {
+		*out = GIT_NOTES_DEFAULT_REF;
+		return 0;
+	}
+
+	return ret;
+}
+
+static int normalize_namespace(const char **notes_ref, git_repository *repo)
+{
+	if (*notes_ref)
+		return 0;
+
+	return note_get_default_ref(notes_ref, repo);
+}
+
+static int retrieve_note_tree_oid(git_oid *tree_oid_out, git_repository *repo, const char *notes_ref)
+{
+	int error = -1;
+	git_commit *commit = NULL;
+	git_oid oid;
+
+	if ((error = git_reference_name_to_oid(&oid, repo, notes_ref)) < 0)
+		goto cleanup;
+
+	if (git_commit_lookup(&commit, repo, &oid) < 0)
+		goto cleanup;
+
+	git_oid_cpy(tree_oid_out, git_commit_tree_oid(commit));
+
+	error = 0;
+
+cleanup:
+	git_commit_free(commit);
+	return error;
+}
+
+int git_note_read(git_note **out, git_repository *repo,
+		  const char *notes_ref, const git_oid *oid)
+{
+	int error;
+	char *target;
+	git_oid sha;
+
+	*out = NULL;
+
+	if (normalize_namespace(&notes_ref, repo) < 0)
+		return -1;
+
+	if ((error = retrieve_note_tree_oid(&sha, repo, notes_ref)) < 0)
+		return error;
+
+	target = git_oid_allocfmt(oid);
+	GITERR_CHECK_ALLOC(target);
+
+	error = note_lookup(out, repo, &sha, target);
+
+	git__free(target);
+	return error;
+}
+
+int git_note_create(
+	git_oid *out, git_repository *repo,
+	git_signature *author, git_signature *committer,
+	const char *notes_ref, const git_oid *oid,
+	const char *note)
+{
+	int error, nparents = 0;
+	char *target;
+	git_oid sha;
+	git_commit *commit = NULL;
+	git_reference *ref;
+
+	if (normalize_namespace(&notes_ref, repo) < 0)
+		return -1;
+
+	error = git_reference_lookup(&ref, repo, notes_ref);
+	if (error < 0 && error != GIT_ENOTFOUND)
+		return error;
+
+	if (!error) {
+		assert(git_reference_type(ref) == GIT_REF_OID);
+
+		/* lookup existing notes tree oid */
+
+		git_oid_cpy(&sha, git_reference_oid(ref));
+		git_reference_free(ref);
+
+		error = git_commit_lookup(&commit, repo, &sha);
+		if (error < 0)
+			return error;
+
+		git_oid_cpy(&sha, git_commit_tree_oid(commit));
+		nparents++;
+	}
+
+	target = git_oid_allocfmt(oid);
+	GITERR_CHECK_ALLOC(target);
+
+	error = note_write(out, repo, author, committer, notes_ref,
+			   note, nparents ? &sha : NULL, target,
+			   nparents, &commit);
+
+	git__free(target);
+	git_commit_free(commit);
+	return error;
+}
+
+int git_note_remove(git_repository *repo, const char *notes_ref,
+		    git_signature *author, git_signature *committer,
+		    const git_oid *oid)
+{
+	int error;
+	char *target;
+	git_oid sha;
+	git_commit *commit;
+	git_reference *ref;
+
+	if (normalize_namespace(&notes_ref, repo) < 0)
+		return -1;
+
+	error = git_reference_lookup(&ref, repo, notes_ref);
+	if (error < 0)
+		return error;
+
+	assert(git_reference_type(ref) == GIT_REF_OID);
+
+	git_oid_cpy(&sha, git_reference_oid(ref));
+	git_reference_free(ref);
+
+	error = git_commit_lookup(&commit, repo, &sha);
+	if (error < 0)
+		return error;
+
+	git_oid_cpy(&sha, git_commit_tree_oid(commit));
+
+	target = git_oid_allocfmt(oid);
+	GITERR_CHECK_ALLOC(target);
+
+	error = note_remove(repo, author, committer, notes_ref,
+			    &sha, target, 1, &commit);
+
+	git__free(target);
+	git_commit_free(commit);
+	return error;
+}
+
+int git_note_default_ref(const char **out, git_repository *repo)
+{
+	assert(repo);
+	return note_get_default_ref(out, repo);
+}
+
+const char * git_note_message(git_note *note)
+{
+	assert(note);
+	return note->message;
+}
+
+const git_oid * git_note_oid(git_note *note)
+{
+	assert(note);
+	return &note->oid;
+}
+
+void git_note_free(git_note *note)
+{
+	if (note == NULL)
+		return;
+
+	git__free(note->message);
+	git__free(note);
+}
+
+static int process_entry_path(
+	const char* entry_path,
+	const git_oid *note_oid,
+	int (*note_cb)(git_note_data *note_data, void *payload),
+	void *payload)
+{
+	int i = 0, j = 0, error = -1, len;
+	git_buf buf = GIT_BUF_INIT;
+	git_note_data note_data;
+
+	if (git_buf_puts(&buf, entry_path) < 0)
+		goto cleanup;
+	
+	len = git_buf_len(&buf);
+
+	while (i < len) {
+		if (buf.ptr[i] == '/') {
+			i++;
+			continue;
+		}
+		
+		if (git__fromhex(buf.ptr[i]) < 0) {
+			/* This is not a note entry */
+			error = 0;
+			goto cleanup;
+		}
+
+		if (i != j)
+			buf.ptr[j] = buf.ptr[i];
+
+		i++;
+		j++;
+	}
+
+	buf.ptr[j] = '\0';
+	buf.size = j;
+
+	if (j != GIT_OID_HEXSZ) {
+		/* This is not a note entry */
+		error = 0;
+		goto cleanup;
+	}
+
+	if (git_oid_fromstr(&note_data.annotated_object_oid, buf.ptr) < 0)
+		return -1;
+
+	git_oid_cpy(&note_data.blob_oid, note_oid);
+
+	error = note_cb(&note_data, payload);
+
+cleanup:
+	git_buf_free(&buf);
+	return error;
+}
+
+int git_note_foreach(
+	git_repository *repo,
+	const char *notes_ref,
+	int (*note_cb)(git_note_data *note_data, void *payload),
+	void *payload)
+{
+	int error = -1;
+	git_oid tree_oid;
+	git_iterator *iter = NULL;
+	git_tree *tree = NULL;
+	const git_index_entry *item;
+
+	if (normalize_namespace(&notes_ref, repo) < 0)
+		return -1;
+
+	if ((error = retrieve_note_tree_oid(&tree_oid, repo, notes_ref)) < 0)
+		goto cleanup;
+
+	if (git_tree_lookup(&tree, repo, &tree_oid) < 0)
+		goto cleanup;
+
+	if (git_iterator_for_tree(&iter, repo, tree) < 0)
+		goto cleanup;
+
+	if (git_iterator_current(iter, &item) < 0)
+		goto cleanup;
+
+	while (item) {
+		if (process_entry_path(item->path, &item->oid, note_cb, payload) < 0)
+			goto cleanup;
+
+		if (git_iterator_advance(iter, &item) < 0)
+			goto cleanup;
+	}
+
+	error = 0;
+
+cleanup:
+	git_iterator_free(iter);
+	git_tree_free(tree);
+	return error;
+}
diff --git a/src/notes.h b/src/notes.h
new file mode 100644
index 000000000..219db1ab0
--- /dev/null
+++ b/src/notes.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_note_h__
+#define INCLUDE_note_h__
+
+#include "common.h"
+
+#include "git2/oid.h"
+
+#define GIT_NOTES_DEFAULT_REF "refs/notes/commits"
+
+#define GIT_NOTES_DEFAULT_MSG_ADD \
+	"Notes added by 'git_note_create' from libgit2"
+
+#define GIT_NOTES_DEFAULT_MSG_RM \
+	"Notes removed by 'git_note_remove' from libgit2"
+
+struct git_note {
+	git_oid oid;
+
+	char *message;
+};
+
+#endif /* INCLUDE_notes_h__ */
diff --git a/src/object.c b/src/object.c
index d14ca8566..d3673eda0 100644
--- a/src/object.c
+++ b/src/object.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #include <stdarg.h>
 
@@ -37,8 +19,8 @@
 static const int OBJECT_BASE_SIZE = 4096;
 
 static struct {
-	const char	*str;   /* type name string */
-	int			loose;  /* valid loose object type flag */
+	const char	*str;	/* type name string */
+	int			loose; /* valid loose object type flag */
 	size_t		size;	/* size in bytes of the object structure */
 } git_objects_table[] = {
 	/* 0 = GIT_OBJ__EXT1 */
@@ -80,47 +62,59 @@ static int create_object(git_object **object_out, git_otype type)
 	case GIT_OBJ_BLOB:
 	case GIT_OBJ_TREE:
 		object = git__malloc(git_object__size(type));
-		if (object == NULL)
-			return GIT_ENOMEM;
+		GITERR_CHECK_ALLOC(object);
 		memset(object, 0x0, git_object__size(type));
 		break;
 
 	default:
-		return git__throw(GIT_EINVALIDTYPE, "The given type is invalid");
+		giterr_set(GITERR_INVALID, "The given type is invalid");
+		return -1;
 	}
 
 	object->type = type;
 
 	*object_out = object;
-	return GIT_SUCCESS;
+	return 0;
 }
 
-int git_object_lookup_prefix(git_object **object_out, git_repository *repo, const git_oid *id, unsigned int len, git_otype type)
+int git_object_lookup_prefix(
+	git_object **object_out,
+	git_repository *repo,
+	const git_oid *id,
+	unsigned int len,
+	git_otype type)
 {
 	git_object *object = NULL;
+	git_odb *odb = NULL;
 	git_odb_object *odb_obj;
-	int error = GIT_SUCCESS;
+	int error = 0;
 
 	assert(repo && object_out && id);
 
 	if (len < GIT_OID_MINPREFIXLEN)
-		return git__throw(GIT_EAMBIGUOUSOIDPREFIX,
-			"Failed to lookup object. Prefix length is lower than %d.", GIT_OID_MINPREFIXLEN);
+		return GIT_EAMBIGUOUS;
+
+	error = git_repository_odb__weakptr(&odb, repo);
+	if (error < 0)
+		return error;
 
 	if (len > GIT_OID_HEXSZ)
 		len = GIT_OID_HEXSZ;
 
-	if (len == GIT_OID_HEXSZ)  {
+	if (len == GIT_OID_HEXSZ) {
 		/* We want to match the full id : we can first look up in the cache,
 		 * since there is no need to check for non ambiguousity
 		 */
 		object = git_cache_get(&repo->objects, id);
 		if (object != NULL) {
-			if (type != GIT_OBJ_ANY && type != object->type)
-				return git__throw(GIT_EINVALIDTYPE, "Failed to lookup object. The given type does not match the type on the ODB");
+			if (type != GIT_OBJ_ANY && type != object->type) {
+				git_object_free(object);
+				giterr_set(GITERR_ODB, "The given type does not match the type in ODB");
+				return GIT_ENOTFOUND;
+			}
 
 			*object_out = object;
-			return GIT_SUCCESS;
+			return 0;
 		}
 
 		/* Object was not found in the cache, let's explore the backends.
@@ -128,7 +122,7 @@ int git_object_lookup_prefix(git_object **object_out, git_repository *repo, cons
 		 * it is the same cost for packed and loose object backends,
 		 * but it may be much more costly for sqlite and hiredis.
 		 */
-		error = git_odb_read(&odb_obj, repo->db, id);
+		error = git_odb_read(&odb_obj, odb, id);
 	} else {
 		git_oid short_oid;
 
@@ -141,28 +135,29 @@ int git_object_lookup_prefix(git_object **object_out, git_repository *repo, cons
 		/* If len < GIT_OID_HEXSZ (a strict short oid was given), we have
 		 * 2 options :
 		 * - We always search in the cache first. If we find that short oid is
-		 *   ambiguous, we can stop. But in all the other cases, we must then
-		 *   explore all the backends (to find an object if there was match,
-		 *   or to check that oid is not ambiguous if we have found 1 match in
-		 *   the cache)
+		 *	ambiguous, we can stop. But in all the other cases, we must then
+		 *	explore all the backends (to find an object if there was match,
+		 *	or to check that oid is not ambiguous if we have found 1 match in
+		 *	the cache)
 		 * - We never explore the cache, go right to exploring the backends
 		 * We chose the latter : we explore directly the backends.
 		 */
-		error = git_odb_read_prefix(&odb_obj, repo->db, &short_oid, len);
+		error = git_odb_read_prefix(&odb_obj, odb, &short_oid, len);
 	}
 
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to lookup object");
+	if (error < 0)
+		return error;
 
 	if (type != GIT_OBJ_ANY && type != odb_obj->raw.type) {
-		git_odb_object_close(odb_obj);
-		return git__throw(GIT_EINVALIDTYPE, "Failed to lookup object. The given type does not match the type on the ODB");
+		git_odb_object_free(odb_obj);
+		giterr_set(GITERR_ODB, "The given type does not match the type on the ODB");
+		return GIT_ENOTFOUND;
 	}
 
 	type = odb_obj->raw.type;
 
-	if ((error = create_object(&object, type)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to lookup object");
+	if (create_object(&object, type) < 0)
+		return -1;
 
 	/* Initialize parent object */
 	git_oid_cpy(&object->cached.oid, &odb_obj->cached.oid);
@@ -189,15 +184,15 @@ int git_object_lookup_prefix(git_object **object_out, git_repository *repo, cons
 		break;
 	}
 
-	git_odb_object_close(odb_obj);
+	git_odb_object_free(odb_obj);
 
-	if (error < GIT_SUCCESS) {
+	if (error < 0) {
 		git_object__free(object);
-		return git__rethrow(error, "Failed to lookup object");
+		return -1;
 	}
 
 	*object_out = git_cache_try_store(&repo->objects, object);
-	return GIT_SUCCESS;
+	return 0;
 }
 
 int git_object_lookup(git_object **object_out, git_repository *repo, const git_oid *id, git_otype type) {
@@ -228,12 +223,12 @@ void git_object__free(void *_obj)
 		break;
 
 	default:
-		free(object);
+		git__free(object);
 		break;
 	}
 }
 
-void git_object_close(git_object *object)
+void git_object_free(git_object *object)
 {
 	if (object == NULL)
 		return;
@@ -297,3 +292,42 @@ size_t git_object__size(git_otype type)
 	return git_objects_table[type].size;
 }
 
+int git_object__resolve_to_type(git_object **obj, git_otype type)
+{
+	int error = 0;
+	git_object *scan, *next;
+
+	if (type == GIT_OBJ_ANY)
+		return 0;
+
+	scan = *obj;
+
+	while (!error && scan && git_object_type(scan) != type) {
+
+		switch (git_object_type(scan)) {
+		case GIT_OBJ_COMMIT:
+		{
+			git_tree *tree = NULL;
+			error = git_commit_tree(&tree, (git_commit *)scan);
+			next = (git_object *)tree;
+			break;
+		}
+
+		case GIT_OBJ_TAG:
+			error = git_tag_target(&next, (git_tag *)scan);
+			break;
+
+		default:
+			giterr_set(GITERR_REFERENCE, "Object does not resolve to type");
+			error = -1;
+			next = NULL;
+			break;
+		}
+
+		git_object_free(scan);
+		scan = next;
+	}
+
+	*obj = scan;
+	return error;
+}
diff --git a/src/odb.c b/src/odb.c
index 8953ec658..a6a18f831 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -1,30 +1,12 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
-#include "git2/zlib.h"
+#include <zlib.h>
 #include "git2/object.h"
 #include "fileops.h"
 #include "hash.h"
@@ -46,45 +28,37 @@ typedef struct
 	int is_alternate;
 } backend_internal;
 
-static int format_object_header(char *hdr, size_t n, git_rawobj *obj)
+static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
 {
-	const char *type_str = git_object_type2string(obj->type);
-	int len = snprintf(hdr, n, "%s %"PRIuZ, type_str, obj->len);
-
-	assert(len > 0);             /* otherwise snprintf() is broken  */
-	assert(((size_t) len) < n);  /* otherwise the caller is broken! */
-
-	if (len < 0 || ((size_t) len) >= n)
-		return git__throw(GIT_ERROR, "Cannot format object header. Length is out of bounds");
+	const char *type_str = git_object_type2string(obj_type);
+	int len = p_snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
+	assert(len > 0 && len <= (int)n);
 	return len+1;
 }
 
-int git_odb__hash_obj(git_oid *id, char *hdr, size_t n, int *len, git_rawobj *obj)
+int git_odb__hashobj(git_oid *id, git_rawobj *obj)
 {
 	git_buf_vec vec[2];
-	int  hdrlen;
+	char header[64];
+	int hdrlen;
 
-	assert(id && hdr && len && obj);
+	assert(id && obj);
 
 	if (!git_object_typeisloose(obj->type))
-		return git__throw(GIT_ERROR, "Failed to hash object. Wrong object type");
-
+		return -1;
 	if (!obj->data && obj->len != 0)
-		return git__throw(GIT_ERROR, "Failed to hash object. No data given");
-
-	if ((hdrlen = format_object_header(hdr, n, obj)) < 0)
-		return git__rethrow(hdrlen, "Failed to hash object");
+		return -1;
 
-	*len = hdrlen;
+	hdrlen = format_object_header(header, sizeof(header), obj->len, obj->type);
 
-	vec[0].data = hdr;
-	vec[0].len  = hdrlen;
+	vec[0].data = header;
+	vec[0].len = hdrlen;
 	vec[1].data = obj->data;
-	vec[1].len  = obj->len;
+	vec[1].len = obj->len;
 
 	git_hash_vec(id, vec, 2);
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 
@@ -104,8 +78,8 @@ static void free_odb_object(void *o)
 	git_odb_object *object = (git_odb_object *)o;
 
 	if (object != NULL) {
-		free(object->raw.data);
-		free(object);
+		git__free(object->raw.data);
+		git__free(object);
 	}
 }
 
@@ -129,15 +103,104 @@ git_otype git_odb_object_type(git_odb_object *object)
 	return object->raw.type;
 }
 
-void git_odb_object_close(git_odb_object *object)
+void git_odb_object_free(git_odb_object *object)
 {
 	git_cached_obj_decref((git_cached_obj *)object, &free_odb_object);
 }
 
+int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type)
+{
+	int hdr_len;
+	char hdr[64], buffer[2048];
+	git_hash_ctx *ctx;
+
+	hdr_len = format_object_header(hdr, sizeof(hdr), size, type);
+
+	ctx = git_hash_new_ctx();
+
+	git_hash_update(ctx, hdr, hdr_len);
+
+	while (size > 0) {
+		ssize_t read_len = read(fd, buffer, sizeof(buffer));
+
+		if (read_len < 0) {
+			git_hash_free_ctx(ctx);
+			giterr_set(GITERR_OS, "Error reading file");
+			return -1;
+		}
+
+		git_hash_update(ctx, buffer, read_len);
+		size -= read_len;
+	}
+
+	git_hash_final(out, ctx);
+	git_hash_free_ctx(ctx);
+
+	return 0;
+}
+
+int git_odb__hashlink(git_oid *out, const char *path)
+{
+	struct stat st;
+	git_off_t size;
+	int result;
+
+	if (git_path_lstat(path, &st) < 0)
+		return -1;
+
+	size = st.st_size;
+
+	if (!git__is_sizet(size)) {
+		giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
+		return -1;
+	}
+
+	if (S_ISLNK(st.st_mode)) {
+		char *link_data;
+		ssize_t read_len;
+
+		link_data = git__malloc((size_t)size);
+		GITERR_CHECK_ALLOC(link_data);
+
+		read_len = p_readlink(path, link_data, (size_t)(size + 1));
+		if (read_len != (ssize_t)size) {
+			giterr_set(GITERR_OS, "Failed to read symlink data for '%s'", path);
+			return -1;
+		}
+
+		result = git_odb_hash(out, link_data, (size_t)size, GIT_OBJ_BLOB);
+		git__free(link_data);
+	} else { 
+		int fd = git_futils_open_ro(path);
+		if (fd < 0)
+			return -1;
+		result = git_odb__hashfd(out, fd, (size_t)size, GIT_OBJ_BLOB);
+		p_close(fd);
+	}
+
+	return result;
+}
+
+int git_odb_hashfile(git_oid *out, const char *path, git_otype type)
+{
+	git_off_t size;
+	int result, fd = git_futils_open_ro(path);
+	if (fd < 0)
+		return fd;
+
+	if ((size = git_futils_filesize(fd)) < 0 || !git__is_sizet(size)) {
+		giterr_set(GITERR_OS, "File size overflow for 32-bit systems");
+		p_close(fd);
+		return -1;
+	}
+
+	result = git_odb__hashfd(out, fd, (size_t)size, type);
+	p_close(fd);
+	return result;
+}
+
 int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
 {
-	char hdr[64];
-	int  hdrlen;
 	git_rawobj raw;
 
 	assert(id);
@@ -146,7 +209,7 @@ int git_odb_hash(git_oid *id, const void *data, size_t len, git_otype type)
 	raw.len = len;
 	raw.type = type;
 
-	return git_odb__hash_obj(id, hdr, sizeof(hdr), &hdrlen, &raw);
+	return git_odb__hashobj(id, &raw);
 }
 
 /**
@@ -171,19 +234,19 @@ static int fake_wstream__write(git_odb_stream *_stream, const char *data, size_t
 	fake_wstream *stream = (fake_wstream *)_stream;
 
 	if (stream->written + len > stream->size)
-		return GIT_ENOMEM;
+		return -1;
 
 	memcpy(stream->buffer + stream->written, data, len);
 	stream->written += len;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static void fake_wstream__free(git_odb_stream *_stream)
 {
 	fake_wstream *stream = (fake_wstream *)_stream;
 
-	free(stream->buffer);
-	free(stream);
+	git__free(stream->buffer);
+	git__free(stream);
 }
 
 static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend, size_t size, git_otype type)
@@ -191,15 +254,14 @@ static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend
 	fake_wstream *stream;
 
 	stream = git__calloc(1, sizeof(fake_wstream));
-	if (stream == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(stream);
 
 	stream->size = size;
 	stream->type = type;
 	stream->buffer = git__malloc(size);
 	if (stream->buffer == NULL) {
-		free(stream);
-		return GIT_ENOMEM;
+		git__free(stream);
+		return -1;
 	}
 
 	stream->stream.backend = backend;
@@ -210,7 +272,7 @@ static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend
 	stream->stream.mode = GIT_STREAM_WRONLY;
 
 	*stream_p = (git_odb_stream *)stream;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 /***********************************************************
@@ -223,8 +285,8 @@ static int init_fake_wstream(git_odb_stream **stream_p, git_odb_backend *backend
 
 static int backend_sort_cmp(const void *a, const void *b)
 {
-	const backend_internal *backend_a = *(const backend_internal **)(a);
-	const backend_internal *backend_b = *(const backend_internal **)(b);
+	const backend_internal *backend_a = (const backend_internal *)(a);
+	const backend_internal *backend_b = (const backend_internal *)(b);
 
 	if (backend_a->is_alternate == backend_b->is_alternate)
 		return (backend_b->priority - backend_a->priority);
@@ -234,25 +296,19 @@ static int backend_sort_cmp(const void *a, const void *b)
 
 int git_odb_new(git_odb **out)
 {
-	int error;
-
 	git_odb *db = git__calloc(1, sizeof(*db));
-	if (!db)
-		return GIT_ENOMEM;
-
-	error = git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object);
-	if (error < GIT_SUCCESS) {
-		free(db);
-		return git__rethrow(error, "Failed to create object database");
-	}
+	GITERR_CHECK_ALLOC(db);
 
-	if ((error = git_vector_init(&db->backends, 4, backend_sort_cmp)) < GIT_SUCCESS) {
-		free(db);
-		return git__rethrow(error, "Failed to create object database");
+	if (git_cache_init(&db->cache, GIT_DEFAULT_CACHE_SIZE, &free_odb_object) < 0 ||
+		git_vector_init(&db->backends, 4, backend_sort_cmp) < 0)
+	{
+		git__free(db);
+		return -1;
 	}
 
 	*out = db;
-	return GIT_SUCCESS;
+	GIT_REFCOUNT_INC(db);
+	return 0;
 }
 
 static int add_backend_internal(git_odb *odb, git_odb_backend *backend, int priority, int is_alternate)
@@ -261,25 +317,24 @@ static int add_backend_internal(git_odb *odb, git_odb_backend *backend, int prio
 
 	assert(odb && backend);
 
-	if (backend->odb != NULL && backend->odb != odb)
-		return git__throw(GIT_EBUSY, "The backend is already owned by another ODB");
+	/* Check if the backend is already owned by another ODB */
+	assert(!backend->odb || backend->odb == odb);
 
 	internal = git__malloc(sizeof(backend_internal));
-	if (internal == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(internal);
 
 	internal->backend = backend;
 	internal->priority = priority;
 	internal->is_alternate = is_alternate;
 
 	if (git_vector_insert(&odb->backends, internal) < 0) {
-		free(internal);
-		return GIT_ENOMEM;
+		git__free(internal);
+		return -1;
 	}
 
 	git_vector_sort(&odb->backends);
 	internal->backend->odb = odb;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 int git_odb_add_backend(git_odb *odb, git_odb_backend *backend, int priority)
@@ -295,130 +350,125 @@ int git_odb_add_alternate(git_odb *odb, git_odb_backend *backend, int priority)
 static int add_default_backends(git_odb *db, const char *objects_dir, int as_alternates)
 {
 	git_odb_backend *loose, *packed;
-	int error;
 
 	/* add the loose object backend */
-	error = git_odb_backend_loose(&loose, objects_dir);
-	if (error < GIT_SUCCESS)
-		return error;
-
-	error = add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to add backend");
+	if (git_odb_backend_loose(&loose, objects_dir, -1, 0) < 0 ||
+		add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates) < 0)
+		return -1;
 
 	/* add the packed file backend */
-	error = git_odb_backend_pack(&packed, objects_dir);
-	if (error < GIT_SUCCESS)
-		return error;
-
-	error = add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to add backend");
+	if (git_odb_backend_pack(&packed, objects_dir) < 0 ||
+		add_backend_internal(db, packed, GIT_PACKED_PRIORITY, as_alternates) < 0)
+		return -1;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static int load_alternates(git_odb *odb, const char *objects_dir)
 {
-	char alternates_path[GIT_PATH_MAX];
-	char *buffer, *alternate;
-
-	gitfo_buf alternates_buf = GITFO_BUF_INIT;
-	int error;
+	git_buf alternates_path = GIT_BUF_INIT;
+	git_buf alternates_buf = GIT_BUF_INIT;
+	char *buffer;
+	const char *alternate;
+	int result = 0;
 
-	git__joinpath(alternates_path, objects_dir, GIT_ALTERNATES_FILE);
+	if (git_buf_joinpath(&alternates_path, objects_dir, GIT_ALTERNATES_FILE) < 0)
+		return -1;
 
-	if (gitfo_exists(alternates_path) < GIT_SUCCESS)
-		return GIT_SUCCESS;
+	if (git_path_exists(alternates_path.ptr) == false) {
+		git_buf_free(&alternates_path);
+		return 0;
+	}
 
-	if (gitfo_read_file(&alternates_buf, alternates_path) < GIT_SUCCESS)
-		return git__throw(GIT_EOSERR, "Failed to add backend. Can't read alternates");
+	if (git_futils_readbuffer(&alternates_buf, alternates_path.ptr) < 0) {
+		git_buf_free(&alternates_path);
+		return -1;
+	}
 
-	buffer = (char *)alternates_buf.data;
-	error = GIT_SUCCESS;
+	buffer = (char *)alternates_buf.ptr;
 
 	/* add each alternate as a new backend; one alternate per line */
 	while ((alternate = git__strtok(&buffer, "\r\n")) != NULL) {
-		char full_path[GIT_PATH_MAX];
-
 		if (*alternate == '\0' || *alternate == '#')
 			continue;
 
 		/* relative path: build based on the current `objects` folder */
 		if (*alternate == '.') {
-			git__joinpath(full_path, objects_dir, alternate);
-			alternate = full_path;
+			if ((result = git_buf_joinpath(&alternates_path, objects_dir, alternate)) < 0)
+				break;
+			alternate = git_buf_cstr(&alternates_path);
 		}
 
-		if ((error = add_default_backends(odb, alternate, 1)) < GIT_SUCCESS)
+		if ((result = add_default_backends(odb, alternate, 1)) < 0)
 			break;
 	}
 
-	gitfo_free_buf(&alternates_buf);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to load alternates");
-	return error;
+	git_buf_free(&alternates_path);
+	git_buf_free(&alternates_buf);
+
+	return result;
 }
 
 int git_odb_open(git_odb **out, const char *objects_dir)
 {
 	git_odb *db;
-	int error;
 
 	assert(out && objects_dir);
 
 	*out = NULL;
 
-	if ((error = git_odb_new(&db)) < 0)
-		return git__rethrow(error, "Failed to open ODB");
-
-	if ((error = add_default_backends(db, objects_dir, 0)) < GIT_SUCCESS)
-		goto cleanup;
+	if (git_odb_new(&db) < 0)
+		return -1;
 
-	if ((error = load_alternates(db, objects_dir)) < GIT_SUCCESS)
-		goto cleanup;
+	if (add_default_backends(db, objects_dir, 0) < 0 ||
+		load_alternates(db, objects_dir) < 0)
+	{
+		git_odb_free(db);
+		return -1;
+	}
 
 	*out = db;
-	return GIT_SUCCESS;
-
-cleanup:
-	git_odb_close(db);
-	return error; /* error already set - pass through */
+	return 0;
 }
 
-void git_odb_close(git_odb *db)
+static void odb_free(git_odb *db)
 {
 	unsigned int i;
 
-	if (db == NULL)
-		return;
-
 	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *backend = internal->backend;
 
 		if (backend->free) backend->free(backend);
-		else free(backend);
+		else git__free(backend);
 
-		free(internal);
+		git__free(internal);
 	}
 
 	git_vector_free(&db->backends);
 	git_cache_free(&db->cache);
-	free(db);
+	git__free(db);
+}
+
+void git_odb_free(git_odb *db)
+{
+	if (db == NULL)
+		return;
+
+	GIT_REFCOUNT_DEC(db, odb_free);
 }
 
 int git_odb_exists(git_odb *db, const git_oid *id)
 {
 	git_odb_object *object;
 	unsigned int i;
-	int found = 0;
+	bool found = false;
 
 	assert(db && id);
 
 	if ((object = git_cache_get(&db->cache, id)) != NULL) {
-		git_odb_object_close(object);
-		return 1;
+		git_odb_object_free(object);
+		return (int)true;
 	}
 
 	for (i = 0; i < db->backends.length && !found; ++i) {
@@ -429,7 +479,7 @@ int git_odb_exists(git_odb *db, const git_oid *id)
 			found = b->exists(b, id);
 	}
 
-	return found;
+	return (int)found;
 }
 
 int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git_oid *id)
@@ -443,8 +493,8 @@ int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git
 	if ((object = git_cache_get(&db->cache, id)) != NULL) {
 		*len_p = object->raw.len;
 		*type_p = object->raw.type;
-		git_odb_object_close(object);
-		return GIT_SUCCESS;
+		git_odb_object_free(object);
+		return 0;
 	}
 
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
@@ -455,20 +505,20 @@ int git_odb_read_header(size_t *len_p, git_otype *type_p, git_odb *db, const git
 			error = b->read_header(len_p, type_p, b, id);
 	}
 
+	if (!error || error == GIT_PASSTHROUGH)
+		return 0;
+
 	/*
 	 * no backend could read only the header.
 	 * try reading the whole object and freeing the contents
 	 */
-	if (error < 0) {
-		if ((error = git_odb_read(&object, db, id)) < GIT_SUCCESS)
-			return error; /* error already set - pass through */
-
-		*len_p = object->raw.len;
-		*type_p = object->raw.type;
-		git_odb_object_close(object);
-	}
+	if ((error = git_odb_read(&object, db, id)) < 0)
+		return error; /* error already set - pass along */
 
-	return GIT_SUCCESS;
+	*len_p = object->raw.len;
+	*type_p = object->raw.type;
+	git_odb_object_free(object);
+	return 0;
 }
 
 int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
@@ -481,7 +531,7 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 
 	*out = git_cache_get(&db->cache, id);
 	if (*out != NULL)
-		return GIT_SUCCESS;
+		return 0;
 
 	for (i = 0; i < db->backends.length && error < 0; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
@@ -491,27 +541,30 @@ int git_odb_read(git_odb_object **out, git_odb *db, const git_oid *id)
 			error = b->read(&raw.data, &raw.len, &raw.type, b, id);
 	}
 
-	if (error == GIT_SUCCESS) {
-		*out = git_cache_try_store(&db->cache, new_odb_object(id, &raw));
-	}
+	/* TODO: If no backends are configured, this returns GIT_ENOTFOUND but
+	 * will never have called giterr_set().
+	 */
 
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to read object");
-	return error;
+	if (error && error != GIT_PASSTHROUGH)
+		return error;
+
+	*out = git_cache_try_store(&db->cache, new_odb_object(id, &raw));
+	return 0;
 }
 
-int git_odb_read_prefix(git_odb_object **out, git_odb *db, const git_oid *short_id, unsigned int len)
+int git_odb_read_prefix(
+	git_odb_object **out, git_odb *db, const git_oid *short_id, unsigned int len)
 {
 	unsigned int i;
 	int error = GIT_ENOTFOUND;
-	git_oid full_oid;
+	git_oid found_full_oid = {{0}};
 	git_rawobj raw;
-	int found = 0;
+	bool found = false;
 
 	assert(out && db);
 
 	if (len < GIT_OID_MINPREFIXLEN)
-		return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to lookup object. Prefix length is lower than %d.", GIT_OID_MINPREFIXLEN);
+		return git_odb__error_ambiguous("prefix length too short");
 
 	if (len > GIT_OID_HEXSZ)
 		len = GIT_OID_HEXSZ;
@@ -519,44 +572,42 @@ int git_odb_read_prefix(git_odb_object **out, git_odb *db, const git_oid *short_
 	if (len == GIT_OID_HEXSZ) {
 		*out = git_cache_get(&db->cache, short_id);
 		if (*out != NULL)
-			return GIT_SUCCESS;
+			return 0;
 	}
 
-	for (i = 0; i < db->backends.length && found < 2; ++i) {
+	for (i = 0; i < db->backends.length; ++i) {
 		backend_internal *internal = git_vector_get(&db->backends, i);
 		git_odb_backend *b = internal->backend;
 
 		if (b->read != NULL) {
+			git_oid full_oid;
 			error = b->read_prefix(&full_oid, &raw.data, &raw.len, &raw.type, b, short_id, len);
-			switch (error) {
-			case GIT_SUCCESS:
-				found++;
-				break;
-			case GIT_ENOTFOUND:
-				break;
-			case GIT_EAMBIGUOUSOIDPREFIX:
-				return git__rethrow(error, "Failed to read object. Ambiguous sha1 prefix");
-			default:
-				return git__rethrow(error, "Failed to read object");
-			}
+			if (error == GIT_ENOTFOUND || error == GIT_PASSTHROUGH)
+				continue;
+
+			if (error)
+				return error;
+
+			if (found && git_oid_cmp(&full_oid, &found_full_oid))
+				return git_odb__error_ambiguous("multiple matches for prefix");
+			found_full_oid = full_oid;
+			found = true;
 		}
 	}
 
-	if (found == 1) {
-		*out = git_cache_try_store(&db->cache, new_odb_object(&full_oid, &raw));
-	} else if (found > 1) {
-		return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to read object. Ambiguous sha1 prefix");
-	} else {
-		return git__throw(GIT_ENOTFOUND, "Failed to read object. Object not found");
-	}
+	if (!found)
+		return git_odb__error_notfound("no match for prefix", short_id);
 
-	return GIT_SUCCESS;
+	*out = git_cache_try_store(&db->cache, new_odb_object(&found_full_oid, &raw));
+	return 0;
 }
 
-int git_odb_write(git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
+int git_odb_write(
+	git_oid *oid, git_odb *db, const void *data, size_t len, git_otype type)
 {
 	unsigned int i;
 	int error = GIT_ERROR;
+	git_odb_stream *stream;
 
 	assert(oid && db);
 
@@ -572,24 +623,25 @@ int git_odb_write(git_oid *oid, git_odb *db, const void *data, size_t len, git_o
 			error = b->write(oid, b, data, len, type);
 	}
 
+	if (!error || error == GIT_PASSTHROUGH)
+		return 0;
+
 	/* if no backends were able to write the object directly, we try a streaming
 	 * write to the backends; just write the whole object into the stream in one
 	 * push */
-	if (error < GIT_SUCCESS) {
-		git_odb_stream *stream;
 
-		if ((error = git_odb_open_wstream(&stream, db, len, type)) == GIT_SUCCESS) {
-			stream->write(stream, data, len);
-			error = stream->finalize_write(oid, stream);
-			stream->free(stream);
-		}
-	}
+	if ((error = git_odb_open_wstream(&stream, db, len, type)) != 0)
+		return error;
 
-	return (error == GIT_SUCCESS) ? GIT_SUCCESS :
-		git__rethrow(error, "Failed to write object");
+	stream->write(stream, data, len);
+	error = stream->finalize_write(oid, stream);
+	stream->free(stream);
+
+	return error;
 }
 
-int git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
+int git_odb_open_wstream(
+	git_odb_stream **stream, git_odb *db, size_t size, git_otype type)
 {
 	unsigned int i;
 	int error = GIT_ERROR;
@@ -610,11 +662,13 @@ int git_odb_open_wstream(git_odb_stream **stream, git_odb *db, size_t size, git_
 			error = init_fake_wstream(stream, b, size, type);
 	}
 
-	return (error == GIT_SUCCESS) ? GIT_SUCCESS :
-		git__rethrow(error, "Failed to open write stream");
+	if (error == GIT_PASSTHROUGH)
+		error = 0;
+
+	return error;
 }
 
-int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid) 
+int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oid)
 {
 	unsigned int i;
 	int error = GIT_ERROR;
@@ -629,7 +683,27 @@ int git_odb_open_rstream(git_odb_stream **stream, git_odb *db, const git_oid *oi
 			error = b->readstream(stream, b, oid);
 	}
 
-	return (error == GIT_SUCCESS) ? GIT_SUCCESS :
-		git__rethrow(error, "Failed to open read stream");
+	if (error == GIT_PASSTHROUGH)
+		error = 0;
+
+	return error;
+}
+
+int git_odb__error_notfound(const char *message, const git_oid *oid)
+{
+	if (oid != NULL) {
+		char oid_str[GIT_OID_HEXSZ + 1];
+		git_oid_tostr(oid_str, sizeof(oid_str), oid);
+		giterr_set(GITERR_ODB, "Object not found - %s (%s)", message, oid_str);
+	} else
+		giterr_set(GITERR_ODB, "Object not found - %s", message);
+
+	return GIT_ENOTFOUND;
+}
+
+int git_odb__error_ambiguous(const char *message)
+{
+	giterr_set(GITERR_ODB, "Ambiguous SHA1 prefix - %s", message);
+	return GIT_EAMBIGUOUS;
 }
 
diff --git a/src/odb.h b/src/odb.h
index f3685834e..263e4c30b 100644
--- a/src/odb.h
+++ b/src/odb.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_odb_h__
 #define INCLUDE_odb_h__
 
@@ -7,12 +13,17 @@
 
 #include "vector.h"
 #include "cache.h"
+#include "posix.h"
+
+#define GIT_OBJECTS_DIR "objects/"
+#define GIT_OBJECT_DIR_MODE 0777
+#define GIT_OBJECT_FILE_MODE 0444
 
 /* DO NOT EXPORT */
 typedef struct {
-	void *data;          /**< Raw, decompressed object data. */
-	size_t len;          /**< Total number of bytes in data. */
-	git_otype type;      /**< Type of this object. */
+	void *data;			/**< Raw, decompressed object data. */
+	size_t len;			/**< Total number of bytes in data. */
+	git_otype type;		/**< Type of this object. */
 } git_rawobj;
 
 /* EXPORT */
@@ -23,11 +34,47 @@ struct git_odb_object {
 
 /* EXPORT */
 struct git_odb {
-	void *_internal;
+	git_refcount rc;
 	git_vector backends;
 	git_cache cache;
 };
 
-int git_odb__hash_obj(git_oid *id, char *hdr, size_t n, int *len, git_rawobj *obj);
+/*
+ * Hash a git_rawobj internally.
+ * The `git_rawobj` is supposed to be previously initialized
+ */
+int git_odb__hashobj(git_oid *id, git_rawobj *obj);
+
+/*
+ * Hash an open file descriptor.
+ * This is a performance call when the contents of a fd need to be hashed,
+ * but the fd is already open and we have the size of the contents.
+ *
+ * Saves us some `stat` calls.
+ *
+ * The fd is never closed, not even on error. It must be opened and closed
+ * by the caller
+ */
+int git_odb__hashfd(git_oid *out, git_file fd, size_t size, git_otype type);
+
+/*
+ * Hash a `path`, assuming it could be a POSIX symlink: if the path is a symlink,
+ * then the raw contents of the symlink will be hashed. Otherwise, this will
+ * fallback to `git_odb__hashfd`.
+ *
+ * The hash type for this call is always `GIT_OBJ_BLOB` because symlinks may only
+ * point to blobs.
+ */
+int git_odb__hashlink(git_oid *out, const char *path);
+
+/*
+ * Generate a GIT_ENOTFOUND error for the ODB.
+ */
+int git_odb__error_notfound(const char *message, const git_oid *oid);
+
+/*
+ * Generate a GIT_EAMBIGUOUS error for the ODB.
+ */
+int git_odb__error_ambiguous(const char *message);
 
 #endif
diff --git a/src/odb_loose.c b/src/odb_loose.c
index deff59ad0..989b03ab2 100644
--- a/src/odb_loose.c
+++ b/src/odb_loose.c
@@ -1,50 +1,31 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
-#include "git2/zlib.h"
+#include <zlib.h>
 #include "git2/object.h"
+#include "git2/oid.h"
 #include "fileops.h"
 #include "hash.h"
 #include "odb.h"
-#include "oid.h"
 #include "delta-apply.h"
 #include "filebuf.h"
 
 #include "git2/odb_backend.h"
 #include "git2/types.h"
 
-typedef struct {  /* object header data */
-	git_otype type;  /* object type */
-	size_t    size;  /* object size */
+typedef struct { /* object header data */
+	git_otype type; /* object type */
+	size_t	size; /* object size */
 } obj_hdr;
 
 typedef struct {
 	git_odb_stream stream;
 	git_filebuf fbuf;
-	int finished;
 } loose_writestream;
 
 typedef struct loose_backend {
@@ -69,41 +50,38 @@ typedef struct {
 } loose_locate_object_state;
 
 
-
 /***********************************************************
  *
  * MISCELANEOUS HELPER FUNCTIONS
  *
  ***********************************************************/
 
-static size_t object_file_name(char *name, size_t n, char *dir, const git_oid *id)
+static int object_file_name(git_buf *name, const char *dir, const git_oid *id)
 {
-	size_t len = strlen(dir);
+	git_buf_sets(name, dir);
 
-	/* check length: 43 = 40 hex sha1 chars + 2 * '/' + '\0' */
-	if (len+43 > n)
-		return len+43;
+	/* expand length for 40 hex sha1 chars + 2 * '/' + '\0' */
+	if (git_buf_grow(name, git_buf_len(name) + GIT_OID_HEXSZ + 3) < 0)
+		return -1;
 
-	/* the object dir: eg $GIT_DIR/objects */
-	strcpy(name, dir);
-	if (name[len-1] != '/')
-		name[len++] = '/';
+	git_path_to_dir(name);
 
 	/* loose object filename: aa/aaa... (41 bytes) */
-	git_oid_pathfmt(&name[len], id);
-	name[len+41] = '\0';
+	git_oid_pathfmt(name->ptr + git_buf_len(name), id);
+	name->size += GIT_OID_HEXSZ + 1;
+	name->ptr[name->size] = '\0';
 
 	return 0;
 }
 
 
-static size_t get_binary_object_header(obj_hdr *hdr, gitfo_buf *obj)
+static size_t get_binary_object_header(obj_hdr *hdr, git_buf *obj)
 {
 	unsigned char c;
-	unsigned char *data = obj->data;
+	unsigned char *data = (unsigned char *)obj->ptr;
 	size_t shift, size, used = 0;
 
-	if (obj->len == 0)
+	if (git_buf_len(obj) == 0)
 		return 0;
 
 	c = data[used++];
@@ -112,7 +90,7 @@ static size_t get_binary_object_header(obj_hdr *hdr, gitfo_buf *obj)
 	size = c & 15;
 	shift = 4;
 	while (c & 0x80) {
-		if (obj->len <= used)
+		if (git_buf_len(obj) <= used)
 			return 0;
 		if (sizeof(size_t) * 8 <= shift)
 			return 0;
@@ -142,7 +120,7 @@ static size_t get_object_header(obj_hdr *hdr, unsigned char *data)
 	if (used == 0)
 		return 0;
 	hdr->type = git_object_string2type(typename);
-	used++;  /* consume the space */
+	used++; /* consume the space */
 
 	/*
 	 * length follows immediately in decimal (without
@@ -182,29 +160,29 @@ static size_t get_object_header(obj_hdr *hdr, unsigned char *data)
 static void init_stream(z_stream *s, void *out, size_t len)
 {
 	memset(s, 0, sizeof(*s));
-	s->next_out  = out;
-	s->avail_out = len;
+	s->next_out = out;
+	s->avail_out = (uInt)len;
 }
 
 static void set_stream_input(z_stream *s, void *in, size_t len)
 {
-	s->next_in  = in;
-	s->avail_in = len;
+	s->next_in = in;
+	s->avail_in = (uInt)len;
 }
 
 static void set_stream_output(z_stream *s, void *out, size_t len)
 {
-	s->next_out  = out;
-	s->avail_out = len;
+	s->next_out = out;
+	s->avail_out = (uInt)len;
 }
 
 
-static int start_inflate(z_stream *s, gitfo_buf *obj, void *out, size_t len)
+static int start_inflate(z_stream *s, git_buf *obj, void *out, size_t len)
 {
 	int status;
 
 	init_stream(s, out, len);
-	set_stream_input(s, obj->data, obj->len);
+	set_stream_input(s, obj->ptr, git_buf_len(obj));
 
 	if ((status = inflateInit(s)) < Z_OK)
 		return status;
@@ -221,10 +199,12 @@ static int finish_inflate(z_stream *s)
 
 	inflateEnd(s);
 
-	if ((status != Z_STREAM_END) || (s->avail_in != 0))
-		return git__throw(GIT_ERROR, "Failed to finish inflation. Stream aborted prematurely");
+	if ((status != Z_STREAM_END) || (s->avail_in != 0)) {
+		giterr_set(GITERR_ZLIB, "Failed to finish ZLib inflation. Stream aborted prematurely");
+		return -1;
+	}
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static int is_zlib_compressed_data(unsigned char *data)
@@ -232,7 +212,7 @@ static int is_zlib_compressed_data(unsigned char *data)
 	unsigned int w;
 
 	w = ((unsigned int)(data[0]) << 8) + data[1];
-	return data[0] == 0x78 && !(w % 31);
+	return (data[0] & 0x8F) == 0x08 && !(w % 31);
 }
 
 static int inflate_buffer(void *in, size_t inlen, void *out, size_t outlen)
@@ -242,27 +222,30 @@ static int inflate_buffer(void *in, size_t inlen, void *out, size_t outlen)
 
 	memset(&zs, 0x0, sizeof(zs));
 
-	zs.next_out  = out;
-	zs.avail_out = outlen;
+	zs.next_out = out;
+	zs.avail_out = (uInt)outlen;
 
-	zs.next_in  = in;
-	zs.avail_in = inlen;
+	zs.next_in = in;
+	zs.avail_in = (uInt)inlen;
 
-	if (inflateInit(&zs) < Z_OK)
-		return git__throw(GIT_ERROR, "Failed to inflate buffer");
+	if (inflateInit(&zs) < Z_OK) {
+		giterr_set(GITERR_ZLIB, "Failed to inflate buffer");
+		return -1;
+	}
 
 	while (status == Z_OK)
 		status = inflate(&zs, Z_FINISH);
 
 	inflateEnd(&zs);
 
-	if ((status != Z_STREAM_END) /*|| (zs.avail_in != 0) */)
-		return git__throw(GIT_ERROR, "Failed to inflate buffer. Stream aborted prematurely");
-
-	if (zs.total_out != outlen)
-		return git__throw(GIT_ERROR, "Failed to inflate buffer. Stream aborted prematurely");
+	if (status != Z_STREAM_END /* || zs.avail_in != 0 */ ||
+		zs.total_out != outlen)
+	{
+		giterr_set(GITERR_ZLIB, "Failed to inflate buffer. Stream aborted prematurely");
+		return -1;
+	}
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr)
@@ -295,7 +278,7 @@ static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr)
 	else {
 		set_stream_output(s, buf + used, hdr->size - used);
 		if (finish_inflate(s)) {
-			free(buf);
+			git__free(buf);
 			return NULL;
 		}
 	}
@@ -309,7 +292,7 @@ static void *inflate_tail(z_stream *s, void *hb, size_t used, obj_hdr *hdr)
  * of loose object data into packs. This format is no longer used, but
  * we must still read it.
  */
-static int inflate_packlike_loose_disk_obj(git_rawobj *out, gitfo_buf *obj)
+static int inflate_packlike_loose_disk_obj(git_rawobj *out, git_buf *obj)
 {
 	unsigned char *in, *buf;
 	obj_hdr hdr;
@@ -319,35 +302,34 @@ static int inflate_packlike_loose_disk_obj(git_rawobj *out, gitfo_buf *obj)
 	 * read the object header, which is an (uncompressed)
 	 * binary encoding of the object type and size.
 	 */
-	if ((used = get_binary_object_header(&hdr, obj)) == 0)
-		return git__throw(GIT_ERROR, "Failed to inflate loose object. Object has no header");
-
-	if (!git_object_typeisloose(hdr.type))
-		return git__throw(GIT_ERROR, "Failed to inflate loose object. Wrong object type");
+	if ((used = get_binary_object_header(&hdr, obj)) == 0 ||
+		!git_object_typeisloose(hdr.type)) {
+		giterr_set(GITERR_ODB, "Failed to inflate loose object.");
+		return -1;
+	}
 
 	/*
 	 * allocate a buffer and inflate the data into it
 	 */
 	buf = git__malloc(hdr.size + 1);
-	if (!buf)
-		return GIT_ENOMEM;
-
-	in  = ((unsigned char *)obj->data) + used;
-	len = obj->len - used;
-	if (inflate_buffer(in, len, buf, hdr.size)) {
-		free(buf);
-		return git__throw(GIT_ERROR, "Failed to inflate loose object. Could not inflate buffer");
+	GITERR_CHECK_ALLOC(buf);
+
+	in = ((unsigned char *)obj->ptr) + used;
+	len = obj->size - used;
+	if (inflate_buffer(in, len, buf, hdr.size) < 0) {
+		git__free(buf);
+		return -1;
 	}
 	buf[hdr.size] = '\0';
 
 	out->data = buf;
-	out->len  = hdr.size;
+	out->len = hdr.size;
 	out->type = hdr.type;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-static int inflate_disk_obj(git_rawobj *out, gitfo_buf *obj)
+static int inflate_disk_obj(git_rawobj *out, git_buf *obj)
 {
 	unsigned char head[64], *buf;
 	z_stream zs;
@@ -357,35 +339,34 @@ static int inflate_disk_obj(git_rawobj *out, gitfo_buf *obj)
 	/*
 	 * check for a pack-like loose object
 	 */
-	if (!is_zlib_compressed_data(obj->data))
+	if (!is_zlib_compressed_data((unsigned char *)obj->ptr))
 		return inflate_packlike_loose_disk_obj(out, obj);
 
 	/*
 	 * inflate the initial part of the io buffer in order
 	 * to parse the object header (type and size).
 	 */
-	if (start_inflate(&zs, obj, head, sizeof(head)) < Z_OK)
-		return git__throw(GIT_ERROR, "Failed to inflate disk object. Could not inflate buffer");
-
-	if ((used = get_object_header(&hdr, head)) == 0)
-		return git__throw(GIT_ERROR, "Failed to inflate disk object. Object has no header");
-
-	if (!git_object_typeisloose(hdr.type))
-		return git__throw(GIT_ERROR, "Failed to inflate disk object. Wrong object type");
+	if (start_inflate(&zs, obj, head, sizeof(head)) < Z_OK ||
+		(used = get_object_header(&hdr, head)) == 0 ||
+		!git_object_typeisloose(hdr.type))
+	{
+		giterr_set(GITERR_ODB, "Failed to inflate disk object.");
+		return -1;
+	}
 
 	/*
 	 * allocate a buffer and inflate the object data into it
 	 * (including the initial sequence in the head buffer).
 	 */
 	if ((buf = inflate_tail(&zs, head, used, &hdr)) == NULL)
-		return GIT_ENOMEM;
+		return -1;
 	buf[hdr.size] = '\0';
 
 	out->data = buf;
-	out->len  = hdr.size;
+	out->len = hdr.size;
 	out->type = hdr.type;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 
@@ -402,29 +383,31 @@ static int inflate_disk_obj(git_rawobj *out, gitfo_buf *obj)
  *
  ***********************************************************/
 
-static int read_loose(git_rawobj *out, const char *loc)
+static int read_loose(git_rawobj *out, git_buf *loc)
 {
 	int error;
-	gitfo_buf obj = GITFO_BUF_INIT;
+	git_buf obj = GIT_BUF_INIT;
 
 	assert(out && loc);
 
+	if (git_buf_oom(loc))
+		return -1;
+
 	out->data = NULL;
-	out->len  = 0;
+	out->len = 0;
 	out->type = GIT_OBJ_BAD;
 
-	if (gitfo_read_file(&obj, loc) < 0)
-		return git__throw(GIT_ENOTFOUND, "Failed to read loose object. File not found");
+	if (!(error = git_futils_readbuffer(&obj, loc->ptr)))
+		error = inflate_disk_obj(out, &obj);
 
-	error = inflate_disk_obj(out, &obj);
-	gitfo_free_buf(&obj);
+	git_buf_free(&obj);
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to read loose object");
+	return error;
 }
 
-static int read_header_loose(git_rawobj *out, const char *loc)
+static int read_header_loose(git_rawobj *out, git_buf *loc)
 {
-	int error = GIT_SUCCESS, z_return = Z_ERRNO, read_bytes;
+	int error = 0, z_return = Z_ERRNO, read_bytes;
 	git_file fd;
 	z_stream zs;
 	obj_hdr header_obj;
@@ -432,129 +415,151 @@ static int read_header_loose(git_rawobj *out, const char *loc)
 
 	assert(out && loc);
 
+	if (git_buf_oom(loc))
+		return -1;
+
 	out->data = NULL;
 
-	if ((fd = gitfo_open(loc, O_RDONLY)) < 0)
-		return git__throw(GIT_ENOTFOUND, "Failed to read loose object header. File not found");
+	if ((fd = git_futils_open_ro(loc->ptr)) < 0)
+		return fd;
 
 	init_stream(&zs, inflated_buffer, sizeof(inflated_buffer));
 
-	if (inflateInit(&zs) < Z_OK) {
-		error = GIT_EZLIB;
-		goto cleanup;
-	}
+	z_return = inflateInit(&zs);
 
-	do {
-		if ((read_bytes = read(fd, raw_buffer, sizeof(raw_buffer))) > 0) {
+	while (z_return == Z_OK) {
+		if ((read_bytes = p_read(fd, raw_buffer, sizeof(raw_buffer))) > 0) {
 			set_stream_input(&zs, raw_buffer, read_bytes);
 			z_return = inflate(&zs, 0);
-		} else {
+		} else
 			z_return = Z_STREAM_END;
-			break;
-		}
-	} while (z_return == Z_OK);
+	}
 
 	if ((z_return != Z_STREAM_END && z_return != Z_BUF_ERROR)
 		|| get_object_header(&header_obj, inflated_buffer) == 0
-		|| git_object_typeisloose(header_obj.type) == 0) {
-		error = GIT_EOBJCORRUPTED;
-		goto cleanup;
+		|| git_object_typeisloose(header_obj.type) == 0)
+	{
+		giterr_set(GITERR_ZLIB, "Failed to read loose object header");
+		error = -1;
+	} else {
+		out->len = header_obj.size;
+		out->type = header_obj.type;
 	}
 
-	out->len  = header_obj.size;
-	out->type = header_obj.type;
-
-cleanup:
 	finish_inflate(&zs);
-	gitfo_close(fd);
+	p_close(fd);
 
-	if (error < GIT_SUCCESS)
-		return git__throw(error, "Failed to read loose object header. Header is corrupted");
-
-	return GIT_SUCCESS;
+	return error;
 }
 
-static int locate_object(char *object_location, loose_backend *backend, const git_oid *oid)
+static int locate_object(
+	git_buf *object_location,
+	loose_backend *backend,
+	const git_oid *oid)
 {
-	object_file_name(object_location, GIT_PATH_MAX, backend->objects_dir, oid);
-	return gitfo_exists(object_location);
+	int error = object_file_name(object_location, backend->objects_dir, oid);
+
+	if (!error && !git_path_exists(object_location->ptr))
+		return GIT_ENOTFOUND;
+
+	return error;
 }
 
 /* Explore an entry of a directory and see if it matches a short oid */
-int fn_locate_object_short_oid(void *state, char *pathbuf) {
+static int fn_locate_object_short_oid(void *state, git_buf *pathbuf) {
 	loose_locate_object_state *sstate = (loose_locate_object_state *)state;
 
-	size_t pathbuf_len = strlen(pathbuf);
-	if (pathbuf_len - sstate->dir_len != GIT_OID_HEXSZ - 2) {
+	if (git_buf_len(pathbuf) - sstate->dir_len != GIT_OID_HEXSZ - 2) {
 		/* Entry cannot be an object. Continue to next entry */
-		return GIT_SUCCESS;
+		return 0;
 	}
 
-	if (!gitfo_exists(pathbuf) && gitfo_isdir(pathbuf)) {
-		/* We are already in the directory matching the 2 first hex characters */
-		if (!git_oid_match_hex(sstate->short_oid_len-2, sstate->short_oid+2, (unsigned char *)pathbuf + sstate->dir_len)) {
+	if (git_path_isdir(pathbuf->ptr) == false) {
+		/* We are already in the directory matching the 2 first hex characters,
+		 * compare the first ncmp characters of the oids */
+		if (!memcmp(sstate->short_oid + 2,
+			(unsigned char *)pathbuf->ptr + sstate->dir_len,
+			sstate->short_oid_len - 2)) {
+
 			if (!sstate->found) {
 				sstate->res_oid[0] = sstate->short_oid[0];
 				sstate->res_oid[1] = sstate->short_oid[1];
-				memcpy(sstate->res_oid+2, pathbuf+sstate->dir_len, GIT_OID_HEXSZ-2);
+				memcpy(sstate->res_oid+2, pathbuf->ptr+sstate->dir_len, GIT_OID_HEXSZ-2);
 			}
 			sstate->found++;
 		}
 	}
+
 	if (sstate->found > 1)
-		return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Ambiguous sha1 prefix within loose objects");
+		return git_odb__error_ambiguous("multiple matches in loose objects");
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 /* Locate an object matching a given short oid */
-static int locate_object_short_oid(char *object_location, git_oid *res_oid, loose_backend *backend, const git_oid *short_oid, unsigned int len)
+static int locate_object_short_oid(
+	git_buf *object_location,
+	git_oid *res_oid,
+	loose_backend *backend,
+	const git_oid *short_oid,
+	unsigned int len)
 {
 	char *objects_dir = backend->objects_dir;
 	size_t dir_len = strlen(objects_dir);
 	loose_locate_object_state state;
 	int error;
 
-	if (dir_len+43 > GIT_PATH_MAX)
-		return git__throw(GIT_ERROR, "Failed to locate object from short oid. Object path too long");
+	/* prealloc memory for OBJ_DIR/xx/ */
+	if (git_buf_grow(object_location, dir_len + 5) < 0)
+		return -1;
 
-	strcpy(object_location, objects_dir);
+	git_buf_sets(object_location, objects_dir);
+	git_path_to_dir(object_location);
 
-	/* Add a separator if not already there */
-	if (object_location[dir_len-1] != '/')
-		object_location[dir_len++] = '/';
+	/* save adjusted position at end of dir so it can be restored later */
+	dir_len = git_buf_len(object_location);
 
 	/* Convert raw oid to hex formatted oid */
 	git_oid_fmt((char *)state.short_oid, short_oid);
+
 	/* Explore OBJ_DIR/xx/ where xx is the beginning of hex formatted short oid */
-	sprintf(object_location+dir_len, "%.2s/", state.short_oid);
+	if (git_buf_printf(object_location, "%.2s/", state.short_oid) < 0)
+		return -1;
 
 	/* Check that directory exists */
-	if (gitfo_exists(object_location) || gitfo_isdir(object_location))
-		return git__throw(GIT_ENOTFOUND, "Failed to locate object from short oid. Object not found");
+	if (git_path_isdir(object_location->ptr) == false)
+		return git_odb__error_notfound("no matching loose object for prefix", short_oid);
 
-	state.dir_len = dir_len+3;
+	state.dir_len = git_buf_len(object_location);
 	state.short_oid_len = len;
 	state.found = 0;
+
 	/* Explore directory to find a unique object matching short_oid */
-	error = gitfo_dirent(object_location, GIT_PATH_MAX, fn_locate_object_short_oid, &state);
-	if (error) {
-		return git__rethrow(error, "Failed to locate object from short oid");
-	}
-	if (!state.found) {
-		return git__throw(GIT_ENOTFOUND, "Failed to locate object from short oid. Object not found");
-	}
+	error = git_path_direach(
+		object_location, fn_locate_object_short_oid, &state);
+	if (error)
+		return error;
+
+	if (!state.found)
+		return git_odb__error_notfound("no matching loose object for prefix", short_oid);
 
 	/* Convert obtained hex formatted oid to raw */
-	error = git_oid_mkstr(res_oid, (char *)state.res_oid);
-	if (error) {
-		return git__rethrow(error, "Failed to locate object from short oid");
-	}
+	error = git_oid_fromstr(res_oid, (char *)state.res_oid);
+	if (error)
+		return error;
 
 	/* Update the location according to the oid obtained */
-	git_oid_pathfmt(object_location+dir_len, res_oid);
 
-	return GIT_SUCCESS;
+	git_buf_truncate(object_location, dir_len);
+	if (git_buf_grow(object_location, dir_len + GIT_OID_HEXSZ + 2) < 0)
+		return -1;
+
+	git_oid_pathfmt(object_location->ptr + dir_len, res_oid);
+
+	object_location->size += GIT_OID_HEXSZ + 1;
+	object_location->ptr[object_location->size] = '\0';
+
+	return 0;
 }
 
 
@@ -573,9 +578,9 @@ static int locate_object_short_oid(char *object_location, git_oid *res_oid, loos
  *
  ***********************************************************/
 
-int loose_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+static int loose_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
 {
-	char object_path[GIT_PATH_MAX];
+	git_buf object_path = GIT_BUF_INIT;
 	git_rawobj raw;
 	int error;
 
@@ -584,39 +589,40 @@ int loose_backend__read_header(size_t *len_p, git_otype *type_p, git_odb_backend
 	raw.len = 0;
 	raw.type = GIT_OBJ_BAD;
 
-	if (locate_object(object_path, (loose_backend *)backend, oid) < 0)
-		return git__throw(GIT_ENOTFOUND, "Failed to read loose backend header. Object not found");
+	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0)
+		error = git_odb__error_notfound("no matching loose object", oid);
+	else if ((error = read_header_loose(&raw, &object_path)) == 0) {
+		*len_p = raw.len;
+		*type_p = raw.type;
+	}
 
-	if ((error = read_header_loose(&raw, object_path)) < GIT_SUCCESS)
-		return error;
+	git_buf_free(&object_path);
 
-	*len_p = raw.len;
-	*type_p = raw.type;
-	return GIT_SUCCESS;
+	return error;
 }
 
-int loose_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+static int loose_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
 {
-	char object_path[GIT_PATH_MAX];
+	git_buf object_path = GIT_BUF_INIT;
 	git_rawobj raw;
-	int error;
+	int error = 0;
 
 	assert(backend && oid);
 
-	if (locate_object(object_path, (loose_backend *)backend, oid) < 0)
-		return git__throw(GIT_ENOTFOUND, "Failed to read loose backend. Object not found");
-
-	if ((error = read_loose(&raw, object_path)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to read loose backend");
+	if (locate_object(&object_path, (loose_backend *)backend, oid) < 0)
+		error = git_odb__error_notfound("no matching loose object", oid);
+	else if ((error = read_loose(&raw, &object_path)) == 0) {
+		*buffer_p = raw.data;
+		*len_p = raw.len;
+		*type_p = raw.type;
+	}
 
-	*buffer_p = raw.data;
-	*len_p = raw.len;
-	*type_p = raw.type;
+	git_buf_free(&object_path);
 
-	return GIT_SUCCESS;
+	return error;
 }
 
-int loose_backend__read_prefix(
+static int loose_backend__read_prefix(
 	git_oid *out_oid,
 	void **buffer_p,
 	size_t *len_p,
@@ -625,82 +631,90 @@ int loose_backend__read_prefix(
 	const git_oid *short_oid,
 	unsigned int len)
 {
+	int error = 0;
+
 	if (len < GIT_OID_MINPREFIXLEN)
-		return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to read loose backend. Prefix length is lower than %d.", GIT_OID_MINPREFIXLEN);
+		error = git_odb__error_ambiguous("prefix length too short");
 
-	if (len >= GIT_OID_HEXSZ) {
+	else if (len >= GIT_OID_HEXSZ) {
 		/* We can fall back to regular read method */
-		int error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
-		if (error == GIT_SUCCESS)
+		error = loose_backend__read(buffer_p, len_p, type_p, backend, short_oid);
+		if (!error)
 			git_oid_cpy(out_oid, short_oid);
-
-		return error;
 	} else {
-		char object_path[GIT_PATH_MAX];
+		git_buf object_path = GIT_BUF_INIT;
 		git_rawobj raw;
-		int error;
 
 		assert(backend && short_oid);
 
-		if ((error = locate_object_short_oid(object_path, out_oid, (loose_backend *)backend, short_oid, len)) < 0) {
-			return git__rethrow(error, "Failed to read loose backend");
+		if ((error = locate_object_short_oid(&object_path, out_oid,
+				(loose_backend *)backend, short_oid, len)) == 0 &&
+			(error = read_loose(&raw, &object_path)) == 0)
+		{
+			*buffer_p = raw.data;
+			*len_p = raw.len;
+			*type_p = raw.type;
 		}
 
-		if ((error = read_loose(&raw, object_path)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to read loose backend");
-
-		*buffer_p = raw.data;
-		*len_p = raw.len;
-		*type_p = raw.type;
+		git_buf_free(&object_path);
 	}
 
-	return GIT_SUCCESS;
+	return error;
 }
 
-int loose_backend__exists(git_odb_backend *backend, const git_oid *oid)
+static int loose_backend__exists(git_odb_backend *backend, const git_oid *oid)
 {
-	char object_path[GIT_PATH_MAX];
+	git_buf object_path = GIT_BUF_INIT;
+	int error;
 
 	assert(backend && oid);
 
-	return locate_object(object_path, (loose_backend *)backend, oid) == GIT_SUCCESS;
+	error = locate_object(&object_path, (loose_backend *)backend, oid);
+
+	git_buf_free(&object_path);
+
+	return !error;
 }
 
-int loose_backend__stream_fwrite(git_oid *oid, git_odb_stream *_stream)
+static int loose_backend__stream_fwrite(git_oid *oid, git_odb_stream *_stream)
 {
 	loose_writestream *stream = (loose_writestream *)_stream;
 	loose_backend *backend = (loose_backend *)_stream->backend;
+	git_buf final_path = GIT_BUF_INIT;
+	int error = 0;
 
-	int error;
-	char final_path[GIT_PATH_MAX];
-
-	if ((error = git_filebuf_hash(oid, &stream->fbuf)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write loose backend");
-
-	if (object_file_name(final_path, sizeof(final_path), backend->objects_dir, oid))
-		return GIT_ENOMEM;
+	if (git_filebuf_hash(oid, &stream->fbuf) < 0 ||
+		object_file_name(&final_path, backend->objects_dir, oid) < 0 ||
+		git_futils_mkpath2file(final_path.ptr, GIT_OBJECT_DIR_MODE) < 0)
+		error = -1;
+	/*
+	 * Don't try to add an existing object to the repository. This
+	 * is what git does and allows us to sidestep the fact that
+	 * we're not allowed to overwrite a read-only file on Windows.
+	 */
+	else if (git_path_exists(final_path.ptr) == true)
+		git_filebuf_cleanup(&stream->fbuf);
+	else
+		error = git_filebuf_commit_at(
+			&stream->fbuf, final_path.ptr, GIT_OBJECT_FILE_MODE);
 
-	if ((error = gitfo_mkdir_2file(final_path)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write loose backend");
+	git_buf_free(&final_path);
 
-	stream->finished = 1;
-	return git_filebuf_commit_at(&stream->fbuf, final_path);
+	return error;
 }
 
-int loose_backend__stream_write(git_odb_stream *_stream, const char *data, size_t len)
+static int loose_backend__stream_write(git_odb_stream *_stream, const char *data, size_t len)
 {
 	loose_writestream *stream = (loose_writestream *)_stream;
 	return git_filebuf_write(&stream->fbuf, data, len);
 }
 
-void loose_backend__stream_free(git_odb_stream *_stream)
+static void loose_backend__stream_free(git_odb_stream *_stream)
 {
 	loose_writestream *stream = (loose_writestream *)_stream;
 
-	if (!stream->finished)
-		git_filebuf_cleanup(&stream->fbuf);
-
-	free(stream);
+	git_filebuf_cleanup(&stream->fbuf);
+	git__free(stream);
 }
 
 static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype obj_type)
@@ -708,22 +722,19 @@ static int format_object_header(char *hdr, size_t n, size_t obj_len, git_otype o
 	const char *type_str = git_object_type2string(obj_type);
 	int len = snprintf(hdr, n, "%s %"PRIuZ, type_str, obj_len);
 
-	assert(len > 0);             /* otherwise snprintf() is broken  */
-	assert(((size_t) len) < n);  /* otherwise the caller is broken! */
+	assert(len > 0);				/* otherwise snprintf() is broken */
+	assert(((size_t)len) < n);		/* otherwise the caller is broken! */
 
-	if (len < 0 || ((size_t) len) >= n)
-		return git__throw(GIT_ERROR, "Failed to format object header. Length is out of bounds");
 	return len+1;
 }
 
-int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, size_t length, git_otype type)
+static int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend, size_t length, git_otype type)
 {
 	loose_backend *backend;
-	loose_writestream *stream;
-
-	char hdr[64], tmp_path[GIT_PATH_MAX];
-	int  hdrlen;
-	int error;
+	loose_writestream *stream = NULL;
+	char hdr[64];
+	git_buf tmp_path = GIT_BUF_INIT;
+	int hdrlen;
 
 	assert(_backend);
 
@@ -731,12 +742,9 @@ int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend
 	*stream_out = NULL;
 
 	hdrlen = format_object_header(hdr, sizeof(hdr), length, type);
-	if (hdrlen < GIT_SUCCESS)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to create loose backend stream. Object is corrupted");
 
 	stream = git__calloc(1, sizeof(loose_writestream));
-	if (stream == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(stream);
 
 	stream->stream.backend = _backend;
 	stream->stream.read = NULL; /* read only */
@@ -745,57 +753,94 @@ int loose_backend__stream(git_odb_stream **stream_out, git_odb_backend *_backend
 	stream->stream.free = &loose_backend__stream_free;
 	stream->stream.mode = GIT_STREAM_WRONLY;
 
-	git__joinpath(tmp_path, backend->objects_dir, "tmp_object");
+	if (git_buf_joinpath(&tmp_path, backend->objects_dir, "tmp_object") < 0 ||
+		git_filebuf_open(&stream->fbuf, tmp_path.ptr,
+			GIT_FILEBUF_HASH_CONTENTS |
+			GIT_FILEBUF_TEMPORARY |
+			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT)) < 0 ||
+		stream->stream.write((git_odb_stream *)stream, hdr, hdrlen) < 0)
+	{
+		git_filebuf_cleanup(&stream->fbuf);
+		git__free(stream);
+		stream = NULL;
+	}
+	git_buf_free(&tmp_path);
+	*stream_out = (git_odb_stream *)stream;
 
-	error = git_filebuf_open(&stream->fbuf, tmp_path,
-		GIT_FILEBUF_HASH_CONTENTS |
-		GIT_FILEBUF_DEFLATE_CONTENTS |
-		GIT_FILEBUF_TEMPORARY);
+	return !stream ? -1 : 0;
+}
 
-	if (error < GIT_SUCCESS) {
-		free(stream);
-		return git__rethrow(error, "Failed to create loose backend stream");
-	}
+static int loose_backend__write(git_oid *oid, git_odb_backend *_backend, const void *data, size_t len, git_otype type)
+{
+	int error = 0, header_len;
+	git_buf final_path = GIT_BUF_INIT;
+	char header[64];
+	git_filebuf fbuf = GIT_FILEBUF_INIT;
+	loose_backend *backend;
 
-	error = stream->stream.write((git_odb_stream *)stream, hdr, hdrlen);
-	if (error < GIT_SUCCESS) {
-		git_filebuf_cleanup(&stream->fbuf);
-		free(stream);
-		return git__rethrow(error, "Failed to create loose backend stream");
+	backend = (loose_backend *)_backend;
+
+	/* prepare the header for the file */
+	header_len = format_object_header(header, sizeof(header), len, type);
+
+	if (git_buf_joinpath(&final_path, backend->objects_dir, "tmp_object") < 0 ||
+		git_filebuf_open(&fbuf, final_path.ptr,
+			GIT_FILEBUF_HASH_CONTENTS |
+			GIT_FILEBUF_TEMPORARY |
+			(backend->object_zlib_level << GIT_FILEBUF_DEFLATE_SHIFT)) < 0)
+	{
+		error = -1;
+		goto cleanup;
 	}
 
-	*stream_out = (git_odb_stream *)stream;
-	return GIT_SUCCESS;
+	git_filebuf_write(&fbuf, header, header_len);
+	git_filebuf_write(&fbuf, data, len);
+	git_filebuf_hash(oid, &fbuf);
+
+	if (object_file_name(&final_path, backend->objects_dir, oid) < 0 ||
+		git_futils_mkpath2file(final_path.ptr, GIT_OBJECT_DIR_MODE) < 0 ||
+		git_filebuf_commit_at(&fbuf, final_path.ptr, GIT_OBJECT_FILE_MODE) < 0)
+		error = -1;
+
+cleanup:
+	if (error < 0)
+		git_filebuf_cleanup(&fbuf);
+	git_buf_free(&final_path);
+	return error;
 }
 
-void loose_backend__free(git_odb_backend *_backend)
+static void loose_backend__free(git_odb_backend *_backend)
 {
 	loose_backend *backend;
 	assert(_backend);
 	backend = (loose_backend *)_backend;
 
-	free(backend->objects_dir);
-	free(backend);
+	git__free(backend->objects_dir);
+	git__free(backend);
 }
 
-int git_odb_backend_loose(git_odb_backend **backend_out, const char *objects_dir)
+int git_odb_backend_loose(
+	git_odb_backend **backend_out,
+	const char *objects_dir,
+	int compression_level,
+	int do_fsync)
 {
 	loose_backend *backend;
 
 	backend = git__calloc(1, sizeof(loose_backend));
-	if (backend == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(backend);
 
 	backend->objects_dir = git__strdup(objects_dir);
-	if (backend->objects_dir == NULL) {
-		free(backend);
-		return GIT_ENOMEM;
-	}
+	GITERR_CHECK_ALLOC(backend->objects_dir);
+
+	if (compression_level < 0)
+		compression_level = Z_BEST_SPEED;
 
-	backend->object_zlib_level = Z_BEST_SPEED;
-	backend->fsync_object_files = 0;
+	backend->object_zlib_level = compression_level;
+	backend->fsync_object_files = do_fsync;
 
 	backend->parent.read = &loose_backend__read;
+	backend->parent.write = &loose_backend__write;
 	backend->parent.read_prefix = &loose_backend__read_prefix;
 	backend->parent.read_header = &loose_backend__read_header;
 	backend->parent.writestream = &loose_backend__stream;
@@ -803,5 +848,5 @@ int git_odb_backend_loose(git_odb_backend **backend_out, const char *objects_dir
 	backend->parent.free = &loose_backend__free;
 
 	*backend_out = (git_odb_backend *)backend;
-	return GIT_SUCCESS;
+	return 0;
 }
diff --git a/src/odb_pack.c b/src/odb_pack.c
index 525cfa429..458f288d9 100644
--- a/src/odb_pack.c
+++ b/src/odb_pack.c
@@ -1,142 +1,38 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
-#include "git2/zlib.h"
+#include <zlib.h>
 #include "git2/repository.h"
+#include "git2/oid.h"
 #include "fileops.h"
 #include "hash.h"
 #include "odb.h"
-#include "oid.h"
 #include "delta-apply.h"
 #include "sha1_lookup.h"
+#include "mwindow.h"
+#include "pack.h"
 
 #include "git2/odb_backend.h"
 
-#define DEFAULT_WINDOW_SIZE \
-	(sizeof(void*) >= 8 \
-		?  1 * 1024 * 1024 * 1024 \
-		: 32 * 1024 * 1024)
-
-#define DEFAULT_MAPPED_LIMIT \
-	((1024L * 1024L) * (sizeof(void*) >= 8 ? 8192 : 256))
-
-#define PACK_SIGNATURE 0x5041434b	/* "PACK" */
-#define PACK_VERSION 2
-#define pack_version_ok(v) ((v) == htonl(2) || (v) == htonl(3))
-struct pack_header {
-	uint32_t hdr_signature;
-	uint32_t hdr_version;
-	uint32_t hdr_entries;
-};
-
-/*
- * The first four bytes of index formats later than version 1 should
- * start with this signature, as all older git binaries would find this
- * value illegal and abort reading the file.
- *
- * This is the case because the number of objects in a packfile
- * cannot exceed 1,431,660,000 as every object would need at least
- * 3 bytes of data and the overall packfile cannot exceed 4 GiB with
- * version 1 of the index file due to the offsets limited to 32 bits.
- * Clearly the signature exceeds this maximum.
- *
- * Very old git binaries will also compare the first 4 bytes to the
- * next 4 bytes in the index and abort with a "non-monotonic index"
- * error if the second 4 byte word is smaller than the first 4
- * byte word.  This would be true in the proposed future index
- * format as idx_signature would be greater than idx_version.
- */
-#define PACK_IDX_SIGNATURE 0xff744f63	/* "\377tOc" */
-
-struct pack_idx_header {
-	uint32_t idx_signature;
-	uint32_t idx_version;
-};
-
-struct pack_window {
-	struct pack_window *next;
-	git_map window_map;
-	off_t offset;
-	unsigned int last_used;
-	unsigned int inuse_cnt;
-};
-
-struct pack_file {
-	struct pack_window *windows;
-	off_t pack_size;
-
-	git_map index_map;
-
-	uint32_t num_objects;
-	uint32_t num_bad_objects;
-	git_oid *bad_object_sha1; /* array of git_oid */
-
-	int index_version;
-	git_time_t mtime;
-	int pack_fd;
-	unsigned pack_local:1, pack_keep:1;
-	git_oid sha1;
-
-	/* something like ".git/objects/pack/xxxxx.pack" */
-	char pack_name[GIT_FLEX_ARRAY]; /* more */
-};
-
-struct pack_entry {
-	off_t offset;
-	git_oid sha1;
-	struct pack_file *p;
-};
-
 struct pack_backend {
 	git_odb_backend parent;
 	git_vector packs;
-	struct pack_file *last_found;
+	struct git_pack_file *last_found;
 	char *pack_folder;
 	time_t pack_folder_mtime;
-
-	size_t window_size; /* needs default value */
-
-	size_t mapped_limit; /* needs default value */
-	size_t peak_mapped;
-	size_t mapped;
-
-	size_t used_ctr;
-
-	unsigned int peak_open_windows;
-	unsigned int open_windows;
-
-	unsigned int mmap_calls;
 };
 
 /**
  * The wonderful tale of a Packed Object lookup query
  * ===================================================
- *   A riveting and epic story of epicness and ASCII
- *          art, presented by yours truly,
- *               Sir Vicent of Marti
+ *	A riveting and epic story of epicness and ASCII
+ *			art, presented by yours truly,
+ *				Sir Vicent of Marti
  *
  *
  *	Chapter 1: Once upon a time...
@@ -147,32 +43,32 @@ struct pack_backend {
  *	| Creates the pack backend structure, initializes the
  *	| callback pointers to our default read() and exist() methods,
  *	| and tries to preload all the known packfiles in the ODB.
- *  |
+ * |
  *	|-# packfile_load_all
- *	  | Tries to find the `pack` folder, if it exists. ODBs without
- *	  | a pack folder are ignored altogether. If there's a `pack` folder
- *	  | we run a `dirent` callback through every file in the pack folder
- *	  | to find our packfiles. The packfiles are then sorted according
- *	  | to a sorting callback.
- * 	  |
- *	  |-# packfile_load__cb
- *	  | | This callback is called from `dirent` with every single file
- *	  | | inside the pack folder. We find the packs by actually locating
- *	  | | their index (ends in ".idx"). From that index, we verify that
- *	  | | the corresponding packfile exists and is valid, and if so, we
- *    | | add it to the pack list.
- *	  | |
- *	  | |-# packfile_check
- *	  |     Make sure that there's a packfile to back this index, and store
- *	  |     some very basic information regarding the packfile itself,
- *	  |     such as the full path, the size, and the modification time.
- *	  |     We don't actually open the packfile to check for internal consistency.
- *    |
- *    |-# packfile_sort__cb
- *        Sort all the preloaded packs according to some specific criteria:
- *        we prioritize the "newer" packs because it's more likely they
- *        contain the objects we are looking for, and we prioritize local
- *        packs over remote ones.
+ *	 | Tries to find the `pack` folder, if it exists. ODBs without
+ *	 | a pack folder are ignored altogether. If there's a `pack` folder
+ *	 | we run a `dirent` callback through every file in the pack folder
+ *	 | to find our packfiles. The packfiles are then sorted according
+ *	 | to a sorting callback.
+ * 	 |
+ *	 |-# packfile_load__cb
+ *	 | | This callback is called from `dirent` with every single file
+ *	 | | inside the pack folder. We find the packs by actually locating
+ *	 | | their index (ends in ".idx"). From that index, we verify that
+ *	 | | the corresponding packfile exists and is valid, and if so, we
+ *	| | add it to the pack list.
+ *	 | |
+ *	 | |-# packfile_check
+ *	 |		Make sure that there's a packfile to back this index, and store
+ *	 |		some very basic information regarding the packfile itself,
+ *	 |		such as the full path, the size, and the modification time.
+ *	 |		We don't actually open the packfile to check for internal consistency.
+ *	|
+ *	|-# packfile_sort__cb
+ *		Sort all the preloaded packs according to some specific criteria:
+ *		we prioritize the "newer" packs because it's more likely they
+ *		contain the objects we are looking for, and we prioritize local
+ *		packs over remote ones.
  *
  *
  *
@@ -180,41 +76,41 @@ struct pack_backend {
  *	A standard packed `exist` query for an OID
  *	--------------------------------------------------
  *
- *  # pack_backend__exists
- *  | Check if the given SHA1 oid exists in any of the packs
- *  | that have been loaded for our ODB.
- *  |
- *  |-# pack_entry_find
- *    | Iterate through all the packs that have been preloaded
- *    | (starting by the pack where the latest object was found)
- *    | to try to find the OID in one of them.
- *    |
- *    |-# pack_entry_find1
- *      | Check the index of an individual pack to see if the SHA1
- *      | OID can be found. If we can find the offset to that SHA1
- *      | inside of the index, that means the object is contained
- *      | inside of the packfile and we can stop searching.
- *      | Before returning, we verify that the packfile behing the
- *      | index we are searching still exists on disk.
- *      |
- *      |-# pack_entry_find_offset
- *      | | Mmap the actual index file to disk if it hasn't been opened
- *      | | yet, and run a binary search through it to find the OID.
- *      | | See <http://book.git-scm.com/7_the_packfile.html> for specifics
- *      | | on the Packfile Index format and how do we find entries in it.
- *      | |
- *      | |-# pack_index_open
- *      |   | Guess the name of the index based on the full path to the
- *      |   | packfile, open it and verify its contents. Only if the index
- *      |   | has not been opened already.
- *      |   |
- *      |   |-# pack_index_check
- *      |       Mmap the index file and do a quick run through the header
- *      |       to guess the index version (right now we support v1 and v2),
- *      |       and to verify that the size of the index makes sense.
- *      |
- *      |-# packfile_open
- *          See `packfile_open` in Chapter 3
+ * # pack_backend__exists
+ * | Check if the given SHA1 oid exists in any of the packs
+ * | that have been loaded for our ODB.
+ * |
+ * |-# pack_entry_find
+ *	| Iterate through all the packs that have been preloaded
+ *	| (starting by the pack where the latest object was found)
+ *	| to try to find the OID in one of them.
+ *	|
+ *	|-# pack_entry_find1
+ *		| Check the index of an individual pack to see if the SHA1
+ *		| OID can be found. If we can find the offset to that SHA1
+ *		| inside of the index, that means the object is contained
+ *		| inside of the packfile and we can stop searching.
+ *		| Before returning, we verify that the packfile behing the
+ *		| index we are searching still exists on disk.
+ *		|
+ *		|-# pack_entry_find_offset
+ *		| | Mmap the actual index file to disk if it hasn't been opened
+ *		| | yet, and run a binary search through it to find the OID.
+ *		| | See <http://book.git-scm.com/7_the_packfile.html> for specifics
+ *		| | on the Packfile Index format and how do we find entries in it.
+ *		| |
+ *		| |-# pack_index_open
+ *		|	| Guess the name of the index based on the full path to the
+ *		|	| packfile, open it and verify its contents. Only if the index
+ *		|	| has not been opened already.
+ *		|	|
+ *		|	|-# pack_index_check
+ *		|		Mmap the index file and do a quick run through the header
+ *		|		to guess the index version (right now we support v1 and v2),
+ *		|		and to verify that the size of the index makes sense.
+ *		|
+ *		|-# packfile_open
+ *			See `packfile_open` in Chapter 3
  *
  *
  *
@@ -226,121 +122,34 @@ struct pack_backend {
  */
 
 
-
- 
 /***********************************************************
  *
  * FORWARD DECLARATIONS
  *
  ***********************************************************/
 
-static void pack_window_free_all(struct pack_backend *backend, struct pack_file *p);
-static int pack_window_contains(struct pack_window *win, off_t offset);
-
-static void pack_window_scan_lru(struct pack_file *p, struct pack_file **lru_p,
-		struct pack_window **lru_w, struct pack_window **lru_l);
-
-static int pack_window_close_lru( struct pack_backend *backend,
-		struct pack_file *current, git_file keep_fd);
-
-static void pack_window_close(struct pack_window **w_cursor);
-
-static unsigned char *pack_window_open( struct pack_backend *backend,
-		struct pack_file *p, struct pack_window **w_cursor, off_t offset,
-		unsigned int *left);
+static void pack_window_free_all(struct pack_backend *backend, struct git_pack_file *p);
+static int pack_window_contains(git_mwindow *win, off_t offset);
 
 static int packfile_sort__cb(const void *a_, const void *b_);
 
-static void pack_index_free(struct pack_file *p);
-
-static int pack_index_check(const char *path,  struct pack_file *p);
-static int pack_index_open(struct pack_file *p);
-
-static struct pack_file *packfile_alloc(int extra);
-static int packfile_open(struct pack_file *p);
-static int packfile_check(struct pack_file **pack_out, const char *path);
-static int packfile_load__cb(void *_data, char *path);
+static int packfile_load__cb(void *_data, git_buf *path);
 static int packfile_refresh_all(struct pack_backend *backend);
 
-static off_t nth_packed_object_offset(const struct pack_file *p, uint32_t n);
+static int pack_entry_find(struct git_pack_entry *e,
+	struct pack_backend *backend, const git_oid *oid);
 
 /* Can find the offset of an object given
  * a prefix of an identifier.
- * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid
- * is ambiguous within the pack.
+ * Sets GIT_EAMBIGUOUS if short oid is ambiguous.
  * This method assumes that len is between
  * GIT_OID_MINPREFIXLEN and GIT_OID_HEXSZ.
  */
-static int pack_entry_find_offset(
-		off_t *offset_out,
-		git_oid *found_oid,
-		struct pack_file *p,
-		const git_oid *short_oid,
-		unsigned int len);
-
-static int pack_entry_find1(
-		struct pack_entry *e,
-		struct pack_file *p,
-		const git_oid *short_oid,
-		unsigned int len);
-
-static int pack_entry_find(struct pack_entry *e,
-		struct pack_backend *backend, const git_oid *oid);
-
-/* Can find the offset of an object given
- * a prefix of an identifier.
- * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid
- * is ambiguous.
- * This method assumes that len is between
- * GIT_OID_MINPREFIXLEN and GIT_OID_HEXSZ.
- */
-static int pack_entry_find_prefix(struct pack_entry *e,
-					struct pack_backend *backend,
-					const git_oid *short_oid,
-					unsigned int len);
-
-static off_t get_delta_base(struct pack_backend *backend,
-		struct pack_file *p, struct pack_window **w_curs,
-		off_t *curpos, git_otype type,
-		off_t delta_obj_offset);
-
-static unsigned long packfile_unpack_header1(
-		size_t *sizep,
-		git_otype *type, 
-		const unsigned char *buf,
-		unsigned long len);
-
-static int packfile_unpack_header(
-		size_t *size_p,
-		git_otype *type_p,
-		struct pack_backend *backend,
-		struct pack_file *p,
-		struct pack_window **w_curs,
-		off_t *curpos);
-
-static int packfile_unpack_compressed(
-		git_rawobj *obj,
-		struct pack_backend *backend,
-		struct pack_file *p,
-		struct pack_window **w_curs,
-		off_t curpos,
-		size_t size,
-		git_otype type);
-
-static int packfile_unpack_delta(
-		git_rawobj *obj,
-		struct pack_backend *backend,
-		struct pack_file *p,
-		struct pack_window **w_curs,
-		off_t curpos,
-		size_t delta_size,
-		git_otype delta_type,
-		off_t obj_offset);
-
-static int packfile_unpack(git_rawobj *obj, struct pack_backend *backend,
-		struct pack_file *p, off_t obj_offset);
-
-
+static int pack_entry_find_prefix(
+	struct git_pack_entry *e,
+	struct pack_backend *backend,
+	const git_oid *short_oid,
+	unsigned int len);
 
 
 
@@ -350,357 +159,32 @@ static int packfile_unpack(git_rawobj *obj, struct pack_backend *backend,
  *
  ***********************************************************/
 
-void pack_window_free_all(struct pack_backend *backend, struct pack_file *p)
+GIT_INLINE(void) pack_window_free_all(struct pack_backend *backend, struct git_pack_file *p)
 {
-	while (p->windows) {
-		struct pack_window *w = p->windows;
-		assert(w->inuse_cnt == 0);
-
-		backend->mapped -= w->window_map.len;
-		backend->open_windows--;
-
-		gitfo_free_map(&w->window_map);
-
-		p->windows = w->next;
-		free(w);
-	}
+	GIT_UNUSED(backend);
+	git_mwindow_free_all(&p->mwf);
 }
 
-GIT_INLINE(int) pack_window_contains(struct pack_window *win, off_t offset)
+GIT_INLINE(int) pack_window_contains(git_mwindow *win, off_t offset)
 {
 	/* We must promise at least 20 bytes (one hash) after the
 	 * offset is available from this window, otherwise the offset
 	 * is not actually in this window and a different window (which
-	 * has that one hash excess) must be used.  This is to support
+	 * has that one hash excess) must be used. This is to support
 	 * the object header and delta base parsing routines below.
 	 */
-	off_t win_off = win->offset;
-	return win_off <= offset
-		&& (offset + 20) <= (off_t)(win_off + win->window_map.len);
-}
-
-static void pack_window_scan_lru(
-	struct pack_file *p,
-	struct pack_file **lru_p,
-	struct pack_window **lru_w,
-	struct pack_window **lru_l)
-{
-	struct pack_window *w, *w_l;
-
-	for (w_l = NULL, w = p->windows; w; w = w->next) {
-		if (!w->inuse_cnt) {
-			if (!*lru_w || w->last_used < (*lru_w)->last_used) {
-				*lru_p = p;
-				*lru_w = w;
-				*lru_l = w_l;
-			}
-		}
-		w_l = w;
-	}
-}
-
-static int pack_window_close_lru(
-		struct pack_backend *backend,
-		struct pack_file *current,
-		git_file keep_fd)
-{
-	struct pack_file *lru_p = NULL;
-	struct pack_window *lru_w = NULL, *lru_l = NULL;
-	size_t i;
-
-	if (current)
-		pack_window_scan_lru(current, &lru_p, &lru_w, &lru_l);
-
-	for (i = 0; i < backend->packs.length; ++i)
-		pack_window_scan_lru(git_vector_get(&backend->packs, i), &lru_p, &lru_w, &lru_l);
-
-	if (lru_p) {
-		backend->mapped -= lru_w->window_map.len;
-		gitfo_free_map(&lru_w->window_map);
-
-		if (lru_l)
-			lru_l->next = lru_w->next;
-		else {
-			lru_p->windows = lru_w->next;
-			if (!lru_p->windows && lru_p->pack_fd != keep_fd) {
-				gitfo_close(lru_p->pack_fd);
-				lru_p->pack_fd = -1;
-			}
-		}
-
-		free(lru_w);
-		backend->open_windows--;
-		return GIT_SUCCESS;
-	}
-
-	return git__throw(GIT_ERROR, "Failed to close pack window");
-}
-
-static void pack_window_close(struct pack_window **w_cursor)
-{
-	struct pack_window *w = *w_cursor;
-	if (w) {
-		w->inuse_cnt--;
-		*w_cursor = NULL;
-	}
-}
-
-static unsigned char *pack_window_open(
-		struct pack_backend *backend,
-		struct pack_file *p,
-		struct pack_window **w_cursor,
-		off_t offset,
-		unsigned int *left)
-{
-	struct pack_window *win = *w_cursor;
-
-	if (p->pack_fd == -1 && packfile_open(p) < GIT_SUCCESS)
-		return NULL;
-
-	/* Since packfiles end in a hash of their content and it's
-	 * pointless to ask for an offset into the middle of that
-	 * hash, and the pack_window_contains function above wouldn't match
-	 * don't allow an offset too close to the end of the file.
-	 */
-	if (offset > (p->pack_size - 20))
-		return NULL;
-
-	if (!win || !pack_window_contains(win, offset)) {
-
-		if (win)
-			win->inuse_cnt--;
-
-		for (win = p->windows; win; win = win->next) {
-			if (pack_window_contains(win, offset))
-				break;
-		}
-
-		if (!win) {
-			size_t window_align = backend->window_size / 2;
-			size_t len;
-
-			win = git__calloc(1, sizeof(*win));
-			win->offset = (offset / window_align) * window_align;
-
-			len = (size_t)(p->pack_size - win->offset);
-			if (len > backend->window_size)
-				len = backend->window_size;
-
-			backend->mapped += len;
-
-			while (backend->mapped_limit < backend->mapped &&
-				pack_window_close_lru(backend, p, p->pack_fd) == GIT_SUCCESS) {}
-
-			if (gitfo_map_ro(&win->window_map, p->pack_fd,
-					win->offset, len) < GIT_SUCCESS)
-				return NULL;
-
-			backend->mmap_calls++;
-			backend->open_windows++;
-
-			if (backend->mapped > backend->peak_mapped)
-				backend->peak_mapped = backend->mapped;
-
-			if (backend->open_windows > backend->peak_open_windows)
-				backend->peak_open_windows = backend->open_windows;
-
-			win->next = p->windows;
-			p->windows = win;
-		}
-	}
-
-	if (win != *w_cursor) {
-		win->last_used = backend->used_ctr++;
-		win->inuse_cnt++;
-		*w_cursor = win;
-	}
-
-	offset -= win->offset;
-	assert(git__is_sizet(offset));
-
-	if (left)
-		*left = win->window_map.len - (size_t)offset;
-
-	return (unsigned char *)win->window_map.data + offset;
+	return git_mwindow_contains(win, offset + 20);
 }
 
-
-
-
-
-
-
-/***********************************************************
- *
- * PACK INDEX METHODS
- *
- ***********************************************************/
-
-static void pack_index_free(struct pack_file *p)
-{
-	if (p->index_map.data) {
-		gitfo_free_map(&p->index_map);
-		p->index_map.data = NULL;
-	}
-}
-
-static int pack_index_check(const char *path,  struct pack_file *p)
-{
-	struct pack_idx_header *hdr;
-	uint32_t version, nr, i, *index;
-
-	void *idx_map;
-	size_t idx_size;
-
-	struct stat st;
-
-	/* TODO: properly open the file without access time */
-	git_file fd = gitfo_open(path, O_RDONLY /*| O_NOATIME */);
-
-	int error;
-
-	if (fd < 0)
-		return git__throw(GIT_EOSERR, "Failed to check index. File missing or corrupted");
-
-	if (gitfo_fstat(fd, &st) < GIT_SUCCESS) {
-		gitfo_close(fd);
-		return git__throw(GIT_EOSERR, "Failed to check index. File appears to be corrupted");
-	}
-
-	if (!git__is_sizet(st.st_size))
-		return GIT_ENOMEM;
-
-	idx_size = (size_t)st.st_size;
-
-	if (idx_size < 4 * 256 + 20 + 20) {
-		gitfo_close(fd);
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Object is corrupted");
-	}
-
-	error = gitfo_map_ro(&p->index_map, fd, 0, idx_size);
-	gitfo_close(fd);
-
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to check index");
-
-	hdr = idx_map = p->index_map.data;
-
-	if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
-		version = ntohl(hdr->idx_version);
-
-		if (version < 2 || version > 2) {
-			gitfo_free_map(&p->index_map);
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Unsupported index version");
-		}
-
-	} else
-		version = 1;
-
-	nr = 0;
-	index = idx_map;
-
-	if (version > 1)
-		index += 2;  /* skip index header */
-
-	for (i = 0; i < 256; i++) {
-		uint32_t n = ntohl(index[i]);
-		if (n < nr) {
-			gitfo_free_map(&p->index_map);
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Index is non-monotonic");
-		}
-		nr = n;
-	}
-
-	if (version == 1) {
-		/*
-		 * Total size:
-		 *  - 256 index entries 4 bytes each
-		 *  - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
-		 *  - 20-byte SHA1 of the packfile
-		 *  - 20-byte SHA1 file checksum
-		 */
-		if (idx_size != 4*256 + nr * 24 + 20 + 20) {
-			gitfo_free_map(&p->index_map);
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Object is corrupted");
-		}
-	} else if (version == 2) {
-		/*
-		 * Minimum size:
-		 *  - 8 bytes of header
-		 *  - 256 index entries 4 bytes each
-		 *  - 20-byte sha1 entry * nr
-		 *  - 4-byte crc entry * nr
-		 *  - 4-byte offset entry * nr
-		 *  - 20-byte SHA1 of the packfile
-		 *  - 20-byte SHA1 file checksum
-		 * And after the 4-byte offset table might be a
-		 * variable sized table containing 8-byte entries
-		 * for offsets larger than 2^31.
-		 */
-		unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
-		unsigned long max_size = min_size;
-
-		if (nr)
-			max_size += (nr - 1)*8;
-
-		if (idx_size < min_size || idx_size > max_size) {
-			gitfo_free_map(&p->index_map);
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to check index. Wrong index size");
-		}
-
-		/* Make sure that off_t is big enough to access the whole pack...
-		 * Is this an issue in libgit2? It shouldn't. */
-		if (idx_size != min_size && (sizeof(off_t) <= 4)) {
-			gitfo_free_map(&p->index_map);
-			return git__throw(GIT_EOSERR, "Failed to check index. off_t not big enough to access the whole pack");
-		}
-	}
-
-	p->index_version = version;
-	p->num_objects = nr;
-	return GIT_SUCCESS;
-}
-
-static int pack_index_open(struct pack_file *p)
-{
-	char *idx_name;
-	int error;
-
-	if (p->index_map.data)
-		return GIT_SUCCESS;
-
-	idx_name = git__strdup(p->pack_name);
-	strcpy(idx_name + strlen(idx_name) - STRLEN(".pack"), ".idx");
-
-	error = pack_index_check(idx_name, p);
-	free(idx_name);
-
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to open index");
-}
-
-
-
-
-
-
-
-
-
-/***********************************************************
- *
- * PACKFILE METHODS
- *
- ***********************************************************/
-
 static int packfile_sort__cb(const void *a_, const void *b_)
 {
-	struct pack_file *a = *((struct pack_file **)a_);
-	struct pack_file *b = *((struct pack_file **)b_);
+	const struct git_pack_file *a = a_;
+	const struct git_pack_file *b = b_;
 	int st;
 
 	/*
 	 * Local packs tend to contain objects specific to our
-	 * variant of the project than remote ones.  In addition,
+	 * variant of the project than remote ones. In addition,
 	 * remote ones could be on a network mounted filesystem.
 	 * Favor local ones for these reasons.
 	 */
@@ -721,174 +205,32 @@ static int packfile_sort__cb(const void *a_, const void *b_)
 	return -1;
 }
 
-static struct pack_file *packfile_alloc(int extra)
-{
-	struct pack_file *p = git__malloc(sizeof(*p) + extra);
-	memset(p, 0, sizeof(*p));
-	p->pack_fd = -1;
-	return p;
-}
-
-
-static void packfile_free(struct pack_backend *backend, struct pack_file *p)
-{
-	assert(p);
-
-	/* clear_delta_base_cache(); */
-	pack_window_free_all(backend, p);
-
-	if (p->pack_fd != -1)
-		gitfo_close(p->pack_fd);
 
-	pack_index_free(p);
 
-	free(p->bad_object_sha1);
-	free(p);
-}
-
-static int packfile_open(struct pack_file *p)
-{
-	struct stat st;
-	struct pack_header hdr;
-	git_oid sha1;
-	unsigned char *idx_sha1;
-
-	if (!p->index_map.data && pack_index_open(p) < GIT_SUCCESS)
-		return git__throw(GIT_ENOTFOUND, "Failed to open packfile. File not found");
-
-	/* TODO: open with noatime */
-	p->pack_fd = gitfo_open(p->pack_name, O_RDONLY);
-	if (p->pack_fd < 0 || gitfo_fstat(p->pack_fd, &st) < GIT_SUCCESS)
-		return git__throw(GIT_EOSERR, "Failed to open packfile. File appears to be corrupted");
-
-	/* If we created the struct before we had the pack we lack size. */
-	if (!p->pack_size) {
-		if (!S_ISREG(st.st_mode))
-			goto cleanup;
-		p->pack_size = (off_t)st.st_size;
-	} else if (p->pack_size != st.st_size)
-		goto cleanup;
-
-#if 0
-	/* We leave these file descriptors open with sliding mmap;
-	 * there is no point keeping them open across exec(), though.
-	 */
-	fd_flag = fcntl(p->pack_fd, F_GETFD, 0);
-	if (fd_flag < 0)
-		return error("cannot determine file descriptor flags");
-
-	fd_flag |= FD_CLOEXEC;
-	if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
-		return GIT_EOSERR;
-#endif
-
-	/* Verify we recognize this pack file format. */
-	if (gitfo_read(p->pack_fd, &hdr, sizeof(hdr)) < GIT_SUCCESS)
-		goto cleanup;
-
-	if (hdr.hdr_signature != htonl(PACK_SIGNATURE))
-		goto cleanup;
-
-	if (!pack_version_ok(hdr.hdr_version))
-		goto cleanup;
-
-	/* Verify the pack matches its index. */
-	if (p->num_objects != ntohl(hdr.hdr_entries))
-		goto cleanup;
-
-	if (gitfo_lseek(p->pack_fd, p->pack_size - GIT_OID_RAWSZ, SEEK_SET) == -1)
-		goto cleanup;
-
-	if (gitfo_read(p->pack_fd, sha1.id, GIT_OID_RAWSZ) < GIT_SUCCESS)
-		goto cleanup;
-
-	idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40;
-
-	if (git_oid_cmp(&sha1, (git_oid *)idx_sha1) != 0)
-		goto cleanup;
-
-	return GIT_SUCCESS;	
-
-cleanup:
-	gitfo_close(p->pack_fd);
-	p->pack_fd = -1;
-	return git__throw(GIT_EPACKCORRUPTED, "Failed to packfile. Pack is corrupted");
-}
-
-static int packfile_check(struct pack_file **pack_out, const char *path)
-{
-	struct stat st;
-	struct pack_file *p;
-	size_t path_len;
-
-	*pack_out = NULL;
-	path_len = strlen(path);
-	p = packfile_alloc(path_len + 2);
-
-	/*
-	 * Make sure a corresponding .pack file exists and that
-	 * the index looks sane.
-	 */
-	path_len -= STRLEN(".idx");
-	if (path_len < 1) {
-		free(p);
-		return git__throw(GIT_ENOTFOUND, "Failed to check packfile. Wrong path name");
-	}
-
-	memcpy(p->pack_name, path, path_len);
-
-	strcpy(p->pack_name + path_len, ".keep");
-	if (gitfo_exists(p->pack_name) == GIT_SUCCESS)
-		p->pack_keep = 1;
-
-	strcpy(p->pack_name + path_len, ".pack");
-	if (gitfo_stat(p->pack_name, &st) < GIT_SUCCESS || !S_ISREG(st.st_mode)) {
-		free(p);
-		return git__throw(GIT_ENOTFOUND, "Failed to check packfile. File not found");
-	}
-
-	/* ok, it looks sane as far as we can check without
-	 * actually mapping the pack file.
-	 */
-	p->pack_size = (off_t)st.st_size;
-	p->pack_local = 1;
-	p->mtime = (git_time_t)st.st_mtime;
-
-	/* see if we can parse the sha1 oid in the packfile name */
-	if (path_len < 40 ||
-		git_oid_mkstr(&p->sha1, path + path_len - GIT_OID_HEXSZ) < GIT_SUCCESS)
-		memset(&p->sha1, 0x0, GIT_OID_RAWSZ);
-
-	*pack_out = p;
-	return GIT_SUCCESS;
-}
-
-static int packfile_load__cb(void *_data, char *path)
+static int packfile_load__cb(void *_data, git_buf *path)
 {
 	struct pack_backend *backend = (struct pack_backend *)_data;
-	struct pack_file *pack;
+	struct git_pack_file *pack;
 	int error;
-	size_t i;
+	unsigned int i;
 
-	if (git__suffixcmp(path, ".idx") != 0)
-		return GIT_SUCCESS; /* not an index */
+	if (git__suffixcmp(path->ptr, ".idx") != 0)
+		return 0; /* not an index */
 
 	for (i = 0; i < backend->packs.length; ++i) {
-		struct pack_file *p = git_vector_get(&backend->packs, i);
-		if (memcmp(p->pack_name, path, strlen(path) - STRLEN(".idx")) == 0)
-			return GIT_SUCCESS;
+		struct git_pack_file *p = git_vector_get(&backend->packs, i);
+		if (memcmp(p->pack_name, git_buf_cstr(path), git_buf_len(path) - strlen(".idx")) == 0)
+			return 0;
 	}
 
-	error = packfile_check(&pack, path);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to load packfile");
-
-	if (git_vector_insert(&backend->packs, pack) < GIT_SUCCESS) {
-		free(pack);
-		return GIT_ENOMEM;
-	}
+	error = git_packfile_check(&pack, path->ptr);
+	if (error == GIT_ENOTFOUND)
+		/* ignore missing .pack file as git does */
+		return 0;
+	else if (error < 0)
+		return error;
 
-	return GIT_SUCCESS;
+	return git_vector_insert(&backend->packs, pack);
 }
 
 static int packfile_refresh_all(struct pack_backend *backend)
@@ -897,534 +239,104 @@ static int packfile_refresh_all(struct pack_backend *backend)
 	struct stat st;
 
 	if (backend->pack_folder == NULL)
-		return GIT_SUCCESS;
+		return 0;
 
-	if (gitfo_stat(backend->pack_folder, &st) < 0 || !S_ISDIR(st.st_mode))
-		return git__throw(GIT_ENOTFOUND, "Failed to refresh packfiles. Backend not found");
+	if (p_stat(backend->pack_folder, &st) < 0 || !S_ISDIR(st.st_mode))
+		return git_odb__error_notfound("failed to refresh packfiles", NULL);
 
 	if (st.st_mtime != backend->pack_folder_mtime) {
-		char path[GIT_PATH_MAX];
-		strcpy(path, backend->pack_folder);
+		git_buf path = GIT_BUF_INIT;
+		git_buf_sets(&path, backend->pack_folder);
 
 		/* reload all packs */
-		error = gitfo_dirent(path, GIT_PATH_MAX, packfile_load__cb, (void *)backend);
-		if (error < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to refresh packfiles");
-
-		git_vector_sort(&backend->packs);
-		backend->pack_folder_mtime = st.st_mtime;
-	}
-
-	return GIT_SUCCESS;
-}
-
-
-
-
-
-
-
-
-/***********************************************************
- *
- * PACKFILE ENTRY SEARCH INTERNALS
- *
- ***********************************************************/
-
-static off_t nth_packed_object_offset(const struct pack_file *p, uint32_t n)
-{
-	const unsigned char *index = p->index_map.data;
-	index += 4 * 256;
-	if (p->index_version == 1) {
-		return ntohl(*((uint32_t *)(index + 24 * n)));
-	} else {
-		uint32_t off;
-		index += 8 + p->num_objects * (20 + 4);
-		off = ntohl(*((uint32_t *)(index + 4 * n)));
-		if (!(off & 0x80000000))
-			return off;
-		index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
-		return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
-				   ntohl(*((uint32_t *)(index + 4)));
-	}
-}
-
-static int pack_entry_find_offset(
-		off_t *offset_out,
-		git_oid *found_oid,
-		struct pack_file *p,
-		const git_oid *short_oid,
-		unsigned int len)
-{
-	const uint32_t *level1_ofs = p->index_map.data;
-	const unsigned char *index = p->index_map.data;
-	unsigned hi, lo, stride;
-	int pos, found = 0;
-	const unsigned char *current;
-
-	*offset_out = 0;
-
-	if (index == NULL) {
-		int error;
-
-		if ((error = pack_index_open(p)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to find offset for pack entry");
-
-		assert(p->index_map.data);
-
-		index = p->index_map.data;
-		level1_ofs = p->index_map.data;
-	}
-
-	if (p->index_version > 1) {
-		level1_ofs += 2;
-		index += 8;
-	}
-
-	index += 4 * 256;
-	hi = ntohl(level1_ofs[(int)short_oid->id[0]]);
-	lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1]));
-
-	if (p->index_version > 1) {
-		stride = 20;
-	} else {
-		stride = 24;
-		index += 4;
-	}
-
-#ifdef INDEX_DEBUG_LOOKUP
-	printf("%02x%02x%02x... lo %u hi %u nr %d\n",
-		short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects);
-#endif
-
-	/* Use git.git lookup code */
-	pos =  sha1_entry_pos(index, stride, 0, lo, hi, p->num_objects, short_oid->id);
-
-	if (pos >= 0) {
-		/* An object matching exactly the oid was found */
-		found = 1;
-		current = index + pos * stride;
-	} else {
-		/* No object was found */
-		/* pos refers to the object with the "closest" oid to short_oid */
-		pos = - 1 - pos;
-		if (pos < (int)p->num_objects) {
-			current = index + pos * stride;
-
-			if (!git_oid_match_raw(len, short_oid->id, current)) {
-				found = 1;
-			}
-		}
-	}
-
-	if (found && pos + 1 < (int)p->num_objects) {
-		/* Check for ambiguousity */
-		const unsigned char *next = current + stride;
-
-		if (!git_oid_match_raw(len, short_oid->id, next)) {
-			found = 2;
-		}
-	}
-
-	if (!found) {
-		return git__throw(GIT_ENOTFOUND, "Failed to find offset for pack entry. Entry not found");
-	} else if (found > 1) {
-		return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to find offset for pack entry. Ambiguous sha1 prefix within pack");
-	} else {
-		*offset_out = nth_packed_object_offset(p, pos);
-		git_oid_mkraw(found_oid, current);
-
-#ifdef INDEX_DEBUG_LOOKUP
-		unsigned char hex_sha1[GIT_OID_HEXSZ + 1];
-		git_oid_fmt(hex_sha1, found_oid);
-		hex_sha1[GIT_OID_HEXSZ] = '\0';
-		printf("found lo=%d %s\n", lo, hex_sha1);
-#endif
-		return GIT_SUCCESS;
-	}
-}
+		error = git_path_direach(&path, packfile_load__cb, (void *)backend);
 
-static int pack_entry_find1(
-		struct pack_entry *e,
-		struct pack_file *p,
-		const git_oid *short_oid,
-		unsigned int len)
-{
-	off_t offset;
-	git_oid found_oid;
-	int error;
+		git_buf_free(&path);
 
-	assert(p);
+		if (error < 0)
+			return error;
 
-	if (len == GIT_OID_HEXSZ && p->num_bad_objects) {
-		unsigned i;
-		for (i = 0; i < p->num_bad_objects; i++)
-			if (git_oid_cmp(short_oid, &p->bad_object_sha1[i]) == 0)
-				return git__throw(GIT_ERROR, "Failed to find pack entry. Bad object found");
+		git_vector_sort(&backend->packs);
+		backend->pack_folder_mtime = st.st_mtime;
 	}
 
-	error = pack_entry_find_offset(&offset, &found_oid, p, short_oid, len);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to find pack entry. Couldn't find offset");
-
-	/* we found a unique entry in the index;
-	 * make sure the packfile backing the index 
-	 * still exists on disk */
-	if (p->pack_fd == -1 && packfile_open(p) < GIT_SUCCESS)
-		return git__throw(GIT_EOSERR, "Failed to find pack entry. Packfile doesn't exist on disk");
-
-	e->offset = offset;
-	e->p = p;
-
-	git_oid_cpy(&e->sha1, &found_oid);
-	return GIT_SUCCESS;
+	return 0;
 }
 
-static int pack_entry_find(struct pack_entry *e, struct pack_backend *backend, const git_oid *oid)
+static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid)
 {
 	int error;
-	size_t i;
+	unsigned int i;
 
-	if ((error = packfile_refresh_all(backend)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to find pack entry");
+	if ((error = packfile_refresh_all(backend)) < 0)
+		return error;
 
 	if (backend->last_found &&
-		pack_entry_find1(e, backend->last_found, oid, GIT_OID_HEXSZ) == GIT_SUCCESS)
-		return GIT_SUCCESS;
+		git_pack_entry_find(e, backend->last_found, oid, GIT_OID_HEXSZ) == 0)
+		return 0;
 
 	for (i = 0; i < backend->packs.length; ++i) {
-		struct pack_file *p;
+		struct git_pack_file *p;
 
 		p = git_vector_get(&backend->packs, i);
 		if (p == backend->last_found)
 			continue;
 
-		if (pack_entry_find1(e, p, oid, GIT_OID_HEXSZ) == GIT_SUCCESS) {
+		if (git_pack_entry_find(e, p, oid, GIT_OID_HEXSZ) == 0) {
 			backend->last_found = p;
-			return GIT_SUCCESS;
+			return 0;
 		}
 	}
 
-	return git__throw(GIT_ENOTFOUND, "Failed to find pack entry");
+	return git_odb__error_notfound("failed to find pack entry", oid);
 }
 
 static int pack_entry_find_prefix(
-	struct pack_entry *e,
+	struct git_pack_entry *e,
 	struct pack_backend *backend,
 	const git_oid *short_oid,
 	unsigned int len)
 {
 	int error;
-	size_t i;
+	unsigned int i;
 	unsigned found = 0;
 
-	if ((error = packfile_refresh_all(backend)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to find pack entry");
+	if ((error = packfile_refresh_all(backend)) < 0)
+		return error;
 
 	if (backend->last_found) {
-		error = pack_entry_find1(e, backend->last_found, short_oid, len);
-		if (error == GIT_EAMBIGUOUSOIDPREFIX) {
-			return git__rethrow(error, "Failed to find pack entry. Ambiguous sha1 prefix");
-		} else if (error == GIT_SUCCESS) {
+		error = git_pack_entry_find(e, backend->last_found, short_oid, len);
+		if (error == GIT_EAMBIGUOUS)
+			return error;
+		if (!error)
 			found = 1;
-		}
 	}
 
 	for (i = 0; i < backend->packs.length; ++i) {
-		struct pack_file *p;
+		struct git_pack_file *p;
 
 		p = git_vector_get(&backend->packs, i);
 		if (p == backend->last_found)
 			continue;
 
-		error = pack_entry_find1(e, p, short_oid, len);
-		if (error == GIT_EAMBIGUOUSOIDPREFIX) {
-			return git__rethrow(error, "Failed to find pack entry. Ambiguous sha1 prefix");
-		} else if (error == GIT_SUCCESS) {
-			found++;
-			if (found > 1)
+		error = git_pack_entry_find(e, p, short_oid, len);
+		if (error == GIT_EAMBIGUOUS)
+			return error;
+		if (!error) {
+			if (++found > 1)
 				break;
 			backend->last_found = p;
 		}
 	}
 
-	if (!found) {
-		return git__rethrow(GIT_ENOTFOUND, "Failed to find pack entry");
-	} else if (found > 1) {
-		return git__rethrow(GIT_EAMBIGUOUSOIDPREFIX, "Failed to find pack entry. Ambiguous sha1 prefix");
-	} else {
-		return GIT_SUCCESS;
-	}
-
-}
-
-
-
-
-
-
-
-
-
-
-
-
-/***********************************************************
- *
- * PACKFILE ENTRY UNPACK INTERNALS
- *
- ***********************************************************/
-
-static unsigned long packfile_unpack_header1(
-		size_t *sizep,
-		git_otype *type, 
-		const unsigned char *buf,
-		unsigned long len)
-{
-	unsigned shift;
-	unsigned long size, c;
-	unsigned long used = 0;
-
-	c = buf[used++];
-	*type = (c >> 4) & 7;
-	size = c & 15;
-	shift = 4;
-	while (c & 0x80) {
-		if (len <= used || bitsizeof(long) <= shift)
-			return 0;
-
-		c = buf[used++];
-		size += (c & 0x7f) << shift;
-		shift += 7;
-	}
-
-	*sizep = (size_t)size;
-	return used;
-}
-
-static int packfile_unpack_header(
-		size_t *size_p,
-		git_otype *type_p,
-		struct pack_backend *backend,
-		struct pack_file *p,
-		struct pack_window **w_curs,
-		off_t *curpos)
-{
-	unsigned char *base;
-	unsigned int left;
-	unsigned long used;
-
-	/* pack_window_open() assures us we have [base, base + 20) available
-	 * as a range that we can look at at.  (Its actually the hash
-	 * size that is assured.)  With our object header encoding
-	 * the maximum deflated object size is 2^137, which is just
-	 * insane, so we know won't exceed what we have been given.
-	 */
-	base = pack_window_open(backend, p, w_curs, *curpos, &left);
-	if (base == NULL)
-		return GIT_ENOMEM;
-
-	used = packfile_unpack_header1(size_p, type_p, base, left);
-
-	if (used == 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Header length is zero");
-
-	*curpos += used;
-	return GIT_SUCCESS;
-}
-
-static int packfile_unpack_compressed(
-		git_rawobj *obj,
-		struct pack_backend *backend,
-		struct pack_file *p,
-		struct pack_window **w_curs,
-		off_t curpos,
-		size_t size,
-		git_otype type)
-{
-	int st;
-	z_stream stream;
-	unsigned char *buffer, *in;
-
-	buffer = git__malloc(size);
-
-	memset(&stream, 0, sizeof(stream));
-	stream.next_out = buffer;
-	stream.avail_out = size + 1;
-
-	st = inflateInit(&stream);
-	if (st != Z_OK) {
-		free(buffer);
-		return git__throw(GIT_EZLIB, "Error in zlib");
-	}
-
-	do {
-		in = pack_window_open(backend, p, w_curs, curpos, &stream.avail_in);
-		stream.next_in = in;
-		st = inflate(&stream, Z_FINISH);
-
-		if (!stream.avail_out)
-			break; /* the payload is larger than it should be */
-
-		curpos += stream.next_in - in;
-	} while (st == Z_OK || st == Z_BUF_ERROR);
-
-	inflateEnd(&stream);
-
-	if ((st != Z_STREAM_END) || stream.total_out != size) {
-		free(buffer);
-		return git__throw(GIT_EZLIB, "Error in zlib");
-	}
-
-	obj->type = type;
-	obj->len = size;
-	obj->data = buffer;
-	return GIT_SUCCESS;
-}
-
-static off_t get_delta_base(
-		struct pack_backend *backend,
-		struct pack_file *p,
-		struct pack_window **w_curs,
-		off_t *curpos,
-		git_otype type,
-		off_t delta_obj_offset)
-{
-	unsigned char *base_info = pack_window_open(backend, p, w_curs, *curpos, NULL);
-	off_t base_offset;
-	git_oid unused;
-
-	/* pack_window_open() assured us we have [base_info, base_info + 20)
-	 * as a range that we can look at without walking off the
-	 * end of the mapped window.  Its actually the hash size
-	 * that is assured.  An OFS_DELTA longer than the hash size
-	 * is stupid, as then a REF_DELTA would be smaller to store.
-	 */
-	if (type == GIT_OBJ_OFS_DELTA) {
-		unsigned used = 0;
-		unsigned char c = base_info[used++];
-		base_offset = c & 127;
-		while (c & 128) {
-			base_offset += 1;
-			if (!base_offset || MSB(base_offset, 7))
-				return 0;  /* overflow */
-			c = base_info[used++];
-			base_offset = (base_offset << 7) + (c & 127);
-		}
-		base_offset = delta_obj_offset - base_offset;
-		if (base_offset <= 0 || base_offset >= delta_obj_offset)
-			return 0;  /* out of bound */
-		*curpos += used;
-	} else if (type == GIT_OBJ_REF_DELTA) {
-		/* The base entry _must_ be in the same pack */
-		if (pack_entry_find_offset(&base_offset, &unused, p, (git_oid *)base_info, GIT_OID_HEXSZ) < GIT_SUCCESS)
-			return git__throw(GIT_EPACKCORRUPTED, "Base entry delta is not in the same pack");
-		*curpos += 20;
-	} else
+	if (!found)
+		return git_odb__error_notfound("no matching pack entry for prefix", short_oid);
+	else if (found > 1)
+		return git_odb__error_ambiguous("found multiple pack entries");
+	else
 		return 0;
-
-	return base_offset;
 }
 
-static int packfile_unpack_delta(
-		git_rawobj *obj,
-		struct pack_backend *backend,
-		struct pack_file *p,
-		struct pack_window **w_curs,
-		off_t curpos,
-		size_t delta_size,
-		git_otype delta_type,
-		off_t obj_offset)
-{
-	off_t base_offset;
-	git_rawobj base, delta;
-	int error;
-
-	base_offset = get_delta_base(backend, p, w_curs, &curpos, delta_type, obj_offset);
-	if (base_offset == 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Delta offset is zero");
-
-	pack_window_close(w_curs);
-	error = packfile_unpack(&base, backend, p, base_offset);
-
-	/* TODO: git.git tries to load the base from other packfiles
-	 * or loose objects */
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Corrupted delta");
-
-	error = packfile_unpack_compressed(&delta, backend, p, w_curs, curpos, delta_size, delta_type);
-	if (error < GIT_SUCCESS) {
-		free(base.data);
-		return git__rethrow(error, "Corrupted delta");
-	}
-
-	obj->type = base.type;
-	error = git__delta_apply(obj,
-			base.data, base.len,
-			delta.data, delta.len);
-
-	free(base.data);
-	free(delta.data);
-
-	/* TODO: we might want to cache this shit. eventually */
-	//add_delta_base_cache(p, base_offset, base, base_size, *type);
-	return error; /* error set by git__delta_apply */
-}
-
-static int packfile_unpack(
-		git_rawobj *obj,
-		struct pack_backend *backend,
-		struct pack_file *p,
-		off_t obj_offset)
-{
-	struct pack_window *w_curs = NULL;
-	off_t curpos = obj_offset;
-	int error;
-
-	size_t size = 0;
-	git_otype type;
-
-	/* 
-	 * TODO: optionally check the CRC on the packfile
-	 */
-
-	obj->data = NULL;
-	obj->len = 0;
-	obj->type = GIT_OBJ_BAD;
-
-	error = packfile_unpack_header(&size, &type, backend, p, &w_curs, &curpos);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to unpack packfile");
-
-	switch (type) {
-	case GIT_OBJ_OFS_DELTA:
-	case GIT_OBJ_REF_DELTA:
-		error = packfile_unpack_delta(
-				obj, backend, p, &w_curs, curpos,
-				size, type, obj_offset);
-		break;
-
-	case GIT_OBJ_COMMIT:
-	case GIT_OBJ_TREE:
-	case GIT_OBJ_BLOB:
-	case GIT_OBJ_TAG:
-		error = packfile_unpack_compressed(
-				obj, backend, p, &w_curs, curpos,
-				size, type);
-		break;
-
-	default:
-		error = GIT_EOBJCORRUPTED;
-		break;
-	}
-
-	pack_window_close(&w_curs);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to unpack packfile");
-}
-
-
-
-
 
 /***********************************************************
  *
@@ -1448,26 +360,24 @@ int pack_backend__read_header(git_rawobj *obj, git_odb_backend *backend, const g
 }
 */
 
-int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
+static int pack_backend__read(void **buffer_p, size_t *len_p, git_otype *type_p, git_odb_backend *backend, const git_oid *oid)
 {
-	struct pack_entry e;
+	struct git_pack_entry e;
 	git_rawobj raw;
 	int error;
 
-	if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to read pack backend");
-
-	if ((error = packfile_unpack(&raw, (struct pack_backend *)backend, e.p, e.offset)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to read pack backend");
+	if ((error = pack_entry_find(&e, (struct pack_backend *)backend, oid)) < 0 ||
+		(error = git_packfile_unpack(&raw, e.p, &e.offset)) < 0)
+		return error;
 
 	*buffer_p = raw.data;
 	*len_p = raw.len;
 	*type_p = raw.type;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-int pack_backend__read_prefix(
+static int pack_backend__read_prefix(
 	git_oid *out_oid,
 	void **buffer_p,
 	size_t *len_p,
@@ -1476,87 +386,77 @@ int pack_backend__read_prefix(
 	const git_oid *short_oid,
 	unsigned int len)
 {
+	int error = 0;
+
 	if (len < GIT_OID_MINPREFIXLEN)
-		return git__throw(GIT_EAMBIGUOUSOIDPREFIX, "Failed to read pack backend. Prefix length is lower than %d.", GIT_OID_MINPREFIXLEN);
+		error = git_odb__error_ambiguous("prefix length too short");
 
-	if (len >= GIT_OID_HEXSZ) {
+	else if (len >= GIT_OID_HEXSZ) {
 		/* We can fall back to regular read method */
-		int error = pack_backend__read(buffer_p, len_p, type_p, backend, short_oid);
-		if (error == GIT_SUCCESS)
+		error = pack_backend__read(buffer_p, len_p, type_p, backend, short_oid);
+		if (!error)
 			git_oid_cpy(out_oid, short_oid);
-
-		return error;
 	} else {
-		struct pack_entry e;
+		struct git_pack_entry e;
 		git_rawobj raw;
-		int error;
-
-		if ((error = pack_entry_find_prefix(&e, (struct pack_backend *)backend, short_oid, len)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to read pack backend");
 
-		if ((error = packfile_unpack(&raw, (struct pack_backend *)backend, e.p, e.offset)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to read pack backend");
-
-		*buffer_p = raw.data;
-		*len_p = raw.len;
-		*type_p = raw.type;
-		git_oid_cpy(out_oid, &e.sha1);
+		if ((error = pack_entry_find_prefix(
+				&e, (struct pack_backend *)backend, short_oid, len)) == 0 &&
+			(error = git_packfile_unpack(&raw, e.p, &e.offset)) == 0)
+		{
+			*buffer_p = raw.data;
+			*len_p = raw.len;
+			*type_p = raw.type;
+			git_oid_cpy(out_oid, &e.sha1);
+		}
 	}
 
-	return GIT_SUCCESS;
+	return error;
 }
 
-int pack_backend__exists(git_odb_backend *backend, const git_oid *oid)
+static int pack_backend__exists(git_odb_backend *backend, const git_oid *oid)
 {
-	struct pack_entry e;
-	return pack_entry_find(&e, (struct pack_backend *)backend, oid) == GIT_SUCCESS;
+	struct git_pack_entry e;
+	return pack_entry_find(&e, (struct pack_backend *)backend, oid) == 0;
 }
 
-void pack_backend__free(git_odb_backend *_backend)
+static void pack_backend__free(git_odb_backend *_backend)
 {
 	struct pack_backend *backend;
-	size_t i;
+	unsigned int i;
 
 	assert(_backend);
 
 	backend = (struct pack_backend *)_backend;
 
 	for (i = 0; i < backend->packs.length; ++i) {
-		struct pack_file *p = git_vector_get(&backend->packs, i);
-		packfile_free(backend, p);
+		struct git_pack_file *p = git_vector_get(&backend->packs, i);
+		packfile_free(p);
 	}
 
 	git_vector_free(&backend->packs);
-	free(backend->pack_folder);
-	free(backend);
+	git__free(backend->pack_folder);
+	git__free(backend);
 }
 
 int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir)
 {
-	struct pack_backend *backend;
-	char path[GIT_PATH_MAX];
+	struct pack_backend *backend = NULL;
+	git_buf path = GIT_BUF_INIT;
 
 	backend = git__calloc(1, sizeof(struct pack_backend));
-	if (backend == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(backend);
 
-	if (git_vector_init(&backend->packs, 8, packfile_sort__cb) < GIT_SUCCESS) {
-		free(backend);
-		return GIT_ENOMEM;
+	if (git_vector_init(&backend->packs, 8, packfile_sort__cb) < 0 ||
+		git_buf_joinpath(&path, objects_dir, "pack") < 0)
+	{
+		git__free(backend);
+		return -1;
 	}
 
-	backend->window_size = DEFAULT_WINDOW_SIZE;
-	backend->mapped_limit = DEFAULT_MAPPED_LIMIT;
-
-	git__joinpath(path, objects_dir, "pack");
-	if (gitfo_isdir(path) == GIT_SUCCESS) {
-		backend->pack_folder = git__strdup(path);
+	if (git_path_isdir(git_buf_cstr(&path)) == true) {
+		backend->pack_folder = git_buf_detach(&path);
 		backend->pack_folder_mtime = 0;
-
-		if (backend->pack_folder == NULL) {
-			free(backend);
-			return GIT_ENOMEM;
-		}
 	}
 
 	backend->parent.read = &pack_backend__read;
@@ -1566,5 +466,8 @@ int git_odb_backend_pack(git_odb_backend **backend_out, const char *objects_dir)
 	backend->parent.free = &pack_backend__free;
 
 	*backend_out = (git_odb_backend *)backend;
-	return GIT_SUCCESS;
+
+	git_buf_free(&path);
+
+	return 0;
 }
diff --git a/src/oid.c b/src/oid.c
index 8dc6903cd..87756010b 100644
--- a/src/oid.c
+++ b/src/oid.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
@@ -29,37 +11,52 @@
 #include <string.h>
 #include <limits.h>
 
-static signed char from_hex[] = {
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 00 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 10 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 20 */
- 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, -1, -1, -1, -1, -1, -1, /* 30 */
--1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 40 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 50 */
--1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 60 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 70 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 80 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 90 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* a0 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* b0 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* c0 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* d0 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* e0 */
--1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* f0 */
-};
 static char to_hex[] = "0123456789abcdef";
 
-int git_oid_mkstr(git_oid *out, const char *str)
+static int oid_error_invalid(const char *msg)
+{
+	giterr_set(GITERR_INVALID, "Unable to parse OID - %s", msg);
+	return -1;
+}
+
+int git_oid_fromstrn(git_oid *out, const char *str, size_t length)
 {
 	size_t p;
-	for (p = 0; p < sizeof(out->id); p++, str += 2) {
-		int v = (from_hex[(unsigned char)str[0]] << 4)
-		       | from_hex[(unsigned char)str[1]];
+	int v;
+
+	if (length < 4)
+		return oid_error_invalid("input too short");
+
+	if (length > GIT_OID_HEXSZ)
+		length = GIT_OID_HEXSZ;
+
+	for (p = 0; p < length - 1; p += 2) {
+		v = (git__fromhex(str[p + 0]) << 4)
+				| git__fromhex(str[p + 1]);
+
 		if (v < 0)
-			return git__throw(GIT_ENOTOID, "Failed to generate sha1. Given string is not a valid sha1 hash");
-		out->id[p] = (unsigned char)v;
+			return oid_error_invalid("contains invalid characters");
+
+		out->id[p / 2] = (unsigned char)v;
+	}
+
+	if (length % 2) {
+		v = (git__fromhex(str[p + 0]) << 4);
+		if (v < 0)
+			return oid_error_invalid("contains invalid characters");
+
+		out->id[p / 2] = (unsigned char)v;
+		p += 2;
 	}
-	return GIT_SUCCESS;
+
+	memset(out->id + p / 2, 0, (GIT_OID_HEXSZ - p) / 2);
+
+	return 0;
+}
+
+int git_oid_fromstr(git_oid *out, const char *str)
+{
+	return git_oid_fromstrn(out, str, GIT_OID_HEXSZ);
 }
 
 GIT_INLINE(char) *fmt_one(char *str, unsigned int val)
@@ -97,14 +94,14 @@ char *git_oid_allocfmt(const git_oid *oid)
 	return str;
 }
 
-char *git_oid_to_string(char *out, size_t n, const git_oid *oid)
+char *git_oid_tostr(char *out, size_t n, const git_oid *oid)
 {
 	char str[GIT_OID_HEXSZ];
 
 	if (!out || n == 0 || !oid)
 		return "";
 
-	n--;  /* allow room for terminating NUL */
+	n--; /* allow room for terminating NUL */
 
 	if (n > 0) {
 		git_oid_fmt(str, oid);
@@ -118,8 +115,9 @@ char *git_oid_to_string(char *out, size_t n, const git_oid *oid)
 	return out;
 }
 
-int git__parse_oid(git_oid *oid, const char **buffer_out,
-		const char *buffer_end, const char *header)
+int git_oid__parse(
+	git_oid *oid, const char **buffer_out,
+	const char *buffer_end, const char *header)
 {
 	const size_t sha_len = GIT_OID_HEXSZ;
 	const size_t header_len = strlen(header);
@@ -127,37 +125,33 @@ int git__parse_oid(git_oid *oid, const char **buffer_out,
 	const char *buffer = *buffer_out;
 
 	if (buffer + (header_len + sha_len + 1) > buffer_end)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse OID. Buffer too small");
+		return -1;
 
 	if (memcmp(buffer, header, header_len) != 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse OID. Buffer and header do not match");
+		return -1;
 
 	if (buffer[header_len + sha_len] != '\n')
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse OID. Buffer not terminated correctly");
+		return -1;
 
-	if (git_oid_mkstr(oid, buffer + header_len) < GIT_SUCCESS)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse OID. Failed to generate sha1");
+	if (git_oid_fromstr(oid, buffer + header_len) < 0)
+		return -1;
 
 	*buffer_out = buffer + (header_len + sha_len + 1);
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-int git__write_oid(git_odb_stream *stream, const char *header, const git_oid *oid)
+void git_oid__writebuf(git_buf *buf, const char *header, const git_oid *oid)
 {
-	char hex_oid[42];
-
-	git_oid_fmt(hex_oid + 1, oid);
-
-	hex_oid[0] = ' ';
-	hex_oid[41] = '\n';
+	char hex_oid[GIT_OID_HEXSZ];
 
-	stream->write(stream, header, strlen(header));
-	stream->write(stream, hex_oid, 42);
-	return GIT_SUCCESS;
+	git_oid_fmt(hex_oid, oid);
+	git_buf_puts(buf, header);
+	git_buf_put(buf, hex_oid, GIT_OID_HEXSZ);
+	git_buf_putc(buf, '\n');
 }
 
-void git_oid_mkraw(git_oid *out, const unsigned char *raw)
+void git_oid_fromraw(git_oid *out, const unsigned char *raw)
 {
 	memcpy(out->id, raw, sizeof(out->id));
 }
@@ -172,30 +166,44 @@ int git_oid_cmp(const git_oid *a, const git_oid *b)
 	return memcmp(a->id, b->id, sizeof(a->id));
 }
 
-
-int git_oid_match_raw(unsigned int len, const unsigned char *a, const unsigned char *b)
+int git_oid_ncmp(const git_oid *oid_a, const git_oid *oid_b, unsigned int len)
 {
-        do {
-                if (*a != *b)
-                        return 1;
-                a++;
-                b++;
-                len -= 2;
-        } while (len > 1);
-        if (len)
-                if ((*a ^ *b) & 0xf0)
-                        return 1;
-        return 0;
+	const unsigned char *a = oid_a->id;
+	const unsigned char *b = oid_b->id;
+
+	do {
+		if (*a != *b)
+			return 1;
+		a++;
+		b++;
+		len -= 2;
+	} while (len > 1);
+
+	if (len)
+		if ((*a ^ *b) & 0xf0)
+			return 1;
+
+	return 0;
 }
 
-int git_oid_match_hex(unsigned int len, const unsigned char *a, const unsigned char *b)
+int git_oid_streq(const git_oid *a, const char *str)
 {
-	return memcmp(a, b, len);
+	git_oid id;
+
+	if (git_oid_fromstr(&id, str) < 0)
+		return -1;
+
+	return git_oid_cmp(a, &id) == 0 ? 0 : -1;
 }
 
-int gid_oid_match(unsigned int len, git_oid *a, git_oid *b)
+int git_oid_iszero(const git_oid *oid_a)
 {
-	return git_oid_match_raw(len, a->id, b->id);
+	const unsigned char *a = oid_a->id;
+	unsigned int i;
+	for (i = 0; i < GIT_OID_RAWSZ; ++i, ++a)
+		if (*a != 0)
+			return 0;
+	return 1;
 }
 
 typedef short node_index;
@@ -213,16 +221,15 @@ struct git_oid_shorten {
 
 static int resize_trie(git_oid_shorten *self, size_t new_size)
 {
-	self->nodes = realloc(self->nodes, new_size * sizeof(trie_node));
-	if (self->nodes == NULL)
-		return GIT_ENOMEM;
+	self->nodes = git__realloc(self->nodes, new_size * sizeof(trie_node));
+	GITERR_CHECK_ALLOC(self->nodes);
 
 	if (new_size > self->size) {
 		memset(&self->nodes[self->size], 0x0, (new_size - self->size) * sizeof(trie_node));
 	}
 
 	self->size = new_size;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static trie_node *push_leaf(git_oid_shorten *os, node_index idx, int push_at, const char *oid)
@@ -231,7 +238,7 @@ static trie_node *push_leaf(git_oid_shorten *os, node_index idx, int push_at, co
 	node_index idx_leaf;
 
 	if (os->node_count >= os->size) {
-		if (resize_trie(os, os->size * 2) < GIT_SUCCESS)
+		if (resize_trie(os, os->size * 2) < 0)
 			return NULL;
 	}
 
@@ -253,27 +260,27 @@ git_oid_shorten *git_oid_shorten_new(size_t min_length)
 {
 	git_oid_shorten *os;
 
-	os = git__malloc(sizeof(git_oid_shorten));
+	assert((size_t)((int)min_length) == min_length);
+
+	os = git__calloc(1, sizeof(git_oid_shorten));
 	if (os == NULL)
 		return NULL;
 
-	memset(os, 0x0, sizeof(git_oid_shorten));
-
-	if (resize_trie(os, 16) < GIT_SUCCESS) {
-		free(os);
+	if (resize_trie(os, 16) < 0) {
+		git__free(os);
 		return NULL;
 	}
 
 	os->node_count = 1;
-	os->min_length = min_length;
+	os->min_length = (int)min_length;
 
 	return os;
 }
 
 void git_oid_shorten_free(git_oid_shorten *os)
 {
-	free(os->nodes);
-	free(os);
+	git__free(os->nodes);
+	git__free(os);
 }
 
 
@@ -293,7 +300,7 @@ void git_oid_shorten_free(git_oid_shorten *os)
  *
  *	- Each normal node points to 16 children (one for each possible
  *	character in the oid). This is *not* stored in an array of
- *	pointers, because in a 64-bit arch this would be sucking 
+ *	pointers, because in a 64-bit arch this would be sucking
  *	16*sizeof(void*) = 128 bytes of memory per node, which is fucking
  *	insane. What we do is store Node Indexes, and use these indexes
  *	to look up each node in the om->index array. These indexes are
@@ -323,21 +330,27 @@ void git_oid_shorten_free(git_oid_shorten *os)
  */
 int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid)
 {
-	int i, is_leaf;
+	int i;
+	bool is_leaf;
 	node_index idx;
 
 	if (os->full)
-		return GIT_ENOMEM;
+		return -1;
+
+	if (text_oid == NULL)
+		return os->min_length;
 
 	idx = 0;
-	is_leaf = 0;
+	is_leaf = false;
 
 	for (i = 0; i < GIT_OID_HEXSZ; ++i) {
-		int c = from_hex[(int)text_oid[i]];
+		int c = git__fromhex(text_oid[i]);
 		trie_node *node;
 
-		if (c == -1)
-			return git__throw(GIT_ENOTOID, "Failed to shorten OID. Invalid hex value");
+		if (c == -1) {
+			giterr_set(GITERR_INVALID, "Unable to shorten OID - invalid hex value");
+			return -1;
+		}
 
 		node = &os->nodes[idx];
 
@@ -347,23 +360,22 @@ int git_oid_shorten_add(git_oid_shorten *os, const char *text_oid)
 			tail = node->tail;
 			node->tail = NULL;
 
-			node = push_leaf(os, idx, from_hex[(int)tail[0]], &tail[1]);
-			if (node == NULL)
-				return GIT_ENOMEM;
+			node = push_leaf(os, idx, git__fromhex(tail[0]), &tail[1]);
+			GITERR_CHECK_ALLOC(node);
 		}
 
 		if (node->children[c] == 0) {
 			if (push_leaf(os, idx, c, &text_oid[i + 1]) == NULL)
-				return GIT_ENOMEM;
+				return -1;
 			break;
 		}
 
 		idx = node->children[c];
-		is_leaf = 0;
+		is_leaf = false;
 
 		if (idx < 0) {
 			node->children[c] = idx = -idx;
-			is_leaf = 1;
+			is_leaf = true;
 		}
 	}
 
diff --git a/src/oid.h b/src/oid.h
deleted file mode 100644
index 1cd2450d6..000000000
--- a/src/oid.h
+++ /dev/null
@@ -1,17 +0,0 @@
-#ifndef INCLUDE_oid_h__
-#define INCLUDE_oid_h__
-
-/**
- * Compare the first ('len'*4) bits of two raw formatted oids.
- * This can be useful for internal use.
- * Return 0 if they match.
- */
-int git_oid_match_raw(unsigned int len, const unsigned char *a, const unsigned char *b);
-
-/**
- * Compare the first 'len' characters of two hex formatted oids.
- * Return 0 if they match.
- */
-int git_oid_match_hex(unsigned int len, const unsigned char *a, const unsigned char *b);
-
-#endif
diff --git a/src/oidmap.h b/src/oidmap.h
new file mode 100644
index 000000000..858268c92
--- /dev/null
+++ b/src/oidmap.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_oidmap_h__
+#define INCLUDE_oidmap_h__
+
+#include "common.h"
+#include "git2/oid.h"
+
+#define kmalloc git__malloc
+#define kcalloc git__calloc
+#define krealloc git__realloc
+#define kfree git__free
+#include "khash.h"
+
+__KHASH_TYPE(oid, const git_oid *, void *);
+typedef khash_t(oid) git_oidmap;
+
+GIT_INLINE(khint_t) hash_git_oid(const git_oid *oid)
+{
+	int i;
+	khint_t h = 0;
+	for (i = 0; i < 20; ++i)
+		h = (h << 5) - h + oid->id[i];
+	return h;
+}
+
+GIT_INLINE(int) hash_git_oid_equal(const git_oid *a, const git_oid *b)
+{
+	return (memcmp(a->id, b->id, sizeof(a->id)) == 0);
+}
+
+#define GIT__USE_OIDMAP \
+	__KHASH_IMPL(oid, static inline, const git_oid *, void *, 1, hash_git_oid, hash_git_oid_equal)
+
+#define git_oidmap_alloc() kh_init(oid)
+#define git_oidmap_free(h) kh_destroy(oid,h), h = NULL
+
+#endif
diff --git a/src/pack.c b/src/pack.c
new file mode 100644
index 000000000..0db1069de
--- /dev/null
+++ b/src/pack.c
@@ -0,0 +1,816 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "odb.h"
+#include "pack.h"
+#include "delta-apply.h"
+#include "sha1_lookup.h"
+#include "mwindow.h"
+#include "fileops.h"
+
+#include "git2/oid.h"
+#include <zlib.h>
+
+static int packfile_open(struct git_pack_file *p);
+static git_off_t nth_packed_object_offset(const struct git_pack_file *p, uint32_t n);
+int packfile_unpack_compressed(
+		git_rawobj *obj,
+		struct git_pack_file *p,
+		git_mwindow **w_curs,
+		git_off_t *curpos,
+		size_t size,
+		git_otype type);
+
+/* Can find the offset of an object given
+ * a prefix of an identifier.
+ * Throws GIT_EAMBIGUOUSOIDPREFIX if short oid
+ * is ambiguous within the pack.
+ * This method assumes that len is between
+ * GIT_OID_MINPREFIXLEN and GIT_OID_HEXSZ.
+ */
+static int pack_entry_find_offset(
+		git_off_t *offset_out,
+		git_oid *found_oid,
+		struct git_pack_file *p,
+		const git_oid *short_oid,
+		unsigned int len);
+
+static int packfile_error(const char *message)
+{
+	giterr_set(GITERR_ODB, "Invalid pack file - %s", message);
+	return -1;
+}
+
+/***********************************************************
+ *
+ * PACK INDEX METHODS
+ *
+ ***********************************************************/
+
+static void pack_index_free(struct git_pack_file *p)
+{
+	if (p->index_map.data) {
+		git_futils_mmap_free(&p->index_map);
+		p->index_map.data = NULL;
+	}
+}
+
+static int pack_index_check(const char *path, struct git_pack_file *p)
+{
+	struct git_pack_idx_header *hdr;
+	uint32_t version, nr, i, *index;
+	void *idx_map;
+	size_t idx_size;
+	struct stat st;
+	int error;
+	/* TODO: properly open the file without access time using O_NOATIME */
+	git_file fd = git_futils_open_ro(path);
+	if (fd < 0)
+		return fd;
+
+	if (p_fstat(fd, &st) < 0 ||
+		!S_ISREG(st.st_mode) ||
+		!git__is_sizet(st.st_size) ||
+		(idx_size = (size_t)st.st_size) < 4 * 256 + 20 + 20)
+	{
+		p_close(fd);
+		giterr_set(GITERR_OS, "Failed to check pack index.");
+		return -1;
+	}
+
+	error = git_futils_mmap_ro(&p->index_map, fd, 0, idx_size);
+
+	p_close(fd);
+
+	if (error < 0)
+		return error;
+
+	hdr = idx_map = p->index_map.data;
+
+	if (hdr->idx_signature == htonl(PACK_IDX_SIGNATURE)) {
+		version = ntohl(hdr->idx_version);
+
+		if (version < 2 || version > 2) {
+			git_futils_mmap_free(&p->index_map);
+			return packfile_error("unsupported index version");
+		}
+
+	} else
+		version = 1;
+
+	nr = 0;
+	index = idx_map;
+
+	if (version > 1)
+		index += 2; /* skip index header */
+
+	for (i = 0; i < 256; i++) {
+		uint32_t n = ntohl(index[i]);
+		if (n < nr) {
+			git_futils_mmap_free(&p->index_map);
+			return packfile_error("index is non-monotonic");
+		}
+		nr = n;
+	}
+
+	if (version == 1) {
+		/*
+		 * Total size:
+		 * - 256 index entries 4 bytes each
+		 * - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
+		 * - 20-byte SHA1 of the packfile
+		 * - 20-byte SHA1 file checksum
+		 */
+		if (idx_size != 4*256 + nr * 24 + 20 + 20) {
+			git_futils_mmap_free(&p->index_map);
+			return packfile_error("index is corrupted");
+		}
+	} else if (version == 2) {
+		/*
+		 * Minimum size:
+		 * - 8 bytes of header
+		 * - 256 index entries 4 bytes each
+		 * - 20-byte sha1 entry * nr
+		 * - 4-byte crc entry * nr
+		 * - 4-byte offset entry * nr
+		 * - 20-byte SHA1 of the packfile
+		 * - 20-byte SHA1 file checksum
+		 * And after the 4-byte offset table might be a
+		 * variable sized table containing 8-byte entries
+		 * for offsets larger than 2^31.
+		 */
+		unsigned long min_size = 8 + 4*256 + nr*(20 + 4 + 4) + 20 + 20;
+		unsigned long max_size = min_size;
+
+		if (nr)
+			max_size += (nr - 1)*8;
+
+		if (idx_size < min_size || idx_size > max_size) {
+			git_futils_mmap_free(&p->index_map);
+			return packfile_error("wrong index size");
+		}
+	}
+
+	p->index_version = version;
+	p->num_objects = nr;
+	return 0;
+}
+
+static int pack_index_open(struct git_pack_file *p)
+{
+	char *idx_name;
+	int error;
+	size_t name_len, offset;
+
+	if (p->index_map.data)
+		return 0;
+
+	idx_name = git__strdup(p->pack_name);
+	GITERR_CHECK_ALLOC(idx_name);
+
+	name_len = strlen(idx_name);
+	offset = name_len - strlen(".pack");
+	assert(offset < name_len); /* make sure no underflow */
+
+	strncpy(idx_name + offset, ".idx", name_len - offset);
+
+	error = pack_index_check(idx_name, p);
+	git__free(idx_name);
+
+	return error;
+}
+
+static unsigned char *pack_window_open(
+		struct git_pack_file *p,
+		git_mwindow **w_cursor,
+		git_off_t offset,
+		unsigned int *left)
+{
+	if (p->mwf.fd == -1 && packfile_open(p) < 0)
+		return NULL;
+
+	/* Since packfiles end in a hash of their content and it's
+	 * pointless to ask for an offset into the middle of that
+	 * hash, and the pack_window_contains function above wouldn't match
+	 * don't allow an offset too close to the end of the file.
+	 */
+	if (offset > (p->mwf.size - 20))
+		return NULL;
+
+	return git_mwindow_open(&p->mwf, w_cursor, offset, 20, left);
+ }
+
+static int packfile_unpack_header1(
+		unsigned long *usedp,
+		size_t *sizep,
+		git_otype *type,
+		const unsigned char *buf,
+		unsigned long len)
+{
+	unsigned shift;
+	unsigned long size, c;
+	unsigned long used = 0;
+
+	c = buf[used++];
+	*type = (c >> 4) & 7;
+	size = c & 15;
+	shift = 4;
+	while (c & 0x80) {
+		if (len <= used)
+			return GIT_EBUFS;
+
+		if (bitsizeof(long) <= shift) {
+			*usedp = 0;
+			return -1;
+		}
+
+		c = buf[used++];
+		size += (c & 0x7f) << shift;
+		shift += 7;
+	}
+
+	*sizep = (size_t)size;
+	*usedp = used;
+	return 0;
+}
+
+int git_packfile_unpack_header(
+		size_t *size_p,
+		git_otype *type_p,
+		git_mwindow_file *mwf,
+		git_mwindow **w_curs,
+		git_off_t *curpos)
+{
+	unsigned char *base;
+	unsigned int left;
+	unsigned long used;
+	int ret;
+
+	/* pack_window_open() assures us we have [base, base + 20) available
+	 * as a range that we can look at at. (Its actually the hash
+	 * size that is assured.) With our object header encoding
+	 * the maximum deflated object size is 2^137, which is just
+	 * insane, so we know won't exceed what we have been given.
+	 */
+//	base = pack_window_open(p, w_curs, *curpos, &left);
+	base = git_mwindow_open(mwf, w_curs, *curpos, 20, &left);
+	if (base == NULL)
+		return GIT_EBUFS;
+
+	ret = packfile_unpack_header1(&used, size_p, type_p, base, left);
+	git_mwindow_close(w_curs);
+	if (ret == GIT_EBUFS)
+		return ret;
+	else if (ret < 0)
+		return packfile_error("header length is zero");
+
+	*curpos += used;
+	return 0;
+}
+
+static int packfile_unpack_delta(
+		git_rawobj *obj,
+		struct git_pack_file *p,
+		git_mwindow **w_curs,
+		git_off_t *curpos,
+		size_t delta_size,
+		git_otype delta_type,
+		git_off_t obj_offset)
+{
+	git_off_t base_offset;
+	git_rawobj base, delta;
+	int error;
+
+	base_offset = get_delta_base(p, w_curs, curpos, delta_type, obj_offset);
+	git_mwindow_close(w_curs);
+	if (base_offset == 0)
+		return packfile_error("delta offset is zero");
+	if (base_offset < 0) /* must actually be an error code */
+		return (int)base_offset;
+
+	error = git_packfile_unpack(&base, p, &base_offset);
+
+	/*
+	 * TODO: git.git tries to load the base from other packfiles
+	 * or loose objects.
+	 *
+	 * We'll need to do this in order to support thin packs.
+	 */
+	if (error < 0)
+		return error;
+
+	error = packfile_unpack_compressed(&delta, p, w_curs, curpos, delta_size, delta_type);
+	git_mwindow_close(w_curs);
+	if (error < 0) {
+		git__free(base.data);
+		return error;
+	}
+
+	obj->type = base.type;
+	error = git__delta_apply(obj, base.data, base.len, delta.data, delta.len);
+
+	git__free(base.data);
+	git__free(delta.data);
+
+	/* TODO: we might want to cache this shit. eventually */
+	//add_delta_base_cache(p, base_offset, base, base_size, *type);
+
+	return error; /* error set by git__delta_apply */
+}
+
+int git_packfile_unpack(
+	git_rawobj *obj,
+	struct git_pack_file *p,
+	git_off_t *obj_offset)
+{
+	git_mwindow *w_curs = NULL;
+	git_off_t curpos = *obj_offset;
+	int error;
+
+	size_t size = 0;
+	git_otype type;
+
+	/*
+	 * TODO: optionally check the CRC on the packfile
+	 */
+
+	obj->data = NULL;
+	obj->len = 0;
+	obj->type = GIT_OBJ_BAD;
+
+	error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos);
+	git_mwindow_close(&w_curs);
+
+	if (error < 0)
+		return error;
+
+	switch (type) {
+	case GIT_OBJ_OFS_DELTA:
+	case GIT_OBJ_REF_DELTA:
+		error = packfile_unpack_delta(
+				obj, p, &w_curs, &curpos,
+				size, type, *obj_offset);
+		break;
+
+	case GIT_OBJ_COMMIT:
+	case GIT_OBJ_TREE:
+	case GIT_OBJ_BLOB:
+	case GIT_OBJ_TAG:
+		error = packfile_unpack_compressed(
+				obj, p, &w_curs, &curpos,
+				size, type);
+		break;
+
+	default:
+		error = packfile_error("invalid packfile type in header");;
+		break;
+	}
+
+	*obj_offset = curpos;
+	return error;
+}
+
+static void *use_git_alloc(void *opaq, unsigned int count, unsigned int size)
+{
+	GIT_UNUSED(opaq);
+	return git__calloc(count, size);
+}
+
+static void use_git_free(void *opaq, void *ptr)
+{
+	GIT_UNUSED(opaq);
+	git__free(ptr);
+}
+
+int packfile_unpack_compressed(
+	git_rawobj *obj,
+	struct git_pack_file *p,
+	git_mwindow **w_curs,
+	git_off_t *curpos,
+	size_t size,
+	git_otype type)
+{
+	int st;
+	z_stream stream;
+	unsigned char *buffer, *in;
+
+	buffer = git__calloc(1, size + 1);
+	GITERR_CHECK_ALLOC(buffer);
+
+	memset(&stream, 0, sizeof(stream));
+	stream.next_out = buffer;
+	stream.avail_out = (uInt)size + 1;
+	stream.zalloc = use_git_alloc;
+	stream.zfree = use_git_free;
+
+	st = inflateInit(&stream);
+	if (st != Z_OK) {
+		git__free(buffer);
+		giterr_set(GITERR_ZLIB, "Failed to inflate packfile");
+
+		return -1;
+	}
+
+	do {
+		in = pack_window_open(p, w_curs, *curpos, &stream.avail_in);
+		stream.next_in = in;
+		st = inflate(&stream, Z_FINISH);
+		git_mwindow_close(w_curs);
+
+		if (!stream.avail_out)
+			break; /* the payload is larger than it should be */
+
+		if (st == Z_BUF_ERROR && in == NULL) {
+			inflateEnd(&stream);
+			git__free(buffer);
+			return GIT_EBUFS;
+		}
+
+		*curpos += stream.next_in - in;
+	} while (st == Z_OK || st == Z_BUF_ERROR);
+
+	inflateEnd(&stream);
+
+	if ((st != Z_STREAM_END) || stream.total_out != size) {
+		git__free(buffer);
+		giterr_set(GITERR_ZLIB, "Failed to inflate packfile");
+		return -1;
+	}
+
+	obj->type = type;
+	obj->len = size;
+	obj->data = buffer;
+	return 0;
+}
+
+/*
+ * curpos is where the data starts, delta_obj_offset is the where the
+ * header starts
+ */
+git_off_t get_delta_base(
+	struct git_pack_file *p,
+	git_mwindow **w_curs,
+	git_off_t *curpos,
+	git_otype type,
+	git_off_t delta_obj_offset)
+{
+	unsigned int left = 0;
+	unsigned char *base_info;
+	git_off_t base_offset;
+	git_oid unused;
+
+	base_info = pack_window_open(p, w_curs, *curpos, &left);
+	/* Assumption: the only reason this would fail is because the file is too small */
+	if (base_info == NULL)
+		return GIT_EBUFS;
+	/* pack_window_open() assured us we have [base_info, base_info + 20)
+	 * as a range that we can look at without walking off the
+	 * end of the mapped window. Its actually the hash size
+	 * that is assured. An OFS_DELTA longer than the hash size
+	 * is stupid, as then a REF_DELTA would be smaller to store.
+	 */
+	if (type == GIT_OBJ_OFS_DELTA) {
+		unsigned used = 0;
+		unsigned char c = base_info[used++];
+		base_offset = c & 127;
+		while (c & 128) {
+			if (left <= used)
+				return GIT_EBUFS;
+			base_offset += 1;
+			if (!base_offset || MSB(base_offset, 7))
+				return 0; /* overflow */
+			c = base_info[used++];
+			base_offset = (base_offset << 7) + (c & 127);
+		}
+		base_offset = delta_obj_offset - base_offset;
+		if (base_offset <= 0 || base_offset >= delta_obj_offset)
+			return 0; /* out of bound */
+		*curpos += used;
+	} else if (type == GIT_OBJ_REF_DELTA) {
+		/* If we have the cooperative cache, search in it first */
+		if (p->has_cache) {
+			int pos;
+			struct git_pack_entry key;
+
+			git_oid_fromraw(&key.sha1, base_info);
+			pos = git_vector_bsearch(&p->cache, &key);
+			if (pos >= 0) {
+				*curpos += 20;
+				return ((struct git_pack_entry *)git_vector_get(&p->cache, pos))->offset;
+			}
+		}
+		/* The base entry _must_ be in the same pack */
+		if (pack_entry_find_offset(&base_offset, &unused, p, (git_oid *)base_info, GIT_OID_HEXSZ) < 0)
+			return packfile_error("base entry delta is not in the same pack");
+		*curpos += 20;
+	} else
+		return 0;
+
+	return base_offset;
+}
+
+/***********************************************************
+ *
+ * PACKFILE METHODS
+ *
+ ***********************************************************/
+
+static struct git_pack_file *packfile_alloc(size_t extra)
+{
+	struct git_pack_file *p = git__calloc(1, sizeof(*p) + extra);
+	if (p != NULL)
+		p->mwf.fd = -1;
+	return p;
+}
+
+
+void packfile_free(struct git_pack_file *p)
+{
+	assert(p);
+
+	/* clear_delta_base_cache(); */
+	git_mwindow_free_all(&p->mwf);
+
+	if (p->mwf.fd != -1)
+		p_close(p->mwf.fd);
+
+	pack_index_free(p);
+
+	git__free(p->bad_object_sha1);
+	git__free(p);
+}
+
+static int packfile_open(struct git_pack_file *p)
+{
+	struct stat st;
+	struct git_pack_header hdr;
+	git_oid sha1;
+	unsigned char *idx_sha1;
+
+	assert(p->index_map.data);
+
+	if (!p->index_map.data && pack_index_open(p) < 0)
+		return git_odb__error_notfound("failed to open packfile", NULL);
+
+	/* TODO: open with noatime */
+	p->mwf.fd = git_futils_open_ro(p->pack_name);
+	if (p->mwf.fd < 0)
+		return p->mwf.fd;
+
+	if (p_fstat(p->mwf.fd, &st) < 0 ||
+		git_mwindow_file_register(&p->mwf) < 0)
+		goto cleanup;
+
+	/* If we created the struct before we had the pack we lack size. */
+	if (!p->mwf.size) {
+		if (!S_ISREG(st.st_mode))
+			goto cleanup;
+		p->mwf.size = (git_off_t)st.st_size;
+	} else if (p->mwf.size != st.st_size)
+		goto cleanup;
+
+#if 0
+	/* We leave these file descriptors open with sliding mmap;
+	 * there is no point keeping them open across exec(), though.
+	 */
+	fd_flag = fcntl(p->mwf.fd, F_GETFD, 0);
+	if (fd_flag < 0)
+		goto cleanup;
+
+	fd_flag |= FD_CLOEXEC;
+	if (fcntl(p->pack_fd, F_SETFD, fd_flag) == -1)
+		goto cleanup;
+#endif
+
+	/* Verify we recognize this pack file format. */
+	if (p_read(p->mwf.fd, &hdr, sizeof(hdr)) < 0 ||
+		hdr.hdr_signature != htonl(PACK_SIGNATURE) ||
+		!pack_version_ok(hdr.hdr_version))
+		goto cleanup;
+
+	/* Verify the pack matches its index. */
+	if (p->num_objects != ntohl(hdr.hdr_entries) ||
+		p_lseek(p->mwf.fd, p->mwf.size - GIT_OID_RAWSZ, SEEK_SET) == -1 ||
+		p_read(p->mwf.fd, sha1.id, GIT_OID_RAWSZ) < 0)
+		goto cleanup;
+
+	idx_sha1 = ((unsigned char *)p->index_map.data) + p->index_map.len - 40;
+
+	if (git_oid_cmp(&sha1, (git_oid *)idx_sha1) == 0)
+		return 0;
+
+cleanup:
+	giterr_set(GITERR_OS, "Invalid packfile '%s'", p->pack_name);
+	p_close(p->mwf.fd);
+	p->mwf.fd = -1;
+	return -1;
+}
+
+int git_packfile_check(struct git_pack_file **pack_out, const char *path)
+{
+	struct stat st;
+	struct git_pack_file *p;
+	size_t path_len;
+
+	*pack_out = NULL;
+	path_len = strlen(path);
+	p = packfile_alloc(path_len + 2);
+	GITERR_CHECK_ALLOC(p);
+
+	/*
+	 * Make sure a corresponding .pack file exists and that
+	 * the index looks sane.
+	 */
+	path_len -= strlen(".idx");
+	if (path_len < 1) {
+		git__free(p);
+		return git_odb__error_notfound("invalid packfile path", NULL);
+	}
+
+	memcpy(p->pack_name, path, path_len);
+
+	strcpy(p->pack_name + path_len, ".keep");
+	if (git_path_exists(p->pack_name) == true)
+		p->pack_keep = 1;
+
+	strcpy(p->pack_name + path_len, ".pack");
+	if (p_stat(p->pack_name, &st) < 0 || !S_ISREG(st.st_mode)) {
+		git__free(p);
+		return git_odb__error_notfound("packfile not found", NULL);
+	}
+
+	/* ok, it looks sane as far as we can check without
+	 * actually mapping the pack file.
+	 */
+	p->mwf.size = st.st_size;
+	p->pack_local = 1;
+	p->mtime = (git_time_t)st.st_mtime;
+
+	/* see if we can parse the sha1 oid in the packfile name */
+	if (path_len < 40 ||
+		git_oid_fromstr(&p->sha1, path + path_len - GIT_OID_HEXSZ) < 0)
+		memset(&p->sha1, 0x0, GIT_OID_RAWSZ);
+
+	*pack_out = p;
+
+	return 0;
+}
+
+/***********************************************************
+ *
+ * PACKFILE ENTRY SEARCH INTERNALS
+ *
+ ***********************************************************/
+
+static git_off_t nth_packed_object_offset(const struct git_pack_file *p, uint32_t n)
+{
+	const unsigned char *index = p->index_map.data;
+	index += 4 * 256;
+	if (p->index_version == 1) {
+		return ntohl(*((uint32_t *)(index + 24 * n)));
+	} else {
+		uint32_t off;
+		index += 8 + p->num_objects * (20 + 4);
+		off = ntohl(*((uint32_t *)(index + 4 * n)));
+		if (!(off & 0x80000000))
+			return off;
+		index += p->num_objects * 4 + (off & 0x7fffffff) * 8;
+		return (((uint64_t)ntohl(*((uint32_t *)(index + 0)))) << 32) |
+					ntohl(*((uint32_t *)(index + 4)));
+	}
+}
+
+static int pack_entry_find_offset(
+	git_off_t *offset_out,
+	git_oid *found_oid,
+	struct git_pack_file *p,
+	const git_oid *short_oid,
+	unsigned int len)
+{
+	const uint32_t *level1_ofs = p->index_map.data;
+	const unsigned char *index = p->index_map.data;
+	unsigned hi, lo, stride;
+	int pos, found = 0;
+	const unsigned char *current = 0;
+
+	*offset_out = 0;
+
+	if (index == NULL) {
+		int error;
+
+		if ((error = pack_index_open(p)) < 0)
+			return error;
+
+		assert(p->index_map.data);
+
+		index = p->index_map.data;
+		level1_ofs = p->index_map.data;
+	}
+
+	if (p->index_version > 1) {
+		level1_ofs += 2;
+		index += 8;
+	}
+
+	index += 4 * 256;
+	hi = ntohl(level1_ofs[(int)short_oid->id[0]]);
+	lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(level1_ofs[(int)short_oid->id[0] - 1]));
+
+	if (p->index_version > 1) {
+		stride = 20;
+	} else {
+		stride = 24;
+		index += 4;
+	}
+
+#ifdef INDEX_DEBUG_LOOKUP
+	printf("%02x%02x%02x... lo %u hi %u nr %d\n",
+		short_oid->id[0], short_oid->id[1], short_oid->id[2], lo, hi, p->num_objects);
+#endif
+
+	/* Use git.git lookup code */
+	pos = sha1_entry_pos(index, stride, 0, lo, hi, p->num_objects, short_oid->id);
+
+	if (pos >= 0) {
+		/* An object matching exactly the oid was found */
+		found = 1;
+		current = index + pos * stride;
+	} else {
+		/* No object was found */
+		/* pos refers to the object with the "closest" oid to short_oid */
+		pos = - 1 - pos;
+		if (pos < (int)p->num_objects) {
+			current = index + pos * stride;
+
+			if (!git_oid_ncmp(short_oid, (const git_oid *)current, len))
+				found = 1;
+		}
+	}
+
+	if (found && len != GIT_OID_HEXSZ && pos + 1 < (int)p->num_objects) {
+		/* Check for ambiguousity */
+		const unsigned char *next = current + stride;
+
+		if (!git_oid_ncmp(short_oid, (const git_oid *)next, len)) {
+			found = 2;
+		}
+	}
+
+	if (!found)
+		return git_odb__error_notfound("failed to find offset for pack entry", short_oid);
+	if (found > 1)
+		return git_odb__error_ambiguous("found multiple offsets for pack entry");
+	*offset_out = nth_packed_object_offset(p, pos);
+	git_oid_fromraw(found_oid, current);
+
+#ifdef INDEX_DEBUG_LOOKUP
+	{
+		unsigned char hex_sha1[GIT_OID_HEXSZ + 1];
+		git_oid_fmt(hex_sha1, found_oid);
+		hex_sha1[GIT_OID_HEXSZ] = '\0';
+		printf("found lo=%d %s\n", lo, hex_sha1);
+	}
+#endif
+	return 0;
+}
+
+int git_pack_entry_find(
+		struct git_pack_entry *e,
+		struct git_pack_file *p,
+		const git_oid *short_oid,
+		unsigned int len)
+{
+	git_off_t offset;
+	git_oid found_oid;
+	int error;
+
+	assert(p);
+
+	if (len == GIT_OID_HEXSZ && p->num_bad_objects) {
+		unsigned i;
+		for (i = 0; i < p->num_bad_objects; i++)
+			if (git_oid_cmp(short_oid, &p->bad_object_sha1[i]) == 0)
+				return packfile_error("bad object found in packfile");
+	}
+
+	error = pack_entry_find_offset(&offset, &found_oid, p, short_oid, len);
+	if (error < 0)
+		return error;
+
+	/* we found a unique entry in the index;
+	 * make sure the packfile backing the index
+	 * still exists on disk */
+	if (p->mwf.fd == -1 && (error = packfile_open(p)) < 0)
+		return error;
+
+	e->offset = offset;
+	e->p = p;
+
+	git_oid_cpy(&e->sha1, &found_oid);
+	return 0;
+}
diff --git a/src/pack.h b/src/pack.h
new file mode 100644
index 000000000..cd7a4d2e1
--- /dev/null
+++ b/src/pack.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_pack_h__
+#define INCLUDE_pack_h__
+
+#include "git2/oid.h"
+
+#include "common.h"
+#include "map.h"
+#include "mwindow.h"
+#include "odb.h"
+
+#define GIT_PACK_FILE_MODE 0444
+
+#define PACK_SIGNATURE 0x5041434b	/* "PACK" */
+#define PACK_VERSION 2
+#define pack_version_ok(v) ((v) == htonl(2) || (v) == htonl(3))
+struct git_pack_header {
+	uint32_t hdr_signature;
+	uint32_t hdr_version;
+	uint32_t hdr_entries;
+};
+
+/*
+ * The first four bytes of index formats later than version 1 should
+ * start with this signature, as all older git binaries would find this
+ * value illegal and abort reading the file.
+ *
+ * This is the case because the number of objects in a packfile
+ * cannot exceed 1,431,660,000 as every object would need at least
+ * 3 bytes of data and the overall packfile cannot exceed 4 GiB with
+ * version 1 of the index file due to the offsets limited to 32 bits.
+ * Clearly the signature exceeds this maximum.
+ *
+ * Very old git binaries will also compare the first 4 bytes to the
+ * next 4 bytes in the index and abort with a "non-monotonic index"
+ * error if the second 4 byte word is smaller than the first 4
+ * byte word. This would be true in the proposed future index
+ * format as idx_signature would be greater than idx_version.
+ */
+
+#define PACK_IDX_SIGNATURE 0xff744f63	/* "\377tOc" */
+
+struct git_pack_idx_header {
+	uint32_t idx_signature;
+	uint32_t idx_version;
+};
+
+struct git_pack_file {
+	git_mwindow_file mwf;
+	git_map index_map;
+
+	uint32_t num_objects;
+	uint32_t num_bad_objects;
+	git_oid *bad_object_sha1; /* array of git_oid */
+
+	int index_version;
+	git_time_t mtime;
+	unsigned pack_local:1, pack_keep:1, has_cache:1;
+	git_oid sha1;
+	git_vector cache;
+
+	/* something like ".git/objects/pack/xxxxx.pack" */
+	char pack_name[GIT_FLEX_ARRAY]; /* more */
+};
+
+struct git_pack_entry {
+	git_off_t offset;
+	git_oid sha1;
+	struct git_pack_file *p;
+};
+
+int git_packfile_unpack_header(
+		size_t *size_p,
+		git_otype *type_p,
+		git_mwindow_file *mwf,
+		git_mwindow **w_curs,
+		git_off_t *curpos);
+
+int git_packfile_unpack(git_rawobj *obj, struct git_pack_file *p, git_off_t *obj_offset);
+int packfile_unpack_compressed(
+	git_rawobj *obj,
+	struct git_pack_file *p,
+	git_mwindow **w_curs,
+	git_off_t *curpos,
+	size_t size,
+	git_otype type);
+
+git_off_t get_delta_base(struct git_pack_file *p, git_mwindow **w_curs,
+		git_off_t *curpos, git_otype type,
+		git_off_t delta_obj_offset);
+
+void packfile_free(struct git_pack_file *p);
+int git_packfile_check(struct git_pack_file **pack_out, const char *path);
+int git_pack_entry_find(
+		struct git_pack_entry *e,
+		struct git_pack_file *p,
+		const git_oid *short_oid,
+		unsigned int len);
+
+#endif
diff --git a/src/path.c b/src/path.c
new file mode 100644
index 000000000..84edf6d89
--- /dev/null
+++ b/src/path.c
@@ -0,0 +1,656 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "path.h"
+#include "posix.h"
+#ifdef GIT_WIN32
+#include "win32/dir.h"
+#include "win32/posix.h"
+#else
+#include <dirent.h>
+#endif
+#include <stdarg.h>
+#include <stdio.h>
+#include <ctype.h>
+
+/*
+ * Based on the Android implementation, BSD licensed.
+ * Check http://android.git.kernel.org/
+ */
+int git_path_basename_r(git_buf *buffer, const char *path)
+{
+	const char *endp, *startp;
+	int len, result;
+
+	/* Empty or NULL string gets treated as "." */
+	if (path == NULL || *path == '\0') {
+		startp = ".";
+		len		= 1;
+		goto Exit;
+	}
+
+	/* Strip trailing slashes */
+	endp = path + strlen(path) - 1;
+	while (endp > path && *endp == '/')
+		endp--;
+
+	/* All slashes becomes "/" */
+	if (endp == path && *endp == '/') {
+		startp = "/";
+		len	= 1;
+		goto Exit;
+	}
+
+	/* Find the start of the base */
+	startp = endp;
+	while (startp > path && *(startp - 1) != '/')
+		startp--;
+
+	/* Cast is safe because max path < max int */
+	len = (int)(endp - startp + 1);
+
+Exit:
+	result = len;
+
+	if (buffer != NULL && git_buf_set(buffer, startp, len) < 0)
+		return -1;
+
+	return result;
+}
+
+/*
+ * Based on the Android implementation, BSD licensed.
+ * Check http://android.git.kernel.org/
+ */
+int git_path_dirname_r(git_buf *buffer, const char *path)
+{
+	const char *endp;
+	int result, len;
+
+	/* Empty or NULL string gets treated as "." */
+	if (path == NULL || *path == '\0') {
+		path = ".";
+		len = 1;
+		goto Exit;
+	}
+
+	/* Strip trailing slashes */
+	endp = path + strlen(path) - 1;
+	while (endp > path && *endp == '/')
+		endp--;
+
+	/* Find the start of the dir */
+	while (endp > path && *endp != '/')
+		endp--;
+
+	/* Either the dir is "/" or there are no slashes */
+	if (endp == path) {
+		path = (*endp == '/') ? "/" : ".";
+		len = 1;
+		goto Exit;
+	}
+
+	do {
+		endp--;
+	} while (endp > path && *endp == '/');
+
+	/* Cast is safe because max path < max int */
+	len = (int)(endp - path + 1);
+
+#ifdef GIT_WIN32
+	/* Mimic unix behavior where '/.git' returns '/': 'C:/.git' will return
+		'C:/' here */
+
+	if (len == 2 && isalpha(path[0]) && path[1] == ':') {
+		len = 3;
+		goto Exit;
+	}
+#endif
+
+Exit:
+	result = len;
+
+	if (buffer != NULL && git_buf_set(buffer, path, len) < 0)
+		return -1;
+
+	return result;
+}
+
+
+char *git_path_dirname(const char *path)
+{
+	git_buf buf = GIT_BUF_INIT;
+	char *dirname;
+
+	git_path_dirname_r(&buf, path);
+	dirname = git_buf_detach(&buf);
+	git_buf_free(&buf); /* avoid memleak if error occurs */
+
+	return dirname;
+}
+
+char *git_path_basename(const char *path)
+{
+	git_buf buf = GIT_BUF_INIT;
+	char *basename;
+
+	git_path_basename_r(&buf, path);
+	basename = git_buf_detach(&buf);
+	git_buf_free(&buf); /* avoid memleak if error occurs */
+
+	return basename;
+}
+
+
+const char *git_path_topdir(const char *path)
+{
+	size_t len;
+	ssize_t i;
+
+	assert(path);
+	len = strlen(path);
+
+	if (!len || path[len - 1] != '/')
+		return NULL;
+
+	for (i = (ssize_t)len - 2; i >= 0; --i)
+		if (path[i] == '/')
+			break;
+
+	return &path[i + 1];
+}
+
+int git_path_root(const char *path)
+{
+	int offset = 0;
+
+#ifdef GIT_WIN32
+	/* Does the root of the path look like a windows drive ? */
+	if (isalpha(path[0]) && (path[1] == ':'))
+		offset += 2;
+
+	/* Are we dealing with a windows network path? */
+	else if ((path[0] == '/' && path[1] == '/') ||
+		(path[0] == '\\' && path[1] == '\\'))
+	{
+		offset += 2;
+
+		/* Skip the computer name segment */
+		while (path[offset] && path[offset] != '/' && path[offset] != '\\')
+			offset++;
+	}
+#endif
+
+	if (path[offset] == '/' || path[offset] == '\\')
+		return offset;
+
+	return -1;	/* Not a real error - signals that path is not rooted */
+}
+
+int git_path_prettify(git_buf *path_out, const char *path, const char *base)
+{
+	char buf[GIT_PATH_MAX];
+
+	assert(path && path_out);
+
+	/* construct path if needed */
+	if (base != NULL && git_path_root(path) < 0) {
+		if (git_buf_joinpath(path_out, base, path) < 0)
+			return -1;
+		path = path_out->ptr;
+	}
+
+	if (p_realpath(path, buf) == NULL) {
+		/* giterr_set resets the errno when dealing with a GITERR_OS kind of error */
+		int error = (errno == ENOENT || errno == ENOTDIR) ? GIT_ENOTFOUND : -1;
+		giterr_set(GITERR_OS, "Failed to resolve path '%s'", path);
+
+		git_buf_clear(path_out);
+		
+		return error;
+	}
+
+	return git_buf_sets(path_out, buf);
+}
+
+int git_path_prettify_dir(git_buf *path_out, const char *path, const char *base)
+{
+	int error = git_path_prettify(path_out, path, base);
+	return (error < 0) ? error : git_path_to_dir(path_out);
+}
+
+int git_path_to_dir(git_buf *path)
+{
+	if (path->asize > 0 &&
+		git_buf_len(path) > 0 &&
+		path->ptr[git_buf_len(path) - 1] != '/')
+		git_buf_putc(path, '/');
+
+	return git_buf_oom(path) ? -1 : 0;
+}
+
+void git_path_string_to_dir(char* path, size_t size)
+{
+	size_t end = strlen(path);
+
+	if (end && path[end - 1] != '/' && end < size) {
+		path[end] = '/';
+		path[end + 1] = '\0';
+	}
+}
+
+int git__percent_decode(git_buf *decoded_out, const char *input)
+{
+	int len, hi, lo, i;
+	assert(decoded_out && input);
+
+	len = (int)strlen(input);
+	git_buf_clear(decoded_out);
+
+	for(i = 0; i < len; i++)
+	{
+		char c = input[i];
+
+		if (c != '%')
+			goto append;
+
+		if (i >= len - 2)
+			goto append;
+
+		hi = git__fromhex(input[i + 1]);
+		lo = git__fromhex(input[i + 2]);
+
+		if (hi < 0 || lo < 0)
+			goto append;
+
+		c = (char)(hi << 4 | lo);
+		i += 2;
+
+append:
+		if (git_buf_putc(decoded_out, c) < 0)
+			return -1;
+	}
+
+	return 0;
+}
+
+static int error_invalid_local_file_uri(const char *uri)
+{
+	giterr_set(GITERR_CONFIG, "'%s' is not a valid local file URI", uri);
+	return -1;
+}
+
+int git_path_fromurl(git_buf *local_path_out, const char *file_url)
+{
+	int offset = 0, len;
+
+	assert(local_path_out && file_url);
+
+	if (git__prefixcmp(file_url, "file://") != 0)
+		return error_invalid_local_file_uri(file_url);
+
+	offset += 7;
+	len = (int)strlen(file_url);
+
+	if (offset < len && file_url[offset] == '/')
+		offset++;
+	else if (offset < len && git__prefixcmp(file_url + offset, "localhost/") == 0)
+		offset += 10;
+	else
+		return error_invalid_local_file_uri(file_url);
+
+	if (offset >= len || file_url[offset] == '/')
+		return error_invalid_local_file_uri(file_url);
+
+#ifndef _MSC_VER
+	offset--;	/* A *nix absolute path starts with a forward slash */
+#endif
+
+	git_buf_clear(local_path_out);
+
+	return git__percent_decode(local_path_out, file_url + offset);
+}
+
+int git_path_walk_up(
+	git_buf *path,
+	const char *ceiling,
+	int (*cb)(void *data, git_buf *),
+	void *data)
+{
+	int error = 0;
+	git_buf iter;
+	ssize_t stop = 0, scan;
+	char oldc = '\0';
+
+	assert(path && cb);
+
+	if (ceiling != NULL) {
+		if (git__prefixcmp(path->ptr, ceiling) == 0)
+			stop = (ssize_t)strlen(ceiling);
+		else
+			stop = git_buf_len(path);
+	}
+	scan = git_buf_len(path);
+
+	iter.ptr = path->ptr;
+	iter.size = git_buf_len(path);
+	iter.asize = path->asize;
+
+	while (scan >= stop) {
+		if ((error = cb(data, &iter)) < 0)
+			break;
+		iter.ptr[scan] = oldc;
+		scan = git_buf_rfind_next(&iter, '/');
+		if (scan >= 0) {
+			scan++;
+			oldc = iter.ptr[scan];
+			iter.size = scan;
+			iter.ptr[scan] = '\0';
+		}
+	}
+
+	if (scan >= 0)
+		iter.ptr[scan] = oldc;
+
+	return error;
+}
+
+bool git_path_exists(const char *path)
+{
+	assert(path);
+	return p_access(path, F_OK) == 0;
+}
+
+bool git_path_isdir(const char *path)
+{
+	struct stat st;
+	if (p_stat(path, &st) < 0)
+		return false;
+
+	return S_ISDIR(st.st_mode) != 0;
+}
+
+bool git_path_isfile(const char *path)
+{
+	struct stat st;
+
+	assert(path);
+	if (p_stat(path, &st) < 0)
+		return false;
+
+	return S_ISREG(st.st_mode) != 0;
+}
+
+int git_path_lstat(const char *path, struct stat *st)
+{
+	int err = 0;
+
+	if (p_lstat(path, st) < 0) {
+		err = (errno == ENOENT) ? GIT_ENOTFOUND : -1;
+		giterr_set(GITERR_OS, "Failed to stat file '%s'", path);
+	}
+
+	return err;
+}
+
+static bool _check_dir_contents(
+	git_buf *dir,
+	const char *sub,
+	bool (*predicate)(const char *))
+{
+	bool result;
+	size_t dir_size = git_buf_len(dir);
+	size_t sub_size = strlen(sub);
+
+	/* leave base valid even if we could not make space for subdir */
+	if (git_buf_try_grow(dir, dir_size + sub_size + 2) < 0)
+		return false;
+
+	/* save excursion */
+	git_buf_joinpath(dir, dir->ptr, sub);
+
+	result = predicate(dir->ptr);
+
+	/* restore path */
+	git_buf_truncate(dir, dir_size);
+	return result;
+}
+
+bool git_path_contains(git_buf *dir, const char *item)
+{
+	return _check_dir_contents(dir, item, &git_path_exists);
+}
+
+bool git_path_contains_dir(git_buf *base, const char *subdir)
+{
+	return _check_dir_contents(base, subdir, &git_path_isdir);
+}
+
+bool git_path_contains_file(git_buf *base, const char *file)
+{
+	return _check_dir_contents(base, file, &git_path_isfile);
+}
+
+int git_path_find_dir(git_buf *dir, const char *path, const char *base)
+{
+	int error;
+
+	if (base != NULL && git_path_root(path) < 0)
+		error = git_buf_joinpath(dir, base, path);
+	else
+		error = git_buf_sets(dir, path);
+
+	if (!error) {
+		char buf[GIT_PATH_MAX];
+		if (p_realpath(dir->ptr, buf) != NULL)
+			error = git_buf_sets(dir, buf);
+	}
+
+	/* call dirname if this is not a directory */
+	if (!error && git_path_isdir(dir->ptr) == false)
+		error = git_path_dirname_r(dir, dir->ptr);
+
+	if (!error)
+		error = git_path_to_dir(dir);
+
+	return error;
+}
+
+int git_path_cmp(
+	const char *name1, size_t len1, int isdir1,
+	const char *name2, size_t len2, int isdir2)
+{
+	size_t len = len1 < len2 ? len1 : len2;
+	int cmp;
+
+	cmp = memcmp(name1, name2, len);
+	if (cmp)
+		return cmp;
+	if (len1 < len2)
+		return (!isdir1 && !isdir2) ? -1 :
+			(isdir1 ? '/' - name2[len1] : name2[len1] - '/');
+	if (len1 > len2)
+		return (!isdir1 && !isdir2) ? 1 :
+			(isdir2 ? name1[len2] - '/' : '/' - name1[len2]);
+	return 0;
+}
+
+/* Taken from git.git */
+GIT_INLINE(int) is_dot_or_dotdot(const char *name)
+{
+	return (name[0] == '.' &&
+		(name[1] == '\0' ||
+		 (name[1] == '.' && name[2] == '\0')));
+}
+
+int git_path_direach(
+	git_buf *path,
+	int (*fn)(void *, git_buf *),
+	void *arg)
+{
+	ssize_t wd_len;
+	DIR *dir;
+	struct dirent *de, *de_buf;
+
+	if (git_path_to_dir(path) < 0)
+		return -1;
+
+	wd_len = git_buf_len(path);
+
+	if ((dir = opendir(path->ptr)) == NULL) {
+		giterr_set(GITERR_OS, "Failed to open directory '%s'", path->ptr);
+		return -1;
+	}
+
+#ifdef __sun
+	de_buf = git__malloc(sizeof(struct dirent) + FILENAME_MAX + 1);
+#else
+	de_buf = git__malloc(sizeof(struct dirent));
+#endif
+
+	while (p_readdir_r(dir, de_buf, &de) == 0 && de != NULL) {
+		int result;
+
+		if (is_dot_or_dotdot(de->d_name))
+			continue;
+
+		if (git_buf_puts(path, de->d_name) < 0) {
+			closedir(dir);
+			git__free(de_buf);
+			return -1;
+		}
+
+		result = fn(arg, path);
+
+		git_buf_truncate(path, wd_len); /* restore path */
+
+		if (result < 0) {
+			closedir(dir);
+			git__free(de_buf);
+			return -1;
+		}
+	}
+
+	closedir(dir);
+	git__free(de_buf);
+	return 0;
+}
+
+int git_path_dirload(
+	const char *path,
+	size_t prefix_len,
+	size_t alloc_extra,
+	git_vector *contents)
+{
+	int error, need_slash;
+	DIR *dir;
+	struct dirent *de, *de_buf;
+	size_t path_len;
+
+	assert(path != NULL && contents != NULL);
+	path_len = strlen(path);
+	assert(path_len > 0 && path_len >= prefix_len);
+
+	if ((dir = opendir(path)) == NULL) {
+		giterr_set(GITERR_OS, "Failed to open directory '%s'", path);
+		return -1;
+	}
+
+#ifdef __sun
+	de_buf = git__malloc(sizeof(struct dirent) + FILENAME_MAX + 1);
+#else
+	de_buf = git__malloc(sizeof(struct dirent));
+#endif
+
+	path += prefix_len;
+	path_len -= prefix_len;
+	need_slash = (path_len > 0 && path[path_len-1] != '/') ? 1 : 0;
+
+	while ((error = p_readdir_r(dir, de_buf, &de)) == 0 && de != NULL) {
+		char *entry_path;
+		size_t entry_len;
+
+		if (is_dot_or_dotdot(de->d_name))
+			continue;
+
+		entry_len = strlen(de->d_name);
+
+		entry_path = git__malloc(
+			path_len + need_slash + entry_len + 1 + alloc_extra);
+		GITERR_CHECK_ALLOC(entry_path);
+
+		if (path_len)
+			memcpy(entry_path, path, path_len);
+		if (need_slash)
+			entry_path[path_len] = '/';
+		memcpy(&entry_path[path_len + need_slash], de->d_name, entry_len);
+		entry_path[path_len + need_slash + entry_len] = '\0';
+
+		if (git_vector_insert(contents, entry_path) < 0) {
+			closedir(dir);
+			git__free(de_buf);
+			return -1;
+		}
+	}
+
+	closedir(dir);
+	git__free(de_buf);
+
+	if (error != 0)
+		giterr_set(GITERR_OS, "Failed to process directory entry in '%s'", path);
+
+	return error;
+}
+
+int git_path_with_stat_cmp(const void *a, const void *b)
+{
+	const git_path_with_stat *psa = a, *psb = b;
+	return git__strcmp_cb(psa->path, psb->path);
+}
+
+int git_path_dirload_with_stat(
+	const char *path,
+	size_t prefix_len,
+	git_vector *contents)
+{
+	int error;
+	unsigned int i;
+	git_path_with_stat *ps;
+	git_buf full = GIT_BUF_INIT;
+
+	if (git_buf_set(&full, path, prefix_len) < 0)
+		return -1;
+
+	error = git_path_dirload(
+		path, prefix_len, sizeof(git_path_with_stat) + 1, contents);
+	if (error < 0) {
+		git_buf_free(&full);
+		return error;
+	}
+
+	git_vector_foreach(contents, i, ps) {
+		size_t path_len = strlen((char *)ps);
+
+		memmove(ps->path, ps, path_len + 1);
+		ps->path_len = path_len;
+
+		if ((error = git_buf_joinpath(&full, full.ptr, ps->path)) < 0 ||
+			(error = git_path_lstat(full.ptr, &ps->st)) < 0)
+			break;
+
+		git_buf_truncate(&full, prefix_len);
+
+		if (S_ISDIR(ps->st.st_mode)) {
+			ps->path[path_len] = '/';
+			ps->path[path_len + 1] = '\0';
+		}
+	}
+
+	git_buf_free(&full);
+
+	return error;
+}
diff --git a/src/path.h b/src/path.h
new file mode 100644
index 000000000..fd76805e5
--- /dev/null
+++ b/src/path.h
@@ -0,0 +1,278 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_path_h__
+#define INCLUDE_path_h__
+
+#include "common.h"
+#include "buffer.h"
+#include "vector.h"
+
+/**
+ * Path manipulation utils
+ *
+ * These are path utilities that munge paths without actually
+ * looking at the real filesystem.
+ */
+
+/*
+ * The dirname() function shall take a pointer to a character string
+ * that contains a pathname, and return a pointer to a string that is a
+ * pathname of the parent directory of that file. Trailing '/' characters
+ * in the path are not counted as part of the path.
+ *
+ * If path does not contain a '/', then dirname() shall return a pointer to
+ * the string ".". If path is a null pointer or points to an empty string,
+ * dirname() shall return a pointer to the string "." .
+ *
+ * The `git_path_dirname` implementation is thread safe. The returned
+ * string must be manually free'd.
+ *
+ * The `git_path_dirname_r` implementation writes the dirname to a `git_buf`
+ * if the buffer pointer is not NULL.
+ * It returns an error code < 0 if there is an allocation error, otherwise
+ * the length of the dirname (which will be > 0).
+ */
+extern char *git_path_dirname(const char *path);
+extern int git_path_dirname_r(git_buf *buffer, const char *path);
+
+/*
+ * This function returns the basename of the file, which is the last
+ * part of its full name given by fname, with the drive letter and
+ * leading directories stripped off. For example, the basename of
+ * c:/foo/bar/file.ext is file.ext, and the basename of a:foo is foo.
+ *
+ * Trailing slashes and backslashes are significant: the basename of
+ * c:/foo/bar/ is an empty string after the rightmost slash.
+ *
+ * The `git_path_basename` implementation is thread safe. The returned
+ * string must be manually free'd.
+ *
+ * The `git_path_basename_r` implementation writes the basename to a `git_buf`.
+ * It returns an error code < 0 if there is an allocation error, otherwise
+ * the length of the basename (which will be >= 0).
+ */
+extern char *git_path_basename(const char *path);
+extern int git_path_basename_r(git_buf *buffer, const char *path);
+
+extern const char *git_path_topdir(const char *path);
+
+/**
+ * Find offset to root of path if path has one.
+ *
+ * This will return a number >= 0 which is the offset to the start of the
+ * path, if the path is rooted (i.e. "/rooted/path" returns 0 and
+ * "c:/windows/rooted/path" returns 2).  If the path is not rooted, this
+ * returns < 0.
+ */
+extern int git_path_root(const char *path);
+
+/**
+ * Ensure path has a trailing '/'.
+ */
+extern int git_path_to_dir(git_buf *path);
+
+/**
+ * Ensure string has a trailing '/' if there is space for it.
+ */
+extern void git_path_string_to_dir(char* path, size_t size);
+
+#ifdef GIT_WIN32
+/**
+ * Convert backslashes in path to forward slashes.
+ */
+GIT_INLINE(void) git_path_mkposix(char *path)
+{
+	while (*path) {
+		if (*path == '\\')
+			*path = '/';
+
+		path++;
+	}
+}
+#else
+#	define git_path_mkposix(p) /* blank */
+#endif
+
+extern int git__percent_decode(git_buf *decoded_out, const char *input);
+
+/**
+ * Extract path from file:// URL.
+ */
+extern int git_path_fromurl(git_buf *local_path_out, const char *file_url);
+
+
+/**
+ * Path filesystem utils
+ *
+ * These are path utilities that actually access the filesystem.
+ */
+
+/**
+ * Check if a file exists and can be accessed.
+ * @return true or false
+ */
+extern bool git_path_exists(const char *path);
+
+/**
+ * Check if the given path points to a directory.
+ * @return true or false
+ */
+extern bool git_path_isdir(const char *path);
+
+/**
+ * Check if the given path points to a regular file.
+ * @return true or false
+ */
+extern bool git_path_isfile(const char *path);
+
+/**
+ * Stat a file and/or link and set error if needed.
+ */
+extern int git_path_lstat(const char *path, struct stat *st);
+
+/**
+ * Check if the parent directory contains the item.
+ *
+ * @param dir Directory to check.
+ * @param item Item that might be in the directory.
+ * @return 0 if item exists in directory, <0 otherwise.
+ */
+extern bool git_path_contains(git_buf *dir, const char *item);
+
+/**
+ * Check if the given path contains the given subdirectory.
+ *
+ * @param parent Directory path that might contain subdir
+ * @param subdir Subdirectory name to look for in parent
+ * @param append_if_exists If true, then subdir will be appended to the parent path if it does exist
+ * @return true if subdirectory exists, false otherwise.
+ */
+extern bool git_path_contains_dir(git_buf *parent, const char *subdir);
+
+/**
+ * Check if the given path contains the given file.
+ *
+ * @param dir Directory path that might contain file
+ * @param file File name to look for in parent
+ * @param append_if_exists If true, then file will be appended to the path if it does exist
+ * @return true if file exists, false otherwise.
+ */
+extern bool git_path_contains_file(git_buf *dir, const char *file);
+
+/**
+ * Clean up path, prepending base if it is not already rooted.
+ */
+extern int git_path_prettify(git_buf *path_out, const char *path, const char *base);
+
+/**
+ * Clean up path, prepending base if it is not already rooted and
+ * appending a slash.
+ */
+extern int git_path_prettify_dir(git_buf *path_out, const char *path, const char *base);
+
+/**
+ * Get a directory from a path.
+ *
+ * If path is a directory, this acts like `git_path_prettify_dir`
+ * (cleaning up path and appending a '/').  If path is a normal file,
+ * this prettifies it, then removed the filename a la dirname and
+ * appends the trailing '/'.  If the path does not exist, it is
+ * treated like a regular filename.
+ */
+extern int git_path_find_dir(git_buf *dir, const char *path, const char *base);
+
+/**
+ * Walk each directory entry, except '.' and '..', calling fn(state).
+ *
+ * @param pathbuf buffer the function reads the initial directory
+ * 		path from, and updates with each successive entry's name.
+ * @param fn function to invoke with each entry. The first arg is
+ *		the input state and the second arg is pathbuf. The function
+ *		may modify the pathbuf, but only by appending new text.
+ * @param state to pass to fn as the first arg.
+ */
+extern int git_path_direach(
+	git_buf *pathbuf,
+	int (*fn)(void *, git_buf *),
+	void *state);
+
+/**
+ * Sort function to order two paths.
+ */
+extern int git_path_cmp(
+	const char *name1, size_t len1, int isdir1,
+	const char *name2, size_t len2, int isdir2);
+
+/**
+ * Invoke callback up path directory by directory until the ceiling is
+ * reached (inclusive of a final call at the root_path).
+ *
+ * Returning anything other than 0 from the callback function
+ * will stop the iteration and propogate the error to the caller.
+ *
+ * @param pathbuf Buffer the function reads the directory from and
+ *		and updates with each successive name.
+ * @param ceiling Prefix of path at which to stop walking up.  If NULL,
+ *      this will walk all the way up to the root.  If not a prefix of
+ *      pathbuf, the callback will be invoked a single time on the
+ *      original input path.
+ * @param fn Function to invoke on each path.  The first arg is the
+ *		input satte and the second arg is the pathbuf.  The function
+ *		should not modify the pathbuf.
+ * @param state Passed to fn as the first ath.
+ */
+extern int git_path_walk_up(
+	git_buf *pathbuf,
+	const char *ceiling,
+	int (*fn)(void *state, git_buf *),
+	void *state);
+
+/**
+ * Load all directory entries (except '.' and '..') into a vector.
+ *
+ * For cases where `git_path_direach()` is not appropriate, this
+ * allows you to load the filenames in a directory into a vector
+ * of strings. That vector can then be sorted, iterated, or whatever.
+ * Remember to free alloc of the allocated strings when you are done.
+ *
+ * @param path The directory to read from.
+ * @param prefix_len When inserting entries, the trailing part of path
+ * 		will be prefixed after this length.  I.e. given path "/a/b" and
+ * 		prefix_len 3, the entries will look like "b/e1", "b/e2", etc.
+ * @param alloc_extra Extra bytes to add to each string allocation in
+ * 		case you want to append anything funny.
+ * @param contents Vector to fill with directory entry names.
+ */
+extern int git_path_dirload(
+	const char *path,
+	size_t prefix_len,
+	size_t alloc_extra,
+	git_vector *contents);
+
+
+typedef struct {
+	struct stat st;
+	size_t      path_len;
+	char        path[GIT_FLEX_ARRAY];
+} git_path_with_stat;
+
+extern int git_path_with_stat_cmp(const void *a, const void *b);
+
+/**
+ * Load all directory entries along with stat info into a vector.
+ *
+ * This is just like git_path_dirload except that each entry in the
+ * vector is a git_path_with_stat structure that contains both the
+ * path and the stat info, plus directories will have a / suffixed
+ * to their path name.
+ */
+extern int git_path_dirload_with_stat(
+	const char *path,
+	size_t prefix_len,
+	git_vector *contents);
+
+#endif
diff --git a/src/pkt.c b/src/pkt.c
new file mode 100644
index 000000000..88510f4b1
--- /dev/null
+++ b/src/pkt.c
@@ -0,0 +1,355 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+
+#include "git2/types.h"
+#include "git2/errors.h"
+#include "git2/refs.h"
+#include "git2/revwalk.h"
+
+#include "pkt.h"
+#include "util.h"
+#include "netops.h"
+#include "posix.h"
+#include "buffer.h"
+
+#include <ctype.h>
+
+#define PKT_LEN_SIZE 4
+static const char pkt_done_str[] = "0009done\n";
+static const char pkt_flush_str[] = "0000";
+static const char pkt_have_prefix[] = "0032have ";
+static const char pkt_want_prefix[] = "0032want ";
+
+static int flush_pkt(git_pkt **out)
+{
+	git_pkt *pkt;
+
+	pkt = git__malloc(sizeof(git_pkt));
+	GITERR_CHECK_ALLOC(pkt);
+
+	pkt->type = GIT_PKT_FLUSH;
+	*out = pkt;
+
+	return 0;
+}
+
+/* the rest of the line will be useful for multi_ack */
+static int ack_pkt(git_pkt **out, const char *line, size_t len)
+{
+	git_pkt *pkt;
+	GIT_UNUSED(line);
+	GIT_UNUSED(len);
+
+	pkt = git__malloc(sizeof(git_pkt));
+	GITERR_CHECK_ALLOC(pkt);
+
+	pkt->type = GIT_PKT_ACK;
+	*out = pkt;
+
+	return 0;
+}
+
+static int nak_pkt(git_pkt **out)
+{
+	git_pkt *pkt;
+
+	pkt = git__malloc(sizeof(git_pkt));
+	GITERR_CHECK_ALLOC(pkt);
+
+	pkt->type = GIT_PKT_NAK;
+	*out = pkt;
+
+	return 0;
+}
+
+static int pack_pkt(git_pkt **out)
+{
+	git_pkt *pkt;
+
+	pkt = git__malloc(sizeof(git_pkt));
+	GITERR_CHECK_ALLOC(pkt);
+
+	pkt->type = GIT_PKT_PACK;
+	*out = pkt;
+
+	return 0;
+}
+
+static int comment_pkt(git_pkt **out, const char *line, size_t len)
+{
+	git_pkt_comment *pkt;
+
+	pkt = git__malloc(sizeof(git_pkt_comment) + len + 1);
+	GITERR_CHECK_ALLOC(pkt);
+
+	pkt->type = GIT_PKT_COMMENT;
+	memcpy(pkt->comment, line, len);
+	pkt->comment[len] = '\0';
+
+	*out = (git_pkt *) pkt;
+
+	return 0;
+}
+
+static int err_pkt(git_pkt **out, const char *line, size_t len)
+{
+	git_pkt_err *pkt;
+
+	/* Remove "ERR " from the line */
+	line += 4;
+	len -= 4;
+	pkt = git__malloc(sizeof(git_pkt_err) + len + 1);
+	GITERR_CHECK_ALLOC(pkt);
+
+	pkt->type = GIT_PKT_ERR;
+	memcpy(pkt->error, line, len);
+	pkt->error[len] = '\0';
+
+	*out = (git_pkt *) pkt;
+
+	return 0;
+}
+
+/*
+ * Parse an other-ref line.
+ */
+static int ref_pkt(git_pkt **out, const char *line, size_t len)
+{
+	int error;
+	git_pkt_ref *pkt;
+
+	pkt = git__malloc(sizeof(git_pkt_ref));
+	GITERR_CHECK_ALLOC(pkt);
+
+	memset(pkt, 0x0, sizeof(git_pkt_ref));
+	pkt->type = GIT_PKT_REF;
+	if ((error = git_oid_fromstr(&pkt->head.oid, line)) < 0)
+		goto error_out;
+
+	/* Check for a bit of consistency */
+	if (line[GIT_OID_HEXSZ] != ' ') {
+		giterr_set(GITERR_NET, "Error parsing pkt-line");
+		error = -1;
+		goto error_out;
+	}
+
+	/* Jump from the name */
+	line += GIT_OID_HEXSZ + 1;
+	len -= (GIT_OID_HEXSZ + 1);
+
+	if (line[len - 1] == '\n')
+		--len;
+
+	pkt->head.name = git__malloc(len + 1);
+	GITERR_CHECK_ALLOC(pkt->head.name);
+
+	memcpy(pkt->head.name, line, len);
+	pkt->head.name[len] = '\0';
+
+	if (strlen(pkt->head.name) < len) {
+		pkt->capabilities = strchr(pkt->head.name, '\0') + 1;
+	}
+
+	*out = (git_pkt *)pkt;
+	return 0;
+
+error_out:
+	git__free(pkt);
+	return error;
+}
+
+static int32_t parse_len(const char *line)
+{
+	char num[PKT_LEN_SIZE + 1];
+	int i, error;
+	int32_t len;
+	const char *num_end;
+
+	memcpy(num, line, PKT_LEN_SIZE);
+	num[PKT_LEN_SIZE] = '\0';
+
+	for (i = 0; i < PKT_LEN_SIZE; ++i) {
+		if (!isxdigit(num[i])) {
+			giterr_set(GITERR_NET, "Found invalid hex digit in length");
+			return -1;
+		}
+	}
+
+	if ((error = git__strtol32(&len, num, &num_end, 16)) < 0)
+		return error;
+
+	return len;
+}
+
+/*
+ * As per the documentation, the syntax is:
+ *
+ * pkt-line	= data-pkt / flush-pkt
+ * data-pkt	= pkt-len pkt-payload
+ * pkt-len		= 4*(HEXDIG)
+ * pkt-payload = (pkt-len -4)*(OCTET)
+ * flush-pkt	= "0000"
+ *
+ * Which means that the first four bytes are the length of the line,
+ * in ASCII hexadecimal (including itself)
+ */
+
+int git_pkt_parse_line(
+	git_pkt **head, const char *line, const char **out, size_t bufflen)
+{
+	int ret;
+	int32_t len;
+
+	/* Not even enough for the length */
+	if (bufflen > 0 && bufflen < PKT_LEN_SIZE)
+		return GIT_EBUFS;
+
+	len = parse_len(line);
+	if (len < 0) {
+		/*
+		 * If we fail to parse the length, it might be because the
+		 * server is trying to send us the packfile already.
+		 */
+		if (bufflen >= 4 && !git__prefixcmp(line, "PACK")) {
+			giterr_clear();
+			*out = line;
+			return pack_pkt(head);
+		}
+
+		return (int)len;
+	}
+
+	/*
+	 * If we were given a buffer length, then make sure there is
+	 * enough in the buffer to satisfy this line
+	 */
+	if (bufflen > 0 && bufflen < (size_t)len)
+		return GIT_EBUFS;
+
+	line += PKT_LEN_SIZE;
+	/*
+	 * TODO: How do we deal with empty lines? Try again? with the next
+	 * line?
+	 */
+	if (len == PKT_LEN_SIZE) {
+		*out = line;
+		return 0;
+	}
+
+	if (len == 0) { /* Flush pkt */
+		*out = line;
+		return flush_pkt(head);
+	}
+
+	len -= PKT_LEN_SIZE; /* the encoded length includes its own size */
+
+	/* Assming the minimal size is actually 4 */
+	if (!git__prefixcmp(line, "ACK"))
+		ret = ack_pkt(head, line, len);
+	else if (!git__prefixcmp(line, "NAK"))
+		ret = nak_pkt(head);
+	else if (!git__prefixcmp(line, "ERR "))
+		ret = err_pkt(head, line, len);
+	else if (*line == '#')
+		ret = comment_pkt(head, line, len);
+	else
+		ret = ref_pkt(head, line, len);
+
+	*out = line + len;
+
+	return ret;
+}
+
+void git_pkt_free(git_pkt *pkt)
+{
+	if (pkt->type == GIT_PKT_REF) {
+		git_pkt_ref *p = (git_pkt_ref *) pkt;
+		git__free(p->head.name);
+	}
+
+	git__free(pkt);
+}
+
+int git_pkt_buffer_flush(git_buf *buf)
+{
+	return git_buf_put(buf, pkt_flush_str, strlen(pkt_flush_str));
+}
+
+static int buffer_want_with_caps(git_remote_head *head, git_transport_caps *caps, git_buf *buf)
+{
+	char capstr[20];
+	char oid[GIT_OID_HEXSZ +1] = {0};
+	unsigned int len;
+
+	if (caps->ofs_delta)
+		strncpy(capstr, GIT_CAP_OFS_DELTA, sizeof(capstr));
+
+	len = (unsigned int)
+		(strlen("XXXXwant ") + GIT_OID_HEXSZ + 1 /* NUL */ +
+		 strlen(capstr) + 1 /* LF */);
+	git_buf_grow(buf, git_buf_len(buf) + len);
+
+	git_oid_fmt(oid, &head->oid);
+	return git_buf_printf(buf, "%04xwant %s%c%s\n", len, oid, 0, capstr);
+}
+
+/*
+ * All "want" packets have the same length and format, so what we do
+ * is overwrite the OID each time.
+ */
+
+int git_pkt_buffer_wants(const git_vector *refs, git_transport_caps *caps, git_buf *buf)
+{
+	unsigned int i = 0;
+	git_remote_head *head;
+
+	if (caps->common) {
+		for (; i < refs->length; ++i) {
+			head = refs->contents[i];
+			if (!head->local)
+				break;
+		}
+
+		if (buffer_want_with_caps(refs->contents[i], caps, buf) < 0)
+			return -1;
+
+		i++;
+	}
+
+	for (; i < refs->length; ++i) {
+		char oid[GIT_OID_HEXSZ];
+
+		head = refs->contents[i];
+		if (head->local)
+			continue;
+
+		git_oid_fmt(oid, &head->oid);
+		git_buf_put(buf, pkt_want_prefix, strlen(pkt_want_prefix));
+		git_buf_put(buf, oid, GIT_OID_HEXSZ);
+		git_buf_putc(buf, '\n');
+		if (git_buf_oom(buf))
+			return -1;
+	}
+
+	return git_pkt_buffer_flush(buf);
+}
+
+int git_pkt_buffer_have(git_oid *oid, git_buf *buf)
+{
+	char oidhex[GIT_OID_HEXSZ + 1];
+
+	memset(oidhex, 0x0, sizeof(oidhex));
+	git_oid_fmt(oidhex, oid);
+	return git_buf_printf(buf, "%s%s\n", pkt_have_prefix, oidhex);
+}
+
+int git_pkt_buffer_done(git_buf *buf)
+{
+	return git_buf_puts(buf, pkt_done_str);
+}
diff --git a/src/pkt.h b/src/pkt.h
new file mode 100644
index 000000000..75442c833
--- /dev/null
+++ b/src/pkt.h
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_pkt_h__
+#define INCLUDE_pkt_h__
+
+#include "common.h"
+#include "transport.h"
+#include "buffer.h"
+#include "posix.h"
+#include "git2/net.h"
+
+enum git_pkt_type {
+	GIT_PKT_CMD,
+	GIT_PKT_FLUSH,
+	GIT_PKT_REF,
+	GIT_PKT_HAVE,
+	GIT_PKT_ACK,
+	GIT_PKT_NAK,
+	GIT_PKT_PACK,
+	GIT_PKT_COMMENT,
+	GIT_PKT_ERR,
+};
+
+/* Used for multi-ack */
+enum git_ack_status {
+	GIT_ACK_NONE,
+	GIT_ACK_CONTINUE,
+	GIT_ACK_COMMON,
+	GIT_ACK_READY
+};
+
+/* This would be a flush pkt */
+typedef struct {
+	enum git_pkt_type type;
+} git_pkt;
+
+struct git_pkt_cmd {
+	enum git_pkt_type type;
+	char *cmd;
+	char *path;
+	char *host;
+};
+
+/* This is a pkt-line with some info in it */
+typedef struct {
+	enum git_pkt_type type;
+	git_remote_head head;
+	char *capabilities;
+} git_pkt_ref;
+
+/* Useful later */
+typedef struct {
+	enum git_pkt_type type;
+	git_oid oid;
+	enum git_ack_status status;
+} git_pkt_ack;
+
+typedef struct {
+	enum git_pkt_type type;
+	char comment[GIT_FLEX_ARRAY];
+} git_pkt_comment;
+
+typedef struct {
+	enum git_pkt_type type;
+	char error[GIT_FLEX_ARRAY];
+} git_pkt_err;
+
+int git_pkt_parse_line(git_pkt **head, const char *line, const char **out, size_t len);
+int git_pkt_buffer_flush(git_buf *buf);
+int git_pkt_send_flush(GIT_SOCKET s);
+int git_pkt_buffer_done(git_buf *buf);
+int git_pkt_buffer_wants(const git_vector *refs, git_transport_caps *caps, git_buf *buf);
+int git_pkt_buffer_have(git_oid *oid, git_buf *buf);
+void git_pkt_free(git_pkt *pkt);
+
+#endif
diff --git a/src/pool.c b/src/pool.c
new file mode 100644
index 000000000..641292d06
--- /dev/null
+++ b/src/pool.c
@@ -0,0 +1,294 @@
+#include "pool.h"
+#ifndef GIT_WIN32
+#include <unistd.h>
+#endif
+
+struct git_pool_page {
+	git_pool_page *next;
+	uint32_t size;
+	uint32_t avail;
+	char data[GIT_FLEX_ARRAY];
+};
+
+#define GIT_POOL_MIN_USABLE	4
+#define GIT_POOL_MIN_PAGESZ	2 * sizeof(void*)
+
+static void *pool_alloc_page(git_pool *pool, uint32_t size);
+static void pool_insert_page(git_pool *pool, git_pool_page *page);
+
+int git_pool_init(
+	git_pool *pool, uint32_t item_size, uint32_t items_per_page)
+{
+	assert(pool);
+
+	if (!item_size)
+		item_size = 1;
+	/* round up item_size for decent object alignment */
+	if (item_size > 4)
+		item_size = (item_size + 7) & ~7;
+	else if (item_size == 3)
+		item_size = 4;
+
+	if (!items_per_page)
+		items_per_page = git_pool__suggest_items_per_page(item_size);
+	if (item_size * items_per_page < GIT_POOL_MIN_PAGESZ)
+		items_per_page = (GIT_POOL_MIN_PAGESZ + item_size - 1) / item_size;
+
+	memset(pool, 0, sizeof(git_pool));
+	pool->item_size = item_size;
+	pool->page_size = item_size * items_per_page;
+
+	return 0;
+}
+
+void git_pool_clear(git_pool *pool)
+{
+	git_pool_page *scan, *next;
+
+	for (scan = pool->open; scan != NULL; scan = next) {
+		next = scan->next;
+		git__free(scan);
+	}
+	pool->open = NULL;
+
+	for (scan = pool->full; scan != NULL; scan = next) {
+		next = scan->next;
+		git__free(scan);
+	}
+	pool->full = NULL;
+
+	pool->free_list = NULL;
+
+	pool->items = 0;
+
+	pool->has_string_alloc     = 0;
+	pool->has_multi_item_alloc = 0;
+	pool->has_large_page_alloc = 0;
+}
+
+void git_pool_swap(git_pool *a, git_pool *b)
+{
+	git_pool temp;
+
+	if (a == b)
+		return;
+
+	memcpy(&temp, a, sizeof(temp));
+	memcpy(a, b, sizeof(temp));
+	memcpy(b, &temp, sizeof(temp));
+}
+
+static void pool_insert_page(git_pool *pool, git_pool_page *page)
+{
+	git_pool_page *scan;
+
+	/* If there are no open pages or this page has the most open space,
+	 * insert it at the beginning of the list.  This is the common case.
+	 */
+	if (pool->open == NULL || pool->open->avail < page->avail) {
+		page->next = pool->open;
+		pool->open = page;
+		return;
+	}
+
+	/* Otherwise insert into sorted position. */
+	for (scan = pool->open;
+		 scan->next && scan->next->avail > page->avail;
+		 scan = scan->next);
+	page->next = scan->next;
+	scan->next = page;
+}
+
+static void *pool_alloc_page(git_pool *pool, uint32_t size)
+{
+	git_pool_page *page;
+	uint32_t alloc_size;
+
+	if (size <= pool->page_size)
+		alloc_size = pool->page_size;
+	else {
+		alloc_size = size;
+		pool->has_large_page_alloc = 1;
+	}
+
+	page = git__calloc(1, alloc_size + sizeof(git_pool_page));
+	if (!page)
+		return NULL;
+
+	page->size  = alloc_size;
+	page->avail = alloc_size - size;
+
+	if (page->avail > 0)
+		pool_insert_page(pool, page);
+	else {
+		page->next = pool->full;
+		pool->full = page;
+	}
+
+	pool->items++;
+
+	return page->data;
+}
+
+GIT_INLINE(void) pool_remove_page(
+	git_pool *pool, git_pool_page *page, git_pool_page *prev)
+{
+	if (prev == NULL)
+		pool->open = page->next;
+	else
+		prev->next = page->next;
+}
+
+void *git_pool_malloc(git_pool *pool, uint32_t items)
+{
+	git_pool_page *scan = pool->open, *prev;
+	uint32_t size = items * pool->item_size;
+	void *ptr = NULL;
+
+	pool->has_string_alloc = 0;
+	if (items > 1)
+		pool->has_multi_item_alloc = 1;
+	else if (pool->free_list != NULL) {
+		ptr = pool->free_list;
+		pool->free_list = *((void **)pool->free_list);
+		return ptr;
+	}
+
+	/* just add a block if there is no open one to accomodate this */
+	if (size >= pool->page_size || !scan || scan->avail < size)
+		return pool_alloc_page(pool, size);
+
+	pool->items++;
+
+	/* find smallest block in free list with space */
+	for (scan = pool->open, prev = NULL;
+		 scan->next && scan->next->avail >= size;
+		 prev = scan, scan = scan->next);
+
+	/* allocate space from the block */
+	ptr = &scan->data[scan->size - scan->avail];
+	scan->avail -= size;
+
+	/* move to full list if there is almost no space left */
+	if (scan->avail < pool->item_size || scan->avail < GIT_POOL_MIN_USABLE) {
+		pool_remove_page(pool, scan, prev);
+		scan->next = pool->full;
+		pool->full = scan;
+	}
+	/* reorder list if block is now smaller than the one after it */
+	else if (scan->next != NULL && scan->next->avail > scan->avail) {
+		pool_remove_page(pool, scan, prev);
+		pool_insert_page(pool, scan);
+	}
+
+	return ptr;
+}
+
+char *git_pool_strndup(git_pool *pool, const char *str, size_t n)
+{
+	void *ptr = NULL;
+
+	assert(pool && str && pool->item_size == sizeof(char));
+
+	if ((ptr = git_pool_malloc(pool, (uint32_t)(n + 1))) != NULL) {
+		memcpy(ptr, str, n);
+		*(((char *)ptr) + n) = '\0';
+	}
+	pool->has_string_alloc = 1;
+
+	return ptr;
+}
+
+char *git_pool_strdup(git_pool *pool, const char *str)
+{
+	assert(pool && str && pool->item_size == sizeof(char));
+
+	return git_pool_strndup(pool, str, strlen(str));
+}
+
+char *git_pool_strcat(git_pool *pool, const char *a, const char *b)
+{
+	void *ptr;
+	size_t len_a, len_b;
+
+	assert(pool && a && b && pool->item_size == sizeof(char));
+
+	len_a = a ? strlen(a) : 0;
+	len_b = b ? strlen(b) : 0;
+
+	if ((ptr = git_pool_malloc(pool, (uint32_t)(len_a + len_b + 1))) != NULL) {
+		if (len_a)
+			memcpy(ptr, a, len_a);
+		if (len_b)
+			memcpy(((char *)ptr) + len_a, b, len_b);
+		*(((char *)ptr) + len_a + len_b) = '\0';
+	}
+	pool->has_string_alloc = 1;
+
+	return ptr;
+}
+
+void git_pool_free(git_pool *pool, void *ptr)
+{
+	assert(pool && ptr && pool->item_size >= sizeof(void*));
+
+	*((void **)ptr) = pool->free_list;
+	pool->free_list = ptr;
+}
+
+uint32_t git_pool__open_pages(git_pool *pool)
+{
+	uint32_t ct = 0;
+	git_pool_page *scan;
+	for (scan = pool->open; scan != NULL; scan = scan->next) ct++;
+	return ct;
+}
+
+uint32_t git_pool__full_pages(git_pool *pool)
+{
+	uint32_t ct = 0;
+	git_pool_page *scan;
+	for (scan = pool->full; scan != NULL; scan = scan->next) ct++;
+	return ct;
+}
+
+bool git_pool__ptr_in_pool(git_pool *pool, void *ptr)
+{
+	git_pool_page *scan;
+	for (scan = pool->open; scan != NULL; scan = scan->next)
+		if ((void *)scan->data <= ptr &&
+			(void *)(((char *)scan->data) + scan->size) > ptr)
+			return true;
+	for (scan = pool->full; scan != NULL; scan = scan->next)
+		if ((void *)scan->data <= ptr &&
+			(void *)(((char *)scan->data) + scan->size) > ptr)
+			return true;
+	return false;
+}
+
+uint32_t git_pool__system_page_size(void)
+{
+	static uint32_t size = 0;
+
+	if (!size) {
+#ifdef GIT_WIN32
+		SYSTEM_INFO info;
+		GetSystemInfo(&info);
+		size = (uint32_t)info.dwPageSize;
+#else
+		size = (uint32_t)sysconf(_SC_PAGE_SIZE);
+#endif
+
+		size -= 2 * sizeof(void *); /* allow space for malloc overhead */
+	}
+
+	return size;
+}
+
+uint32_t git_pool__suggest_items_per_page(uint32_t item_size)
+{
+	uint32_t page_bytes =
+		git_pool__system_page_size() - sizeof(git_pool_page);
+	return page_bytes / item_size;
+}
+
diff --git a/src/pool.h b/src/pool.h
new file mode 100644
index 000000000..54a2861ed
--- /dev/null
+++ b/src/pool.h
@@ -0,0 +1,125 @@
+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_pool_h__
+#define INCLUDE_pool_h__
+
+#include "common.h"
+
+typedef struct git_pool_page git_pool_page;
+
+/**
+ * Chunked allocator.
+ *
+ * A `git_pool` can be used when you want to cheaply allocate
+ * multiple items of the same type and are willing to free them
+ * all together with a single call.  The two most common cases
+ * are a set of fixed size items (such as lots of OIDs) or a
+ * bunch of strings.
+ *
+ * Internally, a `git_pool` allocates pages of memory and then
+ * deals out blocks from the trailing unused portion of each page.
+ * The pages guarantee that the number of actual allocations done
+ * will be much smaller than the number of items needed.
+ *
+ * For examples of how to set up a `git_pool` see `git_pool_init`.
+ */
+typedef struct {
+	git_pool_page *open; /* pages with space left */
+	git_pool_page *full; /* pages with no space left */
+	void *free_list;     /* optional: list of freed blocks */
+	uint32_t item_size;  /* size of single alloc unit in bytes */
+	uint32_t page_size;  /* size of page in bytes */
+	uint32_t items;
+	unsigned has_string_alloc : 1; /* was the strdup function used */
+	unsigned has_multi_item_alloc : 1; /* was items ever > 1 in malloc */
+	unsigned has_large_page_alloc : 1; /* are any pages > page_size */
+} git_pool;
+
+#define GIT_POOL_INIT_STRINGPOOL { 0, 0, 0, 1, 4000, 0, 0, 0, 0 }
+
+/**
+ * Initialize a pool.
+ *
+ * To allocation strings, use like this:
+ *
+ *     git_pool_init(&string_pool, 1, 0);
+ *     my_string = git_pool_strdup(&string_pool, your_string);
+ *
+ * To allocate items of fixed size, use like this:
+ *
+ *     git_pool_init(&pool, sizeof(item), 0);
+ *     my_item = git_pool_malloc(&pool, 1);
+ *
+ * Of course, you can use this in other ways, but those are the
+ * two most common patterns.
+ */
+extern int git_pool_init(
+	git_pool *pool, uint32_t item_size, uint32_t items_per_page);
+
+/**
+ * Free all items in pool
+ */
+extern void git_pool_clear(git_pool *pool);
+
+/**
+ * Swap two pools with one another
+ */
+extern void git_pool_swap(git_pool *a, git_pool *b);
+
+/**
+ * Allocate space for one or more items from a pool.
+ */
+extern void *git_pool_malloc(git_pool *pool, uint32_t items);
+
+/**
+ * Allocate space and duplicate string data into it.
+ *
+ * This is allowed only for pools with item_size == sizeof(char)
+ */
+extern char *git_pool_strndup(git_pool *pool, const char *str, size_t n);
+
+/**
+ * Allocate space and duplicate a string into it.
+ *
+ * This is allowed only for pools with item_size == sizeof(char)
+ */
+extern char *git_pool_strdup(git_pool *pool, const char *str);
+
+/**
+ * Allocate space for the concatenation of two strings.
+ *
+ * This is allowed only for pools with item_size == sizeof(char)
+ */
+extern char *git_pool_strcat(git_pool *pool, const char *a, const char *b);
+
+/**
+ * Push a block back onto the free list for the pool.
+ *
+ * This is allowed only if the item_size is >= sizeof(void*).
+ *
+ * In some cases, it is helpful to "release" an allocated block
+ * for reuse.  Pools don't support a general purpose free, but
+ * they will keep a simple free blocks linked list provided the
+ * native block size is large enough to hold a void pointer
+ */
+extern void git_pool_free(git_pool *pool, void *ptr);
+
+/*
+ * Misc utilities
+ */
+
+extern uint32_t git_pool__open_pages(git_pool *pool);
+
+extern uint32_t git_pool__full_pages(git_pool *pool);
+
+extern bool git_pool__ptr_in_pool(git_pool *pool, void *ptr);
+
+extern uint32_t git_pool__system_page_size(void);
+
+extern uint32_t git_pool__suggest_items_per_page(uint32_t item_size);
+
+#endif
diff --git a/src/posix.c b/src/posix.c
new file mode 100644
index 000000000..a9a6af984
--- /dev/null
+++ b/src/posix.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "posix.h"
+#include "path.h"
+#include <stdio.h>
+#include <ctype.h>
+
+#ifndef GIT_WIN32
+
+int p_open(const char *path, int flags, ...)
+{
+	mode_t mode = 0;
+
+	if (flags & O_CREAT)
+	{
+		va_list arg_list;
+
+		va_start(arg_list, flags);
+		mode = (mode_t)va_arg(arg_list, int);
+		va_end(arg_list);
+	}
+
+	return open(path, flags | O_BINARY, mode);
+}
+
+int p_creat(const char *path, mode_t mode)
+{
+	return open(path, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, mode);
+}
+
+int p_getcwd(char *buffer_out, size_t size)
+{
+	char *cwd_buffer;
+
+	assert(buffer_out && size > 0);
+
+	cwd_buffer = getcwd(buffer_out, size);
+
+	if (cwd_buffer == NULL)
+		return -1;
+
+	git_path_mkposix(buffer_out);
+	git_path_string_to_dir(buffer_out, size); /* append trailing slash */
+
+	return 0;
+}
+
+int p_rename(const char *from, const char *to)
+{
+	if (!link(from, to)) {
+		p_unlink(from);
+		return 0;
+	}
+
+	if (!rename(from, to))
+		return 0;
+
+	return -1;
+}
+
+#endif
+
+int p_read(git_file fd, void *buf, size_t cnt)
+{
+	char *b = buf;
+	while (cnt) {
+		ssize_t r;
+#ifdef GIT_WIN32
+		assert((size_t)((unsigned int)cnt) == cnt);
+		r = read(fd, b, (unsigned int)cnt);
+#else
+		r = read(fd, b, cnt);
+#endif
+		if (r < 0) {
+			if (errno == EINTR || errno == EAGAIN)
+				continue;
+			return -1;
+		}
+		if (!r)
+			break;
+		cnt -= r;
+		b += r;
+	}
+	return (int)(b - (char *)buf);
+}
+
+int p_write(git_file fd, const void *buf, size_t cnt)
+{
+	const char *b = buf;
+	while (cnt) {
+		ssize_t r;
+#ifdef GIT_WIN32
+		assert((size_t)((unsigned int)cnt) == cnt);
+		r = write(fd, b, (unsigned int)cnt);
+#else
+		r = write(fd, b, cnt);
+#endif
+		if (r < 0) {
+			if (errno == EINTR || errno == EAGAIN)
+				continue;
+			return -1;
+		}
+		if (!r) {
+			errno = EPIPE;
+			return -1;
+		}
+		cnt -= r;
+		b += r;
+	}
+	return 0;
+}
diff --git a/src/posix.h b/src/posix.h
new file mode 100644
index 000000000..d020d94ac
--- /dev/null
+++ b/src/posix.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_posix_h__
+#define INCLUDE_posix_h__
+
+#include "common.h"
+#include <fcntl.h>
+#include <time.h>
+
+#define S_IFGITLINK 0160000
+#define S_ISGITLINK(m) (((m) & S_IFMT) == S_IFGITLINK)
+
+#if !defined(O_BINARY)
+#define O_BINARY 0
+#endif
+
+typedef int git_file;
+
+/**
+ * Standard POSIX Methods
+ *
+ * All the methods starting with the `p_` prefix are
+ * direct ports of the standard POSIX methods. 
+ *
+ * Some of the methods are slightly wrapped to provide
+ * saner defaults. Some of these methods are emulated
+ * in Windows platforns.
+ *
+ * Use your manpages to check the docs on these.
+ * Straightforward 
+ */
+
+extern int p_read(git_file fd, void *buf, size_t cnt);
+extern int p_write(git_file fd, const void *buf, size_t cnt);
+
+#define p_fstat(f,b) fstat(f, b)
+#define p_lseek(f,n,w) lseek(f, n, w)
+#define p_close(fd) close(fd)
+#define p_umask(m) umask(m)
+
+extern int p_open(const char *path, int flags, ...);
+extern int p_creat(const char *path, mode_t mode);
+extern int p_getcwd(char *buffer_out, size_t size);
+extern int p_rename(const char *from, const char *to);
+
+#ifndef GIT_WIN32
+
+#define p_stat(p,b) stat(p, b)
+#define p_chdir(p) chdir(p)
+#define p_rmdir(p) rmdir(p)
+#define p_chmod(p,m) chmod(p, m)
+#define p_access(p,m) access(p,m)
+#define p_recv(s,b,l,f) recv(s,b,l,f)
+#define p_send(s,b,l,f) send(s,b,l,f)
+typedef int GIT_SOCKET;
+#define INVALID_SOCKET -1
+
+#else
+
+typedef SOCKET GIT_SOCKET;
+
+#endif
+
+/**
+ * Platform-dependent methods
+ */
+#ifdef GIT_WIN32
+#	include "win32/posix.h"
+#else
+#	include "unix/posix.h"
+#endif
+
+#define p_readdir_r(d,e,r) readdir_r(d,e,r)
+
+#endif
diff --git a/src/ppc/sha1.c b/src/ppc/sha1.c
index ec6a1926d..803b81d0a 100644
--- a/src/ppc/sha1.c
+++ b/src/ppc/sha1.c
@@ -1,17 +1,15 @@
 /*
- * SHA-1 implementation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * Copyright (C) 2005 Paul Mackerras <paulus@samba.org>
- *
- * This version assumes we are running on a big-endian machine.
- * It calls an external sha1_core() to process blocks of 64 bytes.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #include <stdio.h>
 #include <string.h>
 #include "sha1.h"
 
 extern void ppc_sha1_core(uint32_t *hash, const unsigned char *p,
-			  unsigned int nblocks);
+			 unsigned int nblocks);
 
 int ppc_SHA1_Init(ppc_SHA_CTX *c)
 {
diff --git a/src/ppc/sha1.h b/src/ppc/sha1.h
index 70957110c..aca4e5dda 100644
--- a/src/ppc/sha1.h
+++ b/src/ppc/sha1.h
@@ -1,7 +1,8 @@
 /*
- * SHA-1 implementation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * Copyright (C) 2005 Paul Mackerras <paulus@samba.org>
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #include <stdint.h>
 
diff --git a/src/pqueue.c b/src/pqueue.c
index 6307175e3..cb59c13ec 100644
--- a/src/pqueue.c
+++ b/src/pqueue.c
@@ -1,52 +1,36 @@
 /*
- *	BORING COPYRIGHT NOTICE:
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- *	This file is a heavily modified version of the priority queue found
- *	in the Apache project and the libpqueue library.
- *
- *		https://github.com/vy/libpqueue
- *
- *	These are the original authors:
- *
- *		Copyright 2010 Volkan Yazıcı <volkan.yazici@gmail.com>
- *		Copyright 2006-2010 The Apache Software Foundation
- *
- *  This file is licensed under the Apache 2.0 license, which 
- *  supposedly makes it compatible with the GPLv2 that libgit2 uses.
- *
- *  Check the Apache license at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- *  So much licensing trouble for a binary heap. Oh well.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
 #include "pqueue.h"
 
-#define left(i)   ((i) << 1)
-#define right(i)  (((i) << 1) + 1)
+#define left(i)	((i) << 1)
+#define right(i) (((i) << 1) + 1)
 #define parent(i) ((i) >> 1)
 
 int git_pqueue_init(git_pqueue *q, size_t n, git_pqueue_cmp cmppri)
 {
 	assert(q);
 
-    /* Need to allocate n+1 elements since element 0 isn't used. */
-    if ((q->d = malloc((n + 1) * sizeof(void *))) == NULL)
-		return GIT_ENOMEM;
+	/* Need to allocate n+1 elements since element 0 isn't used. */
+	q->d = git__malloc((n + 1) * sizeof(void *));
+	GITERR_CHECK_ALLOC(q->d);
 
-    q->size = 1;
-    q->avail = q->step = (n + 1);  /* see comment above about n+1 */
-    q->cmppri = cmppri;
+	q->size = 1;
+	q->avail = q->step = (n + 1); /* see comment above about n+1 */
+	q->cmppri = cmppri;
 
-    return GIT_SUCCESS;
+	return 0;
 }
 
 
 void git_pqueue_free(git_pqueue *q)
 {
-    free(q->d);
+	git__free(q->d);
 	q->d = NULL;
 }
 
@@ -57,101 +41,101 @@ void git_pqueue_clear(git_pqueue *q)
 
 size_t git_pqueue_size(git_pqueue *q)
 {
-    /* queue element 0 exists but doesn't count since it isn't used. */
-    return (q->size - 1);
+	/* queue element 0 exists but doesn't count since it isn't used. */
+	return (q->size - 1);
 }
 
 
 static void bubble_up(git_pqueue *q, size_t i)
 {
-    size_t parent_node;
-    void *moving_node = q->d[i];
+	size_t parent_node;
+	void *moving_node = q->d[i];
 
-    for (parent_node = parent(i);
-         ((i > 1) && q->cmppri(q->d[parent_node], moving_node));
-         i = parent_node, parent_node = parent(i)) {
-        q->d[i] = q->d[parent_node];
-    }
+	for (parent_node = parent(i);
+			((i > 1) && q->cmppri(q->d[parent_node], moving_node));
+			i = parent_node, parent_node = parent(i)) {
+		q->d[i] = q->d[parent_node];
+	}
 
-    q->d[i] = moving_node;
+	q->d[i] = moving_node;
 }
 
 
 static size_t maxchild(git_pqueue *q, size_t i)
 {
-    size_t child_node = left(i);
+	size_t child_node = left(i);
 
-    if (child_node >= q->size)
-        return 0;
+	if (child_node >= q->size)
+		return 0;
 
-    if ((child_node + 1) < q->size &&
-        q->cmppri(q->d[child_node], q->d[child_node + 1]))
-        child_node++; /* use right child instead of left */
+	if ((child_node + 1) < q->size &&
+		q->cmppri(q->d[child_node], q->d[child_node + 1]))
+		child_node++; /* use right child instead of left */
 
-    return child_node;
+	return child_node;
 }
 
 
 static void percolate_down(git_pqueue *q, size_t i)
 {
-    size_t child_node;
-    void *moving_node = q->d[i];
+	size_t child_node;
+	void *moving_node = q->d[i];
 
-    while ((child_node = maxchild(q, i)) != 0 &&
-           q->cmppri(moving_node, q->d[child_node])) {
-        q->d[i] = q->d[child_node];
-        i = child_node;
-    }
+	while ((child_node = maxchild(q, i)) != 0 &&
+			q->cmppri(moving_node, q->d[child_node])) {
+		q->d[i] = q->d[child_node];
+		i = child_node;
+	}
 
-    q->d[i] = moving_node;
+	q->d[i] = moving_node;
 }
 
 
 int git_pqueue_insert(git_pqueue *q, void *d)
 {
-    void *tmp;
-    size_t i;
-    size_t newsize;
+	void *tmp;
+	size_t i;
+	size_t newsize;
 
-    if (!q) return 1;
+	if (!q) return 1;
 
-    /* allocate more memory if necessary */
-    if (q->size >= q->avail) {
-        newsize = q->size + q->step;
-        if ((tmp = realloc(q->d, sizeof(void *) * newsize)) == NULL)
-            return GIT_ENOMEM;
+	/* allocate more memory if necessary */
+	if (q->size >= q->avail) {
+		newsize = q->size + q->step;
+		tmp = git__realloc(q->d, sizeof(void *) * newsize);
+		GITERR_CHECK_ALLOC(tmp);
 
-        q->d = tmp;
-        q->avail = newsize;
-    }
+		q->d = tmp;
+		q->avail = newsize;
+	}
 
-    /* insert item */
-    i = q->size++;
-    q->d[i] = d;
-    bubble_up(q, i);
+	/* insert item */
+	i = q->size++;
+	q->d[i] = d;
+	bubble_up(q, i);
 
-    return GIT_SUCCESS;
+	return 0;
 }
 
 
 void *git_pqueue_pop(git_pqueue *q)
 {
-    void *head;
+	void *head;
 
-    if (!q || q->size == 1)
-        return NULL;
+	if (!q || q->size == 1)
+		return NULL;
 
-    head = q->d[1];
-    q->d[1] = q->d[--q->size];
-    percolate_down(q, 1);
+	head = q->d[1];
+	q->d[1] = q->d[--q->size];
+	percolate_down(q, 1);
 
-    return head;
+	return head;
 }
 
 
 void *git_pqueue_peek(git_pqueue *q)
 {
-    if (!q || q->size == 1)
-        return NULL;
-    return q->d[1];
+	if (!q || q->size == 1)
+		return NULL;
+	return q->d[1];
 }
diff --git a/src/pqueue.h b/src/pqueue.h
index 7a1394803..a3e1edd1d 100644
--- a/src/pqueue.h
+++ b/src/pqueue.h
@@ -1,24 +1,8 @@
 /*
- *	BORING COPYRIGHT NOTICE:
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- *	This file is a heavily modified version of the priority queue found
- *	in the Apache project and the libpqueue library.
- *
- *		https://github.com/vy/libpqueue
- *
- *	These are the original authors:
- *
- *		Copyright 2010 Volkan Yazıcı <volkan.yazici@gmail.com>
- *		Copyright 2006-2010 The Apache Software Foundation
- *
- *  This file is licensed under the Apache 2.0 license, which 
- *  supposedly makes it compatible with the GPLv2 that libgit2 uses.
- *
- *  Check the Apache license at:
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- *  So much licensing trouble for a binary heap. Oh well.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #ifndef INCLUDE_pqueue_h__
@@ -29,9 +13,9 @@ typedef int (*git_pqueue_cmp)(void *a, void *b);
 
 /** the priority queue handle */
 typedef struct {
-    size_t size, avail, step;
-    git_pqueue_cmp cmppri;
-    void **d;
+	size_t size, avail, step;
+	git_pqueue_cmp cmppri;
+	void **d;
 } git_pqueue;
 
 
@@ -39,7 +23,7 @@ typedef struct {
  * initialize the queue
  *
  * @param n the initial estimate of the number of queue items for which memory
- *          should be preallocated
+ *			should be preallocated
  * @param cmppri the callback function to compare two nodes of the queue
  *
  * @Return the handle or NULL for insufficent memory
diff --git a/src/protocol.c b/src/protocol.c
new file mode 100644
index 000000000..6b3861796
--- /dev/null
+++ b/src/protocol.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "protocol.h"
+#include "pkt.h"
+#include "buffer.h"
+
+int git_protocol_store_refs(git_protocol *p, const char *data, size_t len)
+{
+	git_buf *buf = &p->buf;
+	git_vector *refs = p->refs;
+	int error;
+	const char *line_end, *ptr;
+
+	if (len == 0) { /* EOF */
+		if (git_buf_len(buf) != 0) {
+			giterr_set(GITERR_NET, "Unexpected EOF");
+			return p->error = -1;
+		} else {
+			return 0;
+		}
+	}
+
+	git_buf_put(buf, data, len);
+	ptr = buf->ptr;
+	while (1) {
+		git_pkt *pkt;
+
+		if (git_buf_len(buf) == 0)
+			return 0;
+
+		error = git_pkt_parse_line(&pkt, ptr, &line_end, git_buf_len(buf));
+		if (error == GIT_EBUFS)
+			return 0; /* Ask for more */
+		if (error < 0)
+			return p->error = -1;
+
+		git_buf_consume(buf, line_end);
+
+		if (pkt->type == GIT_PKT_ERR) {
+			giterr_set(GITERR_NET, "Remote error: %s", ((git_pkt_err *)pkt)->error);
+			git__free(pkt);
+			return -1;
+		}
+
+		if (git_vector_insert(refs, pkt) < 0)
+			return p->error = -1;
+
+		if (pkt->type == GIT_PKT_FLUSH)
+			p->flush = 1;
+	}
+
+	return 0;
+}
diff --git a/src/protocol.h b/src/protocol.h
new file mode 100644
index 000000000..a6c3e0735
--- /dev/null
+++ b/src/protocol.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_protocol_h__
+#define INCLUDE_protocol_h__
+
+#include "transport.h"
+#include "buffer.h"
+
+typedef struct {
+	git_transport *transport;
+	git_vector *refs;
+	git_buf buf;
+	int error;
+	unsigned int flush :1;
+} git_protocol;
+
+int git_protocol_store_refs(git_protocol *p, const char *data, size_t len);
+
+#endif
diff --git a/src/reflog.c b/src/reflog.c
new file mode 100644
index 000000000..3ea073e65
--- /dev/null
+++ b/src/reflog.c
@@ -0,0 +1,340 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "reflog.h"
+#include "repository.h"
+#include "filebuf.h"
+#include "signature.h"
+
+static int reflog_init(git_reflog **reflog, git_reference *ref)
+{
+	git_reflog *log;
+
+	*reflog = NULL;
+
+	log = git__calloc(1, sizeof(git_reflog));
+	GITERR_CHECK_ALLOC(log);
+
+	log->ref_name = git__strdup(ref->name);
+	GITERR_CHECK_ALLOC(log->ref_name);
+
+	if (git_vector_init(&log->entries, 0, NULL) < 0) {
+		git__free(log->ref_name);
+		git__free(log);
+		return -1;
+	}
+
+	*reflog = log;
+
+	return 0;
+}
+
+static int reflog_write(const char *log_path, const char *oid_old,
+			const char *oid_new, const git_signature *committer,
+			const char *msg)
+{
+	int error;
+	git_buf log = GIT_BUF_INIT;
+	git_filebuf fbuf = GIT_FILEBUF_INIT;
+	bool trailing_newline = false;
+
+	assert(log_path && oid_old && oid_new && committer);
+
+	if (msg) {
+		const char *newline = strchr(msg, '\n');
+		if (newline) {
+			if (*(newline + 1) == '\0')
+				trailing_newline = true;
+			else {
+				giterr_set(GITERR_INVALID, "Reflog message cannot contain newline");
+				return -1;
+			}
+		}
+	}
+
+	git_buf_puts(&log, oid_old);
+	git_buf_putc(&log, ' ');
+
+	git_buf_puts(&log, oid_new);
+
+	git_signature__writebuf(&log, " ", committer);
+	git_buf_truncate(&log, log.size - 1); /* drop LF */
+
+	if (msg) {
+		git_buf_putc(&log, '\t');
+		git_buf_puts(&log, msg);
+	}
+
+	if (!trailing_newline)
+		git_buf_putc(&log, '\n');
+
+	if (git_buf_oom(&log)) {
+		git_buf_free(&log);
+		return -1;
+	}
+
+	error = git_filebuf_open(&fbuf, log_path, GIT_FILEBUF_APPEND);
+	if (!error) {
+		if ((error = git_filebuf_write(&fbuf, log.ptr, log.size)) < 0)
+			git_filebuf_cleanup(&fbuf);
+		else
+			error = git_filebuf_commit(&fbuf, GIT_REFLOG_FILE_MODE);
+	}
+
+	git_buf_free(&log);
+
+	return error;
+}
+
+static int reflog_parse(git_reflog *log, const char *buf, size_t buf_size)
+{
+	const char *ptr;
+	git_reflog_entry *entry;
+
+#define seek_forward(_increase) do { \
+	if (_increase >= buf_size) { \
+		giterr_set(GITERR_INVALID, "Ran out of data while parsing reflog"); \
+		goto fail; \
+	} \
+	buf += _increase; \
+	buf_size -= _increase; \
+	} while (0)
+
+	while (buf_size > GIT_REFLOG_SIZE_MIN) {
+		entry = git__malloc(sizeof(git_reflog_entry));
+		GITERR_CHECK_ALLOC(entry);
+
+		entry->committer = git__malloc(sizeof(git_signature));
+		GITERR_CHECK_ALLOC(entry->committer);
+
+		if (git_oid_fromstrn(&entry->oid_old, buf, GIT_OID_HEXSZ) < 0)
+			goto fail;
+		seek_forward(GIT_OID_HEXSZ + 1);
+
+		if (git_oid_fromstrn(&entry->oid_cur, buf, GIT_OID_HEXSZ) < 0)
+			goto fail;
+		seek_forward(GIT_OID_HEXSZ + 1);
+
+		ptr = buf;
+
+		/* Seek forward to the end of the signature. */
+		while (*buf && *buf != '\t' && *buf != '\n')
+			seek_forward(1);
+
+		if (git_signature__parse(entry->committer, &ptr, buf + 1, NULL, *buf) < 0)
+			goto fail;
+
+		if (*buf == '\t') {
+			/* We got a message. Read everything till we reach LF. */
+			seek_forward(1);
+			ptr = buf;
+
+			while (*buf && *buf != '\n')
+				seek_forward(1);
+
+			entry->msg = git__strndup(ptr, buf - ptr);
+			GITERR_CHECK_ALLOC(entry->msg);
+		} else
+			entry->msg = NULL;
+
+		while (*buf && *buf == '\n' && buf_size > 1)
+			seek_forward(1);
+
+		if (git_vector_insert(&log->entries, entry) < 0)
+			goto fail;
+	}
+
+	return 0;
+
+#undef seek_forward
+
+fail:
+	if (entry) {
+		git__free(entry->committer);
+		git__free(entry);
+	}
+	return -1;
+}
+
+void git_reflog_free(git_reflog *reflog)
+{
+	unsigned int i;
+	git_reflog_entry *entry;
+
+	for (i=0; i < reflog->entries.length; i++) {
+		entry = git_vector_get(&reflog->entries, i);
+
+		git_signature_free(entry->committer);
+
+		git__free(entry->msg);
+		git__free(entry);
+	}
+
+	git_vector_free(&reflog->entries);
+	git__free(reflog->ref_name);
+	git__free(reflog);
+}
+
+int git_reflog_read(git_reflog **reflog, git_reference *ref)
+{
+	int error;
+	git_buf log_path = GIT_BUF_INIT;
+	git_buf log_file = GIT_BUF_INIT;
+	git_reflog *log = NULL;
+
+	*reflog = NULL;
+
+	if (reflog_init(&log, ref) < 0)
+		return -1;
+
+	error = git_buf_join_n(&log_path, '/', 3,
+		ref->owner->path_repository, GIT_REFLOG_DIR, ref->name);
+
+	if (!error)
+		error = git_futils_readbuffer(&log_file, log_path.ptr);
+
+	if (!error)
+		error = reflog_parse(log, log_file.ptr, log_file.size);
+
+	if (!error)
+		*reflog = log;
+	else
+		git_reflog_free(log);
+
+	git_buf_free(&log_file);
+	git_buf_free(&log_path);
+
+	return error;
+}
+
+int git_reflog_write(git_reference *ref, const git_oid *oid_old,
+				const git_signature *committer, const char *msg)
+{
+	int error;
+	char old[GIT_OID_HEXSZ+1];
+	char new[GIT_OID_HEXSZ+1];
+	git_buf log_path = GIT_BUF_INIT;
+	git_reference *r;
+	const git_oid *oid;
+
+	if ((error = git_reference_resolve(&r, ref)) < 0)
+		return error;
+
+	oid = git_reference_oid(r);
+	if (oid == NULL) {
+		giterr_set(GITERR_REFERENCE,
+			"Failed to write reflog. Cannot resolve reference `%s`", r->name);
+		git_reference_free(r);
+		return -1;
+	}
+
+	git_oid_tostr(new, GIT_OID_HEXSZ+1, oid);
+
+	git_reference_free(r);
+
+	error = git_buf_join_n(&log_path, '/', 3,
+		ref->owner->path_repository, GIT_REFLOG_DIR, ref->name);
+	if (error < 0)
+		goto cleanup;
+
+	if (git_path_exists(log_path.ptr) == false) {
+		error = git_futils_mkpath2file(log_path.ptr, GIT_REFLOG_DIR_MODE);
+	} else if (git_path_isfile(log_path.ptr) == false) {
+		giterr_set(GITERR_REFERENCE,
+			"Failed to write reflog. `%s` is directory", log_path.ptr);
+		error = -1;
+	} else if (oid_old == NULL) {
+		giterr_set(GITERR_REFERENCE,
+			"Failed to write reflog. Old OID cannot be NULL for existing reference");
+		error = -1;
+	}
+	if (error < 0)
+		goto cleanup;
+
+	if (oid_old)
+		git_oid_tostr(old, sizeof(old), oid_old);
+	else
+		p_snprintf(old, sizeof(old), "%0*d", GIT_OID_HEXSZ, 0);
+
+	error = reflog_write(log_path.ptr, old, new, committer, msg);
+
+cleanup:
+	git_buf_free(&log_path);
+	return error;
+}
+
+int git_reflog_rename(git_reference *ref, const char *new_name)
+{
+	int error;
+	git_buf old_path = GIT_BUF_INIT;
+	git_buf new_path = GIT_BUF_INIT;
+
+	if (!git_buf_join_n(&old_path, '/', 3, ref->owner->path_repository,
+			GIT_REFLOG_DIR, ref->name) &&
+		!git_buf_join_n(&new_path, '/', 3, ref->owner->path_repository,
+			GIT_REFLOG_DIR, new_name))
+		error = p_rename(git_buf_cstr(&old_path), git_buf_cstr(&new_path));
+	else
+		error = -1;
+
+	git_buf_free(&old_path);
+	git_buf_free(&new_path);
+
+	return error;
+}
+
+int git_reflog_delete(git_reference *ref)
+{
+	int error;
+	git_buf path = GIT_BUF_INIT;
+
+	error = git_buf_join_n(
+		&path, '/', 3, ref->owner->path_repository, GIT_REFLOG_DIR, ref->name);
+
+	if (!error && git_path_exists(path.ptr))
+		error = p_unlink(path.ptr);
+
+	git_buf_free(&path);
+
+	return error;
+}
+
+unsigned int git_reflog_entrycount(git_reflog *reflog)
+{
+	assert(reflog);
+	return reflog->entries.length;
+}
+
+const git_reflog_entry * git_reflog_entry_byindex(git_reflog *reflog, unsigned int idx)
+{
+	assert(reflog);
+	return git_vector_get(&reflog->entries, idx);
+}
+
+const git_oid * git_reflog_entry_oidold(const git_reflog_entry *entry)
+{
+	assert(entry);
+	return &entry->oid_old;
+}
+
+const git_oid * git_reflog_entry_oidnew(const git_reflog_entry *entry)
+{
+	assert(entry);
+	return &entry->oid_cur;
+}
+
+git_signature * git_reflog_entry_committer(const git_reflog_entry *entry)
+{
+	assert(entry);
+	return entry->committer;
+}
+
+char * git_reflog_entry_msg(const git_reflog_entry *entry)
+{
+	assert(entry);
+	return entry->msg;
+}
diff --git a/src/reflog.h b/src/reflog.h
new file mode 100644
index 000000000..33cf0776c
--- /dev/null
+++ b/src/reflog.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_reflog_h__
+#define INCLUDE_reflog_h__
+
+#include "common.h"
+#include "git2/reflog.h"
+#include "vector.h"
+
+#define GIT_REFLOG_DIR "logs/"
+#define GIT_REFLOG_DIR_MODE 0777
+#define GIT_REFLOG_FILE_MODE 0666
+
+#define GIT_REFLOG_SIZE_MIN (2*GIT_OID_HEXSZ+2+17)
+
+struct git_reflog_entry {
+	git_oid oid_old;
+	git_oid oid_cur;
+
+	git_signature *committer;
+
+	char *msg;
+};
+
+struct git_reflog {
+	char *ref_name;
+	git_vector entries;
+};
+
+#endif /* INCLUDE_reflog_h__ */
diff --git a/src/refs.c b/src/refs.c
index c21c9583d..1ef3e13a4 100644
--- a/src/refs.c
+++ b/src/refs.c
@@ -1,433 +1,332 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "refs.h"
 #include "hash.h"
 #include "repository.h"
 #include "fileops.h"
+#include "pack.h"
+#include "reflog.h"
 
 #include <git2/tag.h>
 #include <git2/object.h>
 
-#define MAX_NESTING_LEVEL 5
-
-typedef struct {
-	git_reference ref;
-	git_oid oid;
-	git_oid peel_target;
-} reference_oid;
+GIT__USE_STRMAP;
 
-typedef struct {
-	git_reference ref;
-	char *target;
-} reference_symbolic;
+#define DEFAULT_NESTING_LEVEL	5
+#define MAX_NESTING_LEVEL		10
 
-static const int default_table_size = 32;
-
-static uint32_t reftable_hash(const void *key, int hash_id)
-{
-	static uint32_t hash_seeds[GIT_HASHTABLE_HASHES] = {
-		2147483647,
-		0x5d20bb23,
-		0x7daaab3c 
-	};
+enum {
+	GIT_PACKREF_HAS_PEEL = 1,
+	GIT_PACKREF_WAS_LOOSE = 2
+};
 
-	return git__hash(key, strlen((const char *)key), hash_seeds[hash_id]);
-}
+struct packref {
+	git_oid oid;
+	git_oid peel;
+	char flags;
+	char name[GIT_FLEX_ARRAY];
+};
 
-static void reference_free(git_reference *reference);
-static int reference_create(git_reference **ref_out, git_repository *repo, const char *name, git_rtype type);
-static int reference_read(gitfo_buf *file_content, time_t *mtime, const char *repo_path, const char *ref_name);
+static int reference_read(
+	git_buf *file_content,
+	time_t *mtime,
+	const char *repo_path,
+	const char *ref_name,
+	int *updated);
 
 /* loose refs */
-static int loose_parse_symbolic(git_reference *ref, gitfo_buf *file_content);
-static int loose_parse_oid(git_reference *ref, gitfo_buf *file_content);
-static int loose_lookup(git_reference **ref_out, git_repository *repo, const char *name, int skip_symbolic);
+static int loose_parse_symbolic(git_reference *ref, git_buf *file_content);
+static int loose_parse_oid(git_oid *ref, git_buf *file_content);
+static int loose_lookup(git_reference *ref);
+static int loose_lookup_to_packfile(struct packref **ref_out,
+	git_repository *repo, const char *name);
 static int loose_write(git_reference *ref);
-static int loose_update(git_reference *ref);
 
 /* packed refs */
-static int packed_parse_peel(reference_oid *tag_ref, const char **buffer_out, const char *buffer_end);
-static int packed_parse_oid(reference_oid **ref_out, git_repository *repo, const char **buffer_out, const char *buffer_end);
+static int packed_parse_peel(struct packref *tag_ref,
+	const char **buffer_out, const char *buffer_end);
+static int packed_parse_oid(struct packref **ref_out,
+	const char **buffer_out, const char *buffer_end);
 static int packed_load(git_repository *repo);
 static int packed_loadloose(git_repository *repository);
-static int packed_write_ref(reference_oid *ref, git_filebuf *file);
-static int packed_find_peel(reference_oid *ref);
+static int packed_write_ref(struct packref *ref, git_filebuf *file);
+static int packed_find_peel(git_repository *repo, struct packref *ref);
 static int packed_remove_loose(git_repository *repo, git_vector *packing_list);
 static int packed_sort(const void *a, const void *b);
+static int packed_lookup(git_reference *ref);
 static int packed_write(git_repository *repo);
 
 /* internal helpers */
-static int reference_create_symbolic(git_reference **ref_out, git_repository *repo, const char *name, const char *target, int force);
-static int reference_create_oid(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id, int force);
-static int reference_rename(git_reference *ref, const char *new_name, int force);
+static int reference_path_available(git_repository *repo,
+	const char *ref, const char *old_ref);
+static int reference_delete(git_reference *ref);
+static int reference_lookup(git_reference *ref);
 
 /* name normalization */
-static int check_valid_ref_char(char ch);
-static int normalize_name(char *buffer_out, const char *name, int is_oid_ref);
+static int normalize_name(char *buffer_out, size_t out_size,
+	const char *name, int is_oid_ref);
 
-/*****************************************
- * Internal methods - Constructor/destructor
- *****************************************/
-static void reference_free(git_reference *reference)
+
+void git_reference_free(git_reference *reference)
 {
 	if (reference == NULL)
 		return;
 
-	if (reference->name)
-		free(reference->name);
+	git__free(reference->name);
+	reference->name = NULL;
 
-	if (reference->type == GIT_REF_SYMBOLIC)
-		free(((reference_symbolic *)reference)->target);
+	if (reference->flags & GIT_REF_SYMBOLIC) {
+		git__free(reference->target.symbolic);
+		reference->target.symbolic = NULL;
+	}
 
-	free(reference);
+	git__free(reference);
 }
 
-static int reference_create(
+static int reference_alloc(
 	git_reference **ref_out,
 	git_repository *repo,
-	const char *name,
-	git_rtype type)
+	const char *name)
 {
-	char normalized[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH];
-	int error = GIT_SUCCESS, size;
 	git_reference *reference = NULL;
 
 	assert(ref_out && repo && name);
 
-	if (type == GIT_REF_SYMBOLIC)
-		size = sizeof(reference_symbolic);
-	else if (type == GIT_REF_OID)
-		size = sizeof(reference_oid);
-	else
-		return git__throw(GIT_EINVALIDARGS,
-			"Invalid reference type. Use either GIT_REF_OID or GIT_REF_SYMBOLIC as type specifier");
-
-	reference = git__malloc(size);
-	if (reference == NULL)
-		return GIT_ENOMEM;
+	reference = git__malloc(sizeof(git_reference));
+	GITERR_CHECK_ALLOC(reference);
 
-	memset(reference, 0x0, size);
+	memset(reference, 0x0, sizeof(git_reference));
 	reference->owner = repo;
-	reference->type = type;
 
-	error = normalize_name(normalized, name, (type & GIT_REF_OID));
-	if (error < GIT_SUCCESS)
-		goto cleanup;
-
-	reference->name = git__strdup(normalized);
-	if (reference->name == NULL) {
-		error = GIT_ENOMEM;
-		goto cleanup;
-	}
+	reference->name = git__strdup(name);
+	GITERR_CHECK_ALLOC(reference->name);
 
 	*ref_out = reference;
-
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create reference");
-
-cleanup:
-	reference_free(reference);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create reference");
+	return 0;
 }
 
-static int reference_read(gitfo_buf *file_content, time_t *mtime, const char *repo_path, const char *ref_name)
+static int reference_read(
+	git_buf *file_content,
+	time_t *mtime,
+	const char *repo_path,
+	const char *ref_name,
+	int *updated)
 {
-	struct stat st;
-	char path[GIT_PATH_MAX];
-
-	/* Determine the full path of the file */
-	git__joinpath(path, repo_path, ref_name);
+	git_buf path = GIT_BUF_INIT;
+	int result;
 
-	if (gitfo_stat(path, &st) < 0 || S_ISDIR(st.st_mode))
-		return git__throw(GIT_ENOTFOUND,
-			"Cannot read reference file '%s'", ref_name);
+	assert(file_content && repo_path && ref_name);
 
-	if (mtime)
-		*mtime = st.st_mtime;
-
-	if (file_content)
-		return gitfo_read_file(file_content, path);
-
-	return GIT_SUCCESS;
-}
-
-
-
-
-/*****************************************
- * Internal methods - Loose references
- *****************************************/
-static int loose_update(git_reference *ref)
-{
-	int error;
-	time_t ref_time;
-	gitfo_buf ref_file = GITFO_BUF_INIT;
-
-	if (ref->type & GIT_REF_PACKED)
-		return packed_load(ref->owner);
-
-	error = reference_read(NULL, &ref_time, ref->owner->path_repository, ref->name);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
-
-	if (ref_time == ref->mtime)
-		return GIT_SUCCESS;
-
-	error = reference_read(&ref_file, &ref->mtime, ref->owner->path_repository, ref->name);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
-
-	if (ref->type == GIT_REF_SYMBOLIC)
-		error = loose_parse_symbolic(ref, &ref_file);
-	else if (ref->type == GIT_REF_OID)
-		error = loose_parse_oid(ref, &ref_file);
-	else
-		error = git__throw(GIT_EOBJCORRUPTED,
-			"Invalid reference type (%d) for loose reference", ref->type);
-
-	gitfo_free_buf(&ref_file);
-
-cleanup:
-	if (error != GIT_SUCCESS) {
-		reference_free(ref);
-		git_hashtable_remove(ref->owner->references.loose_cache, ref->name);
-	}
+	/* Determine the full path of the file */
+	if (git_buf_joinpath(&path, repo_path, ref_name) < 0)
+		return -1;
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to update loose reference");
+	result = git_futils_readbuffer_updated(file_content, path.ptr, mtime, updated);
+	git_buf_free(&path);
+	return result;
 }
 
-static int loose_parse_symbolic(git_reference *ref, gitfo_buf *file_content)
+static int loose_parse_symbolic(git_reference *ref, git_buf *file_content)
 {
-	const unsigned int header_len = strlen(GIT_SYMREF);
+	const unsigned int header_len = (unsigned int)strlen(GIT_SYMREF);
 	const char *refname_start;
 	char *eol;
-	reference_symbolic *ref_sym;
 
-	refname_start = (const char *)file_content->data;
-	ref_sym = (reference_symbolic *)ref;
+	refname_start = (const char *)file_content->ptr;
 
-	if (file_content->len < (header_len + 1))
-		return git__throw(GIT_EOBJCORRUPTED,
-			"Failed to parse loose reference. Object too short");
+	if (git_buf_len(file_content) < header_len + 1)
+		goto corrupt;
 
-	/* 
+	/*
 	 * Assume we have already checked for the header
-	 * before calling this function 
+	 * before calling this function
 	 */
-
 	refname_start += header_len;
 
-	free(ref_sym->target);
-	ref_sym->target = git__strdup(refname_start);
-	if (ref_sym->target == NULL)
-		return GIT_ENOMEM;
+	ref->target.symbolic = git__strdup(refname_start);
+	GITERR_CHECK_ALLOC(ref->target.symbolic);
 
 	/* remove newline at the end of file */
-	eol = strchr(ref_sym->target, '\n');
+	eol = strchr(ref->target.symbolic, '\n');
 	if (eol == NULL)
-		return git__throw(GIT_EOBJCORRUPTED,
-			"Failed to parse loose reference. Missing EOL");
+		goto corrupt;
 
 	*eol = '\0';
 	if (eol[-1] == '\r')
 		eol[-1] = '\0';
 
-	return GIT_SUCCESS;
+	return 0;
+
+corrupt:
+	giterr_set(GITERR_REFERENCE, "Corrupted loose reference file");
+	return -1;
 }
 
-static int loose_parse_oid(git_reference *ref, gitfo_buf *file_content)
+static int loose_parse_oid(git_oid *oid, git_buf *file_content)
 {
-	int error;
-	reference_oid *ref_oid;
 	char *buffer;
 
-	buffer = (char *)file_content->data;
-	ref_oid = (reference_oid *)ref;
+	buffer = (char *)file_content->ptr;
 
 	/* File format: 40 chars (OID) + newline */
-	if (file_content->len < GIT_OID_HEXSZ + 1)
-		return git__throw(GIT_EOBJCORRUPTED,
-			"Failed to parse loose reference. Reference too short");
+	if (git_buf_len(file_content) < GIT_OID_HEXSZ + 1)
+		goto corrupt;
 
-	if ((error = git_oid_mkstr(&ref_oid->oid, buffer)) < GIT_SUCCESS)
-		return git__rethrow(GIT_EOBJCORRUPTED, "Failed to parse loose reference.");
+	if (git_oid_fromstr(oid, buffer) < 0)
+		goto corrupt;
 
 	buffer = buffer + GIT_OID_HEXSZ;
 	if (*buffer == '\r')
 		buffer++;
 
 	if (*buffer != '\n')
-		return git__throw(GIT_EOBJCORRUPTED,
-			"Failed to parse loose reference. Missing EOL");
+		goto corrupt;
 
-	return GIT_SUCCESS;
-}
+	return 0;
 
+corrupt:
+	giterr_set(GITERR_REFERENCE, "Corrupted loose reference file");
+	return -1;
+}
 
-static git_rtype loose_guess_rtype(const char *full_path)
+static git_ref_t loose_guess_rtype(const git_buf *full_path)
 {
-	gitfo_buf ref_file = GITFO_BUF_INIT;
-	git_rtype type;
+	git_buf ref_file = GIT_BUF_INIT;
+	git_ref_t type;
 
 	type = GIT_REF_INVALID;
 
-	if (gitfo_read_file(&ref_file, full_path) == GIT_SUCCESS) {
-		if (git__prefixcmp((const char *)(ref_file.data), GIT_SYMREF) == 0)
+	if (git_futils_readbuffer(&ref_file, full_path->ptr) == 0) {
+		if (git__prefixcmp((const char *)(ref_file.ptr), GIT_SYMREF) == 0)
 			type = GIT_REF_SYMBOLIC;
 		else
 			type = GIT_REF_OID;
 	}
 
-	gitfo_free_buf(&ref_file);
+	git_buf_free(&ref_file);
 	return type;
 }
 
-static int loose_lookup(
-		git_reference **ref_out, 
-		git_repository *repo, 
-		const char *name,
-		int skip_symbolic)
+static int loose_lookup(git_reference *ref)
 {
-	int error = GIT_SUCCESS;
-	gitfo_buf ref_file = GITFO_BUF_INIT;
-	git_reference *ref = NULL;
-	time_t ref_time;
+	int result, updated;
+	git_buf ref_file = GIT_BUF_INIT;
 
-	*ref_out = NULL;
+	result = reference_read(&ref_file, &ref->mtime,
+		ref->owner->path_repository, ref->name, &updated);
 
-	error = reference_read(&ref_file, &ref_time, repo->path_repository, name);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	if (result < 0)
+		return result;
 
-	if (git__prefixcmp((const char *)(ref_file.data), GIT_SYMREF) == 0) {
-		if (skip_symbolic)
-			return GIT_SUCCESS;
+	if (!updated)
+		return 0;
 
-		error = reference_create(&ref, repo, name, GIT_REF_SYMBOLIC);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+	if (ref->flags & GIT_REF_SYMBOLIC) {
+		git__free(ref->target.symbolic);
+		ref->target.symbolic = NULL;
+	}
+
+	ref->flags = 0;
 
-		error = loose_parse_symbolic(ref, &ref_file);
+	if (git__prefixcmp((const char *)(ref_file.ptr), GIT_SYMREF) == 0) {
+		ref->flags |= GIT_REF_SYMBOLIC;
+		result = loose_parse_symbolic(ref, &ref_file);
 	} else {
-		error = reference_create(&ref, repo, name, GIT_REF_OID);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+		ref->flags |= GIT_REF_OID;
+		result = loose_parse_oid(&ref->target.oid, &ref_file);
+	}
+
+	git_buf_free(&ref_file);
+	return result;
+}
 
-		error = loose_parse_oid(ref, &ref_file);
+static int loose_lookup_to_packfile(
+		struct packref **ref_out,
+		git_repository *repo,
+		const char *name)
+{
+	git_buf ref_file = GIT_BUF_INIT;
+	struct packref *ref = NULL;
+	size_t name_len;
+
+	*ref_out = NULL;
+
+	if (reference_read(&ref_file, NULL, repo->path_repository, name, NULL) < 0)
+		return -1;
+
+	name_len = strlen(name);
+	ref = git__malloc(sizeof(struct packref) + name_len + 1);
+	GITERR_CHECK_ALLOC(ref);
+
+	memcpy(ref->name, name, name_len);
+	ref->name[name_len] = 0;
+
+	if (loose_parse_oid(&ref->oid, &ref_file) < 0) {
+		git_buf_free(&ref_file);
+		git__free(ref);
+		return -1;
 	}
 
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	ref->flags = GIT_PACKREF_WAS_LOOSE;
 
-	ref->mtime = ref_time;
 	*ref_out = ref;
-	gitfo_free_buf(&ref_file);
-	return GIT_SUCCESS;
-
-cleanup:
-	gitfo_free_buf(&ref_file);
-	reference_free(ref);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to lookup loose reference");
+	git_buf_free(&ref_file);
+	return 0;
 }
 
 static int loose_write(git_reference *ref)
 {
-	git_filebuf file;
-	char ref_path[GIT_PATH_MAX];
-	int error, contents_size;
-	char *ref_contents = NULL;
+	git_filebuf file = GIT_FILEBUF_INIT;
+	git_buf ref_path = GIT_BUF_INIT;
 	struct stat st;
 
-	assert((ref->type & GIT_REF_PACKED) == 0);
-
-	git__joinpath(ref_path, ref->owner->path_repository, ref->name);
+	if (git_buf_joinpath(&ref_path, ref->owner->path_repository, ref->name) < 0)
+		return -1;
 
-	if ((error = git_filebuf_open(&file, ref_path, GIT_FILEBUF_FORCE)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write loose reference");
+	/* Remove a possibly existing empty directory hierarchy 
+	 * which name would collide with the reference name
+	 */
+	if (git_path_isdir(git_buf_cstr(&ref_path)) && 
+		(git_futils_rmdir_r(git_buf_cstr(&ref_path), GIT_DIRREMOVAL_ONLY_EMPTY_DIRS) < 0)) {
+			git_buf_free(&ref_path);
+			return -1;
+		}
 
-	if (ref->type & GIT_REF_OID) {
-		reference_oid *ref_oid = (reference_oid *)ref;
+	if (git_filebuf_open(&file, ref_path.ptr, GIT_FILEBUF_FORCE) < 0) {
+		git_buf_free(&ref_path);
+		return -1;
+	}
 
-		contents_size = GIT_OID_HEXSZ + 1;
-		ref_contents = git__malloc(contents_size);
-		if (ref_contents == NULL) {
-			error = GIT_ENOMEM;
-			goto unlock;
-		}
+	git_buf_free(&ref_path);
 
-		git_oid_fmt(ref_contents, &ref_oid->oid);
+	if (ref->flags & GIT_REF_OID) {
+		char oid[GIT_OID_HEXSZ + 1];
 
-	} else if (ref->type & GIT_REF_SYMBOLIC) { /* GIT_REF_SYMBOLIC */
-		reference_symbolic *ref_sym = (reference_symbolic *)ref;
+		git_oid_fmt(oid, &ref->target.oid);
+		oid[GIT_OID_HEXSZ] = '\0';
 
-		contents_size = strlen(GIT_SYMREF) + strlen(ref_sym->target) + 1;
-		ref_contents = git__malloc(contents_size);
-		if (ref_contents == NULL) {
-			error = GIT_ENOMEM;
-			goto unlock;
-		}
+		git_filebuf_printf(&file, "%s\n", oid);
 
-		strcpy(ref_contents, GIT_SYMREF);
-		strcat(ref_contents, ref_sym->target);
+	} else if (ref->flags & GIT_REF_SYMBOLIC) {
+		git_filebuf_printf(&file, GIT_SYMREF "%s\n", ref->target.symbolic);
 	} else {
-		error = git__throw(GIT_EOBJCORRUPTED, "Failed to write reference. Invalid reference type");
-		goto unlock;
+		assert(0); /* don't let this happen */
 	}
 
-	/* TODO: win32 carriage return when writing references in Windows? */
-	ref_contents[contents_size - 1] = '\n';
-
-	if ((error = git_filebuf_write(&file, ref_contents, contents_size)) < GIT_SUCCESS)
-		goto unlock;
-
-	error = git_filebuf_commit(&file);
-
-	if (gitfo_stat(ref_path, &st) == GIT_SUCCESS)
+	if (p_stat(ref_path.ptr, &st) == 0)
 		ref->mtime = st.st_mtime;
 
-	free(ref_contents);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to write loose reference");
-
-unlock:
-	git_filebuf_cleanup(&file);
-	free(ref_contents);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to write loose reference");
+	return git_filebuf_commit(&file, GIT_REFS_FILE_MODE);
 }
 
-
-
-
-
-
-/*****************************************
- * Internal methods - Packed references
- *****************************************/
-
 static int packed_parse_peel(
-		reference_oid *tag_ref,
-		const char **buffer_out, 
+		struct packref *tag_ref,
+		const char **buffer_out,
 		const char *buffer_end)
 {
 	const char *buffer = *buffer_out + 1;
@@ -436,185 +335,164 @@ static int packed_parse_peel(
 
 	/* Ensure it's not the first entry of the file */
 	if (tag_ref == NULL)
-		return git__throw(GIT_EPACKEDREFSCORRUPTED, "Failed to parse packed reference. Reference is the first entry of the file");
+		goto corrupt;
 
 	/* Ensure reference is a tag */
-	if (git__prefixcmp(tag_ref->ref.name, GIT_REFS_TAGS_DIR) != 0)
-		return git__throw(GIT_EPACKEDREFSCORRUPTED, "Failed to parse packed reference. Reference is not a tag");
+	if (git__prefixcmp(tag_ref->name, GIT_REFS_TAGS_DIR) != 0)
+		goto corrupt;
 
 	if (buffer + GIT_OID_HEXSZ >= buffer_end)
-		return git__throw(GIT_EPACKEDREFSCORRUPTED, "Failed to parse packed reference. Buffer too small");
+		goto corrupt;
 
 	/* Is this a valid object id? */
-	if (git_oid_mkstr(&tag_ref->peel_target, buffer) < GIT_SUCCESS)
-		return git__throw(GIT_EPACKEDREFSCORRUPTED, "Failed to parse packed reference. Not a valid object ID");
+	if (git_oid_fromstr(&tag_ref->peel, buffer) < 0)
+		goto corrupt;
 
 	buffer = buffer + GIT_OID_HEXSZ;
 	if (*buffer == '\r')
 		buffer++;
 
 	if (*buffer != '\n')
-		return git__throw(GIT_EPACKEDREFSCORRUPTED, "Failed to parse packed reference. Buffer not terminated correctly");
+		goto corrupt;
 
 	*buffer_out = buffer + 1;
-	tag_ref->ref.type |= GIT_REF_HAS_PEEL;
+	return 0;
 
-	return GIT_SUCCESS;
+corrupt:
+	giterr_set(GITERR_REFERENCE, "The packed references file is corrupted");
+	return -1;
 }
 
 static int packed_parse_oid(
-		reference_oid **ref_out,
-		git_repository *repo,
+		struct packref **ref_out,
 		const char **buffer_out,
 		const char *buffer_end)
 {
-	reference_oid *ref = NULL;
+	struct packref *ref = NULL;
 
 	const char *buffer = *buffer_out;
 	const char *refname_begin, *refname_end;
 
-	int error = GIT_SUCCESS;
-	int refname_len;
-	char refname[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH];
+	size_t refname_len;
 	git_oid id;
 
 	refname_begin = (buffer + GIT_OID_HEXSZ + 1);
-	if (refname_begin >= buffer_end ||
-		refname_begin[-1] != ' ') {
-		error = GIT_EPACKEDREFSCORRUPTED;
-		goto cleanup;
-	}
+	if (refname_begin >= buffer_end || refname_begin[-1] != ' ')
+		goto corrupt;
 
 	/* Is this a valid object id? */
-	if ((error = git_oid_mkstr(&id, buffer)) < GIT_SUCCESS)
-		goto cleanup;
+	if (git_oid_fromstr(&id, buffer) < 0)
+		goto corrupt;
 
 	refname_end = memchr(refname_begin, '\n', buffer_end - refname_begin);
-	if (refname_end == NULL) {
-		error = GIT_EPACKEDREFSCORRUPTED;
-		goto cleanup;
-	}
+	if (refname_end == NULL)
+		goto corrupt;
 
-	refname_len = refname_end - refname_begin;
+	if (refname_end[-1] == '\r')
+		refname_end--;
 
-	memcpy(refname, refname_begin, refname_len);
-	refname[refname_len] = 0;
+	refname_len = refname_end - refname_begin;
 
-	if (refname[refname_len - 1] == '\r')
-		refname[refname_len - 1] = 0;
+	ref = git__malloc(sizeof(struct packref) + refname_len + 1);
+	GITERR_CHECK_ALLOC(ref);
 
-	error = reference_create((git_reference **)&ref, repo, refname, GIT_REF_OID);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	memcpy(ref->name, refname_begin, refname_len);
+	ref->name[refname_len] = 0;
 
 	git_oid_cpy(&ref->oid, &id);
-	ref->ref.type |= GIT_REF_PACKED;
+
+	ref->flags = 0;
 
 	*ref_out = ref;
 	*buffer_out = refname_end + 1;
 
-	return GIT_SUCCESS;
+	return 0;
 
-cleanup:
-	reference_free((git_reference *)ref);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to parse OID of packed reference");
+corrupt:
+	git__free(ref);
+	giterr_set(GITERR_REFERENCE, "The packed references file is corrupted");
+	return -1;
 }
 
 static int packed_load(git_repository *repo)
 {
-	int error = GIT_SUCCESS;
-	gitfo_buf packfile = GITFO_BUF_INIT;
+	int result, updated;
+	git_buf packfile = GIT_BUF_INIT;
 	const char *buffer_start, *buffer_end;
 	git_refcache *ref_cache = &repo->references;
 
-	/* already loaded */
-	if (repo->references.packfile != NULL) {
-		time_t packed_time;
-
-		/* check if we can read the time of the index;
-		 * if we can read it and it matches the time of the
-		 * index we had previously loaded, we don't need to do
-		 * anything else.
-		 *
-		 * if we cannot load the time (e.g. the packfile
-		 * has disappeared) or the time is different, we
-		 * have to reload the packfile */
+	/* First we make sure we have allocated the hash table */
+	if (ref_cache->packfile == NULL) {
+		ref_cache->packfile = git_strmap_alloc();
+		GITERR_CHECK_ALLOC(ref_cache->packfile);
+	}
 
-		if (!reference_read(NULL, &packed_time, repo->path_repository, GIT_PACKEDREFS_FILE) &&
-			packed_time == ref_cache->packfile_time)
-			return GIT_SUCCESS;
+	result = reference_read(&packfile, &ref_cache->packfile_time,
+		repo->path_repository, GIT_PACKEDREFS_FILE, &updated);
 
-		git_hashtable_clear(repo->references.packfile);
-	} else {
-		ref_cache->packfile = git_hashtable_alloc(
-			default_table_size, 
-			reftable_hash,
-			(git_hash_keyeq_ptr)strcmp);
-
-		if (ref_cache->packfile == NULL)
-			return GIT_ENOMEM;
+	/*
+	 * If we couldn't find the file, we need to clear the table and
+	 * return. On any other error, we return that error. If everything
+	 * went fine and the file wasn't updated, then there's nothing new
+	 * for us here, so just return. Anything else means we need to
+	 * refresh the packed refs.
+	 */
+	if (result == GIT_ENOTFOUND) {
+		git_strmap_clear(ref_cache->packfile);
+		return 0;
 	}
 
-	/* read the packfile from disk;
-	 * store its modification time to check for future reloads */
-	error = reference_read(
-			&packfile,
-			&ref_cache->packfile_time,
-			repo->path_repository,
-			GIT_PACKEDREFS_FILE);
+	if (result < 0)
+		return -1;
 
-	/* there is no packfile on disk; that's ok */
-	if (error == GIT_ENOTFOUND)
-		return GIT_SUCCESS;
+	if (!updated)
+		return 0;
 
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	/*
+	 * At this point, we want to refresh the packed refs. We already
+	 * have the contents in our buffer.
+	 */
+	git_strmap_clear(ref_cache->packfile);
 
-	buffer_start = (const char *)packfile.data;
-	buffer_end = (const char *)(buffer_start) + packfile.len;
+	buffer_start = (const char *)packfile.ptr;
+	buffer_end = (const char *)(buffer_start) + packfile.size;
 
 	while (buffer_start < buffer_end && buffer_start[0] == '#') {
 		buffer_start = strchr(buffer_start, '\n');
-		if (buffer_start == NULL) {
-			error = GIT_EPACKEDREFSCORRUPTED;
-			goto cleanup;
-		}
+		if (buffer_start == NULL)
+			goto parse_failed;
+
 		buffer_start++;
 	}
 
 	while (buffer_start < buffer_end) {
-		reference_oid *ref = NULL;
+		int err;
+		struct packref *ref = NULL;
 
-		error = packed_parse_oid(&ref, repo, &buffer_start, buffer_end);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+		if (packed_parse_oid(&ref, &buffer_start, buffer_end) < 0)
+			goto parse_failed;
 
 		if (buffer_start[0] == '^') {
-			error = packed_parse_peel(ref, &buffer_start, buffer_end);
-			if (error < GIT_SUCCESS)
-				goto cleanup;
+			if (packed_parse_peel(ref, &buffer_start, buffer_end) < 0)
+				goto parse_failed;
 		}
 
-		error = git_hashtable_insert(ref_cache->packfile, ref->ref.name, ref);
-		if (error < GIT_SUCCESS) {
-			reference_free((git_reference *)ref);
-			goto cleanup;
-		}
+		git_strmap_insert(ref_cache->packfile, ref->name, ref, err);
+		if (err < 0)
+			goto parse_failed;
 	}
 
-	gitfo_free_buf(&packfile);
-	return GIT_SUCCESS;
+	git_buf_free(&packfile);
+	return 0;
 
-cleanup:
-	git_hashtable_free(ref_cache->packfile);
+parse_failed:
+	git_strmap_free(ref_cache->packfile);
 	ref_cache->packfile = NULL;
-	gitfo_free_buf(&packfile);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to load packed references");
+	git_buf_free(&packfile);
+	return -1;
 }
 
 
-
-
 struct dirent_list_data {
 	git_repository *repo;
 	size_t repo_path_len;
@@ -624,52 +502,52 @@ struct dirent_list_data {
 	void *callback_payload;
 };
 
-static int _dirent_loose_listall(void *_data, char *full_path)
+static int _dirent_loose_listall(void *_data, git_buf *full_path)
 {
 	struct dirent_list_data *data = (struct dirent_list_data *)_data;
-	char *file_path = full_path + data->repo_path_len;
+	const char *file_path = full_path->ptr + data->repo_path_len;
 
-	if (gitfo_isdir(full_path) == GIT_SUCCESS)
-		return gitfo_dirent(full_path, GIT_PATH_MAX, _dirent_loose_listall, _data);
+	if (git_path_isdir(full_path->ptr) == true)
+		return git_path_direach(full_path, _dirent_loose_listall, _data);
 
 	/* do not add twice a reference that exists already in the packfile */
 	if ((data->list_flags & GIT_REF_PACKED) != 0 &&
-		git_hashtable_lookup(data->repo->references.packfile, file_path) != NULL)
-		return GIT_SUCCESS;
+		git_strmap_exists(data->repo->references.packfile, file_path))
+		return 0;
 
 	if (data->list_flags != GIT_REF_LISTALL) {
 		if ((data->list_flags & loose_guess_rtype(full_path)) == 0)
-			return GIT_SUCCESS; /* we are filtering out this reference */
+			return 0; /* we are filtering out this reference */
 	}
 
 	return data->callback(file_path, data->callback_payload);
 }
 
-static int _dirent_loose_load(void *data, char *full_path)
+static int _dirent_loose_load(void *data, git_buf *full_path)
 {
 	git_repository *repository = (git_repository *)data;
-	git_reference *reference, *old_ref;
-	char *file_path;
-	int error;
+	void *old_ref = NULL;
+	struct packref *ref;
+	const char *file_path;
+	int err;
 
-	if (gitfo_isdir(full_path) == GIT_SUCCESS)
-		return gitfo_dirent(full_path, GIT_PATH_MAX, _dirent_loose_load, repository);
+	if (git_path_isdir(full_path->ptr) == true)
+		return git_path_direach(full_path, _dirent_loose_load, repository);
 
-	file_path = full_path + strlen(repository->path_repository);
-	error = loose_lookup(&reference, repository, file_path, 1);
-	if (error == GIT_SUCCESS && reference != NULL) {
-		reference->type |= GIT_REF_PACKED;
+	file_path = full_path->ptr + strlen(repository->path_repository);
 
-		if (git_hashtable_insert2(repository->references.packfile, reference->name, reference, (void **)&old_ref) < GIT_SUCCESS) {
-			reference_free(reference);
-			return GIT_ENOMEM;
-		}
+	if (loose_lookup_to_packfile(&ref, repository, file_path) < 0)
+		return -1;
 
-		if (old_ref != NULL)
-			reference_free(old_ref);
+	git_strmap_insert2(
+		repository->references.packfile, ref->name, ref, old_ref, err);
+	if (err < 0) {
+		git__free(ref);
+		return -1;
 	}
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to load loose dirent");
+	git__free(old_ref);
+	return 0;
 }
 
 /*
@@ -680,45 +558,37 @@ static int _dirent_loose_load(void *data, char *full_path)
  */
 static int packed_loadloose(git_repository *repository)
 {
-	char refs_path[GIT_PATH_MAX];
+	git_buf refs_path = GIT_BUF_INIT;
+	int result;
 
 	/* the packfile must have been previously loaded! */
 	assert(repository->references.packfile);
 
-	git__joinpath(refs_path, repository->path_repository, GIT_REFS_DIR);
-
-	/* Remove any loose references from the cache */
-	{
-		const void *GIT_UNUSED(_unused);
-		git_reference *reference;
-
-		GIT_HASHTABLE_FOREACH(repository->references.loose_cache, _unused, reference,
-			reference_free(reference);
-		);
-	}
-
-	git_hashtable_clear(repository->references.loose_cache);
+	if (git_buf_joinpath(&refs_path, repository->path_repository, GIT_REFS_DIR) < 0)
+		return -1;
 
 	/*
 	 * Load all the loose files from disk into the Packfile table.
 	 * This will overwrite any old packed entries with their
-	 * updated loose versions 
+	 * updated loose versions
 	 */
-	return gitfo_dirent(refs_path, GIT_PATH_MAX, _dirent_loose_load, repository);
+	result = git_path_direach(&refs_path, _dirent_loose_load, repository);
+	git_buf_free(&refs_path);
+
+	return result;
 }
 
 /*
  * Write a single reference into a packfile
  */
-static int packed_write_ref(reference_oid *ref, git_filebuf *file)
+static int packed_write_ref(struct packref *ref, git_filebuf *file)
 {
-	int error;
 	char oid[GIT_OID_HEXSZ + 1];
 
 	git_oid_fmt(oid, &ref->oid);
 	oid[GIT_OID_HEXSZ] = 0;
 
-	/* 
+	/*
 	 * For references that peel to an object in the repo, we must
 	 * write the resulting peel on a separate line, e.g.
 	 *
@@ -728,48 +598,48 @@ static int packed_write_ref(reference_oid *ref, git_filebuf *file)
 	 * This obviously only applies to tags.
 	 * The required peels have already been loaded into `ref->peel_target`.
 	 */
-	if (ref->ref.type & GIT_REF_HAS_PEEL) {
+	if (ref->flags & GIT_PACKREF_HAS_PEEL) {
 		char peel[GIT_OID_HEXSZ + 1];
-		git_oid_fmt(peel, &ref->peel_target);
+		git_oid_fmt(peel, &ref->peel);
 		peel[GIT_OID_HEXSZ] = 0;
 
-		error = git_filebuf_printf(file, "%s %s\n^%s\n", oid, ref->ref.name, peel);
+		if (git_filebuf_printf(file, "%s %s\n^%s\n", oid, ref->name, peel) < 0)
+			return -1;
 	} else {
-		error = git_filebuf_printf(file, "%s %s\n", oid, ref->ref.name);
+		if (git_filebuf_printf(file, "%s %s\n", oid, ref->name) < 0)
+			return -1;
 	}
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to write packed reference");
+	return 0;
 }
 
 /*
  * Find out what object this reference resolves to.
  *
- * For references that point to a 'big' tag (e.g. an 
+ * For references that point to a 'big' tag (e.g. an
  * actual tag object on the repository), we need to
  * cache on the packfile the OID of the object to
  * which that 'big tag' is pointing to.
  */
-static int packed_find_peel(reference_oid *ref)
+static int packed_find_peel(git_repository *repo, struct packref *ref)
 {
 	git_object *object;
-	int error;
 
-	if (ref->ref.type & GIT_REF_HAS_PEEL)
-		return GIT_SUCCESS;
+	if (ref->flags & GIT_PACKREF_HAS_PEEL)
+		return 0;
 
 	/*
 	 * Only applies to tags, i.e. references
 	 * in the /refs/tags folder
 	 */
-	if (git__prefixcmp(ref->ref.name, GIT_REFS_TAGS_DIR) != 0)
-		return GIT_SUCCESS;
+	if (git__prefixcmp(ref->name, GIT_REFS_TAGS_DIR) != 0)
+		return 0;
 
 	/*
 	 * Find the tagged object in the repository
 	 */
-	error = git_object_lookup(&object, ref->ref.owner, &ref->oid, GIT_OBJ_ANY);
-	if (error < GIT_SUCCESS)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to find packed reference");
+	if (git_object_lookup(&object, repo, &ref->oid, GIT_OBJ_ANY) < 0)
+		return -1;
 
 	/*
 	 * If the tagged object is a Tag object, we need to resolve it;
@@ -782,8 +652,8 @@ static int packed_find_peel(reference_oid *ref)
 		/*
 		 * Find the object pointed at by this tag
 		 */
-		git_oid_cpy(&ref->peel_target, git_tag_target_oid(tag));
-		ref->ref.type |= GIT_REF_HAS_PEEL;
+		git_oid_cpy(&ref->peel, git_tag_target_oid(tag));
+		ref->flags |= GIT_PACKREF_HAS_PEEL;
 
 		/*
 		 * The reference has now cached the resolved OID, and is
@@ -792,9 +662,8 @@ static int packed_find_peel(reference_oid *ref)
 		 */
 	}
 
-	git_object_close(object);
-
-	return GIT_SUCCESS;
+	git_object_free(object);
+	return 0;
 }
 
 /*
@@ -811,44 +680,45 @@ static int packed_find_peel(reference_oid *ref)
 static int packed_remove_loose(git_repository *repo, git_vector *packing_list)
 {
 	unsigned int i;
-	char full_path[GIT_PATH_MAX];
-	int error = GIT_SUCCESS;
-	git_reference *reference;
+	git_buf full_path = GIT_BUF_INIT;
+	int failed = 0;
 
 	for (i = 0; i < packing_list->length; ++i) {
-		git_reference *ref = git_vector_get(packing_list, i);
+		struct packref *ref = git_vector_get(packing_list, i);
 
-		/* Ensure the packed reference doesn't exist
-		 * in a (more up-to-date?) state as a loose reference
-		 */
-		reference = git_hashtable_lookup(ref->owner->references.loose_cache, ref->name);
-		if (reference != NULL)
+		if ((ref->flags & GIT_PACKREF_WAS_LOOSE) == 0)
 			continue;
 
-		git__joinpath(full_path, repo->path_repository, ref->name);
+		if (git_buf_joinpath(&full_path, repo->path_repository, ref->name) < 0)
+			return -1; /* critical; do not try to recover on oom */
 
-		if (gitfo_exists(full_path) == GIT_SUCCESS &&
-			gitfo_unlink(full_path) < GIT_SUCCESS)
-			error = GIT_EOSERR;
+		if (git_path_exists(full_path.ptr) == true && p_unlink(full_path.ptr) < 0) {
+			if (failed)
+				continue;
+
+			giterr_set(GITERR_REFERENCE,
+				"Failed to remove loose reference '%s' after packing: %s",
+				full_path.ptr, strerror(errno));
+
+			failed = 1;
+		}
 
 		/*
 		 * if we fail to remove a single file, this is *not* good,
 		 * but we should keep going and remove as many as possible.
 		 * After we've removed as many files as possible, we return
 		 * the error code anyway.
-		 *
-		 * TODO: mark this with a very special error code?
-		 * GIT_EFAILTORMLOOSE
 		 */
 	}
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to remove loose packed reference");
+	git_buf_free(&full_path);
+	return failed ? -1 : 0;
 }
 
 static int packed_sort(const void *a, const void *b)
 {
-	const git_reference *ref_a = *(const git_reference **)a;
-	const git_reference *ref_b = *(const git_reference **)b;
+	const struct packref *ref_a = (const struct packref *)a;
+	const struct packref *ref_b = (const struct packref *)b;
 
 	return strcmp(ref_a->name, ref_b->name);
 }
@@ -858,408 +728,450 @@ static int packed_sort(const void *a, const void *b)
  */
 static int packed_write(git_repository *repo)
 {
-	git_filebuf pack_file;
-	int error;
+	git_filebuf pack_file = GIT_FILEBUF_INIT;
 	unsigned int i;
-	char pack_file_path[GIT_PATH_MAX];
-
+	git_buf pack_file_path = GIT_BUF_INIT;
 	git_vector packing_list;
-	size_t total_refs;
+	unsigned int total_refs;
 
 	assert(repo && repo->references.packfile);
 
-	total_refs = repo->references.packfile->key_count;
-	if ((error = git_vector_init(&packing_list, total_refs, packed_sort)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write packed reference");
+	total_refs =
+		(unsigned int)git_strmap_num_entries(repo->references.packfile);
+
+	if (git_vector_init(&packing_list, total_refs, packed_sort) < 0)
+		return -1;
 
 	/* Load all the packfile into a vector */
 	{
-		git_reference *reference;
-		const void *GIT_UNUSED(_unused);
+		struct packref *reference;
 
-		GIT_HASHTABLE_FOREACH(repo->references.packfile, _unused, reference,
-			git_vector_insert(&packing_list, reference);  /* cannot fail: vector already has the right size */
-		);
+		/* cannot fail: vector already has the right size */
+		git_strmap_foreach_value(repo->references.packfile, reference, {
+			git_vector_insert(&packing_list, reference);
+		});
 	}
 
 	/* sort the vector so the entries appear sorted on the packfile */
 	git_vector_sort(&packing_list);
 
 	/* Now we can open the file! */
-	git__joinpath(pack_file_path, repo->path_repository, GIT_PACKEDREFS_FILE);
-	if ((error = git_filebuf_open(&pack_file, pack_file_path, 0)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write packed reference");
+	if (git_buf_joinpath(&pack_file_path, repo->path_repository, GIT_PACKEDREFS_FILE) < 0)
+		goto cleanup_memory;
+
+	if (git_filebuf_open(&pack_file, pack_file_path.ptr, 0) < 0)
+		goto cleanup_packfile;
 
 	/* Packfiles have a header... apparently
 	 * This is in fact not required, but we might as well print it
 	 * just for kicks */
-	if ((error = git_filebuf_printf(&pack_file, "%s\n", GIT_PACKEDREFS_HEADER)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write packed reference");
+	if (git_filebuf_printf(&pack_file, "%s\n", GIT_PACKEDREFS_HEADER) < 0)
+		goto cleanup_packfile;
 
 	for (i = 0; i < packing_list.length; ++i) {
-		reference_oid *ref = (reference_oid *)git_vector_get(&packing_list, i);
-
-		/* only direct references go to the packfile; otherwise
-		 * this is a disaster */
-		assert(ref->ref.type & GIT_REF_OID);
+		struct packref *ref = (struct packref *)git_vector_get(&packing_list, i);
 
-		if ((error = packed_find_peel(ref)) < GIT_SUCCESS)
-			goto cleanup;
+		if (packed_find_peel(repo, ref) < 0)
+			goto cleanup_packfile;
 
-		if ((error = packed_write_ref(ref, &pack_file)) < GIT_SUCCESS)
-			goto cleanup;
+		if (packed_write_ref(ref, &pack_file) < 0)
+			goto cleanup_packfile;
 	}
 
-cleanup:
 	/* if we've written all the references properly, we can commit
 	 * the packfile to make the changes effective */
-	if (error == GIT_SUCCESS) {
-		error = git_filebuf_commit(&pack_file);
-
-		/* when and only when the packfile has been properly written,
-		 * we can go ahead and remove the loose refs */
-		if (error == GIT_SUCCESS) {
-			struct stat st;
+	if (git_filebuf_commit(&pack_file, GIT_PACKEDREFS_FILE_MODE) < 0)
+		goto cleanup_memory;
 
-			error = packed_remove_loose(repo, &packing_list);
+	/* when and only when the packfile has been properly written,
+	 * we can go ahead and remove the loose refs */
+	 if (packed_remove_loose(repo, &packing_list) < 0)
+		 goto cleanup_memory;
 
-			if (gitfo_stat(pack_file_path, &st) == GIT_SUCCESS)
-				repo->references.packfile_time = st.st_mtime;
-		}
-	}
-	else git_filebuf_cleanup(&pack_file);
+	 {
+		struct stat st;
+		if (p_stat(pack_file_path.ptr, &st) == 0)
+			repo->references.packfile_time = st.st_mtime;
+	 }
 
 	git_vector_free(&packing_list);
+	git_buf_free(&pack_file_path);
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to write packed reference");
-}
+	/* we're good now */
+	return 0;
 
-/*****************************************
- * Internal methods - reference creation
- *****************************************/
+cleanup_packfile:
+	git_filebuf_cleanup(&pack_file);
 
-static int reference_create_symbolic(git_reference **ref_out, git_repository *repo, const char *name, const char *target, int force)
-{
-	char normalized[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH];
-	int error = GIT_SUCCESS, updated = 0;
-	git_reference *ref = NULL, *old_ref = NULL;
+cleanup_memory:
+	git_vector_free(&packing_list);
+	git_buf_free(&pack_file_path);
 
-	if (git_reference_lookup(&ref, repo, name) == GIT_SUCCESS && !force)
-		return git__throw(GIT_EEXISTS, "Failed to create symbolic reference. Reference already exists");
+	return -1;
+}
 
-	/*
-	 * If they old ref was of the same type, then we can just update
-	 * it (once we've checked that the target is valid). Otherwise we
-	 * need a new reference because we can't make a symbolic ref out
-	 * of an oid one.
-	 * If if didn't exist, then we need to create a new one anyway.
-     */
-	if (ref && ref->type & GIT_REF_SYMBOLIC){
-		updated = 1;
-	} else {
-		ref = NULL;
-		error = reference_create(&ref, repo, name, GIT_REF_SYMBOLIC);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
-	}
+struct reference_available_t {
+	const char *new_ref;
+	const char *old_ref;
+	int available;
+};
 
-	/* The target can aither be the name of an object id reference or the name of another symbolic reference */
-	error = normalize_name(normalized, target, 0);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+static int _reference_available_cb(const char *ref, void *data)
+{
+	struct reference_available_t *d;
 
-	/* set the target; this will write the reference on disk */
-	error = git_reference_set_target(ref, normalized);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	assert(ref && data);
+	d = (struct reference_available_t *)data;
 
-	/*
-	 * If we didn't update the ref, then we need to insert or replace
-	 * it in the loose cache. If we replaced a ref, free it.
-	 */
-	if (!updated){
-		error = git_hashtable_insert2(repo->references.loose_cache, ref->name, ref, (void **) &old_ref);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+	if (!d->old_ref || strcmp(d->old_ref, ref)) {
+		size_t reflen = strlen(ref);
+		size_t newlen = strlen(d->new_ref);
+		size_t cmplen = reflen < newlen ? reflen : newlen;
+		const char *lead = reflen < newlen ? d->new_ref : ref;
 
-		if(old_ref)
-			reference_free(old_ref);
+		if (!strncmp(d->new_ref, ref, cmplen) && lead[cmplen] == '/') {
+			d->available = 0;
+			return -1;
+		}
 	}
 
-	*ref_out = ref;
-
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create symbolic reference");
-
-cleanup:
-	reference_free(ref);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create symbolic reference");
+	return 0;
 }
 
-static int reference_create_oid(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id, int force)
+static int reference_path_available(
+	git_repository *repo,
+	const char *ref,
+	const char* old_ref)
 {
-	int error = GIT_SUCCESS, updated = 0;
-	git_reference *ref = NULL, *old_ref = NULL;
+	struct reference_available_t data;
 
-	if(git_reference_lookup(&ref, repo, name) == GIT_SUCCESS && !force)
-		return git__throw(GIT_EEXISTS, "Failed to create reference OID. Reference already exists");
+	data.new_ref = ref;
+	data.old_ref = old_ref;
+	data.available = 1;
 
-	/*
-	 * If they old ref was of the same type, then we can just update
-	 * it (once we've checked that the target is valid). Otherwise we
-	 * need a new reference because we can't make a symbolic ref out
-	 * of an oid one.
-	 * If if didn't exist, then we need to create a new one anyway.
-     */
-	if (ref && ref-> type & GIT_REF_OID){
-		updated = 1;
-	} else {
-		ref = NULL;
-		error = reference_create(&ref, repo, name, GIT_REF_OID);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+	if (git_reference_foreach(repo, GIT_REF_LISTALL,
+		_reference_available_cb, (void *)&data) < 0)
+		return -1;
+
+	if (!data.available) {
+		giterr_set(GITERR_REFERENCE,
+			"The path to reference '%s' collides with an existing one", ref);
+		return -1;
 	}
 
-	/* set the oid; this will write the reference on disk */
-	error = git_reference_set_oid(ref, id);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	return 0;
+}
 
-	if(!updated){
-		error = git_hashtable_insert2(repo->references.loose_cache, ref->name, ref, (void **) &old_ref);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+static int reference_exists(int *exists, git_repository *repo, const char *ref_name)
+{
+	git_buf ref_path = GIT_BUF_INIT;
 
-		if(old_ref)
-			reference_free(old_ref);
-	}
+	if (packed_load(repo) < 0)
+		return -1;
 
-	*ref_out = ref;
+	if (git_buf_joinpath(&ref_path, repo->path_repository, ref_name) < 0)
+		return -1;
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create reference OID");
+	if (git_path_isfile(ref_path.ptr) == true ||
+		git_strmap_exists(repo->references.packfile, ref_path.ptr))
+	{
+		*exists = 1;
+	} else {
+		*exists = 0;
+	}
 
-cleanup:
-	reference_free(ref);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create reference OID");
+	git_buf_free(&ref_path);
+	return 0;
 }
 
 /*
- * Rename a reference
- *
- * If the reference is packed, we need to rewrite the
- * packfile to remove the reference from it and create
- * the reference back as a loose one.
+ * Check if a reference could be written to disk, based on:
  *
- * If the reference is loose, we just rename it on
- * the filesystem.
+ *	- Whether a reference with the same name already exists,
+ *	and we are allowing or disallowing overwrites
  *
- * We also need to re-insert the reference on its corresponding
- * in-memory cache, since the caches are indexed by refname.
+ *	- Whether the name of the reference would collide with
+ *	an existing path
  */
-static int reference_rename(git_reference *ref, const char *new_name, int force)
+static int reference_can_write(
+	git_repository *repo,
+	const char *refname,
+	const char *previous_name,
+	int force)
 {
-	int error;
-	char *old_name;
-	char old_path[GIT_PATH_MAX], new_path[GIT_PATH_MAX], normalized_name[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH];
-	git_reference *looked_up_ref, *old_ref = NULL;
+	/* see if the reference shares a path with an existing reference;
+	 * if a path is shared, we cannot create the reference, even when forcing */
+	if (reference_path_available(repo, refname, previous_name) < 0)
+		return -1;
+
+	/* check if the reference actually exists, but only if we are not forcing
+	 * the rename. If we are forcing, it's OK to overwrite */
+	if (!force) {
+		int exists;
+
+		if (reference_exists(&exists, repo, refname) < 0)
+			return -1;
+
+		/* We cannot proceed if the reference already exists and we're not forcing
+		 * the rename; the existing one would be overwritten */
+		if (exists) {
+			giterr_set(GITERR_REFERENCE,
+				"A reference with that name (%s) already exists", refname);
+			return GIT_EEXISTS;
+		}
+	}
+
+	/* FIXME: if the reference exists and we are forcing, do we really need to
+	 * remove the reference first?
+	 *
+	 * Two cases:
+	 *
+	 *	- the reference already exists and is loose: not a problem, the file
+	 *	gets overwritten on disk
+	 *
+	 *	- the reference already exists and is packed: we write a new one as
+	 *	loose, which by all means renders the packed one useless
+	 */
+
+	return 0;
+}
 
-	assert(ref);
 
-	/* Ensure the name is valid */
-	error = normalize_name(normalized_name, new_name, ref->type & GIT_REF_OID);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to rename reference");
+static int packed_lookup(git_reference *ref)
+{
+	struct packref *pack_ref = NULL;
+	git_strmap *packfile_refs;
+	khiter_t pos;
+
+	if (packed_load(ref->owner) < 0)
+		return -1;
+
+	/* maybe the packfile hasn't changed at all, so we don't
+	 * have to re-lookup the reference */
+	if ((ref->flags & GIT_REF_PACKED) &&
+		ref->mtime == ref->owner->references.packfile_time)
+		return 0;
+
+	if (ref->flags & GIT_REF_SYMBOLIC) {
+		git__free(ref->target.symbolic);
+		ref->target.symbolic = NULL;
+	}
 
-	/* Ensure we're not going to overwrite an existing reference
-	   unless the user has allowed us */
-	error = git_reference_lookup(&looked_up_ref, ref->owner, new_name);
-	if (error == GIT_SUCCESS && !force)
-		return git__throw(GIT_EEXISTS, "Failed to rename reference. Reference already exists");
+	/* Look up on the packfile */
+	packfile_refs = ref->owner->references.packfile;
+	pos = git_strmap_lookup_index(packfile_refs, ref->name);
+	if (!git_strmap_valid_index(packfile_refs, pos)) {
+		giterr_set(GITERR_REFERENCE, "Reference '%s' not found", ref->name);
+		return GIT_ENOTFOUND;
+	}
 
-	if (error < GIT_SUCCESS &&
-	    error != GIT_ENOTFOUND)
-		return git__rethrow(error, "Failed to rename reference");
+	pack_ref = git_strmap_value_at(packfile_refs, pos);
 
+	ref->flags = GIT_REF_OID | GIT_REF_PACKED;
+	ref->mtime = ref->owner->references.packfile_time;
+	git_oid_cpy(&ref->target.oid, &pack_ref->oid);
 
-	old_name = ref->name;
-	ref->name = git__strdup(new_name);
+	return 0;
+}
 
-	if (ref->name == NULL) {
-		ref->name = old_name;
-		return GIT_ENOMEM;
+static int reference_lookup(git_reference *ref)
+{
+	int result;
+
+	result = loose_lookup(ref);
+	if (result == 0)
+		return 0;
+
+	/* only try to lookup this reference on the packfile if it
+	 * wasn't found on the loose refs; not if there was a critical error */
+	if (result == GIT_ENOTFOUND) {
+		giterr_clear();
+		result = packed_lookup(ref);
+		if (result == 0)
+			return 0;
 	}
 
-	if (ref->type & GIT_REF_PACKED) {
-		/* write the packfile to disk; note
-		 * that the state of the in-memory cache is not
-		 * consistent, because the reference is indexed
-		 * by its old name but it already has the new one.
-		 * This doesn't affect writing, though, and allows
-		 * us to rollback if writing fails
-		 */
+	/* unexpected error; free the reference */
+	git_reference_free(ref);
+	return result;
+}
 
-		ref->type &= ~GIT_REF_PACKED;
+/*
+ * Delete a reference.
+ * This is an internal method; the reference is removed
+ * from disk or the packfile, but the pointer is not freed
+ */
+static int reference_delete(git_reference *ref)
+{
+	int result;
 
-		/* Create the loose ref under its new name */
-		error = loose_write(ref);
-		if (error < GIT_SUCCESS) {
-			ref->type |= GIT_REF_PACKED;
-			goto cleanup;
+	assert(ref);
+
+	/* If the reference is packed, this is an expensive operation.
+	 * We need to reload the packfile, remove the reference from the
+	 * packing list, and repack */
+	if (ref->flags & GIT_REF_PACKED) {
+		git_strmap *packfile_refs;
+		struct packref *packref;
+		khiter_t pos;
+
+		/* load the existing packfile */
+		if (packed_load(ref->owner) < 0)
+			return -1;
+
+		packfile_refs = ref->owner->references.packfile;
+		pos = git_strmap_lookup_index(packfile_refs, ref->name);
+		if (!git_strmap_valid_index(packfile_refs, pos)) {
+			giterr_set(GITERR_REFERENCE,
+				"Reference %s stopped existing in the packfile", ref->name);
+			return -1;
 		}
 
-		/* Remove from the packfile cache in order to avoid packing it back
-		 * Note : we do not rely on git_reference_delete() because this would
-		 * invalidate the reference.
-		 */
-		git_hashtable_remove(ref->owner->references.packfile, old_name);
+		packref = git_strmap_value_at(packfile_refs, pos);
+		git_strmap_delete_at(packfile_refs, pos);
 
-		/* Recreate the packed-refs file without the reference */
-		error = packed_write(ref->owner);
-		if (error < GIT_SUCCESS)
-			goto rename_loose_to_old_name;
+		git__free(packref);
+		if (packed_write(ref->owner) < 0)
+			return -1;
 
+	/* If the reference is loose, we can just remove the reference
+	 * from the filesystem */
 	} else {
-		git__joinpath(old_path, ref->owner->path_repository, old_name);
-		git__joinpath(new_path, ref->owner->path_repository, ref->name);
+		git_reference *ref_in_pack;
+		git_buf full_path = GIT_BUF_INIT;
 
-		error = gitfo_mv_force(old_path, new_path);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+		if (git_buf_joinpath(&full_path, ref->owner->path_repository, ref->name) < 0)
+			return -1;
 
-		/* Once succesfully renamed, remove from the cache the reference known by its old name*/
-		git_hashtable_remove(ref->owner->references.loose_cache, old_name);
-	}
+		result = p_unlink(full_path.ptr);
+		git_buf_free(&full_path); /* done with path at this point */
 
-	/* Store the renamed reference into the loose ref cache */
-	error = git_hashtable_insert2(ref->owner->references.loose_cache, ref->name, ref, (void **) &old_ref);
+		if (result < 0) {
+			giterr_set(GITERR_OS, "Failed to unlink '%s'", full_path.ptr);
+			return -1;
+		}
 
-	/* If we force-replaced, we need to free the old reference */
-	if(old_ref)
-		reference_free(old_ref);
+		/* When deleting a loose reference, we have to ensure that an older
+		 * packed version of it doesn't exist */
+		if (git_reference_lookup(&ref_in_pack, ref->owner, ref->name) == 0) {
+			assert((ref_in_pack->flags & GIT_REF_PACKED) != 0);
+			return git_reference_delete(ref_in_pack);
+		}
 
-	free(old_name);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to rename reference");
+		giterr_clear();
+	}
 
-cleanup:
-	/* restore the old name if this failed */
-	free(ref->name);
-	ref->name = old_name;
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to rename reference");
-
-rename_loose_to_old_name:
-	/* If we hit this point. Something *bad* happened! Think "Ghostbusters
-	 * crossing the streams" definition of bad.
-	 * Either the packed-refs has been correctly generated and something else
-	 * has gone wrong, or the writing of the new packed-refs has failed, and
-	 * we're stuck with the old one. As a loose ref always takes priority over
-	 * a packed ref, we'll eventually try and rename the generated loose ref to
-	 * its former name. It even that fails, well... we might have lost the reference
-	 * for good. :-/
-	*/
-
-	git__joinpath(old_path, ref->owner->path_repository, ref->name);
-	git__joinpath(new_path, ref->owner->path_repository, old_name);
-
-	/* No error checking. We'll return the initial error */
-	gitfo_mv_force(old_path, new_path);
-
-	/* restore the old name */
-	free(ref->name);
-	ref->name = old_name;
-
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to rename reference");
+	return 0;
 }
 
-/*****************************************
- * External Library API
- *****************************************/
+int git_reference_delete(git_reference *ref)
+{
+	int result = reference_delete(ref);
+	git_reference_free(ref);
+	return result;
+}
 
-/**
- * Constructors
- */
-int git_reference_lookup(git_reference **ref_out, git_repository *repo, const char *name)
+int git_reference_lookup(git_reference **ref_out,
+	git_repository *repo, const char *name)
+{
+	return git_reference_lookup_resolved(ref_out, repo, name, 0);
+}
+
+int git_reference_name_to_oid(
+	git_oid *out, git_repository *repo, const char *name)
 {
 	int error;
-	char normalized_name[GIT_PATH_MAX];
+	git_reference *ref;
+
+	if ((error = git_reference_lookup_resolved(&ref, repo, name, -1)) < 0)
+		return error;
+
+	git_oid_cpy(out, git_reference_oid(ref));
+	git_reference_free(ref);
+	return 0;
+}
+
+int git_reference_lookup_resolved(
+	git_reference **ref_out,
+	git_repository *repo,
+	const char *name,
+	int max_nesting)
+{
+	git_reference *scan;
+	int result, nesting;
 
 	assert(ref_out && repo && name);
 
 	*ref_out = NULL;
 
-	error = normalize_name(normalized_name, name, 0);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to lookup reference");
+	if (max_nesting > MAX_NESTING_LEVEL)
+		max_nesting = MAX_NESTING_LEVEL;
+	else if (max_nesting < 0)
+		max_nesting = DEFAULT_NESTING_LEVEL;
 
-	/* First, check has been previously loaded and cached */
-	*ref_out = git_hashtable_lookup(repo->references.loose_cache, normalized_name);
-	if (*ref_out != NULL)
-		return loose_update(*ref_out);
+	scan = git__calloc(1, sizeof(git_reference));
+	GITERR_CHECK_ALLOC(scan);
 
-	/* Then check if there is a loose file for that reference.*/
-	error = loose_lookup(ref_out, repo, normalized_name, 0);
+	scan->name = git__calloc(GIT_REFNAME_MAX + 1, sizeof(char));
+	GITERR_CHECK_ALLOC(scan->name);
 
-	/* If the file exists, we store it on the cache */
-	if (error == GIT_SUCCESS)
-		return git_hashtable_insert(repo->references.loose_cache, (*ref_out)->name, (*ref_out));
+	if ((result = normalize_name(scan->name, GIT_REFNAME_MAX, name, 0)) < 0) {
+		git_reference_free(scan);
+		return result;
+	}
 
-	/* The loose lookup has failed, but not because the reference wasn't found;
-	 * probably the loose reference is corrupted. this is bad. */
-	if (error != GIT_ENOTFOUND)
-		return git__rethrow(error, "Failed to lookup reference");
+	scan->target.symbolic = git__strdup(scan->name);
+	GITERR_CHECK_ALLOC(scan->target.symbolic);
 
-	/*
-	 * If we cannot find a loose reference, we look into the packfile
-	 * Load the packfile first if it hasn't been loaded 
-	 */
-	/* load all the packed references */
-	error = packed_load(repo);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to lookup reference");
-
-	/* Look up on the packfile */
-	*ref_out = git_hashtable_lookup(repo->references.packfile, normalized_name);
-	if (*ref_out != NULL)
-		return GIT_SUCCESS;
+	scan->owner = repo;
+	scan->flags = GIT_REF_SYMBOLIC;
 
-	/* The reference doesn't exist anywhere */
-	return git__throw(GIT_ENOTFOUND, "Failed to lookup reference. Reference doesn't exist");
-}
+	for (nesting = max_nesting;
+		 nesting >= 0 && (scan->flags & GIT_REF_SYMBOLIC) != 0;
+		 nesting--)
+	{
+		if (nesting != max_nesting)
+			strncpy(scan->name, scan->target.symbolic, GIT_REFNAME_MAX);
 
-int git_reference_create_symbolic(git_reference **ref_out, git_repository *repo, const char *name, const char *target)
-{
-	return reference_create_symbolic(ref_out, repo, name, target, 0);
-}
+		scan->mtime = 0;
 
-int git_reference_create_symbolic_f(git_reference **ref_out, git_repository *repo, const char *name, const char *target)
-{
-	return reference_create_symbolic(ref_out, repo, name, target, 1);
-}
+		if ((result = reference_lookup(scan)) < 0)
+			return result; /* lookup git_reference_free on scan already */
+	}
 
-int git_reference_create_oid(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id)
-{
-	return reference_create_oid(ref_out, repo, name, id, 0);
-}
+	if ((scan->flags & GIT_REF_OID) == 0 && max_nesting != 0) {
+		giterr_set(GITERR_REFERENCE,
+			"Cannot resolve reference (>%u levels deep)", max_nesting);
+		git_reference_free(scan);
+		return -1;
+	}
 
-int git_reference_create_oid_f(git_reference **ref_out, git_repository *repo, const char *name, const git_oid *id)
-{
-	return reference_create_oid(ref_out, repo, name, id, 1);
+	*ref_out = scan;
+	return 0;
 }
 
 /**
  * Getters
  */
-git_rtype git_reference_type(git_reference *ref)
+git_ref_t git_reference_type(git_reference *ref)
 {
 	assert(ref);
 
-	if (ref->type & GIT_REF_OID)
+	if (ref->flags & GIT_REF_OID)
 		return GIT_REF_OID;
 
-	if (ref->type & GIT_REF_SYMBOLIC)
+	if (ref->flags & GIT_REF_SYMBOLIC)
 		return GIT_REF_SYMBOLIC;
 
 	return GIT_REF_INVALID;
 }
 
+int git_reference_is_packed(git_reference *ref)
+{
+	assert(ref);
+	return !!(ref->flags & GIT_REF_PACKED);
+}
+
 const char *git_reference_name(git_reference *ref)
 {
 	assert(ref);
@@ -1276,110 +1188,129 @@ const git_oid *git_reference_oid(git_reference *ref)
 {
 	assert(ref);
 
-	if ((ref->type & GIT_REF_OID) == 0)
+	if ((ref->flags & GIT_REF_OID) == 0)
 		return NULL;
 
-	if (loose_update(ref) < GIT_SUCCESS)
-		return NULL;
-
-	return &((reference_oid *)ref)->oid;
+	return &ref->target.oid;
 }
 
 const char *git_reference_target(git_reference *ref)
 {
 	assert(ref);
 
-	if ((ref->type & GIT_REF_SYMBOLIC) == 0)
+	if ((ref->flags & GIT_REF_SYMBOLIC) == 0)
 		return NULL;
 
-	if (loose_update(ref) < GIT_SUCCESS)
-		return NULL;
+	return ref->target.symbolic;
+}
+
+int git_reference_create_symbolic(
+	git_reference **ref_out,
+	git_repository *repo,
+	const char *name,
+	const char *target,
+	int force)
+{
+	char normalized[GIT_REFNAME_MAX];
+	git_reference *ref = NULL;
+
+	if (normalize_name(normalized, sizeof(normalized), name, 0) < 0)
+		return -1;
+
+	if (reference_can_write(repo, normalized, NULL, force) < 0)
+		return -1;
+
+	if (reference_alloc(&ref, repo, normalized) < 0)
+		return -1;
+
+	ref->flags |= GIT_REF_SYMBOLIC;
 
-	return ((reference_symbolic *)ref)->target;
+	/* set the target; this will normalize the name automatically
+	 * and write the reference on disk */
+	if (git_reference_set_target(ref, target) < 0) {
+		git_reference_free(ref);
+		return -1;
+	}
+	if (ref_out == NULL) {
+		git_reference_free(ref);
+	} else {
+		*ref_out = ref;
+	}
+
+	return 0;
 }
 
-/**
- * Setters
- */
+int git_reference_create_oid(
+	git_reference **ref_out,
+	git_repository *repo,
+	const char *name,
+	const git_oid *id,
+	int force)
+{
+	git_reference *ref = NULL;
+	char normalized[GIT_REFNAME_MAX];
+
+	if (normalize_name(normalized, sizeof(normalized), name, 1) < 0)
+		return -1;
+
+	if (reference_can_write(repo, normalized, NULL, force) < 0)
+		return -1;
 
+	if (reference_alloc(&ref, repo, name) < 0)
+		return -1;
+
+	ref->flags |= GIT_REF_OID;
+
+	/* set the oid; this will write the reference on disk */
+	if (git_reference_set_oid(ref, id) < 0) {
+		git_reference_free(ref);
+		return -1;
+	}
+
+	if (ref_out == NULL) {
+		git_reference_free(ref);
+	} else {
+		*ref_out = ref;
+	}
+
+	return 0;
+}
 /*
  * Change the OID target of a reference.
  *
- * For loose references, just change the oid in memory
- * and overwrite the file in disk.
+ * For both loose and packed references, just change
+ * the oid in memory and (over)write the file in disk.
  *
- * For packed files, this is not pretty:
- * For performance reasons, we write the new reference
- * loose on disk (it replaces the old on the packfile),
- * but we cannot invalidate the pointer to the reference,
- * and most importantly, the `packfile` object must stay
- * consistent with the representation of the packfile
- * on disk. This is what we need to:
- *
- * 1. Copy the reference
- * 2. Change the oid on the original
- * 3. Write the original to disk
- * 4. Write the original to the loose cache
- * 5. Replace the original with the copy (old reference) in the packfile cache
+ * We do not repack packed references because of performance
+ * reasons.
  */
 int git_reference_set_oid(git_reference *ref, const git_oid *id)
 {
-	reference_oid *ref_oid;
-	reference_oid *ref_old = NULL;
-	int error = GIT_SUCCESS;
-
-	if ((ref->type & GIT_REF_OID) == 0)
-		return git__throw(GIT_EINVALIDREFSTATE, "Failed to set OID target of reference. Not an OID reference");
+	git_odb *odb = NULL;
 
-	ref_oid = (reference_oid *)ref;
+	if ((ref->flags & GIT_REF_OID) == 0) {
+		giterr_set(GITERR_REFERENCE, "Cannot set OID on symbolic reference");
+		return -1;
+	}
 
 	assert(ref->owner);
 
+	if (git_repository_odb__weakptr(&odb, ref->owner) < 0)
+		return -1;
+
 	/* Don't let the user create references to OIDs that
 	 * don't exist in the ODB */
-	if (!git_odb_exists(git_repository_database(ref->owner), id))
-		return git__throw(GIT_ENOTFOUND, "Failed to set OID target of reference. OID doesn't exist in ODB");
-
-	/* duplicate the reference;
-	 * this copy will stay on the packfile cache */
-	if (ref->type & GIT_REF_PACKED) {
-		ref_old = git__malloc(sizeof(reference_oid));
-		if (ref_old == NULL)
-			return GIT_ENOMEM;
-
-		ref_old->ref.name = git__strdup(ref->name);
-		if (ref_old->ref.name == NULL) {
-			free(ref_old);
-			return GIT_ENOMEM;
-		}
-	}
-
-	git_oid_cpy(&ref_oid->oid, id);
-	ref->type &= ~GIT_REF_HAS_PEEL;
-
-	error = loose_write(ref); 
-	if (error < GIT_SUCCESS)
-		goto cleanup;
-
-	if (ref->type & GIT_REF_PACKED) {
-		/* insert the original on the loose cache */
-		error = git_hashtable_insert(ref->owner->references.loose_cache, ref->name, ref);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
-
-		ref->type &= ~GIT_REF_PACKED;
-
-		/* replace the original in the packfile with the copy */
-		error = git_hashtable_insert(ref->owner->references.packfile, ref_old->ref.name, ref_old);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+	if (!git_odb_exists(odb, id)) {
+		giterr_set(GITERR_REFERENCE,
+			"Target OID for the reference doesn't exist on the repository");
+		return -1;
 	}
 
-	return GIT_SUCCESS;
+	/* Update the OID value on `ref` */
+	git_oid_cpy(&ref->target.oid, id);
 
-cleanup:
-	reference_free((git_reference *)ref_old);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to set OID target of reference");
+	/* Write back to disk */
+	return loose_write(ref);
 }
 
 /*
@@ -1391,149 +1322,177 @@ cleanup:
  */
 int git_reference_set_target(git_reference *ref, const char *target)
 {
-	reference_symbolic *ref_sym;
+	char normalized[GIT_REFNAME_MAX];
 
-	if ((ref->type & GIT_REF_SYMBOLIC) == 0)
-		return git__throw(GIT_EINVALIDREFSTATE, "Failed to set reference target. Not a symbolic reference");
+	if ((ref->flags & GIT_REF_SYMBOLIC) == 0) {
+		giterr_set(GITERR_REFERENCE,
+			"Cannot set symbolic target on a direct reference");
+		return -1;
+	}
 
-	ref_sym = (reference_symbolic *)ref;
+	if (normalize_name(normalized, sizeof(normalized), target, 0))
+		return -1;
 
-	free(ref_sym->target);
-	ref_sym->target = git__strdup(target);
-	if (ref_sym->target == NULL)
-		return GIT_ENOMEM;
+	git__free(ref->target.symbolic);
+	ref->target.symbolic = git__strdup(normalized);
+	GITERR_CHECK_ALLOC(ref->target.symbolic);
 
 	return loose_write(ref);
 }
 
-/**
- * Other
- */
-
-/*
- * Delete a reference.
- *
- * If the reference is packed, this is an expensive
- * operation. We need to remove the reference from
- * the memory cache and then rewrite the whole pack
- *
- * If the reference is loose, we remove it on
- * the filesystem and update the in-memory cache
- * accordingly. We also make sure that an older version
- * of it doesn't exist as a packed reference. If this
- * is the case, this packed reference is removed as well.
- *
- * This obviously invalidates the `ref` pointer.
- */
-int git_reference_delete(git_reference *ref)
+int git_reference_rename(git_reference *ref, const char *new_name, int force)
 {
-	int error;
-	git_reference *reference;
+	int result;
+	git_buf aux_path = GIT_BUF_INIT;
+	char normalized[GIT_REFNAME_MAX];
 
-	assert(ref);
+	const char *head_target = NULL;
+	git_reference *head = NULL;
 
-	if (ref->type & GIT_REF_PACKED) {
-		/* load the existing packfile */
-		if ((error = packed_load(ref->owner)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to delete reference");
-		
-		git_hashtable_remove(ref->owner->references.packfile, ref->name);
-		error = packed_write(ref->owner);
+	if (normalize_name(normalized, sizeof(normalized),
+		new_name, ref->flags & GIT_REF_OID) < 0)
+		return -1;
+
+	if (reference_can_write(ref->owner, normalized, ref->name, force) < 0)
+		return -1;
+
+	/* Initialize path now so we won't get an allocation failure once
+	 * we actually start removing things. */
+	if (git_buf_joinpath(&aux_path, ref->owner->path_repository, new_name) < 0)
+		return -1;
+
+	/*
+	 * Now delete the old ref and remove an possibly existing directory
+	 * named `new_name`. Note that using the internal `reference_delete`
+	 * method deletes the ref from disk but doesn't free the pointer, so
+	 * we can still access the ref's attributes for creating the new one
+	 */
+	if (reference_delete(ref) < 0)
+		goto cleanup;
+
+	/*
+	 * Finally we can create the new reference.
+	 */
+	if (ref->flags & GIT_REF_SYMBOLIC) {
+		result = git_reference_create_symbolic(
+			NULL, ref->owner, new_name, ref->target.symbolic, force);
 	} else {
-		char full_path[GIT_PATH_MAX];
-		git__joinpath(full_path, ref->owner->path_repository, ref->name);
-		git_hashtable_remove(ref->owner->references.loose_cache, ref->name);
-		error = gitfo_unlink(full_path);
-		if (error < GIT_SUCCESS)
-			goto cleanup;
+		result = git_reference_create_oid(
+			NULL, ref->owner, new_name, &ref->target.oid, force);
+	}
 
-		/* When deleting a loose reference, we have to ensure that an older
-		 * packed version of it doesn't exist
-		 */
-		if (!git_reference_lookup(&reference, ref->owner, ref->name)) {
-			assert((reference->type & GIT_REF_PACKED) != 0);
-			error = git_reference_delete(reference);
-		}
+	if (result < 0)
+		goto rollback;
+
+	/*
+	 * Check if we have to update HEAD.
+	 */
+	if (git_reference_lookup(&head, ref->owner, GIT_HEAD_FILE) < 0) {
+		giterr_set(GITERR_REFERENCE,
+			"Failed to update HEAD after renaming reference");
+		goto cleanup;
 	}
 
-cleanup:
-	reference_free(ref);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to delete reference");
-}
+	head_target = git_reference_target(head);
 
-int git_reference_rename(git_reference *ref, const char *new_name)
-{
-	return reference_rename(ref, new_name, 0);
-}
+	if (head_target && !strcmp(head_target, ref->name)) {
+		if (git_reference_create_symbolic(&head, ref->owner, "HEAD", new_name, 1) < 0) {
+			giterr_set(GITERR_REFERENCE,
+				"Failed to update HEAD after renaming reference");
+			goto cleanup;
+		}
+	}
 
-int git_reference_rename_f(git_reference *ref, const char *new_name)
-{
-	return reference_rename(ref, new_name, 1);
-}
+	/*
+	 * Rename the reflog file.
+	 */
+	if (git_buf_join_n(&aux_path, '/', 3, ref->owner->path_repository, GIT_REFLOG_DIR, ref->name) < 0)
+		goto cleanup;
 
-int git_reference_resolve(git_reference **resolved_ref, git_reference *ref)
-{
-	git_repository *repo;
-	int error, i;
+	if (git_path_exists(aux_path.ptr) == true) {
+		if (git_reflog_rename(ref, new_name) < 0)
+			goto cleanup;
+	} else {
+		giterr_clear();
+	}
 
-	assert(resolved_ref && ref);
-	*resolved_ref = NULL;
+	/*
+	 * Change the name of the reference given by the user.
+	 */
+	git__free(ref->name);
+	ref->name = git__strdup(new_name);
 
-	if ((error = loose_update(ref)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to resolve reference");
-	
-	repo = ref->owner;
+	/* The reference is no longer packed */
+	ref->flags &= ~GIT_REF_PACKED;
 
-	for (i = 0; i < MAX_NESTING_LEVEL; ++i) {
-		reference_symbolic *ref_sym;
+	git_reference_free(head);
+	git_buf_free(&aux_path);
+	return 0;
 
-		*resolved_ref = ref;
+cleanup:
+	git_reference_free(head);
+	git_buf_free(&aux_path);
+	return -1;
 
-		if (ref->type & GIT_REF_OID)
-			return GIT_SUCCESS;
+rollback:
+	/*
+	 * Try to create the old reference again, ignore failures
+	 */
+	if (ref->flags & GIT_REF_SYMBOLIC)
+		git_reference_create_symbolic(
+			NULL, ref->owner, ref->name, ref->target.symbolic, 0);
+	else
+		git_reference_create_oid(
+			NULL, ref->owner, ref->name, &ref->target.oid, 0);
 
-		ref_sym = (reference_symbolic *)ref;
-		if ((error = git_reference_lookup(&ref, repo, ref_sym->target)) < GIT_SUCCESS)
-			return error;
-	}
+	/* The reference is no longer packed */
+	ref->flags &= ~GIT_REF_PACKED;
 
-	return git__throw(GIT_ENOMEM, "Failed to resolve reference. Reference is too nested");
+	git_buf_free(&aux_path);
+	return -1;
 }
 
-int git_reference_packall(git_repository *repo)
+int git_reference_resolve(git_reference **ref_out, git_reference *ref)
 {
-	int error;
-
-	/* load the existing packfile */
-	if ((error = packed_load(repo)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to pack references");
+	if (ref->flags & GIT_REF_OID)
+		return git_reference_lookup(ref_out, ref->owner, ref->name);
+	else
+		return git_reference_lookup_resolved(ref_out, ref->owner, ref->target.symbolic, -1);
+}
 
-	/* update it in-memory with all the loose references */
-	if ((error = packed_loadloose(repo)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to pack references");
+int git_reference_packall(git_repository *repo)
+{
+	if (packed_load(repo) < 0 || /* load the existing packfile */
+		packed_loadloose(repo) < 0 || /* add all the loose refs */
+		packed_write(repo) < 0) /* write back to disk */
+		return -1;
 
-	/* write it back to disk */
-	return packed_write(repo);
+	return 0;
 }
 
-int git_reference_listcb(git_repository *repo, unsigned int list_flags, int (*callback)(const char *, void *), void *payload)
+int git_reference_foreach(
+	git_repository *repo,
+	unsigned int list_flags,
+	int (*callback)(const char *, void *),
+	void *payload)
 {
-	int error;
+	int result;
 	struct dirent_list_data data;
-	char refs_path[GIT_PATH_MAX];
+	git_buf refs_path = GIT_BUF_INIT;
 
 	/* list all the packed references first */
 	if (list_flags & GIT_REF_PACKED) {
 		const char *ref_name;
-		void *GIT_UNUSED(_unused);
+		void *ref;
+		GIT_UNUSED(ref);
 
-		if ((error = packed_load(repo)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to list references");
+		if (packed_load(repo) < 0)
+			return -1;
 
-		GIT_HASHTABLE_FOREACH(repo->references.packfile, ref_name, _unused,
-			if ((error = callback(ref_name, payload)) < GIT_SUCCESS)
-				return git__throw(error, "Failed to list references. User callback failed");
-		);
+		git_strmap_foreach(repo->references.packfile, ref_name, ref, {
+			if (callback(ref_name, payload) < 0)
+				return 0;
+		});
 	}
 
 	/* now list the loose references, trying not to
@@ -1545,19 +1504,25 @@ int git_reference_listcb(git_repository *repo, unsigned int list_flags, int (*ca
 	data.callback = callback;
 	data.callback_payload = payload;
 
+	if (git_buf_joinpath(&refs_path, repo->path_repository, GIT_REFS_DIR) < 0)
+		return -1;
+
+	result = git_path_direach(&refs_path, _dirent_loose_listall, &data);
+	git_buf_free(&refs_path);
 
-	git__joinpath(refs_path, repo->path_repository, GIT_REFS_DIR);
-	return gitfo_dirent(refs_path, GIT_PATH_MAX, _dirent_loose_listall, &data);
+	return result;
 }
 
-int cb__reflist_add(const char *ref, void *data)
+static int cb__reflist_add(const char *ref, void *data)
 {
 	return git_vector_insert((git_vector *)data, git__strdup(ref));
 }
 
-int git_reference_listall(git_strarray *array, git_repository *repo, unsigned int list_flags)
+int git_reference_list(
+	git_strarray *array,
+	git_repository *repo,
+	unsigned int list_flags)
 {
-	int error;
 	git_vector ref_list;
 
 	assert(array && repo);
@@ -1565,73 +1530,44 @@ int git_reference_listall(git_strarray *array, git_repository *repo, unsigned in
 	array->strings = NULL;
 	array->count = 0;
 
-	if (git_vector_init(&ref_list, 8, NULL) < GIT_SUCCESS)
-		return GIT_ENOMEM;
-
-	error = git_reference_listcb(repo, list_flags, &cb__reflist_add, (void *)&ref_list);
+	if (git_vector_init(&ref_list, 8, NULL) < 0)
+		return -1;
 
-	if (error < GIT_SUCCESS) {
+	if (git_reference_foreach(
+			repo, list_flags, &cb__reflist_add, (void *)&ref_list) < 0) {
 		git_vector_free(&ref_list);
-		return error;
+		return -1;
 	}
 
 	array->strings = (char **)ref_list.contents;
 	array->count = ref_list.length;
-	return GIT_SUCCESS;
+	return 0;
 }
 
-
-
-
-/*****************************************
- * Init/free (repository API)
- *****************************************/
-int git_repository__refcache_init(git_refcache *refs)
+int git_reference_reload(git_reference *ref)
 {
-	assert(refs);
-
-	refs->loose_cache = git_hashtable_alloc(
-		default_table_size,
-		reftable_hash,
-		(git_hash_keyeq_ptr)strcmp);
-
-	/* packfile loaded lazily */
-	refs->packfile = NULL;
-
-	return (refs->loose_cache) ? GIT_SUCCESS : GIT_ENOMEM;
+	return reference_lookup(ref);
 }
 
 void git_repository__refcache_free(git_refcache *refs)
 {
-	git_reference *reference;
-	const void *GIT_UNUSED(_unused);
-
 	assert(refs);
 
-	GIT_HASHTABLE_FOREACH(refs->loose_cache, _unused, reference,
-		reference_free(reference);
-	);
-
-	git_hashtable_free(refs->loose_cache);
-
 	if (refs->packfile) {
-		GIT_HASHTABLE_FOREACH(refs->packfile, _unused, reference,
-			reference_free(reference);
-		);
+		struct packref *reference;
+
+		git_strmap_foreach_value(refs->packfile, reference, {
+			git__free(reference);
+		});
 
-		git_hashtable_free(refs->packfile);
+		git_strmap_free(refs->packfile);
 	}
 }
 
-
-
-/*****************************************
- * Name normalization
- *****************************************/
-static int check_valid_ref_char(char ch)
+static int is_valid_ref_char(char ch)
 {
-	if (ch <= ' ')
-		return GIT_ERROR;
+	if ((unsigned) ch <= ' ')
+		return 0;
 
 	switch (ch) {
 	case '~':
@@ -1641,48 +1577,53 @@ static int check_valid_ref_char(char ch)
 	case '?':
 	case '[':
 	case '*':
-		return GIT_ERROR;
-		break;
-
+		return 0;
 	default:
-		return GIT_SUCCESS;
+		return 1;
 	}
 }
 
-static int normalize_name(char *buffer_out, const char *name, int is_oid_ref)
+static int normalize_name(
+	char *buffer_out,
+	size_t out_size,
+	const char *name,
+	int is_oid_ref)
 {
 	const char *name_end, *buffer_out_start;
-	char *current;
+	const char *current;
 	int contains_a_slash = 0;
 
 	assert(name && buffer_out);
 
 	buffer_out_start = buffer_out;
-	current = (char *)name;
+	current = name;
 	name_end = name + strlen(name);
 
+	/* Terminating null byte */
+	out_size--;
+
 	/* A refname can not be empty */
 	if (name_end == name)
-		return git__throw(GIT_EINVALIDREFNAME, "Failed to normalize name. Reference name is empty");
+		goto invalid_name;
 
 	/* A refname can not end with a dot or a slash */
 	if (*(name_end - 1) == '.' || *(name_end - 1) == '/')
-		return git__throw(GIT_EINVALIDREFNAME, "Failed to normalize name. Reference name ends with dot or slash");
+		goto invalid_name;
 
-	while (current < name_end) {
-		if (check_valid_ref_char(*current))
-			return git__throw(GIT_EINVALIDREFNAME, "Failed to normalize name. Reference name contains invalid characters");
+	while (current < name_end && out_size) {
+		if (!is_valid_ref_char(*current))
+			goto invalid_name;
 
 		if (buffer_out > buffer_out_start) {
 			char prev = *(buffer_out - 1);
 
 			/* A refname can not start with a dot nor contain a double dot */
 			if (*current == '.' && ((prev == '.') || (prev == '/')))
-				return git__throw(GIT_EINVALIDREFNAME, "Failed to normalize name. Reference name starts with a dot or contains a double dot");
+				goto invalid_name;
 
 			/* '@{' is forbidden within a refname */
 			if (*current == '{' && prev == '@')
-				return git__throw(GIT_EINVALIDREFNAME, "Failed to normalize name. Reference name contains '@{'");
+				goto invalid_name;
 
 			/* Prevent multiple slashes from being added to the output */
 			if (*current == '/' && prev == '/') {
@@ -1695,17 +1636,25 @@ static int normalize_name(char *buffer_out, const char *name, int is_oid_ref)
 			contains_a_slash = 1;
 
 		*buffer_out++ = *current++;
+		out_size--;
 	}
 
+	if (!out_size)
+		goto invalid_name;
+
 	/* Object id refname have to contain at least one slash, except
 	 * for HEAD in a detached state or MERGE_HEAD if we're in the
 	 * middle of a merge */
-	if (is_oid_ref && !contains_a_slash && (strcmp(name, GIT_HEAD_FILE) && strcmp(name, GIT_MERGE_HEAD_FILE)))
-		return git__throw(GIT_EINVALIDREFNAME, "Failed to normalize name. Reference name contains no slashes");
+	if (is_oid_ref &&
+		!contains_a_slash &&
+		strcmp(name, GIT_HEAD_FILE) != 0 &&
+		strcmp(name, GIT_MERGE_HEAD_FILE) != 0 &&
+		strcmp(name, GIT_FETCH_HEAD_FILE) != 0)
+		goto invalid_name;
 
 	/* A refname can not end with ".lock" */
 	if (!git__suffixcmp(name, GIT_FILELOCK_EXTENSION))
-		return git__throw(GIT_EINVALIDREFNAME, "Failed to normalize name. Reference name ends with '.lock'");
+		goto invalid_name;
 
 	*buffer_out = '\0';
 
@@ -1715,19 +1664,44 @@ static int normalize_name(char *buffer_out, const char *name, int is_oid_ref)
 	 */
 	if (is_oid_ref && !(git__prefixcmp(buffer_out_start, GIT_REFS_DIR) ||
 		strcmp(buffer_out_start, GIT_HEAD_FILE)))
-		return git__throw(GIT_EINVALIDREFNAME, "Failed to normalize name. Reference name does not start with 'refs/'");
+		goto invalid_name;
+
+	return 0;
 
-	return GIT_SUCCESS;
+invalid_name:
+	giterr_set(GITERR_REFERENCE, "The given reference name is not valid");
+	return -1;
 }
 
-int git_reference__normalize_name(char *buffer_out, const char *name)
+int git_reference__normalize_name(
+	char *buffer_out,
+	size_t out_size,
+	const char *name)
 {
-	return normalize_name(buffer_out, name, 0);
+	return normalize_name(buffer_out, out_size, name, 0);
 }
 
-int git_reference__normalize_name_oid(char *buffer_out, const char *name)
+int git_reference__normalize_name_oid(
+	char *buffer_out,
+	size_t out_size,
+	const char *name)
 {
-	return normalize_name(buffer_out, name, 1);
+	return normalize_name(buffer_out, out_size, name, 1);
 }
 
+#define GIT_REF_TYPEMASK (GIT_REF_OID | GIT_REF_SYMBOLIC)
+
+int git_reference_cmp(git_reference *ref1, git_reference *ref2)
+{
+	assert(ref1 && ref2);
+
+	/* let's put symbolic refs before OIDs */
+	if ((ref1->flags & GIT_REF_TYPEMASK) != (ref2->flags & GIT_REF_TYPEMASK))
+		return (ref1->flags & GIT_REF_SYMBOLIC) ? -1 : 1;
+
+	if (ref1->flags & GIT_REF_SYMBOLIC)
+		return strcmp(ref1->target.symbolic, ref2->target.symbolic);
+
+	return git_oid_cmp(&ref1->target.oid, &ref2->target.oid);
+}
 
diff --git a/src/refs.h b/src/refs.h
index b8f3e2f6d..369e91e1c 100644
--- a/src/refs.h
+++ b/src/refs.h
@@ -1,43 +1,81 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_refs_h__
 #define INCLUDE_refs_h__
 
 #include "common.h"
 #include "git2/oid.h"
 #include "git2/refs.h"
-#include "hashtable.h"
+#include "strmap.h"
 
 #define GIT_REFS_DIR "refs/"
 #define GIT_REFS_HEADS_DIR GIT_REFS_DIR "heads/"
 #define GIT_REFS_TAGS_DIR GIT_REFS_DIR "tags/"
 #define GIT_REFS_REMOTES_DIR GIT_REFS_DIR "remotes/"
+#define GIT_REFS_DIR_MODE 0777
+#define GIT_REFS_FILE_MODE 0666
+
+#define GIT_RENAMED_REF_FILE GIT_REFS_DIR "RENAMED-REF"
 
 #define GIT_SYMREF "ref: "
 #define GIT_PACKEDREFS_FILE "packed-refs"
 #define GIT_PACKEDREFS_HEADER "# pack-refs with: peeled "
-#define MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH 100
+#define GIT_PACKEDREFS_FILE_MODE 0666
 
 #define GIT_HEAD_FILE "HEAD"
+#define GIT_FETCH_HEAD_FILE "FETCH_HEAD"
 #define GIT_MERGE_HEAD_FILE "MERGE_HEAD"
 #define GIT_REFS_HEADS_MASTER_FILE GIT_REFS_HEADS_DIR "master"
 
+#define GIT_REFNAME_MAX 1024
+
 struct git_reference {
+	unsigned int flags;
 	git_repository *owner;
 	char *name;
-	unsigned int type;
 	time_t mtime;
+
+	union {
+		git_oid oid;
+		char *symbolic;
+	} target;
 };
 
 typedef struct {
-	git_hashtable *packfile;
-	git_hashtable *loose_cache;
+	git_strmap *packfile;
 	time_t packfile_time;
 } git_refcache;
 
-
 void git_repository__refcache_free(git_refcache *refs);
-int git_repository__refcache_init(git_refcache *refs);
 
-int git_reference__normalize_name(char *buffer_out, const char *name);
-int git_reference__normalize_name_oid(char *buffer_out, const char *name);
+int git_reference__normalize_name(char *buffer_out, size_t out_size, const char *name);
+int git_reference__normalize_name_oid(char *buffer_out, size_t out_size, const char *name);
+
+/**
+ * Lookup a reference by name and try to resolve to an OID.
+ *
+ * You can control how many dereferences this will attempt to resolve the
+ * reference with the `max_deref` parameter, or pass -1 to use a sane
+ * default.  If you pass 0 for `max_deref`, this will not attempt to resolve
+ * the reference.  For any value of `max_deref` other than 0, not
+ * successfully resolving the reference will be reported as an error.
+
+ * The generated reference must be freed by the user.
+ *
+ * @param reference_out Pointer to the looked-up reference
+ * @param repo The repository to look up the reference
+ * @param name The long name for the reference (e.g. HEAD, ref/heads/master, refs/tags/v0.1.0, ...)
+ * @param max_deref Maximum number of dereferences to make of symbolic refs, 0 means simple lookup, < 0 means use default reasonable value
+ * @return 0 on success or < 0 on error; not being able to resolve the reference is an error unless 0 was passed for max_deref
+ */
+int git_reference_lookup_resolved(
+	git_reference **reference_out,
+	git_repository *repo,
+	const char *name,
+	int max_deref);
 
 #endif
diff --git a/src/refspec.c b/src/refspec.c
new file mode 100644
index 000000000..b6b1158b7
--- /dev/null
+++ b/src/refspec.c
@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "git2/errors.h"
+
+#include "common.h"
+#include "refspec.h"
+#include "util.h"
+#include "posix.h"
+
+int git_refspec_parse(git_refspec *refspec, const char *str)
+{
+	char *delim;
+
+	memset(refspec, 0x0, sizeof(git_refspec));
+
+	if (*str == '+') {
+		refspec->force = 1;
+		str++;
+	}
+
+	delim = strchr(str, ':');
+	if (delim == NULL) {
+		refspec->src = git__strdup(str);
+		GITERR_CHECK_ALLOC(refspec->src);
+		return 0;
+	}
+
+	refspec->src = git__strndup(str, delim - str);
+	GITERR_CHECK_ALLOC(refspec->src);
+
+	refspec->dst = git__strdup(delim + 1);
+	if (refspec->dst == NULL) {
+		git__free(refspec->src);
+		refspec->src = NULL;
+		return -1;
+	}
+
+	return 0;
+}
+
+const char *git_refspec_src(const git_refspec *refspec)
+{
+	return refspec == NULL ? NULL : refspec->src;
+}
+
+const char *git_refspec_dst(const git_refspec *refspec)
+{
+	return refspec == NULL ? NULL : refspec->dst;
+}
+
+int git_refspec_force(const git_refspec *refspec)
+{
+	assert(refspec);
+
+	return refspec->force;
+}
+
+int git_refspec_src_matches(const git_refspec *refspec, const char *refname)
+{
+	if (refspec == NULL || refspec->src == NULL)
+		return false;
+
+	return (p_fnmatch(refspec->src, refname, 0) == 0);
+}
+
+int git_refspec_transform(char *out, size_t outlen, const git_refspec *spec, const char *name)
+{
+	size_t baselen, namelen;
+
+	baselen = strlen(spec->dst);
+	if (outlen <= baselen) {
+		giterr_set(GITERR_INVALID, "Reference name too long");
+		return GIT_EBUFS;
+	}
+
+	/*
+	 * No '*' at the end means that it's mapped to one specific local
+	 * branch, so no actual transformation is needed.
+	 */
+	if (spec->dst[baselen - 1] != '*') {
+		memcpy(out, spec->dst, baselen + 1); /* include '\0' */
+		return 0;
+	}
+
+	/* There's a '*' at the end, so remove its length */
+	baselen--;
+
+	/* skip the prefix, -1 is for the '*' */
+	name += strlen(spec->src) - 1;
+
+	namelen = strlen(name);
+
+	if (outlen <= baselen + namelen) {
+		giterr_set(GITERR_INVALID, "Reference name too long");
+		return GIT_EBUFS;
+	}
+
+	memcpy(out, spec->dst, baselen);
+	memcpy(out + baselen, name, namelen + 1);
+
+	return 0;
+}
+
+int git_refspec_transform_r(git_buf *out, const git_refspec *spec, const char *name)
+{
+	if (git_buf_sets(out, spec->dst) < 0)
+		return -1;
+
+	/*
+	 * No '*' at the end means that it's mapped to one specific local
+	 * branch, so no actual transformation is needed.
+	 */
+	if (git_buf_len(out) > 0 && out->ptr[git_buf_len(out) - 1] != '*')
+		return 0;
+
+	git_buf_truncate(out, git_buf_len(out) - 1); /* remove trailing '*' */
+	git_buf_puts(out, name + strlen(spec->src) - 1);
+
+	if (git_buf_oom(out))
+		return -1;
+
+	return 0;
+}
+
diff --git a/src/refspec.h b/src/refspec.h
new file mode 100644
index 000000000..2db504910
--- /dev/null
+++ b/src/refspec.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_refspec_h__
+#define INCLUDE_refspec_h__
+
+#include "git2/refspec.h"
+#include "buffer.h"
+
+struct git_refspec {
+	struct git_refspec *next;
+	char *src;
+	char *dst;
+	unsigned int force :1,
+		pattern :1,
+		matching :1;
+};
+
+int git_refspec_parse(struct git_refspec *refspec, const char *str);
+
+/**
+ * Transform a reference to its target following the refspec's rules,
+ * and writes the results into a git_buf.
+ *
+ * @param out where to store the target name
+ * @param spec the refspec
+ * @param name the name of the reference to transform
+ * @return 0 or error if buffer allocation fails
+ */
+int git_refspec_transform_r(git_buf *out, const git_refspec *spec, const char *name);
+
+#endif
diff --git a/src/remote.c b/src/remote.c
new file mode 100644
index 000000000..8d6076107
--- /dev/null
+++ b/src/remote.c
@@ -0,0 +1,524 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "git2/remote.h"
+#include "git2/config.h"
+#include "git2/types.h"
+
+#include "config.h"
+#include "repository.h"
+#include "remote.h"
+#include "fetch.h"
+#include "refs.h"
+
+#include <regex.h>
+
+static int refspec_parse(git_refspec *refspec, const char *str)
+{
+	char *delim;
+
+	memset(refspec, 0x0, sizeof(git_refspec));
+
+	if (*str == '+') {
+		refspec->force = 1;
+		str++;
+	}
+
+	delim = strchr(str, ':');
+	if (delim == NULL) {
+		giterr_set(GITERR_NET, "Invalid refspec, missing ':'");
+		return -1;
+	}
+
+	refspec->src = git__strndup(str, delim - str);
+	GITERR_CHECK_ALLOC(refspec->src);
+
+	refspec->dst = git__strdup(delim + 1);
+	GITERR_CHECK_ALLOC(refspec->dst);
+
+	return 0;
+}
+
+static int parse_remote_refspec(git_config *cfg, git_refspec *refspec, const char *var)
+{
+	int error;
+	const char *val;
+
+	if ((error = git_config_get_string(&val, cfg, var)) < 0)
+		return error;
+
+	return refspec_parse(refspec, val);
+}
+
+int git_remote_new(git_remote **out, git_repository *repo, const char *name, const char *url, const char *fetch)
+{
+	git_remote *remote;
+
+	/* name is optional */
+	assert(out && repo && url);
+
+	remote = git__malloc(sizeof(git_remote));
+	GITERR_CHECK_ALLOC(remote);
+
+	memset(remote, 0x0, sizeof(git_remote));
+	remote->repo = repo;
+	remote->check_cert = 1;
+
+	if (git_vector_init(&remote->refs, 32, NULL) < 0)
+		return -1;
+
+	remote->url = git__strdup(url);
+	GITERR_CHECK_ALLOC(remote->url);
+
+	if (name != NULL) {
+		remote->name = git__strdup(name);
+		GITERR_CHECK_ALLOC(remote->name);
+	}
+
+	if (fetch != NULL) {
+		if (refspec_parse(&remote->fetch, fetch) < 0)
+			goto on_error;
+	}
+
+	*out = remote;
+	return 0;
+
+on_error:
+	git_remote_free(remote);
+	return -1;
+}
+
+int git_remote_load(git_remote **out, git_repository *repo, const char *name)
+{
+	git_remote *remote;
+	git_buf buf = GIT_BUF_INIT;
+	const char *val;
+	int error = 0;
+	git_config *config;
+
+	assert(out && repo && name);
+
+	if (git_repository_config__weakptr(&config, repo) < 0)
+		return -1;
+
+	remote = git__malloc(sizeof(git_remote));
+	GITERR_CHECK_ALLOC(remote);
+
+	memset(remote, 0x0, sizeof(git_remote));
+	remote->check_cert = 1;
+	remote->name = git__strdup(name);
+	GITERR_CHECK_ALLOC(remote->name);
+
+	if (git_vector_init(&remote->refs, 32, NULL) < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	if (git_buf_printf(&buf, "remote.%s.url", name) < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	if ((error = git_config_get_string(&val, config, git_buf_cstr(&buf))) < 0)
+		goto cleanup;
+
+	remote->repo = repo;
+	remote->url = git__strdup(val);
+	GITERR_CHECK_ALLOC(remote->url);
+
+	git_buf_clear(&buf);
+	if (git_buf_printf(&buf, "remote.%s.fetch", name) < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	error = parse_remote_refspec(config, &remote->fetch, git_buf_cstr(&buf));
+	if (error == GIT_ENOTFOUND)
+		error = 0;
+
+	if (error < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	git_buf_clear(&buf);
+	if (git_buf_printf(&buf, "remote.%s.push", name) < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	error = parse_remote_refspec(config, &remote->push, git_buf_cstr(&buf));
+	if (error == GIT_ENOTFOUND)
+		error = 0;
+
+	if (error < 0) {
+		error = -1;
+		goto cleanup;
+	}
+
+	*out = remote;
+
+cleanup:
+	git_buf_free(&buf);
+
+	if (error < 0)
+		git_remote_free(remote);
+
+	return error;
+}
+
+int git_remote_save(const git_remote *remote)
+{
+	git_config *config;
+	git_buf buf = GIT_BUF_INIT, value = GIT_BUF_INIT;
+
+	if (git_repository_config__weakptr(&config, remote->repo) < 0)
+		return -1;
+
+	if (git_buf_printf(&buf, "remote.%s.%s", remote->name, "url") < 0)
+		return -1;
+
+	if (git_config_set_string(config, git_buf_cstr(&buf), remote->url) < 0) {
+		git_buf_free(&buf);
+		return -1;
+	}
+
+	if (remote->fetch.src != NULL && remote->fetch.dst != NULL) {
+		git_buf_clear(&buf);
+		git_buf_clear(&value);
+		git_buf_printf(&buf, "remote.%s.fetch", remote->name);
+		if (remote->fetch.force)
+			git_buf_putc(&value, '+');
+		git_buf_printf(&value, "%s:%s", remote->fetch.src, remote->fetch.dst);
+		if (git_buf_oom(&buf) || git_buf_oom(&value))
+			return -1;
+
+		if (git_config_set_string(config, git_buf_cstr(&buf), git_buf_cstr(&value)) < 0)
+			goto on_error;
+	}
+
+	if (remote->push.src != NULL && remote->push.dst != NULL) {
+		git_buf_clear(&buf);
+		git_buf_clear(&value);
+		git_buf_printf(&buf, "remote.%s.push", remote->name);
+		if (remote->push.force)
+			git_buf_putc(&value, '+');
+		git_buf_printf(&value, "%s:%s", remote->push.src, remote->push.dst);
+		if (git_buf_oom(&buf) || git_buf_oom(&value))
+			return -1;
+
+		if (git_config_set_string(config, git_buf_cstr(&buf), git_buf_cstr(&value)) < 0)
+			goto on_error;
+	}
+
+	git_buf_free(&buf);
+	git_buf_free(&value);
+
+	return 0;
+
+on_error:
+	git_buf_free(&buf);
+	git_buf_free(&value);
+	return -1;
+}
+
+const char *git_remote_name(git_remote *remote)
+{
+	assert(remote);
+	return remote->name;
+}
+
+const char *git_remote_url(git_remote *remote)
+{
+	assert(remote);
+	return remote->url;
+}
+
+int git_remote_set_fetchspec(git_remote *remote, const char *spec)
+{
+	git_refspec refspec;
+
+	assert(remote && spec);
+
+	if (refspec_parse(&refspec, spec) < 0)
+		return -1;
+
+	git__free(remote->fetch.src);
+	git__free(remote->fetch.dst);
+	remote->fetch.src = refspec.src;
+	remote->fetch.dst = refspec.dst;
+
+	return 0;
+}
+
+const git_refspec *git_remote_fetchspec(git_remote *remote)
+{
+	assert(remote);
+	return &remote->fetch;
+}
+
+int git_remote_set_pushspec(git_remote *remote, const char *spec)
+{
+	git_refspec refspec;
+
+	assert(remote && spec);
+
+	if (refspec_parse(&refspec, spec) < 0)
+		return -1;
+
+	git__free(remote->push.src);
+	git__free(remote->push.dst);
+	remote->push.src = refspec.src;
+	remote->push.dst = refspec.dst;
+
+	return 0;
+}
+
+const git_refspec *git_remote_pushspec(git_remote *remote)
+{
+	assert(remote);
+	return &remote->push;
+}
+
+int git_remote_connect(git_remote *remote, int direction)
+{
+	git_transport *t;
+
+	assert(remote);
+
+	if (git_transport_new(&t, remote->url) < 0)
+		return -1;
+
+	t->check_cert = remote->check_cert;
+	if (t->connect(t, direction) < 0) {
+		goto on_error;
+	}
+
+	remote->transport = t;
+
+	return 0;
+
+on_error:
+	t->free(t);
+	return -1;
+}
+
+int git_remote_ls(git_remote *remote, git_headlist_cb list_cb, void *payload)
+{
+	assert(remote);
+
+	if (!remote->transport || !remote->transport->connected) {
+		giterr_set(GITERR_NET, "The remote is not connected");
+		return -1;
+	}
+
+	return remote->transport->ls(remote->transport, list_cb, payload);
+}
+
+int git_remote_download(git_remote *remote, git_off_t *bytes, git_indexer_stats *stats)
+{
+	int error;
+
+	assert(remote && bytes && stats);
+
+	if ((error = git_fetch_negotiate(remote)) < 0)
+		return error;
+
+	return git_fetch_download_pack(remote, bytes, stats);
+}
+
+int git_remote_update_tips(git_remote *remote, int (*cb)(const char *refname, const git_oid *a, const git_oid *b))
+{
+	int error = 0;
+	unsigned int i = 0;
+	git_buf refname = GIT_BUF_INIT;
+	git_oid old;
+	git_vector *refs = &remote->refs;
+	git_remote_head *head;
+	git_reference *ref;
+	struct git_refspec *spec = &remote->fetch;
+
+	assert(remote);
+
+	if (refs->length == 0)
+		return 0;
+
+	/* HEAD is only allowed to be the first in the list */
+	head = refs->contents[0];
+	if (!strcmp(head->name, GIT_HEAD_FILE)) {
+		if (git_reference_create_oid(&ref, remote->repo, GIT_FETCH_HEAD_FILE, &head->oid, 1) < 0)
+			return -1;
+
+		i = 1;
+		git_reference_free(ref);
+	}
+
+	for (; i < refs->length; ++i) {
+		head = refs->contents[i];
+
+		if (git_refspec_transform_r(&refname, spec, head->name) < 0)
+			goto on_error;
+
+		error = git_reference_name_to_oid(&old, remote->repo, refname.ptr);
+		if (error < 0 && error != GIT_ENOTFOUND)
+			goto on_error;
+
+		if (error == GIT_ENOTFOUND)
+			memset(&old, 0, GIT_OID_RAWSZ);
+
+		if (!git_oid_cmp(&old, &head->oid))
+			continue;
+
+		if (git_reference_create_oid(&ref, remote->repo, refname.ptr, &head->oid, 1) < 0)
+			break;
+
+		git_reference_free(ref);
+
+		if (cb != NULL) {
+			if (cb(refname.ptr, &old, &head->oid) < 0)
+				goto on_error;
+		}
+	}
+
+	git_buf_free(&refname);
+	return 0;
+
+on_error:
+	git_buf_free(&refname);
+	return -1;
+
+}
+
+int git_remote_connected(git_remote *remote)
+{
+	assert(remote);
+	return remote->transport == NULL ? 0 : remote->transport->connected;
+}
+
+void git_remote_disconnect(git_remote *remote)
+{
+	assert(remote);
+
+	if (remote->transport != NULL && remote->transport->connected)
+			remote->transport->close(remote->transport);
+}
+
+void git_remote_free(git_remote *remote)
+{
+	if (remote == NULL)
+		return;
+
+	if (remote->transport != NULL) {
+		git_remote_disconnect(remote);
+
+		remote->transport->free(remote->transport);
+		remote->transport = NULL;
+	}
+
+	git_vector_free(&remote->refs);
+
+	git__free(remote->fetch.src);
+	git__free(remote->fetch.dst);
+	git__free(remote->push.src);
+	git__free(remote->push.dst);
+	git__free(remote->url);
+	git__free(remote->name);
+	git__free(remote);
+}
+
+struct cb_data {
+	git_vector *list;
+	regex_t *preg;
+};
+
+static int remote_list_cb(const char *name, const char *value, void *data_)
+{
+	struct cb_data *data = (struct cb_data *)data_;
+	size_t nmatch = 2;
+	regmatch_t pmatch[2];
+	GIT_UNUSED(value);
+
+	if (!regexec(data->preg, name, nmatch, pmatch, 0)) {
+		char *remote_name = git__strndup(&name[pmatch[1].rm_so], pmatch[1].rm_eo - pmatch[1].rm_so);
+		GITERR_CHECK_ALLOC(remote_name);
+
+		if (git_vector_insert(data->list, remote_name) < 0)
+			return -1;
+	}
+
+	return 0;
+}
+
+int git_remote_list(git_strarray *remotes_list, git_repository *repo)
+{
+	git_config *cfg;
+	git_vector list;
+	regex_t preg;
+	struct cb_data data;
+	int error;
+
+	if (git_repository_config__weakptr(&cfg, repo) < 0)
+		return -1;
+
+	if (git_vector_init(&list, 4, NULL) < 0)
+		return -1;
+
+	if (regcomp(&preg, "^remote\\.(.*)\\.url$", REG_EXTENDED) < 0) {
+		giterr_set(GITERR_OS, "Remote catch regex failed to compile");
+		return -1;
+	}
+
+	data.list = &list;
+	data.preg = &preg;
+	error = git_config_foreach(cfg, remote_list_cb, &data);
+	regfree(&preg);
+	if (error < 0) {
+		size_t i;
+		char *elem;
+		git_vector_foreach(&list, i, elem) {
+			git__free(elem);
+		}
+
+		git_vector_free(&list);
+		return error;
+	}
+
+	remotes_list->strings = (char **)list.contents;
+	remotes_list->count = list.length;
+
+	return 0;
+}
+
+int git_remote_add(git_remote **out, git_repository *repo, const char *name, const char *url)
+{
+	git_buf buf = GIT_BUF_INIT;
+
+	if (git_buf_printf(&buf, "+refs/heads/*:refs/remotes/%s/*", name) < 0)
+		return -1;
+
+	if (git_remote_new(out, repo, name, url, git_buf_cstr(&buf)) < 0)
+		goto on_error;
+
+	git_buf_free(&buf);
+
+	if (git_remote_save(*out) < 0)
+		goto on_error;
+
+	return 0;
+
+on_error:
+	git_buf_free(&buf);
+	git_remote_free(*out);
+	return -1;
+}
+
+void git_remote_check_cert(git_remote *remote, int check)
+{
+	assert(remote);
+
+	remote->check_cert = check;
+}
diff --git a/src/remote.h b/src/remote.h
new file mode 100644
index 000000000..0949ad434
--- /dev/null
+++ b/src/remote.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_remote_h__
+#define INCLUDE_remote_h__
+
+#include "refspec.h"
+#include "transport.h"
+#include "repository.h"
+
+struct git_remote {
+	char *name;
+	char *url;
+	git_vector refs;
+	struct git_refspec fetch;
+	struct git_refspec push;
+	git_transport *transport;
+	git_repository *repo;
+	unsigned int need_pack:1,
+		check_cert;
+};
+
+#endif
diff --git a/src/repository.c b/src/repository.c
index 32ca8dd79..718170839 100644
--- a/src/repository.c
+++ b/src/repository.c
@@ -1,28 +1,11 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 #include <stdarg.h>
+#include <ctype.h>
 
 #include "git2/object.h"
 
@@ -32,447 +15,930 @@
 #include "tag.h"
 #include "blob.h"
 #include "fileops.h"
-
+#include "config.h"
 #include "refs.h"
 
 #define GIT_OBJECTS_INFO_DIR GIT_OBJECTS_DIR "info/"
 #define GIT_OBJECTS_PACK_DIR GIT_OBJECTS_DIR "pack/"
 
+#define GIT_FILE_CONTENT_PREFIX "gitdir:"
+
 #define GIT_BRANCH_MASTER "master"
 
-typedef struct {
-	char *path_repository;
-	unsigned is_bare:1, has_been_reinit:1;
-} repo_init;
+#define GIT_REPO_VERSION 0
+
+static void drop_odb(git_repository *repo)
+{
+	if (repo->_odb != NULL) {
+		GIT_REFCOUNT_OWN(repo->_odb, NULL);
+		git_odb_free(repo->_odb);
+		repo->_odb = NULL;
+	}
+}
+
+static void drop_config(git_repository *repo)
+{
+	if (repo->_config != NULL) {
+		GIT_REFCOUNT_OWN(repo->_config, NULL);
+		git_config_free(repo->_config);
+		repo->_config = NULL;
+	}
+
+	git_repository__cvar_cache_clear(repo);
+}
+
+static void drop_index(git_repository *repo)
+{
+	if (repo->_index != NULL) {
+		GIT_REFCOUNT_OWN(repo->_index, NULL);
+		git_index_free(repo->_index);
+		repo->_index = NULL;
+	}
+}
+
+void git_repository_free(git_repository *repo)
+{
+	if (repo == NULL)
+		return;
+
+	git_cache_free(&repo->objects);
+	git_repository__refcache_free(&repo->references);
+	git_attr_cache_flush(repo);
+	git_submodule_config_free(repo);
+
+	git__free(repo->path_repository);
+	git__free(repo->workdir);
+
+	drop_config(repo);
+	drop_index(repo);
+	drop_odb(repo);
+
+	git__free(repo);
+}
 
 /*
  * Git repository open methods
  *
  * Open a repository object from its path
  */
-static int assign_repository_dirs(
-		git_repository *repo,
-		const char *git_dir,
-		const char *git_object_directory,
-		const char *git_index_file,
-		const char *git_work_tree)
+static bool valid_repository_path(git_buf *repository_path)
 {
-	char path_aux[GIT_PATH_MAX];
-	int error = GIT_SUCCESS;
+	/* Check OBJECTS_DIR first, since it will generate the longest path name */
+	if (git_path_contains_dir(repository_path, GIT_OBJECTS_DIR) == false)
+		return false;
 
-	assert(repo);
+	/* Ensure HEAD file exists */
+	if (git_path_contains_file(repository_path, GIT_HEAD_FILE) == false)
+		return false;
 
-	if (git_dir == NULL)
-		return git__throw(GIT_ENOTFOUND, "Failed to open repository. Git dir not found");
+	if (git_path_contains_dir(repository_path, GIT_REFS_DIR)  == false)
+		return false;
+
+	return true;
+}
 
-	error = gitfo_prettify_dir_path(path_aux, sizeof(path_aux), git_dir);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to open repository");
+static git_repository *repository_alloc(void)
+{
+	git_repository *repo = git__malloc(sizeof(git_repository));
+	if (!repo)
+		return NULL;
 
-	/* store GIT_DIR */
-	repo->path_repository = git__strdup(path_aux);
-	if (repo->path_repository == NULL)
-		return GIT_ENOMEM;
+	memset(repo, 0x0, sizeof(git_repository));
 
-	/* path to GIT_OBJECT_DIRECTORY */
-	if (git_object_directory == NULL)
-		git__joinpath(path_aux, repo->path_repository, GIT_OBJECTS_DIR);
-	else {
-		error = gitfo_prettify_dir_path(path_aux, sizeof(path_aux), git_object_directory);
-		if (error < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to open repository");
+	if (git_cache_init(&repo->objects, GIT_DEFAULT_CACHE_SIZE, &git_object__free) < 0) {
+		git__free(repo);
+		return NULL;
 	}
 
-	/* Store GIT_OBJECT_DIRECTORY */
-	repo->path_odb = git__strdup(path_aux);
-	if (repo->path_odb == NULL)
-		return GIT_ENOMEM;
+	/* set all the entries in the cvar cache to `unset` */
+	git_repository__cvar_cache_clear(repo);
 
-	/* path to GIT_WORK_TREE */
-	if (git_work_tree == NULL)
-		repo->is_bare = 1;
+	return repo;
+}
+
+static int load_config_data(git_repository *repo)
+{
+	int is_bare;
+	git_config *config;
+
+	if (git_repository_config__weakptr(&config, repo) < 0)
+		return -1;
+
+	/* Try to figure out if it's bare, default to non-bare if it's not set */
+	if (git_config_get_bool(&is_bare, config, "core.bare") < 0)
+		repo->is_bare = 0;
+	else
+		repo->is_bare = is_bare;
+
+	return 0;
+}
+
+static int load_workdir(git_repository *repo, git_buf *parent_path)
+{
+	int         error;
+	git_config *config;
+	const char *worktree;
+	git_buf     worktree_buf = GIT_BUF_INIT;
+
+	if (repo->is_bare)
+		return 0;
+
+	if (git_repository_config__weakptr(&config, repo) < 0)
+		return -1;
+
+	error = git_config_get_string(&worktree, config, "core.worktree");
+	if (!error && worktree != NULL)
+		repo->workdir = git__strdup(worktree);
+	else if (error != GIT_ENOTFOUND)
+		return error;
 	else {
-		error = gitfo_prettify_dir_path(path_aux, sizeof(path_aux), git_work_tree);
-		if (error < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to open repository");
-
-		/* Store GIT_WORK_TREE */
-		repo->path_workdir = git__strdup(path_aux);
-		if (repo->path_workdir == NULL)
-			return GIT_ENOMEM;
-
-		/* Path to GIT_INDEX_FILE */
-		if (git_index_file == NULL)
-			git__joinpath(path_aux, repo->path_repository, GIT_INDEX_FILE);
+		giterr_clear();
+
+		if (parent_path && git_path_isdir(parent_path->ptr))
+			repo->workdir = git_buf_detach(parent_path);
 		else {
-			error = gitfo_prettify_file_path(path_aux, sizeof(path_aux), git_index_file);
-			if (error < GIT_SUCCESS)
-				return git__rethrow(error, "Failed to open repository");
+			git_path_dirname_r(&worktree_buf, repo->path_repository);
+			git_path_to_dir(&worktree_buf);
+			repo->workdir = git_buf_detach(&worktree_buf);
 		}
+	}
+
+	GITERR_CHECK_ALLOC(repo->workdir);
+
+	return 0;
+}
+
+/*
+ * This function returns furthest offset into path where a ceiling dir
+ * is found, so we can stop processing the path at that point.
+ *
+ * Note: converting this to use git_bufs instead of GIT_PATH_MAX buffers on
+ * the stack could remove directories name limits, but at the cost of doing
+ * repeated malloc/frees inside the loop below, so let's not do it now.
+ */
+static int find_ceiling_dir_offset(
+	const char *path,
+	const char *ceiling_directories)
+{
+	char buf[GIT_PATH_MAX + 1];
+	char buf2[GIT_PATH_MAX + 1];
+	const char *ceil, *sep;
+	size_t len, max_len = 0, min_len;
+
+	assert(path);
+
+	min_len = (size_t)(git_path_root(path) + 1);
+
+	if (ceiling_directories == NULL || min_len == 0)
+		return (int)min_len;
+
+	for (sep = ceil = ceiling_directories; *sep; ceil = sep + 1) {
+		for (sep = ceil; *sep && *sep != GIT_PATH_LIST_SEPARATOR; sep++);
+		len = sep - ceil;
+
+		if (len == 0 || len >= sizeof(buf) || git_path_root(ceil) == -1)
+			continue;
+
+		strncpy(buf, ceil, len);
+		buf[len] = '\0';
 
-		/* store GIT_INDEX_FILE */
-		repo->path_index = git__strdup(path_aux);
-		if (repo->path_index == NULL)
-			return GIT_ENOMEM;
+		if (p_realpath(buf, buf2) == NULL)
+			continue;
+
+		len = strlen(buf2);
+		if (len > 0 && buf2[len-1] == '/')
+			buf[--len] = '\0';
+
+		if (!strncmp(path, buf2, len) &&
+			path[len] == '/' &&
+			len > max_len)
+		{
+			max_len = len;
+		}
 	}
-	
-	return GIT_SUCCESS;
+
+	return (int)(max_len <= min_len ? min_len : max_len);
 }
 
-static int check_repository_dirs(git_repository *repo)
+/*
+ * Read the contents of `file_path` and set `path_out` to the repo dir that
+ * it points to.  Before calling, set `path_out` to the base directory that
+ * should be used if the contents of `file_path` are a relative path.
+ */
+static int read_gitfile(git_buf *path_out, const char *file_path)
 {
-	char path_aux[GIT_PATH_MAX];
+	int     error = 0;
+	git_buf file = GIT_BUF_INIT;
+	size_t  prefix_len = strlen(GIT_FILE_CONTENT_PREFIX);
 
-	if (gitfo_isdir(repo->path_repository) < GIT_SUCCESS)
-		return git__throw(GIT_ENOTAREPO, "`%s` is not a folder", repo->path_repository);
+	assert(path_out && file_path);
 
-	/* Ensure GIT_OBJECT_DIRECTORY exists */
-	if (gitfo_isdir(repo->path_odb) < GIT_SUCCESS)
-		return git__throw(GIT_ENOTAREPO, "`%s` does not exist", repo->path_odb);
+	if (git_futils_readbuffer(&file, file_path) < 0)
+		return -1;
 
-	/* Ensure HEAD file exists */
-	git__joinpath(path_aux, repo->path_repository, GIT_HEAD_FILE);
-	if (gitfo_exists(path_aux) < 0)
-		return git__throw(GIT_ENOTAREPO, "HEAD file is missing");
+	git_buf_rtrim(&file);
 
-	return GIT_SUCCESS;
+	if (file.size <= prefix_len ||
+		memcmp(file.ptr, GIT_FILE_CONTENT_PREFIX, prefix_len) != 0)
+	{
+		giterr_set(GITERR_REPOSITORY, "The `.git` file at '%s' is malformed", file_path);
+		error = -1;
+	}
+	else if ((error = git_path_dirname_r(path_out, file_path)) >= 0) {
+		const char *gitlink = ((const char *)file.ptr) + prefix_len;
+		while (*gitlink && git__isspace(*gitlink)) gitlink++;
+		error = git_path_prettify_dir(path_out, gitlink, path_out->ptr);
+	}
+
+	git_buf_free(&file);
+	return error;
 }
 
-static int guess_repository_dirs(git_repository *repo, const char *repository_path)
+static int find_repo(
+	git_buf *repo_path,
+	git_buf *parent_path,
+	const char *start_path,
+	uint32_t flags,
+	const char *ceiling_dirs)
 {
-	char buffer[GIT_PATH_MAX];
-	const char *path_work_tree = NULL;
+	int error;
+	git_buf path = GIT_BUF_INIT;
+	struct stat st;
+	dev_t initial_device = 0;
+	bool try_with_dot_git = false;
+	int ceiling_offset;
+
+	git_buf_free(repo_path);
+
+	if ((error = git_path_prettify_dir(&path, start_path, NULL)) < 0)
+		return error;
+
+	ceiling_offset = find_ceiling_dir_offset(path.ptr, ceiling_dirs);
+
+	if ((error = git_buf_joinpath(&path, path.ptr, DOT_GIT)) < 0)
+		return error;
+
+	while (!error && !git_buf_len(repo_path)) {
+		if (p_stat(path.ptr, &st) == 0) {
+			/* check that we have not crossed device boundaries */
+			if (initial_device == 0)
+				initial_device = st.st_dev;
+			else if (st.st_dev != initial_device &&
+				(flags & GIT_REPOSITORY_OPEN_CROSS_FS) == 0)
+				break;
+
+			if (S_ISDIR(st.st_mode)) {
+				if (valid_repository_path(&path)) {
+					git_path_to_dir(&path);
+					git_buf_set(repo_path, path.ptr, path.size);
+					break;
+				}
+			}
+			else if (S_ISREG(st.st_mode)) {
+				git_buf repo_link = GIT_BUF_INIT;
+
+				if (!(error = read_gitfile(&repo_link, path.ptr))) {
+					if (valid_repository_path(&repo_link))
+						git_buf_swap(repo_path, &repo_link);
+
+					git_buf_free(&repo_link);
+					break;
+				}
+				git_buf_free(&repo_link);
+			}
+		}
 
-	/* Git directory name */
-	if (git__basename_r(buffer, sizeof(buffer), repository_path) < 0)
-		return git__throw(GIT_EINVALIDPATH, "Unable to parse folder name from `%s`", repository_path);
+		/* move up one directory level */
+		if (git_path_dirname_r(&path, path.ptr) < 0) {
+			error = -1;
+			break;
+		}
 
-	if (strcmp(buffer, DOT_GIT) == 0) {
-		/* Path to working dir */
-		if (git__dirname_r(buffer, sizeof(buffer), repository_path) < 0)
-			return git__throw(GIT_EINVALIDPATH, "Unable to parse parent folder name from `%s`", repository_path);
-		path_work_tree = buffer;
+		if (try_with_dot_git) {
+			/* if we tried original dir with and without .git AND either hit
+			 * directory ceiling or NO_SEARCH was requested, then be done.
+			 */
+			if (path.ptr[ceiling_offset] == '\0' ||
+				(flags & GIT_REPOSITORY_OPEN_NO_SEARCH) != 0)
+				break;
+			/* otherwise look first for .git item */
+			error = git_buf_joinpath(&path, path.ptr, DOT_GIT);
+		}
+		try_with_dot_git = !try_with_dot_git;
 	}
 
-	return assign_repository_dirs(repo, repository_path, NULL, NULL, path_work_tree);
+	if (!error && parent_path != NULL) {
+		if (!git_buf_len(repo_path))
+			git_buf_clear(parent_path);
+		else {
+			git_path_dirname_r(parent_path, path.ptr);
+			git_path_to_dir(parent_path);
+		}
+		if (git_buf_oom(parent_path))
+			return -1;
+	}
+
+	git_buf_free(&path);
+
+	if (!git_buf_len(repo_path) && !error) {
+		giterr_set(GITERR_REPOSITORY,
+			"Could not find repository from '%s'", start_path);
+		error = GIT_ENOTFOUND;
+	}
+
+	return error;
 }
 
-static git_repository *repository_alloc()
+int git_repository_open_ext(
+	git_repository **repo_ptr,
+	const char *start_path,
+	uint32_t flags,
+	const char *ceiling_dirs)
 {
 	int error;
+	git_buf path = GIT_BUF_INIT, parent = GIT_BUF_INIT;
+	git_repository *repo;
 
-	git_repository *repo = git__malloc(sizeof(git_repository));
-	if (!repo)
-		return NULL;
+	*repo_ptr = NULL;
 
-	memset(repo, 0x0, sizeof(git_repository));
+	if ((error = find_repo(&path, &parent, start_path, flags, ceiling_dirs)) < 0)
+		return error;
 
-	error = git_cache_init(&repo->objects, GIT_DEFAULT_CACHE_SIZE, &git_object__free);
-	if (error < GIT_SUCCESS) {
-		free(repo);
-		return NULL;
+	repo = repository_alloc();
+	GITERR_CHECK_ALLOC(repo);
+
+	repo->path_repository = git_buf_detach(&path);
+	GITERR_CHECK_ALLOC(repo->path_repository);
+
+	if ((error = load_config_data(repo)) < 0 ||
+		(error = load_workdir(repo, &parent)) < 0)
+	{
+		git_repository_free(repo);
+		return error;
 	}
 
-	if (git_repository__refcache_init(&repo->references) < GIT_SUCCESS) {
-		free(repo);
-		return NULL;
+	git_buf_free(&parent);
+	*repo_ptr = repo;
+	return 0;
+}
+
+int git_repository_open(git_repository **repo_out, const char *path)
+{
+	return git_repository_open_ext(
+		repo_out, path, GIT_REPOSITORY_OPEN_NO_SEARCH, NULL);
+}
+
+int git_repository_discover(
+	char *repository_path,
+	size_t size,
+	const char *start_path,
+	int across_fs,
+	const char *ceiling_dirs)
+{
+	git_buf path = GIT_BUF_INIT;
+	uint32_t flags = across_fs ? GIT_REPOSITORY_OPEN_CROSS_FS : 0;
+	int error;
+
+	assert(start_path && repository_path && size > 0);
+
+	*repository_path = '\0';
+
+	if ((error = find_repo(&path, NULL, start_path, flags, ceiling_dirs)) < 0)
+		return error != GIT_ENOTFOUND ? -1 : error;
+
+	if (size < (size_t)(path.size + 1)) {
+		giterr_set(GITERR_REPOSITORY,
+			"The given buffer is too long to store the discovered path");
+		git_buf_free(&path);
+		return -1;
 	}
 
-	return repo;
+	/* success: we discovered a repository */
+	git_buf_copy_cstr(repository_path, size, &path);
+	git_buf_free(&path);
+	return 0;
 }
 
-static int init_odb(git_repository *repo)
+static int load_config(
+	git_config **out,
+	git_repository *repo,
+	const char *global_config_path,
+	const char *system_config_path)
 {
-	return git_odb_open(&repo->db, repo->path_odb);	/* TODO: Move odb.c to new error handling */
+	git_buf config_path = GIT_BUF_INIT;
+	git_config *cfg = NULL;
+
+	assert(repo && out);
+
+	if (git_config_new(&cfg) < 0)
+		return -1;
+
+	if (git_buf_joinpath(
+		&config_path, repo->path_repository, GIT_CONFIG_FILENAME_INREPO) < 0)
+		goto on_error;
+
+	if (git_config_add_file_ondisk(cfg, config_path.ptr, 3) < 0)
+		goto on_error;
+
+	git_buf_free(&config_path);
+
+	if (global_config_path != NULL) {
+		if (git_config_add_file_ondisk(cfg, global_config_path, 2) < 0)
+			goto on_error;
+	}
+
+	if (system_config_path != NULL) {
+		if (git_config_add_file_ondisk(cfg, system_config_path, 1) < 0)
+			goto on_error;
+	}
+
+	*out = cfg;
+	return 0;
+
+on_error:
+	git_buf_free(&config_path);
+	git_config_free(cfg);
+	*out = NULL;
+	return -1;
 }
 
-int git_repository_open3(git_repository **repo_out,
-		const char *git_dir,
-		git_odb *object_database,
-		const char *git_index_file,
-		const char *git_work_tree)
+int git_repository_config__weakptr(git_config **out, git_repository *repo)
 {
-	git_repository *repo;
-	int error = GIT_SUCCESS;
+	if (repo->_config == NULL) {
+		git_buf global_buf = GIT_BUF_INIT, system_buf = GIT_BUF_INIT;
+		int res;
 
-	assert(repo_out);
+		const char *global_config_path = NULL;
+		const char *system_config_path = NULL;
 
-	if (object_database == NULL)
-		return git__throw(GIT_EINVALIDARGS, "Failed to open repository. `object_database` can't be null");
+		if (git_config_find_global_r(&global_buf) == 0)
+			global_config_path = global_buf.ptr;
 
-	repo = repository_alloc();
-	if (repo == NULL)
-		return GIT_ENOMEM;
+		if (git_config_find_system_r(&system_buf) == 0)
+			system_config_path = system_buf.ptr;
 
-	error = assign_repository_dirs(repo, 
-			git_dir, 
-			NULL,
-			git_index_file,
-			git_work_tree);
+		res = load_config(&repo->_config, repo, global_config_path, system_config_path);
 
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+		git_buf_free(&global_buf);
+		git_buf_free(&system_buf);
 
-	error = check_repository_dirs(repo);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+		if (res < 0)
+			return -1;
 
-	repo->db = object_database;
+		GIT_REFCOUNT_OWN(repo->_config, repo);
+	}
 
-	*repo_out = repo;
-	return GIT_SUCCESS;
+	*out = repo->_config;
+	return 0;
+}
 
-cleanup:
-	git_repository_free(repo);
-	return git__rethrow(error, "Failed to open repository");
+int git_repository_config(git_config **out, git_repository *repo)
+{
+	if (git_repository_config__weakptr(out, repo) < 0)
+		return -1;
+
+	GIT_REFCOUNT_INC(*out);
+	return 0;
 }
 
+void git_repository_set_config(git_repository *repo, git_config *config)
+{
+	assert(repo && config);
+
+	drop_config(repo);
+
+	repo->_config = config;
+	GIT_REFCOUNT_OWN(repo->_config, repo);
+}
 
-int git_repository_open2(git_repository **repo_out,
-		const char *git_dir,
-		const char *git_object_directory,
-		const char *git_index_file,
-		const char *git_work_tree)
+int git_repository_odb__weakptr(git_odb **out, git_repository *repo)
 {
-	git_repository *repo;
-	int error = GIT_SUCCESS;
+	assert(repo && out);
 
-	assert(repo_out);
+	if (repo->_odb == NULL) {
+		git_buf odb_path = GIT_BUF_INIT;
+		int res;
 
-	repo = repository_alloc();
-	if (repo == NULL)
-		return GIT_ENOMEM;
+		if (git_buf_joinpath(&odb_path, repo->path_repository, GIT_OBJECTS_DIR) < 0)
+			return -1;
 
-	error = assign_repository_dirs(repo,
-			git_dir, 
-			git_object_directory,
-			git_index_file,
-			git_work_tree);
+		res = git_odb_open(&repo->_odb, odb_path.ptr);
+		git_buf_free(&odb_path); /* done with path */
 
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+		if (res < 0)
+			return -1;
 
-	error = check_repository_dirs(repo);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+		GIT_REFCOUNT_OWN(repo->_odb, repo);
+	}
 
-	error = init_odb(repo);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	*out = repo->_odb;
+	return 0;
+}
 
-	*repo_out = repo;
-	return GIT_SUCCESS;
+int git_repository_odb(git_odb **out, git_repository *repo)
+{
+	if (git_repository_odb__weakptr(out, repo) < 0)
+		return -1;
 
-cleanup:
-	git_repository_free(repo);
-	return git__rethrow(error, "Failed to open repository");
+	GIT_REFCOUNT_INC(*out);
+	return 0;
 }
 
-int git_repository_open(git_repository **repo_out, const char *path)
+void git_repository_set_odb(git_repository *repo, git_odb *odb)
 {
-	git_repository *repo;
-	int error = GIT_SUCCESS;
+	assert(repo && odb);
 
-	assert(repo_out && path);
+	drop_odb(repo);
 
-	repo = repository_alloc();
-	if (repo == NULL)
-		return GIT_ENOMEM;
+	repo->_odb = odb;
+	GIT_REFCOUNT_OWN(repo->_odb, repo);
+	GIT_REFCOUNT_INC(odb);
+}
 
-	error = guess_repository_dirs(repo, path);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+int git_repository_index__weakptr(git_index **out, git_repository *repo)
+{
+	assert(out && repo);
 
-	error = check_repository_dirs(repo);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	if (repo->_index == NULL) {
+		int res;
+		git_buf index_path = GIT_BUF_INIT;
 
-	error = init_odb(repo);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+		if (git_buf_joinpath(&index_path, repo->path_repository, GIT_INDEX_FILE) < 0)
+			return -1;
 
-	*repo_out = repo;
-	return GIT_SUCCESS;
+		res = git_index_open(&repo->_index, index_path.ptr);
+		git_buf_free(&index_path); /* done with path */
 
-cleanup:
-	git_repository_free(repo);
-	return git__rethrow(error, "Failed to open repository");
+		if (res < 0)
+			return -1;
+
+		GIT_REFCOUNT_OWN(repo->_index, repo);
+	}
+
+	*out = repo->_index;
+	return 0;
 }
 
-void git_repository_free(git_repository *repo)
+int git_repository_index(git_index **out, git_repository *repo)
 {
-	if (repo == NULL)
-		return;
+	if (git_repository_index__weakptr(out, repo) < 0)
+		return -1;
 
-	git_cache_free(&repo->objects);
-	git_repository__refcache_free(&repo->references);
+	GIT_REFCOUNT_INC(*out);
+	return 0;
+}
 
-	free(repo->path_workdir);
-	free(repo->path_index);
-	free(repo->path_repository);
-	free(repo->path_odb);
+void git_repository_set_index(git_repository *repo, git_index *index)
+{
+	assert(repo && index);
 
-	if (repo->db != NULL)
-		git_odb_close(repo->db);
+	drop_index(repo);
 
-	free(repo);
+	repo->_index = index;
+	GIT_REFCOUNT_OWN(repo->_index, repo);
+	GIT_REFCOUNT_INC(index);
 }
 
-git_odb *git_repository_database(git_repository *repo)
+static int check_repositoryformatversion(git_repository *repo)
 {
-	assert(repo);
-	return repo->db;
+	git_config *config;
+	int version;
+
+	if (git_repository_config__weakptr(&config, repo) < 0)
+		return -1;
+
+	if (git_config_get_int32(&version, config, "core.repositoryformatversion") < 0)
+		return -1;
+
+	if (GIT_REPO_VERSION < version) {
+		giterr_set(GITERR_REPOSITORY,
+			"Unsupported repository version %d. Only versions up to %d are supported.",
+			version, GIT_REPO_VERSION);
+		return -1;
+	}
+
+	return 0;
 }
 
-static int repo_init_reinit(repo_init *results)
+static int repo_init_reinit(git_repository **repo_out, const char *repository_path, int is_bare)
 {
-	/* TODO: reinit the repository */
-	results->has_been_reinit = 1;
-	return git__throw(GIT_ENOTIMPLEMENTED, "Failed to reinitialize the repository. This feature is not yet implemented");
+	git_repository *repo = NULL;
+
+	GIT_UNUSED(is_bare);
+
+	if (git_repository_open(&repo, repository_path) < 0)
+		return -1;
+
+	if (check_repositoryformatversion(repo) < 0) {
+		git_repository_free(repo);
+		return -1;
+	}
+
+	/* TODO: reinitialize the templates */
+
+	*repo_out = repo;
+	return 0;
 }
 
-static int repo_init_createhead(git_repository *repo)
+static int repo_init_createhead(const char *git_dir)
 {
-	git_reference *head_reference;
-	return git_reference_create_symbolic(&head_reference, repo, GIT_HEAD_FILE, GIT_REFS_HEADS_MASTER_FILE);	/* TODO: finalize moving refs.c to new error handling */
+	git_buf ref_path = GIT_BUF_INIT;
+	git_filebuf ref = GIT_FILEBUF_INIT;
+
+	if (git_buf_joinpath(&ref_path, git_dir, GIT_HEAD_FILE) < 0 ||
+		git_filebuf_open(&ref, ref_path.ptr, 0) < 0 ||
+		git_filebuf_printf(&ref, "ref: refs/heads/master\n") < 0 ||
+		git_filebuf_commit(&ref, GIT_REFS_FILE_MODE) < 0)
+		return -1;
+
+	git_buf_free(&ref_path);
+	return 0;
 }
 
-static int repo_init_check_head_existence(char * repository_path)
+static bool is_chmod_supported(const char *file_path)
 {
-	char temp_path[GIT_PATH_MAX];
+	struct stat st1, st2;
+	static int _is_supported = -1;
+
+	if (_is_supported > -1)
+		return _is_supported;
+
+	if (p_stat(file_path, &st1) < 0)
+		return false;
+
+	if (p_chmod(file_path, st1.st_mode ^ S_IXUSR) < 0)
+		return false;
+
+	if (p_stat(file_path, &st2) < 0)
+		return false;
 
-	git__joinpath(temp_path, repository_path, GIT_HEAD_FILE);
-	return gitfo_exists(temp_path);
+	_is_supported = (st1.st_mode != st2.st_mode);
+	return _is_supported;
 }
 
-static int repo_init_structure(repo_init *results)
+static bool is_filesystem_case_insensitive(const char *gitdir_path)
 {
-	const int mode = 0755; /* or 0777 ? */
-	int error;
+	git_buf path = GIT_BUF_INIT;
+	static int _is_insensitive = -1;
+
+	if (_is_insensitive > -1)
+		return _is_insensitive;
 
-	char temp_path[GIT_PATH_MAX];
-	char *git_dir = results->path_repository;
+	if (git_buf_joinpath(&path, gitdir_path, "CoNfIg") < 0)
+		goto cleanup;
 
-	if (gitfo_mkdir_recurs(git_dir, mode))
-		return git__throw(GIT_ERROR, "Failed to initialize repository structure. Could not mkdir");
+	_is_insensitive = git_path_exists(git_buf_cstr(&path));
 
-	/* Creates the '/objects/info/' directory */
-	git__joinpath(temp_path, git_dir, GIT_OBJECTS_INFO_DIR);
-	error = gitfo_mkdir_recurs(temp_path, mode);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to initialize repository structure");
+cleanup:
+	git_buf_free(&path);
+	return _is_insensitive;
+}
 
-	/* Creates the '/objects/pack/' directory */
-	git__joinpath(temp_path, git_dir, GIT_OBJECTS_PACK_DIR);
-	error = gitfo_mkdir(temp_path, mode);
-	if (error < GIT_SUCCESS)
-		return git__throw(error, "Unable to create `%s` folder", temp_path);
+static int repo_init_config(const char *git_dir, bool is_bare, bool is_reinit)
+{
+	git_buf cfg_path = GIT_BUF_INIT;
+	git_config *config = NULL;
+
+#define SET_REPO_CONFIG(type, name, val) {\
+	if (git_config_set_##type(config, name, val) < 0) { \
+		git_buf_free(&cfg_path); \
+		git_config_free(config); \
+		return -1; } \
+}
 
-	/* Creates the '/refs/heads/' directory */
-	git__joinpath(temp_path, git_dir, GIT_REFS_HEADS_DIR);
-	error = gitfo_mkdir_recurs(temp_path, mode);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to initialize repository structure");
+	if (git_buf_joinpath(&cfg_path, git_dir, GIT_CONFIG_FILENAME_INREPO) < 0)
+		return -1;
 
-	/* Creates the '/refs/tags/' directory */
-	git__joinpath(temp_path, git_dir, GIT_REFS_TAGS_DIR);
-	error = gitfo_mkdir(temp_path, mode);
-	if (error < GIT_SUCCESS)
-		return git__throw(error, "Unable to create `%s` folder", temp_path);
+	if (git_config_open_ondisk(&config, git_buf_cstr(&cfg_path)) < 0) {
+		git_buf_free(&cfg_path);
+		return -1;
+	}
 
-	/* TODO: what's left? templates? */
+	SET_REPO_CONFIG(bool, "core.bare", is_bare);
+	SET_REPO_CONFIG(int32, "core.repositoryformatversion", GIT_REPO_VERSION);
+	SET_REPO_CONFIG(bool, "core.filemode", is_chmod_supported(git_buf_cstr(&cfg_path)));
+	
+	if (!is_reinit && is_filesystem_case_insensitive(git_dir))
+		SET_REPO_CONFIG(bool, "core.ignorecase", true);
+	/* TODO: what other defaults? */
 
-	return GIT_SUCCESS;
+	git_buf_free(&cfg_path);
+	git_config_free(config);
+	return 0;
 }
 
-static int repo_init_find_dir(repo_init *results, const char* path)
+#define GIT_HOOKS_DIR "hooks/"
+#define GIT_HOOKS_DIR_MODE 0755
+
+#define GIT_HOOKS_README_FILE GIT_HOOKS_DIR "README.sample"
+#define GIT_HOOKS_README_MODE 0755
+#define GIT_HOOKS_README_CONTENT \
+"#!/bin/sh\n"\
+"#\n"\
+"# Place appropriately named executable hook scripts into this directory\n"\
+"# to intercept various actions that git takes.  See `git help hooks` for\n"\
+"# more information.\n"
+
+#define GIT_INFO_DIR "info/"
+#define GIT_INFO_DIR_MODE 0755
+
+#define GIT_INFO_EXCLUDE_FILE GIT_INFO_DIR "exclude"
+#define GIT_INFO_EXCLUDE_MODE 0644
+#define GIT_INFO_EXCLUDE_CONTENT \
+"# File patterns to ignore; see `git help ignore` for more information.\n"\
+"# Lines that start with '#' are comments.\n"
+
+#define GIT_DESC_FILE "description"
+#define GIT_DESC_MODE 0644
+#define GIT_DESC_CONTENT "Unnamed repository; edit this file 'description' to name the repository.\n"
+
+static int repo_write_template(
+	const char *git_dir, const char *file, mode_t mode, const char *content)
 {
-	char temp_path[GIT_PATH_MAX];
-	int error = GIT_SUCCESS;
+	git_buf path = GIT_BUF_INIT;
+	int fd, error = 0;
+
+	if (git_buf_joinpath(&path, git_dir, file) < 0)
+		return -1;
+
+	fd = p_open(git_buf_cstr(&path), O_WRONLY | O_CREAT | O_EXCL, mode);
 
-	error = gitfo_prettify_dir_path(temp_path, sizeof(temp_path), path);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to find directory to initialize repository");
+	if (fd >= 0) {
+		error = p_write(fd, content, strlen(content));
 
-	if (!results->is_bare) {
-		git__joinpath(temp_path, temp_path, GIT_DIR);
+		p_close(fd);
 	}
+	else if (errno != EEXIST)
+		error = fd;
 
-	results->path_repository = git__strdup(temp_path);
-	if (results->path_repository == NULL)
-		return GIT_ENOMEM;
+	git_buf_free(&path);
 
-	return GIT_SUCCESS;
+	if (error)
+		giterr_set(GITERR_OS,
+			"Failed to initialize repository with template '%s'", file);
+
+	return error;
+}
+
+static int repo_init_structure(const char *git_dir, int is_bare)
+{
+	int i;
+	struct { const char *dir; mode_t mode; } dirs[] = {
+		{ GIT_OBJECTS_INFO_DIR, GIT_OBJECT_DIR_MODE }, /* '/objects/info/' */
+		{ GIT_OBJECTS_PACK_DIR, GIT_OBJECT_DIR_MODE }, /* '/objects/pack/' */
+		{ GIT_REFS_HEADS_DIR, GIT_REFS_DIR_MODE },     /* '/refs/heads/' */
+		{ GIT_REFS_TAGS_DIR, GIT_REFS_DIR_MODE },      /* '/refs/tags/' */
+		{ GIT_HOOKS_DIR, GIT_HOOKS_DIR_MODE },         /* '/hooks/' */
+		{ GIT_INFO_DIR, GIT_INFO_DIR_MODE },           /* '/info/' */
+		{ NULL, 0 }
+	};
+	struct { const char *file; mode_t mode; const char *content; } tmpl[] = {
+		{ GIT_DESC_FILE, GIT_DESC_MODE, GIT_DESC_CONTENT },
+		{ GIT_HOOKS_README_FILE, GIT_HOOKS_README_MODE, GIT_HOOKS_README_CONTENT },
+		{ GIT_INFO_EXCLUDE_FILE, GIT_INFO_EXCLUDE_MODE, GIT_INFO_EXCLUDE_CONTENT },
+		{ NULL, 0, NULL }
+	};
+
+	/* Make the base directory */
+	if (git_futils_mkdir_r(git_dir, NULL, is_bare ? GIT_BARE_DIR_MODE : GIT_DIR_MODE) < 0)
+		return -1;
+
+	/* Hides the ".git" directory */
+	if (!is_bare) {
+#ifdef GIT_WIN32
+		if (p_hide_directory__w32(git_dir) < 0) {
+			giterr_set(GITERR_REPOSITORY,
+				"Failed to mark Git repository folder as hidden");
+			return -1;
+		}
+#endif
+	}
+
+	/* Make subdirectories as needed */
+	for (i = 0; dirs[i].dir != NULL; ++i) {
+		if (git_futils_mkdir_r(dirs[i].dir, git_dir, dirs[i].mode) < 0)
+			return -1;
+	}
+
+	/* Make template files as needed */
+	for (i = 0; tmpl[i].file != NULL; ++i) {
+		if (repo_write_template(
+				git_dir, tmpl[i].file, tmpl[i].mode, tmpl[i].content) < 0)
+			return -1;
+	}
+
+	return 0;
 }
 
 int git_repository_init(git_repository **repo_out, const char *path, unsigned is_bare)
 {
-	int error = GIT_SUCCESS;
-	git_repository *repo = NULL;
-	repo_init results;
-	
-	assert(repo_out && path);
+	git_buf repository_path = GIT_BUF_INIT;
+	bool is_reinit;
+	int result = -1;
 
-	results.path_repository = NULL;
-	results.is_bare = is_bare;
+	assert(repo_out && path);
 
-	error = repo_init_find_dir(&results, path);
-	if (error < GIT_SUCCESS)
+	if (git_buf_joinpath(&repository_path, path, is_bare ? "" : GIT_DIR) < 0)
 		goto cleanup;
 
-	if (!repo_init_check_head_existence(results.path_repository))
-		return repo_init_reinit(&results);
+	is_reinit = git_path_isdir(repository_path.ptr) && valid_repository_path(&repository_path);
 
-	error = repo_init_structure(&results);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+	if (is_reinit) {
+		if (repo_init_reinit(repo_out, repository_path.ptr, is_bare) < 0)
+			goto cleanup;
 
-	repo = repository_alloc();
-	if (repo == NULL) {
-		error = GIT_ENOMEM;
-		goto cleanup;
+		result = repo_init_config(repository_path.ptr, is_bare, is_reinit);
 	}
 
-	error = guess_repository_dirs(repo, results.path_repository);
-	if (error < GIT_SUCCESS)
+	if (repo_init_structure(repository_path.ptr, is_bare) < 0 ||
+		repo_init_config(repository_path.ptr, is_bare, is_reinit) < 0 || 
+		repo_init_createhead(repository_path.ptr) < 0 ||
+		git_repository_open(repo_out, repository_path.ptr) < 0) {
 		goto cleanup;
+	}
 
-	assert(repo->is_bare == is_bare);
+	result = 0;
 
-	error = init_odb(repo);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+cleanup:
+	git_buf_free(&repository_path);
+	return result;
+}
 
-	error = repo_init_createhead(repo);
-	if (error < GIT_SUCCESS)
-		goto cleanup;
+int git_repository_head_detached(git_repository *repo)
+{
+	git_reference *ref;
+	git_odb *odb = NULL;
+	int exists;
 
-	/* should never fail */
-	assert(check_repository_dirs(repo) == GIT_SUCCESS);
+	if (git_repository_odb__weakptr(&odb, repo) < 0)
+		return -1;
 
-	free(results.path_repository);
-	*repo_out = repo;
-	return GIT_SUCCESS;
+	if (git_reference_lookup(&ref, repo, GIT_HEAD_FILE) < 0)
+		return -1;
 
-cleanup:
-	free(results.path_repository);
-	git_repository_free(repo);
-	return git__rethrow(error, "Failed to (re)init the repository `%s`", path);
+	if (git_reference_type(ref) == GIT_REF_SYMBOLIC) {
+		git_reference_free(ref);
+		return 0;
+	}
+
+	exists = git_odb_exists(odb, git_reference_oid(ref));
+
+	git_reference_free(ref);
+	return exists;
+}
+
+int git_repository_head(git_reference **head_out, git_repository *repo)
+{
+	return git_reference_lookup_resolved(head_out, repo, GIT_HEAD_FILE, -1);
+}
+
+int git_repository_head_orphan(git_repository *repo)
+{
+	git_reference *ref = NULL;
+	int error;
+
+	error = git_repository_head(&ref, repo);
+	git_reference_free(ref);
+
+	if (error == GIT_ENOTFOUND)
+		return 1;
+
+	if (error < 0)
+		return -1;
+
+	return 0;
 }
 
 int git_repository_is_empty(git_repository *repo)
 {
-	git_reference *head, *branch;
+	git_reference *head = NULL, *branch = NULL;
 	int error;
 
-	error = git_reference_lookup(&head, repo, "HEAD");
-	if (error < GIT_SUCCESS)
-		return git__throw(error, "Failed to determine the emptiness of the repository. An error occured while retrieving the HEAD reference");
+	if (git_reference_lookup(&head, repo, "HEAD") < 0)
+		return -1;
+
+	if (git_reference_type(head) != GIT_REF_SYMBOLIC) {
+		git_reference_free(head);
+		return 0;
+	}
+
+	if (strcmp(git_reference_target(head), "refs/heads/master") != 0) {
+		git_reference_free(head);
+		return 0;
+	}
+
+	error = git_reference_resolve(&branch, head);
+
+	git_reference_free(head);
+	git_reference_free(branch);
+
+	if (error == GIT_ENOTFOUND)
+		return 1;
 
-	if (git_reference_type(head) != GIT_REF_SYMBOLIC)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to determine the emptiness of the repository. HEAD is probably in detached state");
+	if (error < 0)
+		return -1;
 
-	return git_reference_resolve(&branch, head) == GIT_SUCCESS ? 0 : 1;
+	return 0;
 }
 
 const char *git_repository_path(git_repository *repo)
@@ -484,7 +950,27 @@ const char *git_repository_path(git_repository *repo)
 const char *git_repository_workdir(git_repository *repo)
 {
 	assert(repo);
-	return repo->path_workdir;
+
+	if (repo->is_bare)
+		return NULL;
+
+	return repo->workdir;
+}
+
+int git_repository_set_workdir(git_repository *repo, const char *workdir)
+{
+	git_buf path = GIT_BUF_INIT;
+
+	assert(repo && workdir);
+
+	if (git_path_prettify_dir(&path, workdir, NULL) < 0)
+		return -1;
+
+	git__free(repo->workdir);
+
+	repo->workdir = git_buf_detach(&path);
+	repo->is_bare = 0;
+	return 0;
 }
 
 int git_repository_is_bare(git_repository *repo)
@@ -492,3 +978,23 @@ int git_repository_is_bare(git_repository *repo)
 	assert(repo);
 	return repo->is_bare;
 }
+
+int git_repository_head_tree(git_tree **tree, git_repository *repo)
+{
+	git_oid head_oid;
+	git_object *obj = NULL;
+
+	if (git_reference_name_to_oid(&head_oid, repo, GIT_HEAD_FILE) < 0) {
+		/* cannot resolve HEAD - probably brand new repo */
+		giterr_clear();
+		*tree = NULL;
+		return 0;
+	}
+
+	if (git_object_lookup(&obj, repo, &head_oid, GIT_OBJ_ANY) < 0 ||
+		git_object__resolve_to_type(&obj, GIT_OBJ_TREE) < 0)
+		return -1;
+
+	*tree = (git_tree *)obj;
+	return 0;
+}
diff --git a/src/repository.h b/src/repository.h
index bcf9b2bc8..91c69a655 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_repository_h__
 #define INCLUDE_repository_h__
 
@@ -7,16 +13,62 @@
 #include "git2/repository.h"
 #include "git2/object.h"
 
-#include "hashtable.h"
 #include "index.h"
 #include "cache.h"
 #include "refs.h"
+#include "buffer.h"
+#include "odb.h"
+#include "attr.h"
+#include "strmap.h"
 
 #define DOT_GIT ".git"
 #define GIT_DIR DOT_GIT "/"
-#define GIT_OBJECTS_DIR "objects/"
-#define GIT_INDEX_FILE "index"
+#define GIT_DIR_MODE 0755
+#define GIT_BARE_DIR_MODE 0777
 
+/** Cvar cache identifiers */
+typedef enum {
+	GIT_CVAR_AUTO_CRLF = 0, /* core.autocrlf */
+	GIT_CVAR_EOL, /* core.eol */
+	GIT_CVAR_CACHE_MAX
+} git_cvar_cached;
+
+/**
+ * CVAR value enumerations
+ *
+ * These are the values that are actually stored in the cvar cache, instead
+ * of their string equivalents. These values are internal and symbolic;
+ * make sure that none of them is set to `-1`, since that is the unique
+ * identifier for "not cached"
+ */
+typedef enum {
+	/* The value hasn't been loaded from the cache yet */
+	GIT_CVAR_NOT_CACHED = -1,
+
+	/* core.safecrlf: false, 'fail', 'warn' */
+	GIT_SAFE_CRLF_FALSE = 0,
+	GIT_SAFE_CRLF_FAIL = 1,
+	GIT_SAFE_CRLF_WARN = 2,
+
+	/* core.autocrlf: false, true, 'input; */
+	GIT_AUTO_CRLF_FALSE = 0,
+	GIT_AUTO_CRLF_TRUE = 1,
+	GIT_AUTO_CRLF_INPUT = 2,
+	GIT_AUTO_CRLF_DEFAULT = GIT_AUTO_CRLF_FALSE,
+
+	/* core.eol: unset, 'crlf', 'lf', 'native' */
+	GIT_EOL_UNSET = 0,
+	GIT_EOL_CRLF = 1,
+	GIT_EOL_LF = 2,
+#ifdef GIT_WIN32
+	GIT_EOL_NATIVE = GIT_EOL_CRLF,
+#else
+	GIT_EOL_NATIVE = GIT_EOL_LF,
+#endif
+	GIT_EOL_DEFAULT = GIT_EOL_NATIVE
+} git_cvar_value;
+
+/** Base git object for inheritance */
 struct git_object {
 	git_cached_obj cached;
 	git_repository *repo;
@@ -24,25 +76,63 @@ struct git_object {
 };
 
 struct git_repository {
-	git_odb *db;
+	git_odb *_odb;
+	git_config *_config;
+	git_index *_index;
 
 	git_cache objects;
 	git_refcache references;
+	git_attr_cache attrcache;
+	git_strmap *submodules;
 
 	char *path_repository;
-	char *path_index;
-	char *path_odb;
-	char *path_workdir;
+	char *workdir;
 
 	unsigned is_bare:1;
 	unsigned int lru_counter;
+
+	git_cvar_value cvar_cache[GIT_CVAR_CACHE_MAX];
 };
 
 /* fully free the object; internal method, do not
  * export */
 void git_object__free(void *object);
 
-int git__parse_oid(git_oid *oid, const char **buffer_out, const char *buffer_end, const char *header);
-int git__write_oid(git_odb_stream *src, const char *header, const git_oid *oid);
+int git_object__resolve_to_type(git_object **obj, git_otype type);
+
+int git_oid__parse(git_oid *oid, const char **buffer_out, const char *buffer_end, const char *header);
+void git_oid__writebuf(git_buf *buf, const char *header, const git_oid *oid);
+
+GIT_INLINE(git_attr_cache *) git_repository_attr_cache(git_repository *repo)
+{
+	return &repo->attrcache;
+}
+
+int git_repository_head_tree(git_tree **tree, git_repository *repo);
+
+/*
+ * Weak pointers to repository internals.
+ *
+ * The returned pointers do not need to be freed. Do not keep
+ * permanent references to these (i.e. between API calls), since they may
+ * become invalidated if the user replaces a repository internal.
+ */
+int git_repository_config__weakptr(git_config **out, git_repository *repo);
+int git_repository_odb__weakptr(git_odb **out, git_repository *repo);
+int git_repository_index__weakptr(git_index **out, git_repository *repo);
+
+/*
+ * CVAR cache 
+ *
+ * Efficient access to the most used config variables of a repository.
+ * The cache is cleared everytime the config backend is replaced.
+ */
+int git_repository__cvar(int *out, git_repository *repo, git_cvar_cached cvar);
+void git_repository__cvar_cache_clear(git_repository *repo);
+
+/*
+ * Submodule cache
+ */
+extern void git_submodule_config_free(git_repository *repo);
 
 #endif
diff --git a/src/revwalk.c b/src/revwalk.c
index b64b0e2c0..e64d93f20 100644
--- a/src/revwalk.c
+++ b/src/revwalk.c
@@ -1,35 +1,28 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
 #include "commit.h"
 #include "odb.h"
-#include "hashtable.h"
 #include "pqueue.h"
+#include "pool.h"
+#include "oidmap.h"
 
 #include "git2/revwalk.h"
+#include "git2/merge.h"
+
+#include <regex.h>
+
+GIT__USE_OIDMAP;
+
+#define PARENT1  (1 << 0)
+#define PARENT2  (1 << 1)
+#define RESULT   (1 << 2)
+#define STALE    (1 << 3)
 
 typedef struct commit_object {
 	git_oid oid;
@@ -37,7 +30,8 @@ typedef struct commit_object {
 	unsigned int seen:1,
 			 uninteresting:1,
 			 topo_delay:1,
-			 parsed:1;
+			 parsed:1,
+			 flags : 4;
 
 	unsigned short in_degree;
 	unsigned short out_degree;
@@ -52,8 +46,10 @@ typedef struct commit_list {
 
 struct git_revwalk {
 	git_repository *repo;
+	git_odb *odb;
 
-	git_hashtable *commits;
+	git_oidmap *commits;
+	git_pool commit_pool;
 
 	commit_list *iterator_topo;
 	commit_list *iterator_rand;
@@ -63,110 +59,102 @@ struct git_revwalk {
 	int (*get_next)(commit_object **, git_revwalk *);
 	int (*enqueue)(git_revwalk *, commit_object *);
 
-	git_vector memory_alloc;
-	size_t chunk_size;
-
 	unsigned walking:1;
 	unsigned int sorting;
+
+	/* merge base calculation */
+	commit_object *one;
+	git_vector twos;
 };
 
-commit_list *commit_list_insert(commit_object *item, commit_list **list_p)
+static int commit_time_cmp(void *a, void *b)
+{
+	commit_object *commit_a = (commit_object *)a;
+	commit_object *commit_b = (commit_object *)b;
+
+	return (commit_a->time < commit_b->time);
+}
+
+static commit_list *commit_list_insert(commit_object *item, commit_list **list_p)
 {
 	commit_list *new_list = git__malloc(sizeof(commit_list));
-	new_list->item = item;
-	new_list->next = *list_p;
+	if (new_list != NULL) {
+		new_list->item = item;
+		new_list->next = *list_p;
+	}
 	*list_p = new_list;
 	return new_list;
 }
 
-void commit_list_free(commit_list **list_p)
+static commit_list *commit_list_insert_by_date(commit_object *item, commit_list **list_p)
+{
+	commit_list **pp = list_p;
+	commit_list *p;
+
+	while ((p = *pp) != NULL) {
+		if (commit_time_cmp(p->item, item) < 0)
+			break;
+
+		pp = &p->next;
+	}
+
+	return commit_list_insert(item, pp);
+}
+static void commit_list_free(commit_list **list_p)
 {
 	commit_list *list = *list_p;
 
 	while (list) {
 		commit_list *temp = list;
 		list = temp->next;
-		free(temp);
+		git__free(temp);
 	}
 
 	*list_p = NULL;
 }
 
-commit_object *commit_list_pop(commit_list **stack)
+static commit_object *commit_list_pop(commit_list **stack)
 {
 	commit_list *top = *stack;
 	commit_object *item = top ? top->item : NULL;
 
 	if (top) {
 		*stack = top->next;
-		free(top);
+		git__free(top);
 	}
 	return item;
 }
 
-static int commit_time_cmp(void *a, void *b)
-{
-	commit_object *commit_a = (commit_object *)a;
-	commit_object *commit_b = (commit_object *)b;
-
-	return (commit_a->time < commit_b->time);
-}
-
-static uint32_t object_table_hash(const void *key, int hash_id)
-{
-	uint32_t r;
-	git_oid *id;
-
-	id = (git_oid *)key;
-	memcpy(&r, id->id + (hash_id * sizeof(uint32_t)), sizeof(r));
-	return r;
-}
-
-#define COMMITS_PER_CHUNK 128
-#define CHUNK_STEP 64
-#define PARENTS_PER_COMMIT ((CHUNK_STEP - sizeof(commit_object)) / sizeof(commit_object *))
-
-static int alloc_chunk(git_revwalk *walk)
-{
-	void *chunk;
-
-	chunk = git__calloc(COMMITS_PER_CHUNK, CHUNK_STEP);
-	if (chunk == NULL)
-		return GIT_ENOMEM;
-
-	walk->chunk_size = 0;
-	return git_vector_insert(&walk->memory_alloc, chunk);
-}
+#define PARENTS_PER_COMMIT	2
+#define COMMIT_ALLOC \
+	(sizeof(commit_object) + PARENTS_PER_COMMIT * sizeof(commit_object *))
 
 static commit_object *alloc_commit(git_revwalk *walk)
 {
-	unsigned char *chunk;
-
-	if (walk->chunk_size == COMMITS_PER_CHUNK)
-		alloc_chunk(walk);
-
-	chunk = git_vector_get(&walk->memory_alloc, walk->memory_alloc.length - 1);
-	chunk += (walk->chunk_size * CHUNK_STEP);
-	walk->chunk_size++;
-
-	return (commit_object *)chunk;
+	return (commit_object *)git_pool_malloc(&walk->commit_pool, COMMIT_ALLOC);
 }
 
-static commit_object **alloc_parents(commit_object *commit, size_t n_parents)
+static commit_object **alloc_parents(
+	git_revwalk *walk, commit_object *commit, size_t n_parents)
 {
 	if (n_parents <= PARENTS_PER_COMMIT)
-		return (commit_object **)((unsigned char *)commit + sizeof(commit_object));
+		return (commit_object **)((char *)commit + sizeof(commit_object));
 
-	return git__malloc(n_parents * sizeof(commit_object *));
+	return (commit_object **)git_pool_malloc(
+		&walk->commit_pool, (uint32_t)(n_parents * sizeof(commit_object *)));
 }
 
 
 static commit_object *commit_lookup(git_revwalk *walk, const git_oid *oid)
 {
 	commit_object *commit;
+	khiter_t pos;
+	int ret;
 
-	if ((commit = git_hashtable_lookup(walk->commits, oid)) != NULL)
-		return commit;
+	/* lookup and reserve space if not already present */
+	pos = kh_get(oid, walk->commits, oid);
+	if (pos != kh_end(walk->commits))
+		return kh_value(walk->commits, pos);
 
 	commit = alloc_commit(walk);
 	if (commit == NULL)
@@ -174,66 +162,70 @@ static commit_object *commit_lookup(git_revwalk *walk, const git_oid *oid)
 
 	git_oid_cpy(&commit->oid, oid);
 
-	if (git_hashtable_insert(walk->commits, &commit->oid, commit) < GIT_SUCCESS) {
-		free(commit);
-		return NULL;
-	}
+	pos = kh_put(oid, walk->commits, &commit->oid, &ret);
+	assert(ret != 0);
+	kh_value(walk->commits, pos) = commit;
 
 	return commit;
 }
 
 static int commit_quick_parse(git_revwalk *walk, commit_object *commit, git_rawobj *raw)
 {
-	const int parent_len = STRLEN("parent ") + GIT_OID_HEXSZ + 1;
+	const size_t parent_len = strlen("parent ") + GIT_OID_HEXSZ + 1;
 
 	unsigned char *buffer = raw->data;
 	unsigned char *buffer_end = buffer + raw->len;
 	unsigned char *parents_start;
 
 	int i, parents = 0;
-	long commit_time;
+	int commit_time;
 
-	buffer += STRLEN("tree ") + GIT_OID_HEXSZ + 1;
+	buffer += strlen("tree ") + GIT_OID_HEXSZ + 1;
 
 	parents_start = buffer;
-	while (buffer + parent_len < buffer_end && memcmp(buffer, "parent ", STRLEN("parent ")) == 0) {
+	while (buffer + parent_len < buffer_end && memcmp(buffer, "parent ", strlen("parent ")) == 0) {
 		parents++;
 		buffer += parent_len;
 	}
 
-	commit->parents = alloc_parents(commit, parents);
-	if (commit->parents == NULL)
-		return GIT_ENOMEM;
+	commit->parents = alloc_parents(walk, commit, parents);
+	GITERR_CHECK_ALLOC(commit->parents);
 
 	buffer = parents_start;
 	for (i = 0; i < parents; ++i) {
 		git_oid oid;
 
-		if (git_oid_mkstr(&oid, (char *)buffer + STRLEN("parent ")) < GIT_SUCCESS)
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to parse commit. Parent object is corrupted");
+		if (git_oid_fromstr(&oid, (char *)buffer + strlen("parent ")) < 0)
+			return -1;
 
 		commit->parents[i] = commit_lookup(walk, &oid);
 		if (commit->parents[i] == NULL)
-			return GIT_ENOMEM;
+			return -1;
 
 		buffer += parent_len;
 	}
 
 	commit->out_degree = (unsigned short)parents;
 
-	if ((buffer = memchr(buffer, '\n', buffer_end - buffer)) == NULL)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse commit. Object is corrupted");
+	if ((buffer = memchr(buffer, '\n', buffer_end - buffer)) == NULL) {
+		giterr_set(GITERR_ODB, "Failed to parse commit. Object is corrupted");
+		return -1;
+	}
 
 	buffer = memchr(buffer, '>', buffer_end - buffer);
-	if (buffer == NULL)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse commit. Can't find author");
+	if (buffer == NULL) {
+		giterr_set(GITERR_ODB, "Failed to parse commit. Can't find author");
+		return -1;
+	}
 
-	if (git__strtol32(&commit_time, (char *)buffer + 2, NULL, 10) < GIT_SUCCESS)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse commit. Can't parse commit time");
+	if (git__strtol32(&commit_time, (char *)buffer + 2, NULL, 10) < 0) {
+		giterr_set(GITERR_ODB, "Failed to parse commit. Can't parse commit time");
+		return -1;
+	}
 
 	commit->time = (time_t)commit_time;
 	commit->parsed = 1;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 static int commit_parse(git_revwalk *walk, commit_object *commit)
@@ -242,19 +234,159 @@ static int commit_parse(git_revwalk *walk, commit_object *commit)
 	int error;
 
 	if (commit->parsed)
-		return GIT_SUCCESS;
+		return 0;
 
-	if ((error = git_odb_read(&obj, walk->repo->db, &commit->oid)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to parse commit. Can't read object");
+	if ((error = git_odb_read(&obj, walk->odb, &commit->oid)) < 0)
+		return error;
 
 	if (obj->raw.type != GIT_OBJ_COMMIT) {
-		git_odb_object_close(obj);
-		return git__throw(GIT_EOBJTYPE, "Failed to parse commit. Object is no commit object");
+		git_odb_object_free(obj);
+		giterr_set(GITERR_INVALID, "Failed to parse commit. Object is no commit object");
+		return -1;
 	}
 
 	error = commit_quick_parse(walk, commit, &obj->raw);
-	git_odb_object_close(obj);
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to parse commit");
+	git_odb_object_free(obj);
+	return error;
+}
+
+static int interesting(git_pqueue *list)
+{
+	unsigned int i;
+	for (i = 1; i < git_pqueue_size(list); i++) {
+		commit_object *commit = list->d[i];
+		if ((commit->flags & STALE) == 0)
+			return 1;
+	}
+
+	return 0;
+}
+
+static int merge_bases_many(commit_list **out, git_revwalk *walk, commit_object *one, git_vector *twos)
+{
+	int error;
+	unsigned int i;
+	commit_object *two;
+	commit_list *result = NULL, *tmp = NULL;
+	git_pqueue list;
+
+	/* if the commit is repeated, we have a our merge base already */
+	git_vector_foreach(twos, i, two) {
+		if (one == two)
+			return commit_list_insert(one, out) ? 0 : -1;
+	}
+
+	if (git_pqueue_init(&list, twos->length * 2, commit_time_cmp) < 0)
+		return -1;
+
+	if (commit_parse(walk, one) < 0)
+	    return -1;
+
+	one->flags |= PARENT1;
+	if (git_pqueue_insert(&list, one) < 0)
+		return -1;
+
+	git_vector_foreach(twos, i, two) {
+		commit_parse(walk, two);
+		two->flags |= PARENT2;
+		if (git_pqueue_insert(&list, two) < 0)
+			return -1;
+	}
+
+	/* as long as there are non-STALE commits */
+	while (interesting(&list)) {
+		commit_object *commit;
+		int flags;
+
+		commit = git_pqueue_pop(&list);
+
+		flags = commit->flags & (PARENT1 | PARENT2 | STALE);
+		if (flags == (PARENT1 | PARENT2)) {
+			if (!(commit->flags & RESULT)) {
+				commit->flags |= RESULT;
+				if (commit_list_insert(commit, &result) == NULL)
+					return -1;
+			}
+			/* we mark the parents of a merge stale */
+			flags |= STALE;
+		}
+
+		for (i = 0; i < commit->out_degree; i++) {
+			commit_object *p = commit->parents[i];
+			if ((p->flags & flags) == flags)
+				continue;
+
+			if ((error = commit_parse(walk, p)) < 0)
+				return error;
+
+			p->flags |= flags;
+			if (git_pqueue_insert(&list, p) < 0)
+				return -1;
+		}
+	}
+
+	git_pqueue_free(&list);
+
+	/* filter out any stale commits in the results */
+	tmp = result;
+	result = NULL;
+
+	while (tmp) {
+		struct commit_list *next = tmp->next;
+		if (!(tmp->item->flags & STALE))
+			if (commit_list_insert_by_date(tmp->item, &result) == NULL)
+				return -1;
+
+		git__free(tmp);
+		tmp = next;
+	}
+
+	*out = result;
+	return 0;
+}
+
+int git_merge_base(git_oid *out, git_repository *repo, git_oid *one, git_oid *two)
+{
+	git_revwalk *walk;
+	git_vector list;
+	commit_list *result = NULL;
+	commit_object *commit;
+	void *contents[1];
+
+	if (git_revwalk_new(&walk, repo) < 0)
+		return -1;
+
+	commit = commit_lookup(walk, two);
+	if (commit == NULL)
+		goto on_error;
+
+	/* This is just one value, so we can do it on the stack */
+	memset(&list, 0x0, sizeof(git_vector));
+	contents[0] = commit;
+	list.length = 1;
+	list.contents = contents;
+
+	commit = commit_lookup(walk, one);
+	if (commit == NULL)
+		goto on_error;
+
+	if (merge_bases_many(&result, walk, commit, &list) < 0)
+		goto on_error;
+
+	if (!result) {
+		git_revwalk_free(walk);
+		return GIT_ENOTFOUND;
+	}
+
+	git_oid_cpy(out, &result->item->oid);
+	commit_list_free(&result);
+	git_revwalk_free(walk);
+
+	return 0;
+
+on_error:
+	git_revwalk_free(walk);
+	return -1;
 }
 
 static void mark_uninteresting(commit_object *commit)
@@ -264,25 +396,29 @@ static void mark_uninteresting(commit_object *commit)
 
 	commit->uninteresting = 1;
 
+	/* This means we've reached a merge base, so there's no need to walk any more */
+	if ((commit->flags & (RESULT | STALE)) == RESULT)
+		return;
+
 	for (i = 0; i < commit->out_degree; ++i)
 		if (!commit->parents[i]->uninteresting)
 			mark_uninteresting(commit->parents[i]);
 }
 
-static int process_commit(git_revwalk *walk, commit_object *commit)
+static int process_commit(git_revwalk *walk, commit_object *commit, int hide)
 {
 	int error;
 
+	if (hide)
+		mark_uninteresting(commit);
+
 	if (commit->seen)
-		return GIT_SUCCESS;
+		return 0;
 
 	commit->seen = 1;
 
-	if ((error = commit_parse(walk, commit)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to process commit");
-
-	if (commit->uninteresting)
-		mark_uninteresting(commit);
+	if ((error = commit_parse(walk, commit)) < 0)
+		return error;
 
 	return walk->enqueue(walk, commit);
 }
@@ -290,13 +426,12 @@ static int process_commit(git_revwalk *walk, commit_object *commit)
 static int process_commit_parents(git_revwalk *walk, commit_object *commit)
 {
 	unsigned short i;
-	int error = GIT_SUCCESS;
+	int error = 0;
 
-	for (i = 0; i < commit->out_degree && error == GIT_SUCCESS; ++i) {
-		error = process_commit(walk, commit->parents[i]);
-	}
+	for (i = 0; i < commit->out_degree && !error; ++i)
+		error = process_commit(walk, commit->parents[i], commit->uninteresting);
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to process commit parents");
+	return error;
 }
 
 static int push_commit(git_revwalk *walk, const git_oid *oid, int uninteresting)
@@ -305,11 +440,17 @@ static int push_commit(git_revwalk *walk, const git_oid *oid, int uninteresting)
 
 	commit = commit_lookup(walk, oid);
 	if (commit == NULL)
-		return git__throw(GIT_ENOTFOUND, "Failed to push commit. Object not found");
+		return -1; /* error already reported by failed lookup */
 
 	commit->uninteresting = uninteresting;
+	if (walk->one == NULL && !uninteresting) {
+		walk->one = commit;
+	} else {
+		if (git_vector_insert(&walk->twos, commit) < 0)
+			return -1;
+	}
 
-	return process_commit(walk, commit);
+	return 0;
 }
 
 int git_revwalk_push(git_revwalk *walk, const git_oid *oid)
@@ -318,12 +459,122 @@ int git_revwalk_push(git_revwalk *walk, const git_oid *oid)
 	return push_commit(walk, oid, 0);
 }
 
+
 int git_revwalk_hide(git_revwalk *walk, const git_oid *oid)
 {
 	assert(walk && oid);
 	return push_commit(walk, oid, 1);
 }
 
+static int push_ref(git_revwalk *walk, const char *refname, int hide)
+{
+	git_oid oid;
+
+	if (git_reference_name_to_oid(&oid, walk->repo, refname) < 0)
+		return -1;
+
+	return push_commit(walk, &oid, hide);
+}
+
+struct push_cb_data {
+	git_revwalk *walk;
+	const char *glob;
+	int hide;
+};
+
+static int push_glob_cb(const char *refname, void *data_)
+{
+	struct push_cb_data *data = (struct push_cb_data *)data_;
+
+	if (!p_fnmatch(data->glob, refname, 0))
+		return push_ref(data->walk, refname, data->hide);
+
+	return 0;
+}
+
+static int push_glob(git_revwalk *walk, const char *glob, int hide)
+{
+	git_buf buf = GIT_BUF_INIT;
+	struct push_cb_data data;
+	regex_t preg;
+
+	assert(walk && glob);
+
+	/* refs/ is implied if not given in the glob */
+	if (strncmp(glob, GIT_REFS_DIR, strlen(GIT_REFS_DIR))) {
+		git_buf_printf(&buf, GIT_REFS_DIR "%s", glob);
+	} else {
+		git_buf_puts(&buf, glob);
+	}
+
+	/* If no '?', '*' or '[' exist, we append '/ *' to the glob */
+	memset(&preg, 0x0, sizeof(regex_t));
+	if (regcomp(&preg, "[?*[]", REG_EXTENDED)) {
+		giterr_set(GITERR_OS, "Regex failed to compile");
+		git_buf_free(&buf);
+		return -1;
+	}
+
+	if (regexec(&preg, glob, 0, NULL, 0))
+		git_buf_puts(&buf, "/*");
+
+	if (git_buf_oom(&buf))
+		goto on_error;
+
+	data.walk = walk;
+	data.glob = git_buf_cstr(&buf);
+	data.hide = hide;
+
+	if (git_reference_foreach(
+			walk->repo, GIT_REF_LISTALL, push_glob_cb, &data) < 0)
+		goto on_error;
+
+	regfree(&preg);
+	git_buf_free(&buf);
+	return 0;
+
+on_error:
+	regfree(&preg);
+	git_buf_free(&buf);
+	return -1;
+}
+
+int git_revwalk_push_glob(git_revwalk *walk, const char *glob)
+{
+	assert(walk && glob);
+	return push_glob(walk, glob, 0);
+}
+
+int git_revwalk_hide_glob(git_revwalk *walk, const char *glob)
+{
+	assert(walk && glob);
+	return push_glob(walk, glob, 1);
+}
+
+int git_revwalk_push_head(git_revwalk *walk)
+{
+	assert(walk);
+	return push_ref(walk, GIT_HEAD_FILE, 0);
+}
+
+int git_revwalk_hide_head(git_revwalk *walk)
+{
+	assert(walk);
+	return push_ref(walk, GIT_HEAD_FILE, 1);
+}
+
+int git_revwalk_push_ref(git_revwalk *walk, const char *refname)
+{
+	assert(walk && refname);
+	return push_ref(walk, refname, 0);
+}
+
+int git_revwalk_hide_ref(git_revwalk *walk, const char *refname)
+{
+	assert(walk && refname);
+	return push_ref(walk, refname, 1);
+}
+
 static int revwalk_enqueue_timesort(git_revwalk *walk, commit_object *commit)
 {
 	return git_pqueue_insert(&walk->iterator_time, commit);
@@ -331,7 +582,7 @@ static int revwalk_enqueue_timesort(git_revwalk *walk, commit_object *commit)
 
 static int revwalk_enqueue_unsorted(git_revwalk *walk, commit_object *commit)
 {
-	return commit_list_insert(commit, &walk->iterator_rand) ? GIT_SUCCESS : GIT_ENOMEM;
+	return commit_list_insert(commit, &walk->iterator_rand) ? 0 : -1;
 }
 
 static int revwalk_next_timesort(commit_object **object_out, git_revwalk *walk)
@@ -340,16 +591,16 @@ static int revwalk_next_timesort(commit_object **object_out, git_revwalk *walk)
 	commit_object *next;
 
 	while ((next = git_pqueue_pop(&walk->iterator_time)) != NULL) {
-		if ((error = process_commit_parents(walk, next)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to load next revision");
+		if ((error = process_commit_parents(walk, next)) < 0)
+			return error;
 
 		if (!next->uninteresting) {
 			*object_out = next;
-			return GIT_SUCCESS;
+			return 0;
 		}
 	}
 
-	return git__throw(GIT_EREVWALKOVER, "Failed to load next revision");
+	return GIT_REVWALKOVER;
 }
 
 static int revwalk_next_unsorted(commit_object **object_out, git_revwalk *walk)
@@ -358,16 +609,16 @@ static int revwalk_next_unsorted(commit_object **object_out, git_revwalk *walk)
 	commit_object *next;
 
 	while ((next = commit_list_pop(&walk->iterator_rand)) != NULL) {
-		if ((error = process_commit_parents(walk, next)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to load next revision");
+		if ((error = process_commit_parents(walk, next)) < 0)
+			return error;
 
 		if (!next->uninteresting) {
 			*object_out = next;
-			return GIT_SUCCESS;
+			return 0;
 		}
 	}
 
-	return git__throw(GIT_EREVWALKOVER, "Failed to load next revision");
+	return GIT_REVWALKOVER;
 }
 
 static int revwalk_next_toposort(commit_object **object_out, git_revwalk *walk)
@@ -378,7 +629,7 @@ static int revwalk_next_toposort(commit_object **object_out, git_revwalk *walk)
 	for (;;) {
 		next = commit_list_pop(&walk->iterator_topo);
 		if (next == NULL)
-			return git__throw(GIT_EREVWALKOVER, "Failed to load next revision");
+			return GIT_REVWALKOVER;
 
 		if (next->in_degree > 0) {
 			next->topo_delay = 1;
@@ -390,58 +641,83 @@ static int revwalk_next_toposort(commit_object **object_out, git_revwalk *walk)
 
 			if (--parent->in_degree == 0 && parent->topo_delay) {
 				parent->topo_delay = 0;
-				commit_list_insert(parent, &walk->iterator_topo);
+				if (commit_list_insert(parent, &walk->iterator_topo) == NULL)
+					return -1;
 			}
 		}
 
 		*object_out = next;
-		return GIT_SUCCESS;
+		return 0;
 	}
 }
 
 static int revwalk_next_reverse(commit_object **object_out, git_revwalk *walk)
 {
 	*object_out = commit_list_pop(&walk->iterator_reverse);
-	return *object_out ? GIT_SUCCESS : GIT_EREVWALKOVER;
+	return *object_out ? 0 : GIT_REVWALKOVER;
 }
 
 
 static int prepare_walk(git_revwalk *walk)
 {
 	int error;
-	commit_object *next;
+	unsigned int i;
+	commit_object *next, *two;
+	commit_list *bases = NULL;
+
+	/*
+	 * If walk->one is NULL, there were no positive references,
+	 * so we know that the walk is already over.
+	 */
+	if (walk->one == NULL)
+		return GIT_REVWALKOVER;
+
+	/* first figure out what the merge bases are */
+	if (merge_bases_many(&bases, walk, walk->one, &walk->twos) < 0)
+		return -1;
+
+	commit_list_free(&bases);
+	if (process_commit(walk, walk->one, walk->one->uninteresting) < 0)
+		return -1;
+
+	git_vector_foreach(&walk->twos, i, two) {
+		if (process_commit(walk, two, two->uninteresting) < 0)
+			return -1;
+	}
 
 	if (walk->sorting & GIT_SORT_TOPOLOGICAL) {
 		unsigned short i;
 
-		while ((error = walk->get_next(&next, walk)) == GIT_SUCCESS) {
+		while ((error = walk->get_next(&next, walk)) == 0) {
 			for (i = 0; i < next->out_degree; ++i) {
 				commit_object *parent = next->parents[i];
 				parent->in_degree++;
 			}
 
-			commit_list_insert(next, &walk->iterator_topo);
+			if (commit_list_insert(next, &walk->iterator_topo) == NULL)
+				return -1;
 		}
 
-		if (error != GIT_EREVWALKOVER)
-			return git__rethrow(error, "Failed to prepare revision walk");
+		if (error != GIT_REVWALKOVER)
+			return error;
 
 		walk->get_next = &revwalk_next_toposort;
 	}
 
 	if (walk->sorting & GIT_SORT_REVERSE) {
 
-		while ((error = walk->get_next(&next, walk)) == GIT_SUCCESS)
-			commit_list_insert(next, &walk->iterator_reverse);
+		while ((error = walk->get_next(&next, walk)) == 0)
+			if (commit_list_insert(next, &walk->iterator_reverse) == NULL)
+				return -1;
 
-		if (error != GIT_EREVWALKOVER)
-			return git__rethrow(error, "Failed to prepare revision walk");
+		if (error != GIT_REVWALKOVER)
+			return error;
 
 		walk->get_next = &revwalk_next_reverse;
 	}
 
 	walk->walking = 1;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 
@@ -453,60 +729,46 @@ int git_revwalk_new(git_revwalk **revwalk_out, git_repository *repo)
 	git_revwalk *walk;
 
 	walk = git__malloc(sizeof(git_revwalk));
-	if (walk == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(walk);
 
 	memset(walk, 0x0, sizeof(git_revwalk));
 
-	walk->commits = git_hashtable_alloc(64,
-			object_table_hash,
-			(git_hash_keyeq_ptr)git_oid_cmp);
-
-	if (walk->commits == NULL) {
-		free(walk);
-		return GIT_ENOMEM;
-	}
+	walk->commits = git_oidmap_alloc();
+	GITERR_CHECK_ALLOC(walk->commits);
 
-	git_pqueue_init(&walk->iterator_time, 8, commit_time_cmp);
-	git_vector_init(&walk->memory_alloc, 8, NULL);
-	alloc_chunk(walk);
+	if (git_pqueue_init(&walk->iterator_time, 8, commit_time_cmp) < 0 ||
+		git_vector_init(&walk->twos, 4, NULL) < 0 ||
+		git_pool_init(&walk->commit_pool, 1,
+			git_pool__suggest_items_per_page(COMMIT_ALLOC) * COMMIT_ALLOC) < 0)
+		return -1;
 
 	walk->get_next = &revwalk_next_unsorted;
 	walk->enqueue = &revwalk_enqueue_unsorted;
 
 	walk->repo = repo;
 
+	if (git_repository_odb(&walk->odb, repo) < 0) {
+		git_revwalk_free(walk);
+		return -1;
+	}
+
 	*revwalk_out = walk;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 void git_revwalk_free(git_revwalk *walk)
 {
-	unsigned int i;
-	const void *GIT_UNUSED(_unused);
-	commit_object *commit;
-
 	if (walk == NULL)
 		return;
 
 	git_revwalk_reset(walk);
+	git_odb_free(walk->odb);
 
-	/* if the parent has more than PARENTS_PER_COMMIT parents,
-	 * we had to allocate a separate array for those parents.
-	 * make sure it's being free'd */
-	GIT_HASHTABLE_FOREACH(walk->commits, _unused, commit, {
-		if (commit->out_degree > PARENTS_PER_COMMIT)
-			free(commit->parents);
-	});
-
-	git_hashtable_free(walk->commits);
+	git_oidmap_free(walk->commits);
+	git_pool_clear(&walk->commit_pool);
 	git_pqueue_free(&walk->iterator_time);
-
-	for (i = 0; i < walk->memory_alloc.length; ++i)
-		free(git_vector_get(&walk->memory_alloc, i));
-
-	git_vector_free(&walk->memory_alloc);
-	free(walk);
+	git_vector_free(&walk->twos);
+	git__free(walk);
 }
 
 git_repository *git_revwalk_repository(git_revwalk *walk)
@@ -541,38 +803,43 @@ int git_revwalk_next(git_oid *oid, git_revwalk *walk)
 	assert(walk && oid);
 
 	if (!walk->walking) {
-		if ((error = prepare_walk(walk)) < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to load next revision");
+		if ((error = prepare_walk(walk)) < 0)
+			return error;
 	}
 
 	error = walk->get_next(&next, walk);
-	if (error < GIT_SUCCESS) {
-		if (error == GIT_EREVWALKOVER)
-			git_revwalk_reset(walk);
-		return git__rethrow(error, "Failed to load next revision");
+
+	if (error == GIT_REVWALKOVER) {
+		git_revwalk_reset(walk);
+		return GIT_REVWALKOVER;
 	}
 
-	git_oid_cpy(oid, &next->oid);
-	return GIT_SUCCESS;
+	if (!error)
+		git_oid_cpy(oid, &next->oid);
+
+	return error;
 }
 
 void git_revwalk_reset(git_revwalk *walk)
 {
-	const void *GIT_UNUSED(_unused);
 	commit_object *commit;
 
 	assert(walk);
 
-	GIT_HASHTABLE_FOREACH(walk->commits, _unused, commit,
+	kh_foreach_value(walk->commits, commit, {
 		commit->seen = 0;
 		commit->in_degree = 0;
 		commit->topo_delay = 0;
-	);
+		commit->uninteresting = 0;
+		});
 
 	git_pqueue_clear(&walk->iterator_time);
 	commit_list_free(&walk->iterator_topo);
 	commit_list_free(&walk->iterator_rand);
 	commit_list_free(&walk->iterator_reverse);
 	walk->walking = 0;
+
+	walk->one = NULL;
+	git_vector_clear(&walk->twos);
 }
 
diff --git a/src/block-sha1/sha1.c b/src/sha1.c
index 8c1460102..8aaedeb8f 100644
--- a/src/block-sha1/sha1.c
+++ b/src/sha1.c
@@ -1,9 +1,8 @@
 /*
- * SHA1 routine optimized to do word accesses rather than byte accesses,
- * and to avoid unnecessary copies into the context array.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * This was initially based on the Mozilla SHA1 implementation, although
- * none of the original Mozilla code remains.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
@@ -53,11 +52,11 @@
  */
 
 #if defined(__i386__) || defined(__x86_64__)
-  #define setW(x, val) (*(volatile unsigned int *)&W(x) = (val))
+ #define setW(x, val) (*(volatile unsigned int *)&W(x) = (val))
 #elif defined(__GNUC__) && defined(__arm__)
-  #define setW(x, val) do { W(x) = (val); __asm__("":::"memory"); } while (0)
+ #define setW(x, val) do { W(x) = (val); __asm__("":::"memory"); } while (0)
 #else
-  #define setW(x, val) (W(x) = (val))
+ #define setW(x, val) (W(x) = (val))
 #endif
 
 /*
@@ -68,27 +67,27 @@
  */
 
 #if defined(__i386__) || defined(__x86_64__) || \
-    defined(_M_IX86) || defined(_M_X64) || \
-    defined(__ppc__) || defined(__ppc64__) || \
-    defined(__powerpc__) || defined(__powerpc64__) || \
-    defined(__s390__) || defined(__s390x__)
+	defined(_M_IX86) || defined(_M_X64) || \
+	defined(__ppc__) || defined(__ppc64__) || \
+	defined(__powerpc__) || defined(__powerpc64__) || \
+	defined(__s390__) || defined(__s390x__)
 
-#define get_be32(p)	ntohl(*(unsigned int *)(p))
+#define get_be32(p)	ntohl(*(const unsigned int *)(p))
 #define put_be32(p, v)	do { *(unsigned int *)(p) = htonl(v); } while (0)
 
 #else
 
 #define get_be32(p)	( \
-	(*((unsigned char *)(p) + 0) << 24) | \
-	(*((unsigned char *)(p) + 1) << 16) | \
-	(*((unsigned char *)(p) + 2) <<  8) | \
-	(*((unsigned char *)(p) + 3) <<  0) )
+	(*((const unsigned char *)(p) + 0) << 24) | \
+	(*((const unsigned char *)(p) + 1) << 16) | \
+	(*((const unsigned char *)(p) + 2) << 8) | \
+	(*((const unsigned char *)(p) + 3) << 0) )
 #define put_be32(p, v)	do { \
 	unsigned int __v = (v); \
 	*((unsigned char *)(p) + 0) = __v >> 24; \
 	*((unsigned char *)(p) + 1) = __v >> 16; \
-	*((unsigned char *)(p) + 2) = __v >>  8; \
-	*((unsigned char *)(p) + 3) = __v >>  0; } while (0)
+	*((unsigned char *)(p) + 2) = __v >> 8; \
+	*((unsigned char *)(p) + 3) = __v >> 0; } while (0)
 
 #endif
 
@@ -107,11 +106,11 @@
 	E += TEMP + SHA_ROL(A,5) + (fn) + (constant); \
 	B = SHA_ROR(B, 2); } while (0)
 
-#define T_0_15(t, A, B, C, D, E)  SHA_ROUND(t, SHA_SRC, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
+#define T_0_15(t, A, B, C, D, E) SHA_ROUND(t, SHA_SRC, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
 #define T_16_19(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
 #define T_20_39(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) , 0x6ed9eba1, A, B, C, D, E )
 #define T_40_59(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, ((B&C)+(D&(B^C))) , 0x8f1bbcdc, A, B, C, D, E )
-#define T_60_79(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) ,  0xca62c1d6, A, B, C, D, E )
+#define T_60_79(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (B^C^D) , 0xca62c1d6, A, B, C, D, E )
 
 static void blk_SHA1_Block(blk_SHA_CTX *ctx, const unsigned int *data)
 {
@@ -233,7 +232,7 @@ void git__blk_SHA1_Init(blk_SHA_CTX *ctx)
 	ctx->H[4] = 0xc3d2e1f0;
 }
 
-void git__blk_SHA1_Update(blk_SHA_CTX *ctx, const void *data, unsigned long len)
+void git__blk_SHA1_Update(blk_SHA_CTX *ctx, const void *data, size_t len)
 {
 	unsigned int lenW = ctx->size & 63;
 
@@ -243,7 +242,7 @@ void git__blk_SHA1_Update(blk_SHA_CTX *ctx, const void *data, unsigned long len)
 	if (lenW) {
 		unsigned int left = 64 - lenW;
 		if (len < left)
-			left = len;
+			left = (unsigned int)len;
 		memcpy(lenW + (char *)ctx->W, data, left);
 		lenW = (lenW + left) & 63;
 		len -= left;
diff --git a/src/block-sha1/sha1.h b/src/sha1.h
index 558d6aece..93a244d76 100644
--- a/src/block-sha1/sha1.h
+++ b/src/sha1.h
@@ -1,9 +1,8 @@
 /*
- * SHA1 routine optimized to do word accesses rather than byte accesses,
- * and to avoid unnecessary copies into the context array.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * This was initially based on the Mozilla SHA1 implementation, although
- * none of the original Mozilla code remains.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 typedef struct {
@@ -13,7 +12,7 @@ typedef struct {
 } blk_SHA_CTX;
 
 void git__blk_SHA1_Init(blk_SHA_CTX *ctx);
-void git__blk_SHA1_Update(blk_SHA_CTX *ctx, const void *dataIn, unsigned long len);
+void git__blk_SHA1_Update(blk_SHA_CTX *ctx, const void *dataIn, size_t len);
 void git__blk_SHA1_Final(unsigned char hashout[20], blk_SHA_CTX *ctx);
 
 #define SHA_CTX		blk_SHA_CTX
diff --git a/src/sha1_lookup.c b/src/sha1_lookup.c
index f4a3c42cc..096da1739 100644
--- a/src/sha1_lookup.c
+++ b/src/sha1_lookup.c
@@ -1,27 +1,8 @@
 /*
- * This file is basically taken from git code.
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include <stdio.h>
@@ -33,71 +14,71 @@
  * Conventional binary search loop looks like this:
  *
  *	unsigned lo, hi;
- *      do {
- *              unsigned mi = (lo + hi) / 2;
- *              int cmp = "entry pointed at by mi" minus "target";
- *              if (!cmp)
- *                      return (mi is the wanted one)
- *              if (cmp > 0)
- *                      hi = mi; "mi is larger than target"
- *              else
- *                      lo = mi+1; "mi is smaller than target"
- *      } while (lo < hi);
+ *		do {
+ *				unsigned mi = (lo + hi) / 2;
+ *				int cmp = "entry pointed at by mi" minus "target";
+ *				if (!cmp)
+ *						return (mi is the wanted one)
+ *				if (cmp > 0)
+ *						hi = mi; "mi is larger than target"
+ *				else
+ *						lo = mi+1; "mi is smaller than target"
+ *		} while (lo < hi);
  *
  * The invariants are:
  *
  * - When entering the loop, lo points at a slot that is never
- *   above the target (it could be at the target), hi points at a
- *   slot that is guaranteed to be above the target (it can never
- *   be at the target).
+ *	above the target (it could be at the target), hi points at a
+ *	slot that is guaranteed to be above the target (it can never
+ *	be at the target).
  *
  * - We find a point 'mi' between lo and hi (mi could be the same
- *   as lo, but never can be as same as hi), and check if it hits
- *   the target.  There are three cases:
+ *	as lo, but never can be as same as hi), and check if it hits
+ *	the target. There are three cases:
  *
- *    - if it is a hit, we are happy.
+ *	- if it is a hit, we are happy.
  *
- *    - if it is strictly higher than the target, we set it to hi,
- *      and repeat the search.
+ *	- if it is strictly higher than the target, we set it to hi,
+ *		and repeat the search.
  *
- *    - if it is strictly lower than the target, we update lo to
- *      one slot after it, because we allow lo to be at the target.
+ *	- if it is strictly lower than the target, we update lo to
+ *		one slot after it, because we allow lo to be at the target.
  *
- *   If the loop exits, there is no matching entry.
+ *	If the loop exits, there is no matching entry.
  *
  * When choosing 'mi', we do not have to take the "middle" but
  * anywhere in between lo and hi, as long as lo <= mi < hi is
- * satisfied.  When we somehow know that the distance between the
+ * satisfied. When we somehow know that the distance between the
  * target and lo is much shorter than the target and hi, we could
  * pick mi that is much closer to lo than the midway.
  *
  * Now, we can take advantage of the fact that SHA-1 is a good hash
  * function, and as long as there are enough entries in the table, we
- * can expect uniform distribution.  An entry that begins with for
+ * can expect uniform distribution. An entry that begins with for
  * example "deadbeef..." is much likely to appear much later than in
- * the midway of the table.  It can reasonably be expected to be near
+ * the midway of the table. It can reasonably be expected to be near
  * 87% (222/256) from the top of the table.
  *
- * However, we do not want to pick "mi" too precisely.  If the entry at
+ * However, we do not want to pick "mi" too precisely. If the entry at
  * the 87% in the above example turns out to be higher than the target
  * we are looking for, we would end up narrowing the search space down
  * only by 13%, instead of 50% we would get if we did a simple binary
- * search.  So we would want to hedge our bets by being less aggressive.
+ * search. So we would want to hedge our bets by being less aggressive.
  *
  * The table at "table" holds at least "nr" entries of "elem_size"
- * bytes each.  Each entry has the SHA-1 key at "key_offset".  The
- * table is sorted by the SHA-1 key of the entries.  The caller wants
+ * bytes each. Each entry has the SHA-1 key at "key_offset". The
+ * table is sorted by the SHA-1 key of the entries. The caller wants
  * to find the entry with "key", and knows that the entry at "lo" is
  * not higher than the entry it is looking for, and that the entry at
  * "hi" is higher than the entry it is looking for.
  */
 int sha1_entry_pos(const void *table,
-		   size_t elem_size,
-		   size_t key_offset,
-		   unsigned lo, unsigned hi, unsigned nr,
-		   const unsigned char *key)
+			size_t elem_size,
+			size_t key_offset,
+			unsigned lo, unsigned hi, unsigned nr,
+			const unsigned char *key)
 {
-	const unsigned char *base = table;
+	const unsigned char *base = (const unsigned char*)table;
 	const unsigned char *hi_key, *lo_key;
 	unsigned ofs_0;
 
@@ -157,7 +138,7 @@ int sha1_entry_pos(const void *table,
 		 * end up narrowing the search space by a smaller
 		 * amount (i.e. the distance between 'mi' and 'hi')
 		 * than what we would have (i.e. about half of 'lo'
-		 * and 'hi').  Hedge our bets to pick 'mi' less
+		 * and 'hi'). Hedge our bets to pick 'mi' less
 		 * aggressively, i.e. make 'mi' a bit closer to the
 		 * middle than we would otherwise pick.
 		 */
@@ -173,11 +154,12 @@ int sha1_entry_pos(const void *table,
 #ifdef INDEX_DEBUG_LOOKUP
 		printf("lo %u hi %u rg %u mi %u ", lo, hi, range, mi);
 		printf("ofs %u lov %x, hiv %x, kyv %x\n",
-		       ofs_0, lov, hiv, kyv);
+				ofs_0, lov, hiv, kyv);
 #endif
 
 		if (!(lo <= mi && mi < hi)) {
-			return git__throw(GIT_ERROR, "Assertion failure. Binary search invariant is false");
+			giterr_set(GITERR_INVALID, "Assertion failure. Binary search invariant is false");
+			return -1;
 		}
 
 		mi_key = base + elem_size * mi + key_offset;
@@ -192,5 +174,5 @@ int sha1_entry_pos(const void *table,
 			lo_key = mi_key + elem_size;
 		}
 	} while (lo < hi);
-	return -lo-1;
+	return -((int)lo)-1;
 }
diff --git a/src/sha1_lookup.h b/src/sha1_lookup.h
index 5caa2f5ed..cd40a9d57 100644
--- a/src/sha1_lookup.h
+++ b/src/sha1_lookup.h
@@ -1,12 +1,18 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_sha1_lookup_h__
 #define INCLUDE_sha1_lookup_h__
 
 #include <stdlib.h>
 
 int sha1_entry_pos(const void *table,
-		   size_t elem_size,
-		   size_t key_offset,
-		   unsigned lo, unsigned hi, unsigned nr,
-		   const unsigned char *key);
+			size_t elem_size,
+			size_t key_offset,
+			unsigned lo, unsigned hi, unsigned nr,
+			const unsigned char *key);
 
 #endif
diff --git a/src/signature.c b/src/signature.c
index 51a2fff47..7d329c4c9 100644
--- a/src/signature.c
+++ b/src/signature.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
@@ -33,48 +15,111 @@ void git_signature_free(git_signature *sig)
 	if (sig == NULL)
 		return;
 
-	free(sig->name);
-	free(sig->email);
-	free(sig);
+	git__free(sig->name);
+	sig->name = NULL;
+	git__free(sig->email);
+	sig->email = NULL;
+	git__free(sig);
 }
 
-git_signature *git_signature_new(const char *name, const char *email, git_time_t time, int offset)
+static const char *skip_leading_spaces(const char *buffer, const char *buffer_end)
 {
+	while (*buffer == ' ' && buffer < buffer_end)
+		buffer++;
+
+	return buffer;
+}
+
+static const char *skip_trailing_spaces(const char *buffer_start, const char *buffer_end)
+{
+	while (*buffer_end == ' ' && buffer_end > buffer_start)
+		buffer_end--;
+
+	return buffer_end;
+}
+
+static int signature_error(const char *msg)
+{
+	giterr_set(GITERR_INVALID, "Failed to parse signature - %s", msg);
+	return -1;
+}
+
+static int process_trimming(const char *input, char **storage, const char *input_end, int fail_when_empty)
+{
+	const char *left, *right;
+	size_t trimmed_input_length;
+
+	assert(storage);
+
+	left = skip_leading_spaces(input, input_end);
+	right = skip_trailing_spaces(input, input_end - 1);
+
+	if (right < left) {
+		if (fail_when_empty)
+			return signature_error("input is either empty of contains only spaces");
+
+		right = left - 1;
+	}
+
+	trimmed_input_length = right - left + 1;
+
+	*storage = git__malloc(trimmed_input_length + 1);
+	GITERR_CHECK_ALLOC(*storage);
+
+	memcpy(*storage, left, trimmed_input_length);
+	(*storage)[trimmed_input_length] = 0;
+
+	return 0;
+}
+
+int git_signature_new(git_signature **sig_out, const char *name, const char *email, git_time_t time, int offset)
+{
+	int error;
 	git_signature *p = NULL;
 
-	if ((p = git__malloc(sizeof(git_signature))) == NULL)
-		goto cleanup;
+	assert(name && email);
+
+	*sig_out = NULL;
+
+	p = git__calloc(1, sizeof(git_signature));
+	GITERR_CHECK_ALLOC(p);
+
+	if ((error = process_trimming(name, &p->name, name + strlen(name), 1)) < 0 ||
+		(error = process_trimming(email, &p->email, email + strlen(email), 1)) < 0)
+	{
+		git_signature_free(p);
+		return error;
+	}
 
-	p->name = git__strdup(name);
-	p->email = git__strdup(email);
 	p->when.time = time;
 	p->when.offset = offset;
 
-	if (p->name == NULL || p->email == NULL)
-		goto cleanup;
+	*sig_out = p;
 
-	return p;
-
-cleanup:
-	git_signature_free(p);
-	return NULL;
+	return 0;
 }
 
 git_signature *git_signature_dup(const git_signature *sig)
 {
-	return git_signature_new(sig->name, sig->email, sig->when.time, sig->when.offset);
+	git_signature *new;
+	if (git_signature_new(&new, sig->name, sig->email, sig->when.time, sig->when.offset) < 0)
+		return NULL;
+	return new;
 }
 
-git_signature *git_signature_now(const char *name, const char *email)
+int git_signature_now(git_signature **sig_out, const char *name, const char *email)
 {
 	time_t now;
 	time_t offset;
 	struct tm *utc_tm, *local_tm;
+	git_signature *sig;
 
 #ifndef GIT_WIN32
 	struct tm _utc, _local;
 #endif
 
+	*sig_out = NULL;
+
 	time(&now);
 
 	/**
@@ -96,163 +141,196 @@ git_signature *git_signature_now(const char *name, const char *email)
 	if (local_tm->tm_isdst)
 		offset += 60;
 
-	return git_signature_new(name, email, now, (int)offset);
+	if (git_signature_new(&sig, name, email, now, (int)offset) < 0)
+		return -1;
+
+	*sig_out = sig;
+
+	return 0;
 }
 
-static int parse_timezone_offset(const char *buffer, long *offset_out)
+static int timezone_error(const char *msg)
 {
-	long offset, dec_offset;
-	int mins, hours;
+	giterr_set(GITERR_INVALID, "Failed to parse TZ offset - %s", msg);
+	return -1;
+}
+
+static int parse_timezone_offset(const char *buffer, int *offset_out)
+{
+	int dec_offset;
+	int mins, hours, offset;
 
 	const char *offset_start;
 	const char *offset_end;
 
-	//we are sure that *buffer == ' ' 
-	offset_start = buffer + 1;
+	offset_start = buffer;
 
 	if (*offset_start == '\n') {
 		*offset_out = 0;
-		return GIT_SUCCESS;
+		return 0;
 	}
 
 	if (offset_start[0] != '-' && offset_start[0] != '+')
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse TZ offset. It doesn't start with '+' or '-'");
+		return timezone_error("does not start with '+' or '-'");
 
 	if (offset_start[1] < '0' || offset_start[1] > '9')
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse TZ offset.");
+		return timezone_error("expected initial digit");
 
-	if (git__strtol32(&dec_offset, offset_start + 1, &offset_end, 10) < GIT_SUCCESS)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse TZ offset. It isn't a number");
+	if (git__strtol32(&dec_offset, offset_start + 1, &offset_end, 10) < 0)
+		return timezone_error("not a valid number");
 
 	if (offset_end - offset_start != 5)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse TZ offset. Invalid length");
+		return timezone_error("invalid length");
 
 	if (dec_offset > 1400)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse TZ offset. Value too large");
+		return timezone_error("value too large");
 
 	hours = dec_offset / 100;
 	mins = dec_offset % 100;
 
-	if (hours > 14)	// see http://www.worldtimezone.com/faq.html 
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse TZ offset. Hour value too large");
+	if (hours > 14)	// see http://www.worldtimezone.com/faq.html
+		return timezone_error("hour value too large");
 
 	if (mins > 59)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse TZ offset. Minute value too large");
+		return timezone_error("minutes value too large");
 
 	offset = (hours * 60) + mins;
 
 	if (offset_start[0] == '-')
 		offset *= -1;
-	
+
 	*offset_out = offset;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
+static int process_next_token(const char **buffer_out, char **storage,
+	const char *token_end, const char *right_boundary)
+{
+	int error = process_trimming(*buffer_out, storage, token_end, 0);
+	if (error < 0)
+		return error;
 
-int git_signature__parse(git_signature *sig, const char **buffer_out,
-		const char *buffer_end, const char *header)
+	*buffer_out = token_end + 1;
+
+	if (*buffer_out > right_boundary)
+		return signature_error("signature is too short");
+
+	return 0;
+}
+
+static const char *scan_for_previous_token(const char *buffer, const char *left_boundary)
 {
-	const size_t header_len = strlen(header);
+	const char *start;
 
-	int name_length, email_length;
-	const char *buffer = *buffer_out;
-	const char *line_end, *name_end, *email_end;
-	long offset = 0, time;
+	if (buffer <= left_boundary)
+		return NULL;
 
-	memset(sig, 0x0, sizeof(git_signature));
+	start = skip_trailing_spaces(left_boundary, buffer);
 
-	line_end = memchr(buffer, '\n', buffer_end - buffer);
-	if (!line_end)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. No newline found");;
+	/* Search for previous occurence of space */
+	while (start[-1] != ' ' && start > left_boundary)
+		start--;
 
-	if (buffer + (header_len + 1) > line_end)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Signature too short");
+	return start;
+}
 
-	if (memcmp(buffer, header, header_len) != 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Expected prefix '%s' doesn't match actual", header);
+static int parse_time(git_time_t *time_out, const char *buffer)
+{
+	int time;
+	int error;
 
-	buffer += header_len;
+	if (*buffer == '+' || *buffer == '-') {
+		giterr_set(GITERR_INVALID, "Failed while parsing time. '%s' actually looks like a timezone offset.", buffer);
+		return -1;
+	}
 
-	/* Parse name */
-	if ((name_end = strstr(buffer, " <")) == NULL)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Can't find e-mail start");
+	error = git__strtol32(&time, buffer, &buffer, 10);
 
-	name_length = name_end - buffer;
-	if (name_length <= 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Missing tagger name");
+	if (!error)
+		*time_out = (git_time_t)time;
 
-	sig->name = git__malloc(name_length + 1);
-	if (sig->name == NULL)
-		return GIT_ENOMEM;
+	return error;
+}
 
-	memcpy(sig->name, buffer, name_length);
-	sig->name[name_length] = 0;
-	buffer = name_end + 2;
+int git_signature__parse(git_signature *sig, const char **buffer_out,
+		const char *buffer_end, const char *header, char ender)
+{
+	const char *buffer = *buffer_out;
+	const char *line_end, *name_end, *email_end, *tz_start, *time_start;
+	int error = 0;
+
+	memset(sig, 0x0, sizeof(git_signature));
+
+	if ((line_end = memchr(buffer, ender, buffer_end - buffer)) == NULL)
+		return signature_error("no newline given");
+
+	if (header) {
+		const size_t header_len = strlen(header);
+
+		if (memcmp(buffer, header, header_len) != 0)
+			return signature_error("expected prefix doesn't match actual");
+
+		buffer += header_len;
+	}
+
+	if (buffer > line_end)
+		return signature_error("signature too short");
 
-	if (buffer >= line_end)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Ended unexpectedly");
+	if ((name_end = strchr(buffer, '<')) == NULL)
+		return signature_error("character '<' not allowed in signature");
 
-	/* Parse email */
-	if ((email_end = strstr(buffer, "> ")) == NULL)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Can't find e-mail end");
+	if ((email_end = strchr(name_end, '>')) == NULL)
+		return signature_error("character '>' not allowed in signature");
 
-	email_length = email_end - buffer;
-	sig->email = git__malloc(email_length + 1);
-	if (sig->name == NULL)
-		return GIT_ENOMEM;
+	if (email_end < name_end)
+		return signature_error("malformed e-mail");
 
-	memcpy(sig->email, buffer, email_length);
-	sig->email[email_length] = 0;
-	buffer = email_end + 2;
+	error = process_next_token(&buffer, &sig->name, name_end, line_end);
+	if (error < 0)
+		return error;
 
-	if (buffer >= line_end)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Ended unexpectedly");
+	error = process_next_token(&buffer, &sig->email, email_end, line_end);
+	if (error < 0)
+		return error;
 
-	/* verify email */
-	if (strpbrk(sig->email, "><\n ") != NULL)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Malformed e-mail");
+	tz_start = scan_for_previous_token(line_end - 1, buffer);
 
-	if (git__strtol32(&time, buffer, &buffer, 10) < GIT_SUCCESS)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Timestamp isn't a number");
+	if (tz_start == NULL)
+		goto clean_exit;	/* No timezone nor date */
 
-	sig->when.time = (time_t)time;
+	time_start = scan_for_previous_token(tz_start - 1, buffer);
+	if (time_start == NULL || parse_time(&sig->when.time, time_start) < 0) {
+		/* The tz_start might point at the time */
+		parse_time(&sig->when.time, tz_start);
+		goto clean_exit;
+	}
 
-	if (parse_timezone_offset(buffer, &offset) < GIT_SUCCESS)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse signature. Could not parse timezone offset");
-	
-	sig->when.offset = offset;
+	if (parse_timezone_offset(tz_start, &sig->when.offset) < 0) {
+		sig->when.time = 0; /* Bogus timezone, we reset the time */
+	}
 
-	*buffer_out = (line_end + 1);
-	return GIT_SUCCESS;
+clean_exit:
+	*buffer_out = line_end + 1;
+	return 0;
 }
 
-int git_signature__write(char **signature, const char *header, const git_signature *sig)
+void git_signature__writebuf(git_buf *buf, const char *header, const git_signature *sig)
 {
 	int offset, hours, mins;
-	char sig_buffer[2048];
-	int sig_buffer_len;
 	char sign;
 
 	offset = sig->when.offset;
 	sign = (sig->when.offset < 0) ? '-' : '+';
-	
+
 	if (offset < 0)
 		offset = -offset;
 
 	hours = offset / 60;
 	mins = offset % 60;
 
-	sig_buffer_len = snprintf(sig_buffer, sizeof(sig_buffer),
-			"%s %s <%s> %u %c%02d%02d\n",
-			header, sig->name, sig->email,
+	git_buf_printf(buf, "%s%s <%s> %u %c%02d%02d\n",
+			header ? header : "", sig->name, sig->email,
 			(unsigned)sig->when.time, sign, hours, mins);
-
-	if (sig_buffer_len < 0 || (size_t)sig_buffer_len > sizeof(sig_buffer))
-		return GIT_ENOMEM;
-
-	*signature = git__strdup(sig_buffer);
-	return sig_buffer_len;
 }
 
-
diff --git a/src/signature.h b/src/signature.h
index feba6578d..97b3a055e 100644
--- a/src/signature.h
+++ b/src/signature.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_signature_h__
 #define INCLUDE_signature_h__
 
@@ -6,7 +12,7 @@
 #include "repository.h"
 #include <time.h>
 
-int git_signature__parse(git_signature *sig, const char **buffer_out, const char *buffer_end, const char *header);
-int git_signature__write(char **signature, const char *header, const git_signature *sig);
+int git_signature__parse(git_signature *sig, const char **buffer_out, const char *buffer_end, const char *header, char ender);
+void git_signature__writebuf(git_buf *buf, const char *header, const git_signature *sig);
 
 #endif
diff --git a/src/status.c b/src/status.c
new file mode 100644
index 000000000..e9ad3cfe4
--- /dev/null
+++ b/src/status.c
@@ -0,0 +1,242 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "git2.h"
+#include "fileops.h"
+#include "hash.h"
+#include "vector.h"
+#include "tree.h"
+#include "git2/status.h"
+#include "repository.h"
+#include "ignore.h"
+
+#include "git2/diff.h"
+#include "diff.h"
+
+static unsigned int index_delta2status(git_delta_t index_status)
+{
+	unsigned int st = GIT_STATUS_CURRENT;
+
+	switch (index_status) {
+	case GIT_DELTA_ADDED:
+	case GIT_DELTA_COPIED:
+	case GIT_DELTA_RENAMED:
+		st = GIT_STATUS_INDEX_NEW;
+		break;
+	case GIT_DELTA_DELETED:
+		st = GIT_STATUS_INDEX_DELETED;
+		break;
+	case GIT_DELTA_MODIFIED:
+		st = GIT_STATUS_INDEX_MODIFIED;
+		break;
+	default:
+		break;
+	}
+
+	return st;
+}
+
+static unsigned int workdir_delta2status(git_delta_t workdir_status)
+{
+	unsigned int st = GIT_STATUS_CURRENT;
+
+	switch (workdir_status) {
+	case GIT_DELTA_ADDED:
+	case GIT_DELTA_COPIED:
+	case GIT_DELTA_RENAMED:
+	case GIT_DELTA_UNTRACKED:
+		st = GIT_STATUS_WT_NEW;
+		break;
+	case GIT_DELTA_DELETED:
+		st = GIT_STATUS_WT_DELETED;
+		break;
+	case GIT_DELTA_MODIFIED:
+		st = GIT_STATUS_WT_MODIFIED;
+		break;
+	case GIT_DELTA_IGNORED:
+		st = GIT_STATUS_IGNORED;
+		break;
+	default:
+		break;
+	}
+
+	return st;
+}
+
+int git_status_foreach_ext(
+	git_repository *repo,
+	const git_status_options *opts,
+	int (*cb)(const char *, unsigned int, void *),
+	void *cbdata)
+{
+	int err = 0, cmp;
+	git_diff_options diffopt;
+	git_diff_list *idx2head = NULL, *wd2idx = NULL;
+	git_tree *head = NULL;
+	git_status_show_t show =
+		opts ? opts->show : GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
+	git_diff_delta *i2h, *w2i;
+	unsigned int i, j, i_max, j_max;
+
+	assert(show <= GIT_STATUS_SHOW_INDEX_THEN_WORKDIR);
+
+	if ((err = git_repository_head_tree(&head, repo)) < 0)
+		return err;
+
+	memset(&diffopt, 0, sizeof(diffopt));
+	memcpy(&diffopt.pathspec, &opts->pathspec, sizeof(diffopt.pathspec));
+
+	if ((opts->flags & GIT_STATUS_OPT_INCLUDE_UNTRACKED) != 0)
+		diffopt.flags = diffopt.flags | GIT_DIFF_INCLUDE_UNTRACKED;
+	if ((opts->flags & GIT_STATUS_OPT_INCLUDE_IGNORED) != 0)
+		diffopt.flags = diffopt.flags | GIT_DIFF_INCLUDE_IGNORED;
+	if ((opts->flags & GIT_STATUS_OPT_INCLUDE_UNMODIFIED) != 0)
+		diffopt.flags = diffopt.flags | GIT_DIFF_INCLUDE_UNMODIFIED;
+	if ((opts->flags & GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS) != 0)
+		diffopt.flags = diffopt.flags | GIT_DIFF_RECURSE_UNTRACKED_DIRS;
+	/* TODO: support EXCLUDE_SUBMODULES flag */
+
+	if (show != GIT_STATUS_SHOW_WORKDIR_ONLY &&
+		(err = git_diff_index_to_tree(repo, &diffopt, head, &idx2head)) < 0)
+		goto cleanup;
+
+	if (show != GIT_STATUS_SHOW_INDEX_ONLY &&
+		(err = git_diff_workdir_to_index(repo, &diffopt, &wd2idx)) < 0)
+		goto cleanup;
+
+	if (show == GIT_STATUS_SHOW_INDEX_THEN_WORKDIR) {
+		for (i = 0; !err && i < idx2head->deltas.length; i++) {
+			i2h = GIT_VECTOR_GET(&idx2head->deltas, i);
+			err = cb(i2h->old_file.path, index_delta2status(i2h->status), cbdata);
+		}
+		git_diff_list_free(idx2head);
+		idx2head = NULL;
+	}
+
+	i_max = idx2head ? idx2head->deltas.length : 0;
+	j_max = wd2idx   ? wd2idx->deltas.length   : 0;
+
+	for (i = 0, j = 0; !err && (i < i_max || j < j_max); ) {
+		i2h = idx2head ? GIT_VECTOR_GET(&idx2head->deltas,i) : NULL;
+		w2i = wd2idx   ? GIT_VECTOR_GET(&wd2idx->deltas,j)   : NULL;
+
+		cmp = !w2i ? -1 : !i2h ? 1 : strcmp(i2h->old_file.path, w2i->old_file.path);
+
+		if (cmp < 0) {
+			err = cb(i2h->old_file.path, index_delta2status(i2h->status), cbdata);
+			i++;
+		} else if (cmp > 0) {
+			err = cb(w2i->old_file.path, workdir_delta2status(w2i->status), cbdata);
+			j++;
+		} else {
+			err = cb(i2h->old_file.path, index_delta2status(i2h->status) |
+					 workdir_delta2status(w2i->status), cbdata);
+			i++; j++;
+		}
+	}
+
+cleanup:
+	git_tree_free(head);
+	git_diff_list_free(idx2head);
+	git_diff_list_free(wd2idx);
+	return err;
+}
+
+int git_status_foreach(
+	git_repository *repo,
+	int (*callback)(const char *, unsigned int, void *),
+	void *payload)
+{
+	git_status_options opts;
+
+	memset(&opts, 0, sizeof(opts));
+	opts.show  = GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
+	opts.flags = GIT_STATUS_OPT_INCLUDE_IGNORED |
+		GIT_STATUS_OPT_INCLUDE_UNTRACKED |
+		GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS;
+
+	return git_status_foreach_ext(repo, &opts, callback, payload);
+}
+
+struct status_file_info {
+	unsigned int count;
+	unsigned int status;
+	char *expected;
+};
+
+static int get_one_status(const char *path, unsigned int status, void *data)
+{
+	struct status_file_info *sfi = data;
+
+	sfi->count++;
+	sfi->status = status;
+
+	if (sfi->count > 1 || strcmp(sfi->expected, path) != 0) {
+		giterr_set(GITERR_INVALID,
+			"Ambiguous path '%s' given to git_status_file", sfi->expected);
+		return -1;
+	}
+
+	return 0;
+}
+
+int git_status_file(
+	unsigned int *status_flags,
+	git_repository *repo,
+	const char *path)
+{
+	int error;
+	git_status_options opts;
+	struct status_file_info sfi;
+
+	assert(status_flags && repo && path);
+
+	memset(&sfi, 0, sizeof(sfi));
+	if ((sfi.expected = git__strdup(path)) == NULL)
+		return -1;
+
+	memset(&opts, 0, sizeof(opts));
+	opts.show  = GIT_STATUS_SHOW_INDEX_AND_WORKDIR;
+	opts.flags = GIT_STATUS_OPT_INCLUDE_IGNORED |
+		GIT_STATUS_OPT_INCLUDE_UNTRACKED |
+		GIT_STATUS_OPT_RECURSE_UNTRACKED_DIRS |
+		GIT_STATUS_OPT_INCLUDE_UNMODIFIED;
+	opts.pathspec.count = 1;
+	opts.pathspec.strings = &sfi.expected;
+
+	error = git_status_foreach_ext(repo, &opts, get_one_status, &sfi);
+
+	if (!error && !sfi.count) {
+		giterr_set(GITERR_INVALID,
+			"Attempt to get status of nonexistent file '%s'", path);
+		error = GIT_ENOTFOUND;
+	}
+
+	*status_flags = sfi.status;
+
+	git__free(sfi.expected);
+
+	return error;
+}
+
+int git_status_should_ignore(
+	int *ignored,
+	git_repository *repo,
+	const char *path)
+{
+	int error;
+	git_ignores ignores;
+
+	if (git_ignore__for_path(repo, path, &ignores) < 0)
+		return -1;
+
+	error = git_ignore__lookup(&ignores, path, ignored);
+	git_ignore__free(&ignores);
+	return error;
+}
+
diff --git a/src/strmap.h b/src/strmap.h
new file mode 100644
index 000000000..da5ca0dba
--- /dev/null
+++ b/src/strmap.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_strmap_h__
+#define INCLUDE_strmap_h__
+
+#include "common.h"
+
+#define kmalloc git__malloc
+#define kcalloc git__calloc
+#define krealloc git__realloc
+#define kfree git__free
+#include "khash.h"
+
+__KHASH_TYPE(str, const char *, void *);
+typedef khash_t(str) git_strmap;
+
+#define GIT__USE_STRMAP \
+	__KHASH_IMPL(str, static inline, const char *, void *, 1, kh_str_hash_func, kh_str_hash_equal)
+
+#define git_strmap_alloc()  kh_init(str)
+#define git_strmap_free(h)  kh_destroy(str, h), h = NULL
+#define git_strmap_clear(h) kh_clear(str, h)
+
+#define git_strmap_num_entries(h) kh_size(h)
+
+#define git_strmap_lookup_index(h, k)  kh_get(str, h, k)
+#define git_strmap_valid_index(h, idx) (idx != kh_end(h))
+
+#define git_strmap_exists(h, k) (kh_get(str, h, k) != kh_end(h))
+
+#define git_strmap_value_at(h, idx)        kh_val(h, idx)
+#define git_strmap_set_value_at(h, idx, v) kh_val(h, idx) = v
+#define git_strmap_delete_at(h, idx)       kh_del(str, h, idx)
+
+#define git_strmap_insert(h, key, val, rval) do { \
+	khiter_t __pos = kh_put(str, h, key, &rval); \
+	if (rval >= 0) { \
+		if (rval == 0) kh_key(h, __pos) = key; \
+		kh_val(h, __pos) = val; \
+	} } while (0)
+
+#define git_strmap_insert2(h, key, val, oldv, rval) do { \
+	khiter_t __pos = kh_put(str, h, key, &rval); \
+	if (rval >= 0) { \
+		if (rval == 0) { \
+			oldv = kh_val(h, __pos); \
+			kh_key(h, __pos) = key; \
+		} else { oldv = NULL; } \
+		kh_val(h, __pos) = val; \
+	} } while (0)
+
+#define git_strmap_delete(h, key) do { \
+	khiter_t __pos = git_strmap_lookup_index(h, key); \
+	if (git_strmap_valid_index(h, __pos)) \
+		git_strmap_delete_at(h, __pos); } while (0)
+
+#define git_strmap_foreach		kh_foreach
+#define git_strmap_foreach_value	kh_foreach_value
+
+#endif
diff --git a/src/submodule.c b/src/submodule.c
new file mode 100644
index 000000000..3c07e657d
--- /dev/null
+++ b/src/submodule.c
@@ -0,0 +1,387 @@
+/*
+ * Copyright (C) 2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "git2/config.h"
+#include "git2/types.h"
+#include "git2/repository.h"
+#include "git2/index.h"
+#include "git2/submodule.h"
+#include "buffer.h"
+#include "vector.h"
+#include "posix.h"
+#include "config_file.h"
+#include "config.h"
+#include "repository.h"
+
+static git_cvar_map _sm_update_map[] = {
+	{GIT_CVAR_STRING, "checkout", GIT_SUBMODULE_UPDATE_CHECKOUT},
+	{GIT_CVAR_STRING, "rebase", GIT_SUBMODULE_UPDATE_REBASE},
+	{GIT_CVAR_STRING, "merge", GIT_SUBMODULE_UPDATE_MERGE}
+};
+
+static git_cvar_map _sm_ignore_map[] = {
+	{GIT_CVAR_STRING, "all", GIT_SUBMODULE_IGNORE_ALL},
+	{GIT_CVAR_STRING, "dirty", GIT_SUBMODULE_IGNORE_DIRTY},
+	{GIT_CVAR_STRING, "untracked", GIT_SUBMODULE_IGNORE_UNTRACKED},
+	{GIT_CVAR_STRING, "none", GIT_SUBMODULE_IGNORE_NONE}
+};
+
+static inline khint_t str_hash_no_trailing_slash(const char *s)
+{
+	khint_t h;
+
+	for (h = 0; *s; ++s)
+		if (s[1] || *s != '/')
+			h = (h << 5) - h + *s;
+
+	return h;
+}
+
+static inline int str_equal_no_trailing_slash(const char *a, const char *b)
+{
+	size_t alen = a ? strlen(a) : 0;
+	size_t blen = b ? strlen(b) : 0;
+
+	if (alen && a[alen] == '/')
+		alen--;
+	if (blen && b[blen] == '/')
+		blen--;
+
+	return (alen == blen && strncmp(a, b, alen) == 0);
+}
+
+__KHASH_IMPL(str, static inline, const char *, void *, 1, str_hash_no_trailing_slash, str_equal_no_trailing_slash);
+
+static git_submodule *submodule_alloc(const char *name)
+{
+	git_submodule *sm = git__calloc(1, sizeof(git_submodule));
+	if (sm == NULL)
+		return sm;
+
+	sm->path = sm->name = git__strdup(name);
+	if (!sm->name) {
+		git__free(sm);
+		return NULL;
+	}
+
+	return sm;
+}
+
+static void submodule_release(git_submodule *sm, int decr)
+{
+	if (!sm)
+		return;
+
+	sm->refcount -= decr;
+
+	if (sm->refcount == 0) {
+		if (sm->name != sm->path)
+			git__free(sm->path);
+		git__free(sm->name);
+		git__free(sm->url);
+		git__free(sm);
+	}
+}
+
+static int submodule_from_entry(
+	git_strmap *smcfg, git_index_entry *entry)
+{
+	git_submodule *sm;
+	void *old_sm;
+	khiter_t pos;
+	int error;
+
+	pos = git_strmap_lookup_index(smcfg, entry->path);
+
+	if (git_strmap_valid_index(smcfg, pos))
+		sm = git_strmap_value_at(smcfg, pos);
+	else
+		sm = submodule_alloc(entry->path);
+
+	git_oid_cpy(&sm->oid, &entry->oid);
+
+	if (strcmp(sm->path, entry->path) != 0) {
+		if (sm->path != sm->name) {
+			git__free(sm->path);
+			sm->path = sm->name;
+		}
+		sm->path = git__strdup(entry->path);
+		if (!sm->path)
+			goto fail;
+	}
+
+	git_strmap_insert2(smcfg, sm->path, sm, old_sm, error);
+	if (error < 0)
+		goto fail;
+	sm->refcount++;
+
+	if (old_sm && ((git_submodule *)old_sm) != sm) {
+		/* TODO: log warning about multiple entrys for same submodule path */
+		submodule_release(old_sm, 1);
+	}
+
+	return 0;
+
+fail:
+	submodule_release(sm, 0);
+	return -1;
+}
+
+static int submodule_from_config(
+	const char *key, const char *value, void *data)
+{
+	git_strmap *smcfg = data;
+	const char *namestart;
+	const char *property;
+	git_buf name = GIT_BUF_INIT;
+	git_submodule *sm;
+	void *old_sm = NULL;
+	bool is_path;
+	khiter_t pos;
+	int error;
+
+	if (git__prefixcmp(key, "submodule.") != 0)
+		return 0;
+
+	namestart = key + strlen("submodule.");
+	property  = strrchr(namestart, '.');
+	if (property == NULL)
+		return 0;
+	property++;
+	is_path = (strcmp(property, "path") == 0);
+
+	if (git_buf_set(&name, namestart, property - namestart - 1) < 0)
+		return -1;
+
+	pos = git_strmap_lookup_index(smcfg, name.ptr);
+	if (!git_strmap_valid_index(smcfg, pos) && is_path)
+		pos = git_strmap_lookup_index(smcfg, value);
+	if (!git_strmap_valid_index(smcfg, pos))
+		sm = submodule_alloc(name.ptr);
+	else
+		sm = git_strmap_value_at(smcfg, pos);
+	if (!sm)
+		goto fail;
+
+	if (strcmp(sm->name, name.ptr) != 0) {
+		assert(sm->path == sm->name);
+		sm->name = git_buf_detach(&name);
+
+		git_strmap_insert2(smcfg, sm->name, sm, old_sm, error);
+		if (error < 0)
+			goto fail;
+		sm->refcount++;
+	}
+	else if (is_path && strcmp(sm->path, value) != 0) {
+		assert(sm->path == sm->name);
+		sm->path = git__strdup(value);
+		if (sm->path == NULL)
+			goto fail;
+
+		git_strmap_insert2(smcfg, sm->path, sm, old_sm, error);
+		if (error < 0)
+			goto fail;
+		sm->refcount++;
+	}
+	git_buf_free(&name);
+
+	if (old_sm && ((git_submodule *)old_sm) != sm) {
+		/* TODO: log warning about multiple submodules with same path */
+		submodule_release(old_sm, 1);
+	}
+
+	if (is_path)
+		return 0;
+
+	/* copy other properties into submodule entry */
+	if (strcmp(property, "url") == 0) {
+		if (sm->url) {
+			git__free(sm->url);
+			sm->url = NULL;
+		}
+		if ((sm->url = git__strdup(value)) == NULL)
+			goto fail;
+	}
+	else if (strcmp(property, "update") == 0) {
+		int val;
+		if (git_config_lookup_map_value(
+			_sm_update_map, ARRAY_SIZE(_sm_update_map), value, &val) < 0) {
+			giterr_set(GITERR_INVALID,
+				"Invalid value for submodule update property: '%s'", value);
+			goto fail;
+		}
+		sm->update = (git_submodule_update_t)val;
+	}
+	else if (strcmp(property, "fetchRecurseSubmodules") == 0) {
+		if (git__parse_bool(&sm->fetch_recurse, value) < 0) {
+			giterr_set(GITERR_INVALID,
+				"Invalid value for submodule 'fetchRecurseSubmodules' property: '%s'", value);
+			goto fail;
+		}
+	}
+	else if (strcmp(property, "ignore") == 0) {
+		int val;
+		if (git_config_lookup_map_value(
+			_sm_ignore_map, ARRAY_SIZE(_sm_ignore_map), value, &val) < 0) {
+			giterr_set(GITERR_INVALID,
+				"Invalid value for submodule ignore property: '%s'", value);
+			goto fail;
+		}
+		sm->ignore = (git_submodule_ignore_t)val;
+	}
+	/* ignore other unknown submodule properties */
+
+	return 0;
+
+fail:
+	submodule_release(sm, 0);
+	git_buf_free(&name);
+	return -1;
+}
+
+static int load_submodule_config(git_repository *repo)
+{
+	int error;
+	git_index *index;
+	unsigned int i, max_i;
+	git_oid gitmodules_oid;
+	git_strmap *smcfg;
+	struct git_config_file *mods = NULL;
+
+	if (repo->submodules)
+		return 0;
+
+	/* submodule data is kept in a hashtable with each submodule stored
+	 * under both its name and its path.  These are usually the same, but
+	 * that is not guaranteed.
+	 */
+	smcfg = git_strmap_alloc();
+	GITERR_CHECK_ALLOC(smcfg);
+
+	/* scan index for gitmodules (and .gitmodules entry) */
+	if ((error = git_repository_index__weakptr(&index, repo)) < 0)
+		goto cleanup;
+	memset(&gitmodules_oid, 0, sizeof(gitmodules_oid));
+	max_i = git_index_entrycount(index);
+
+	for (i = 0; i < max_i; i++) {
+		git_index_entry *entry = git_index_get(index, i);
+		if (S_ISGITLINK(entry->mode)) {
+			if ((error = submodule_from_entry(smcfg, entry)) < 0)
+				goto cleanup;
+		}
+		else if (strcmp(entry->path, ".gitmodules") == 0)
+			git_oid_cpy(&gitmodules_oid, &entry->oid);
+	}
+
+	/* load .gitmodules from workdir if it exists */
+	if (git_repository_workdir(repo) != NULL) {
+		/* look in workdir for .gitmodules */
+		git_buf path = GIT_BUF_INIT;
+		if (!git_buf_joinpath(
+				&path, git_repository_workdir(repo), ".gitmodules") &&
+			git_path_isfile(path.ptr))
+		{
+			if (!(error = git_config_file__ondisk(&mods, path.ptr)))
+				error = git_config_file_open(mods);
+		}
+		git_buf_free(&path);
+	}
+
+	/* load .gitmodules from object cache if not in workdir */
+	if (!error && mods == NULL && !git_oid_iszero(&gitmodules_oid)) {
+		/* TODO: is it worth loading gitmodules from object cache? */
+	}
+
+	/* process .gitmodules info */
+	if (!error && mods != NULL)
+		error = git_config_file_foreach(mods, submodule_from_config, smcfg);
+
+	/* store submodule config in repo */
+	if (!error)
+		repo->submodules = smcfg;
+
+cleanup:
+	if (mods != NULL)
+		git_config_file_free(mods);
+	if (error)
+		git_strmap_free(smcfg);
+	return error;
+}
+
+void git_submodule_config_free(git_repository *repo)
+{
+	git_strmap *smcfg = repo->submodules;
+	git_submodule *sm;
+
+	repo->submodules = NULL;
+
+	if (smcfg == NULL)
+		return;
+
+	git_strmap_foreach_value(smcfg, sm, {
+		submodule_release(sm,1);
+	});
+	git_strmap_free(smcfg);
+}
+
+static int submodule_cmp(const void *a, const void *b)
+{
+	return strcmp(((git_submodule *)a)->name, ((git_submodule *)b)->name);
+}
+
+int git_submodule_foreach(
+	git_repository *repo,
+	int (*callback)(const char *name, void *payload),
+	void *payload)
+{
+	int error;
+	git_submodule *sm;
+	git_vector seen = GIT_VECTOR_INIT;
+	seen._cmp = submodule_cmp;
+
+	if ((error = load_submodule_config(repo)) < 0)
+		return error;
+
+	git_strmap_foreach_value(repo->submodules, sm, {
+		/* usually the following will not come into play */
+		if (sm->refcount > 1) {
+			if (git_vector_bsearch(&seen, sm) != GIT_ENOTFOUND)
+				continue;
+			if ((error = git_vector_insert(&seen, sm)) < 0)
+				break;
+		}
+
+		if ((error = callback(sm->name, payload)) < 0)
+			break;
+	});
+
+	git_vector_free(&seen);
+
+	return error;
+}
+
+int git_submodule_lookup(
+	git_submodule **sm_ptr, /* NULL allowed if user only wants to test */
+	git_repository *repo,
+	const char *name)       /* trailing slash is allowed */
+{
+	khiter_t pos;
+
+	if (load_submodule_config(repo) < 0)
+		return -1;
+
+	pos = git_strmap_lookup_index(repo->submodules, name);
+	if (!git_strmap_valid_index(repo->submodules, pos))
+		return GIT_ENOTFOUND;
+
+	if (sm_ptr)
+		*sm_ptr = git_strmap_value_at(repo->submodules, pos);
+
+	return 0;
+}
diff --git a/src/t03-data.h b/src/t03-data.h
deleted file mode 100644
index a4b73fec3..000000000
--- a/src/t03-data.h
+++ /dev/null
@@ -1,344 +0,0 @@
-
-typedef struct object_data {
-    char *id;     /* object id (sha1)                          */
-    char *dir;    /* object store (fan-out) directory name     */
-    char *file;   /* object store filename                     */
-} object_data;
-
-static object_data commit = {
-    "3d7f8a6af076c8c3f20071a8935cdbe8228594d1",
-    "test-objects/3d",
-    "test-objects/3d/7f8a6af076c8c3f20071a8935cdbe8228594d1",
-};
-
-static unsigned char commit_data[] = {
-    0x74, 0x72, 0x65, 0x65, 0x20, 0x64, 0x66, 0x66,
-    0x32, 0x64, 0x61, 0x39, 0x30, 0x62, 0x32, 0x35,
-    0x34, 0x65, 0x31, 0x62, 0x65, 0x62, 0x38, 0x38,
-    0x39, 0x64, 0x31, 0x66, 0x31, 0x66, 0x31, 0x32,
-    0x38, 0x38, 0x62, 0x65, 0x31, 0x38, 0x30, 0x33,
-    0x37, 0x38, 0x32, 0x64, 0x66, 0x0a, 0x61, 0x75,
-    0x74, 0x68, 0x6f, 0x72, 0x20, 0x41, 0x20, 0x55,
-    0x20, 0x54, 0x68, 0x6f, 0x72, 0x20, 0x3c, 0x61,
-    0x75, 0x74, 0x68, 0x6f, 0x72, 0x40, 0x65, 0x78,
-    0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f,
-    0x6d, 0x3e, 0x20, 0x31, 0x32, 0x32, 0x37, 0x38,
-    0x31, 0x34, 0x32, 0x39, 0x37, 0x20, 0x2b, 0x30,
-    0x30, 0x30, 0x30, 0x0a, 0x63, 0x6f, 0x6d, 0x6d,
-    0x69, 0x74, 0x74, 0x65, 0x72, 0x20, 0x43, 0x20,
-    0x4f, 0x20, 0x4d, 0x69, 0x74, 0x74, 0x65, 0x72,
-    0x20, 0x3c, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
-    0x74, 0x65, 0x72, 0x40, 0x65, 0x78, 0x61, 0x6d,
-    0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x3e,
-    0x20, 0x31, 0x32, 0x32, 0x37, 0x38, 0x31, 0x34,
-    0x32, 0x39, 0x37, 0x20, 0x2b, 0x30, 0x30, 0x30,
-    0x30, 0x0a, 0x0a, 0x41, 0x20, 0x6f, 0x6e, 0x65,
-    0x2d, 0x6c, 0x69, 0x6e, 0x65, 0x20, 0x63, 0x6f,
-    0x6d, 0x6d, 0x69, 0x74, 0x20, 0x73, 0x75, 0x6d,
-    0x6d, 0x61, 0x72, 0x79, 0x0a, 0x0a, 0x54, 0x68,
-    0x65, 0x20, 0x62, 0x6f, 0x64, 0x79, 0x20, 0x6f,
-    0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f,
-    0x6d, 0x6d, 0x69, 0x74, 0x20, 0x6d, 0x65, 0x73,
-    0x73, 0x61, 0x67, 0x65, 0x2c, 0x20, 0x63, 0x6f,
-    0x6e, 0x74, 0x61, 0x69, 0x6e, 0x69, 0x6e, 0x67,
-    0x20, 0x66, 0x75, 0x72, 0x74, 0x68, 0x65, 0x72,
-    0x20, 0x65, 0x78, 0x70, 0x6c, 0x61, 0x6e, 0x61,
-    0x74, 0x69, 0x6f, 0x6e, 0x0a, 0x6f, 0x66, 0x20,
-    0x74, 0x68, 0x65, 0x20, 0x70, 0x75, 0x72, 0x70,
-    0x6f, 0x73, 0x65, 0x20, 0x6f, 0x66, 0x20, 0x74,
-    0x68, 0x65, 0x20, 0x63, 0x68, 0x61, 0x6e, 0x67,
-    0x65, 0x73, 0x20, 0x69, 0x6e, 0x74, 0x72, 0x6f,
-    0x64, 0x75, 0x63, 0x65, 0x64, 0x20, 0x62, 0x79,
-    0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f, 0x6d,
-    0x6d, 0x69, 0x74, 0x2e, 0x0a, 0x0a, 0x53, 0x69,
-    0x67, 0x6e, 0x65, 0x64, 0x2d, 0x6f, 0x66, 0x2d,
-    0x62, 0x79, 0x3a, 0x20, 0x41, 0x20, 0x55, 0x20,
-    0x54, 0x68, 0x6f, 0x72, 0x20, 0x3c, 0x61, 0x75,
-    0x74, 0x68, 0x6f, 0x72, 0x40, 0x65, 0x78, 0x61,
-    0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d,
-    0x3e, 0x0a,
-};
-
-static git_rawobj commit_obj = {
-	commit_data,
-	sizeof(commit_data),
-	GIT_OBJ_COMMIT
-};
-
-static object_data tree = {
-    "dff2da90b254e1beb889d1f1f1288be1803782df",
-    "test-objects/df",
-    "test-objects/df/f2da90b254e1beb889d1f1f1288be1803782df",
-};
-
-static unsigned char tree_data[] = {
-    0x31, 0x30, 0x30, 0x36, 0x34, 0x34, 0x20, 0x6f,
-    0x6e, 0x65, 0x00, 0x8b, 0x13, 0x78, 0x91, 0x79,
-    0x1f, 0xe9, 0x69, 0x27, 0xad, 0x78, 0xe6, 0x4b,
-    0x0a, 0xad, 0x7b, 0xde, 0xd0, 0x8b, 0xdc, 0x31,
-    0x30, 0x30, 0x36, 0x34, 0x34, 0x20, 0x73, 0x6f,
-    0x6d, 0x65, 0x00, 0xfd, 0x84, 0x30, 0xbc, 0x86,
-    0x4c, 0xfc, 0xd5, 0xf1, 0x0e, 0x55, 0x90, 0xf8,
-    0xa4, 0x47, 0xe0, 0x1b, 0x94, 0x2b, 0xfe, 0x31,
-    0x30, 0x30, 0x36, 0x34, 0x34, 0x20, 0x74, 0x77,
-    0x6f, 0x00, 0x78, 0x98, 0x19, 0x22, 0x61, 0x3b,
-    0x2a, 0xfb, 0x60, 0x25, 0x04, 0x2f, 0xf6, 0xbd,
-    0x87, 0x8a, 0xc1, 0x99, 0x4e, 0x85, 0x31, 0x30,
-    0x30, 0x36, 0x34, 0x34, 0x20, 0x7a, 0x65, 0x72,
-    0x6f, 0x00, 0xe6, 0x9d, 0xe2, 0x9b, 0xb2, 0xd1,
-    0xd6, 0x43, 0x4b, 0x8b, 0x29, 0xae, 0x77, 0x5a,
-    0xd8, 0xc2, 0xe4, 0x8c, 0x53, 0x91,
-};
-
-static git_rawobj tree_obj = {
-	tree_data,
-	sizeof(tree_data),
-	GIT_OBJ_TREE
-};
-
-static object_data tag = {
-    "09d373e1dfdc16b129ceec6dd649739911541e05",
-    "test-objects/09",
-    "test-objects/09/d373e1dfdc16b129ceec6dd649739911541e05",
-};
-
-static unsigned char tag_data[] = {
-    0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x20, 0x33,
-    0x64, 0x37, 0x66, 0x38, 0x61, 0x36, 0x61, 0x66,
-    0x30, 0x37, 0x36, 0x63, 0x38, 0x63, 0x33, 0x66,
-    0x32, 0x30, 0x30, 0x37, 0x31, 0x61, 0x38, 0x39,
-    0x33, 0x35, 0x63, 0x64, 0x62, 0x65, 0x38, 0x32,
-    0x32, 0x38, 0x35, 0x39, 0x34, 0x64, 0x31, 0x0a,
-    0x74, 0x79, 0x70, 0x65, 0x20, 0x63, 0x6f, 0x6d,
-    0x6d, 0x69, 0x74, 0x0a, 0x74, 0x61, 0x67, 0x20,
-    0x76, 0x30, 0x2e, 0x30, 0x2e, 0x31, 0x0a, 0x74,
-    0x61, 0x67, 0x67, 0x65, 0x72, 0x20, 0x43, 0x20,
-    0x4f, 0x20, 0x4d, 0x69, 0x74, 0x74, 0x65, 0x72,
-    0x20, 0x3c, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74,
-    0x74, 0x65, 0x72, 0x40, 0x65, 0x78, 0x61, 0x6d,
-    0x70, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x3e,
-    0x20, 0x31, 0x32, 0x32, 0x37, 0x38, 0x31, 0x34,
-    0x32, 0x39, 0x37, 0x20, 0x2b, 0x30, 0x30, 0x30,
-    0x30, 0x0a, 0x0a, 0x54, 0x68, 0x69, 0x73, 0x20,
-    0x69, 0x73, 0x20, 0x74, 0x68, 0x65, 0x20, 0x74,
-    0x61, 0x67, 0x20, 0x6f, 0x62, 0x6a, 0x65, 0x63,
-    0x74, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x72, 0x65,
-    0x6c, 0x65, 0x61, 0x73, 0x65, 0x20, 0x76, 0x30,
-    0x2e, 0x30, 0x2e, 0x31, 0x0a,
-};
-
-static git_rawobj tag_obj = {
-	tag_data,
-	sizeof(tag_data),
-	GIT_OBJ_TAG
-};
-
-static object_data zero = {
-    "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391",
-    "test-objects/e6",
-    "test-objects/e6/9de29bb2d1d6434b8b29ae775ad8c2e48c5391",
-};
-
-static unsigned char zero_data[] = {
-    0x00  /* dummy data */
-};
-
-static git_rawobj zero_obj = {
-	zero_data,
-	0,
-	GIT_OBJ_BLOB
-};
-
-static object_data one = {
-    "8b137891791fe96927ad78e64b0aad7bded08bdc",
-    "test-objects/8b",
-    "test-objects/8b/137891791fe96927ad78e64b0aad7bded08bdc",
-};
-
-static unsigned char one_data[] = {
-    0x0a,
-};
-
-static git_rawobj one_obj = {
-	one_data,
-	sizeof(one_data),
-	GIT_OBJ_BLOB
-};
-
-static object_data two = {
-    "78981922613b2afb6025042ff6bd878ac1994e85",
-    "test-objects/78",
-    "test-objects/78/981922613b2afb6025042ff6bd878ac1994e85",
-};
-
-static unsigned char two_data[] = {
-    0x61, 0x0a,
-};
-
-static git_rawobj two_obj = {
-	two_data,
-	sizeof(two_data),
-	GIT_OBJ_BLOB
-};
-
-static object_data some = {
-    "fd8430bc864cfcd5f10e5590f8a447e01b942bfe",
-    "test-objects/fd",
-    "test-objects/fd/8430bc864cfcd5f10e5590f8a447e01b942bfe",
-};
-
-static unsigned char some_data[] = {
-    0x2f, 0x2a, 0x0a, 0x20, 0x2a, 0x20, 0x54, 0x68,
-    0x69, 0x73, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x20,
-    0x69, 0x73, 0x20, 0x66, 0x72, 0x65, 0x65, 0x20,
-    0x73, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65,
-    0x3b, 0x20, 0x79, 0x6f, 0x75, 0x20, 0x63, 0x61,
-    0x6e, 0x20, 0x72, 0x65, 0x64, 0x69, 0x73, 0x74,
-    0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x20, 0x69,
-    0x74, 0x20, 0x61, 0x6e, 0x64, 0x2f, 0x6f, 0x72,
-    0x20, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x79, 0x0a,
-    0x20, 0x2a, 0x20, 0x69, 0x74, 0x20, 0x75, 0x6e,
-    0x64, 0x65, 0x72, 0x20, 0x74, 0x68, 0x65, 0x20,
-    0x74, 0x65, 0x72, 0x6d, 0x73, 0x20, 0x6f, 0x66,
-    0x20, 0x74, 0x68, 0x65, 0x20, 0x47, 0x4e, 0x55,
-    0x20, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c,
-    0x20, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x20,
-    0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x2c,
-    0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
-    0x20, 0x32, 0x2c, 0x0a, 0x20, 0x2a, 0x20, 0x61,
-    0x73, 0x20, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x73,
-    0x68, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x74,
-    0x68, 0x65, 0x20, 0x46, 0x72, 0x65, 0x65, 0x20,
-    0x53, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65,
-    0x20, 0x46, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x74,
-    0x69, 0x6f, 0x6e, 0x2e, 0x0a, 0x20, 0x2a, 0x0a,
-    0x20, 0x2a, 0x20, 0x49, 0x6e, 0x20, 0x61, 0x64,
-    0x64, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74,
-    0x6f, 0x20, 0x74, 0x68, 0x65, 0x20, 0x70, 0x65,
-    0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e,
-    0x73, 0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65,
-    0x20, 0x47, 0x4e, 0x55, 0x20, 0x47, 0x65, 0x6e,
-    0x65, 0x72, 0x61, 0x6c, 0x20, 0x50, 0x75, 0x62,
-    0x6c, 0x69, 0x63, 0x20, 0x4c, 0x69, 0x63, 0x65,
-    0x6e, 0x73, 0x65, 0x2c, 0x0a, 0x20, 0x2a, 0x20,
-    0x74, 0x68, 0x65, 0x20, 0x61, 0x75, 0x74, 0x68,
-    0x6f, 0x72, 0x73, 0x20, 0x67, 0x69, 0x76, 0x65,
-    0x20, 0x79, 0x6f, 0x75, 0x20, 0x75, 0x6e, 0x6c,
-    0x69, 0x6d, 0x69, 0x74, 0x65, 0x64, 0x20, 0x70,
-    0x65, 0x72, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f,
-    0x6e, 0x20, 0x74, 0x6f, 0x20, 0x6c, 0x69, 0x6e,
-    0x6b, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6f,
-    0x6d, 0x70, 0x69, 0x6c, 0x65, 0x64, 0x0a, 0x20,
-    0x2a, 0x20, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f,
-    0x6e, 0x20, 0x6f, 0x66, 0x20, 0x74, 0x68, 0x69,
-    0x73, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x20, 0x69,
-    0x6e, 0x74, 0x6f, 0x20, 0x63, 0x6f, 0x6d, 0x62,
-    0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73,
-    0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x6f, 0x74,
-    0x68, 0x65, 0x72, 0x20, 0x70, 0x72, 0x6f, 0x67,
-    0x72, 0x61, 0x6d, 0x73, 0x2c, 0x0a, 0x20, 0x2a,
-    0x20, 0x61, 0x6e, 0x64, 0x20, 0x74, 0x6f, 0x20,
-    0x64, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75,
-    0x74, 0x65, 0x20, 0x74, 0x68, 0x6f, 0x73, 0x65,
-    0x20, 0x63, 0x6f, 0x6d, 0x62, 0x69, 0x6e, 0x61,
-    0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20, 0x77, 0x69,
-    0x74, 0x68, 0x6f, 0x75, 0x74, 0x20, 0x61, 0x6e,
-    0x79, 0x20, 0x72, 0x65, 0x73, 0x74, 0x72, 0x69,
-    0x63, 0x74, 0x69, 0x6f, 0x6e, 0x0a, 0x20, 0x2a,
-    0x20, 0x63, 0x6f, 0x6d, 0x69, 0x6e, 0x67, 0x20,
-    0x66, 0x72, 0x6f, 0x6d, 0x20, 0x74, 0x68, 0x65,
-    0x20, 0x75, 0x73, 0x65, 0x20, 0x6f, 0x66, 0x20,
-    0x74, 0x68, 0x69, 0x73, 0x20, 0x66, 0x69, 0x6c,
-    0x65, 0x2e, 0x20, 0x20, 0x28, 0x54, 0x68, 0x65,
-    0x20, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c,
-    0x20, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x20,
-    0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x0a,
-    0x20, 0x2a, 0x20, 0x72, 0x65, 0x73, 0x74, 0x72,
-    0x69, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x20,
-    0x64, 0x6f, 0x20, 0x61, 0x70, 0x70, 0x6c, 0x79,
-    0x20, 0x69, 0x6e, 0x20, 0x6f, 0x74, 0x68, 0x65,
-    0x72, 0x20, 0x72, 0x65, 0x73, 0x70, 0x65, 0x63,
-    0x74, 0x73, 0x3b, 0x20, 0x66, 0x6f, 0x72, 0x20,
-    0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2c,
-    0x20, 0x74, 0x68, 0x65, 0x79, 0x20, 0x63, 0x6f,
-    0x76, 0x65, 0x72, 0x0a, 0x20, 0x2a, 0x20, 0x6d,
-    0x6f, 0x64, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74,
-    0x69, 0x6f, 0x6e, 0x20, 0x6f, 0x66, 0x20, 0x74,
-    0x68, 0x65, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x2c,
-    0x20, 0x61, 0x6e, 0x64, 0x20, 0x64, 0x69, 0x73,
-    0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f,
-    0x6e, 0x20, 0x77, 0x68, 0x65, 0x6e, 0x20, 0x6e,
-    0x6f, 0x74, 0x20, 0x6c, 0x69, 0x6e, 0x6b, 0x65,
-    0x64, 0x20, 0x69, 0x6e, 0x74, 0x6f, 0x0a, 0x20,
-    0x2a, 0x20, 0x61, 0x20, 0x63, 0x6f, 0x6d, 0x62,
-    0x69, 0x6e, 0x65, 0x64, 0x20, 0x65, 0x78, 0x65,
-    0x63, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x2e,
-    0x29, 0x0a, 0x20, 0x2a, 0x0a, 0x20, 0x2a, 0x20,
-    0x54, 0x68, 0x69, 0x73, 0x20, 0x66, 0x69, 0x6c,
-    0x65, 0x20, 0x69, 0x73, 0x20, 0x64, 0x69, 0x73,
-    0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x64,
-    0x20, 0x69, 0x6e, 0x20, 0x74, 0x68, 0x65, 0x20,
-    0x68, 0x6f, 0x70, 0x65, 0x20, 0x74, 0x68, 0x61,
-    0x74, 0x20, 0x69, 0x74, 0x20, 0x77, 0x69, 0x6c,
-    0x6c, 0x20, 0x62, 0x65, 0x20, 0x75, 0x73, 0x65,
-    0x66, 0x75, 0x6c, 0x2c, 0x20, 0x62, 0x75, 0x74,
-    0x0a, 0x20, 0x2a, 0x20, 0x57, 0x49, 0x54, 0x48,
-    0x4f, 0x55, 0x54, 0x20, 0x41, 0x4e, 0x59, 0x20,
-    0x57, 0x41, 0x52, 0x52, 0x41, 0x4e, 0x54, 0x59,
-    0x3b, 0x20, 0x77, 0x69, 0x74, 0x68, 0x6f, 0x75,
-    0x74, 0x20, 0x65, 0x76, 0x65, 0x6e, 0x20, 0x74,
-    0x68, 0x65, 0x20, 0x69, 0x6d, 0x70, 0x6c, 0x69,
-    0x65, 0x64, 0x20, 0x77, 0x61, 0x72, 0x72, 0x61,
-    0x6e, 0x74, 0x79, 0x20, 0x6f, 0x66, 0x0a, 0x20,
-    0x2a, 0x20, 0x4d, 0x45, 0x52, 0x43, 0x48, 0x41,
-    0x4e, 0x54, 0x41, 0x42, 0x49, 0x4c, 0x49, 0x54,
-    0x59, 0x20, 0x6f, 0x72, 0x20, 0x46, 0x49, 0x54,
-    0x4e, 0x45, 0x53, 0x53, 0x20, 0x46, 0x4f, 0x52,
-    0x20, 0x41, 0x20, 0x50, 0x41, 0x52, 0x54, 0x49,
-    0x43, 0x55, 0x4c, 0x41, 0x52, 0x20, 0x50, 0x55,
-    0x52, 0x50, 0x4f, 0x53, 0x45, 0x2e, 0x20, 0x20,
-    0x53, 0x65, 0x65, 0x20, 0x74, 0x68, 0x65, 0x20,
-    0x47, 0x4e, 0x55, 0x0a, 0x20, 0x2a, 0x20, 0x47,
-    0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x20, 0x50,
-    0x75, 0x62, 0x6c, 0x69, 0x63, 0x20, 0x4c, 0x69,
-    0x63, 0x65, 0x6e, 0x73, 0x65, 0x20, 0x66, 0x6f,
-    0x72, 0x20, 0x6d, 0x6f, 0x72, 0x65, 0x20, 0x64,
-    0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x2e, 0x0a,
-    0x20, 0x2a, 0x0a, 0x20, 0x2a, 0x20, 0x59, 0x6f,
-    0x75, 0x20, 0x73, 0x68, 0x6f, 0x75, 0x6c, 0x64,
-    0x20, 0x68, 0x61, 0x76, 0x65, 0x20, 0x72, 0x65,
-    0x63, 0x65, 0x69, 0x76, 0x65, 0x64, 0x20, 0x61,
-    0x20, 0x63, 0x6f, 0x70, 0x79, 0x20, 0x6f, 0x66,
-    0x20, 0x74, 0x68, 0x65, 0x20, 0x47, 0x4e, 0x55,
-    0x20, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x6c,
-    0x20, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x20,
-    0x4c, 0x69, 0x63, 0x65, 0x6e, 0x73, 0x65, 0x0a,
-    0x20, 0x2a, 0x20, 0x61, 0x6c, 0x6f, 0x6e, 0x67,
-    0x20, 0x77, 0x69, 0x74, 0x68, 0x20, 0x74, 0x68,
-    0x69, 0x73, 0x20, 0x70, 0x72, 0x6f, 0x67, 0x72,
-    0x61, 0x6d, 0x3b, 0x20, 0x73, 0x65, 0x65, 0x20,
-    0x74, 0x68, 0x65, 0x20, 0x66, 0x69, 0x6c, 0x65,
-    0x20, 0x43, 0x4f, 0x50, 0x59, 0x49, 0x4e, 0x47,
-    0x2e, 0x20, 0x20, 0x49, 0x66, 0x20, 0x6e, 0x6f,
-    0x74, 0x2c, 0x20, 0x77, 0x72, 0x69, 0x74, 0x65,
-    0x20, 0x74, 0x6f, 0x0a, 0x20, 0x2a, 0x20, 0x74,
-    0x68, 0x65, 0x20, 0x46, 0x72, 0x65, 0x65, 0x20,
-    0x53, 0x6f, 0x66, 0x74, 0x77, 0x61, 0x72, 0x65,
-    0x20, 0x46, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x74,
-    0x69, 0x6f, 0x6e, 0x2c, 0x20, 0x35, 0x31, 0x20,
-    0x46, 0x72, 0x61, 0x6e, 0x6b, 0x6c, 0x69, 0x6e,
-    0x20, 0x53, 0x74, 0x72, 0x65, 0x65, 0x74, 0x2c,
-    0x20, 0x46, 0x69, 0x66, 0x74, 0x68, 0x20, 0x46,
-    0x6c, 0x6f, 0x6f, 0x72, 0x2c, 0x0a, 0x20, 0x2a,
-    0x20, 0x42, 0x6f, 0x73, 0x74, 0x6f, 0x6e, 0x2c,
-    0x20, 0x4d, 0x41, 0x20, 0x30, 0x32, 0x31, 0x31,
-    0x30, 0x2d, 0x31, 0x33, 0x30, 0x31, 0x2c, 0x20,
-    0x55, 0x53, 0x41, 0x2e, 0x0a, 0x20, 0x2a, 0x2f,
-    0x0a,
-};
-
-static git_rawobj some_obj = {
-	some_data,
-	sizeof(some_data),
-	GIT_OBJ_BLOB
-};
diff --git a/src/tag.c b/src/tag.c
index 905d129ef..63424f530 100644
--- a/src/tag.c
+++ b/src/tag.c
@@ -1,32 +1,15 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
 #include "commit.h"
 #include "tag.h"
 #include "signature.h"
+#include "message.h"
 #include "git2/object.h"
 #include "git2/repository.h"
 #include "git2/signature.h"
@@ -34,9 +17,9 @@
 void git_tag__free(git_tag *tag)
 {
 	git_signature_free(tag->tagger);
-	free(tag->message);
-	free(tag->tag_name);
-	free(tag);
+	git__free(tag->message);
+	git__free(tag->tag_name);
+	git__free(tag);
 }
 
 const git_oid *git_tag_id(git_tag *c)
@@ -79,24 +62,32 @@ const char *git_tag_message(git_tag *t)
 	return t->message;
 }
 
-static int parse_tag_buffer(git_tag *tag, const char *buffer, const char *buffer_end)
+static int tag_error(const char *str)
+{
+	giterr_set(GITERR_TAG, "Failed to parse tag. %s", str);
+	return -1;
+}
+
+int git_tag__parse_buffer(git_tag *tag, const char *buffer, size_t length)
 {
 	static const char *tag_types[] = {
 		NULL, "commit\n", "tree\n", "blob\n", "tag\n"
 	};
 
-	unsigned int i, text_len;
+	unsigned int i;
+	size_t text_len;
 	char *search;
-	int error;
 
-	if ((error = git__parse_oid(&tag->target, &buffer, buffer_end, "object ")) < 0)
-		return git__rethrow(error, "Failed to parse tag. Object field invalid");
+	const char *buffer_end = buffer + length;
+
+	if (git_oid__parse(&tag->target, &buffer, buffer_end, "object ") < 0)
+		return tag_error("Object field invalid");
 
 	if (buffer + 5 >= buffer_end)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tag. Object too short");
+		return tag_error("Object too short");
 
 	if (memcmp(buffer, "type ", 5) != 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tag. Type field not found");
+		return tag_error("Type field not found");
 	buffer += 5;
 
 	tag->type = GIT_OBJ_BAD;
@@ -105,7 +96,7 @@ static int parse_tag_buffer(git_tag *tag, const char *buffer, const char *buffer
 		size_t type_length = strlen(tag_types[i]);
 
 		if (buffer + type_length >= buffer_end)
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tag. Object too short");
+			return tag_error("Object too short");
 
 		if (memcmp(buffer, tag_types[i], type_length) == 0) {
 			tag->type = i;
@@ -115,262 +106,289 @@ static int parse_tag_buffer(git_tag *tag, const char *buffer, const char *buffer
 	}
 
 	if (tag->type == GIT_OBJ_BAD)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tag. Invalid object type");
+		return tag_error("Invalid object type");
 
 	if (buffer + 4 >= buffer_end)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tag. Object too short");
+		return tag_error("Object too short");
 
 	if (memcmp(buffer, "tag ", 4) != 0)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tag. Tag field not found");
+		return tag_error("Tag field not found");
 
 	buffer += 4;
 
 	search = memchr(buffer, '\n', buffer_end - buffer);
 	if (search == NULL)
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tag. Object too short");
+		return tag_error("Object too short");
 
 	text_len = search - buffer;
 
 	tag->tag_name = git__malloc(text_len + 1);
-	if (tag->tag_name == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(tag->tag_name);
 
 	memcpy(tag->tag_name, buffer, text_len);
 	tag->tag_name[text_len] = '\0';
 
 	buffer = search + 1;
 
-	tag->tagger = git__malloc(sizeof(git_signature));
-	if (tag->tagger == NULL)
-		return GIT_ENOMEM;
+	tag->tagger = NULL;
+	if (*buffer != '\n') {
+		tag->tagger = git__malloc(sizeof(git_signature));
+		GITERR_CHECK_ALLOC(tag->tagger);
 
-	if ((error = git_signature__parse(tag->tagger, &buffer, buffer_end, "tagger ")) != 0) {
-		free(tag->tag_name);
-		git_signature_free(tag->tagger);
-		return git__rethrow(error, "Failed to parse tag");
+		if (git_signature__parse(tag->tagger, &buffer, buffer_end, "tagger ", '\n') < 0)
+			return -1;
 	}
 
 	if( *buffer != '\n' )
-		return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tag. No new line before message");
+		return tag_error("No new line before message");
 
 	text_len = buffer_end - ++buffer;
 
 	tag->message = git__malloc(text_len + 1);
-	if (tag->message == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(tag->message);
 
 	memcpy(tag->message, buffer, text_len);
 	tag->message[text_len] = '\0';
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-static int retreive_tag_reference(git_reference **tag_reference_out, char *ref_name_out, git_repository *repo, const char *tag_name)
+static int retrieve_tag_reference(
+	git_reference **tag_reference_out,
+	git_buf *ref_name_out,
+	git_repository *repo,
+	const char *tag_name)
 {
 	git_reference *tag_ref;
 	int error;
 
-	git__joinpath(ref_name_out, GIT_REFS_TAGS_DIR, tag_name);
-	error = git_reference_lookup(&tag_ref, repo, ref_name_out);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to retrieve tag reference");
+	*tag_reference_out = NULL;
+
+	if (git_buf_joinpath(ref_name_out, GIT_REFS_TAGS_DIR, tag_name) < 0)
+		return -1;
+
+	error = git_reference_lookup(&tag_ref, repo, ref_name_out->ptr);
+	if (error < 0)
+		return error; /* Be it not foundo or corrupted */
 
 	*tag_reference_out = tag_ref;
 
-	return GIT_SUCCESS;
+	return 0;
+}
+
+static int retrieve_tag_reference_oid(
+	git_oid *oid,
+	git_buf *ref_name_out,
+	git_repository *repo,
+	const char *tag_name)
+{
+	if (git_buf_joinpath(ref_name_out, GIT_REFS_TAGS_DIR, tag_name) < 0)
+		return -1;
+
+	return git_reference_name_to_oid(oid, repo, ref_name_out->ptr);
 }
 
-static int tag_create(
+static int write_tag_annotation(
 		git_oid *oid,
 		git_repository *repo,
 		const char *tag_name,
-		const git_oid *target,
-		git_otype target_type,
+		const git_object *target,
 		const git_signature *tagger,
-		const char *message,
-		int allow_ref_overwrite)
+		const char *message)
 {
-	size_t final_size = 0;
-	git_odb_stream *stream;
-
-	const char *type_str;
-	char *tagger_str;
-	git_reference *new_ref;
-
-	char ref_name[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH];
-
-	int type_str_len, tag_name_len, tagger_str_len, message_len;
-	int error, should_update_ref = 0;
-
-	/** Ensure the tag name doesn't conflict with an already existing 
-	    reference unless overwriting has explictly been requested **/
-	error = retreive_tag_reference(&new_ref, ref_name, repo, tag_name);
-
-	switch (error) {
-	case GIT_SUCCESS:
-		if (!allow_ref_overwrite)
-			return GIT_EEXISTS;	
-		should_update_ref = 1;
-		
-		/* Fall trough */
-
-	case GIT_ENOTFOUND: 
-		break;
-
-	default:
-		return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create tag");
-	}
-
-	if (!git_odb_exists(repo->db, target))
-		return git__throw(GIT_ENOTFOUND, "Failed to create tag. Object to tag doesn't exist");
-
-	/* Try to find out what the type is */
-	if (target_type == GIT_OBJ_ANY) {
-		size_t _unused;
-		error = git_odb_read_header(&_unused, &target_type, repo->db, target);
-		if (error < GIT_SUCCESS)
-			return git__rethrow(error, "Failed to create tag");
-	}
-
-	type_str = git_object_type2string(target_type);
-
-	tagger_str_len = git_signature__write(&tagger_str, "tagger", tagger);
+	git_buf tag = GIT_BUF_INIT, cleaned_message = GIT_BUF_INIT;
+	git_odb *odb;
 
-	type_str_len = strlen(type_str);
-	tag_name_len = strlen(tag_name);
-	message_len = strlen(message);
+	git_oid__writebuf(&tag, "object ", git_object_id(target));
+	git_buf_printf(&tag, "type %s\n", git_object_type2string(git_object_type(target)));
+	git_buf_printf(&tag, "tag %s\n", tag_name);
+	git_signature__writebuf(&tag, "tagger ", tagger);
+	git_buf_putc(&tag, '\n');
 
-	final_size += GIT_OID_LINE_LENGTH("object");
-	final_size += STRLEN("type ") + type_str_len + 1;
-	final_size += STRLEN("tag ") + tag_name_len + 1;
-	final_size += tagger_str_len;
-	final_size += 1 + message_len;
+	/* Remove comments by default */
+	if (git_message_prettify(&cleaned_message, message, 1) < 0)
+		goto on_error;
 
-	if ((error = git_odb_open_wstream(&stream, repo->db, final_size, GIT_OBJ_TAG)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to create tag");
+	if (git_buf_puts(&tag, git_buf_cstr(&cleaned_message)) < 0)
+		goto on_error;
 
-	git__write_oid(stream, "object", target);
+	git_buf_free(&cleaned_message);
 
-	stream->write(stream, "type ", STRLEN("type "));
-	stream->write(stream, type_str, type_str_len);
+	if (git_repository_odb__weakptr(&odb, repo) < 0)
+		goto on_error;
 
-	stream->write(stream, "\ntag ", STRLEN("\ntag "));
-	stream->write(stream, tag_name, tag_name_len);
-	stream->write(stream, "\n", 1);
+	if (git_odb_write(oid, odb, tag.ptr, tag.size, GIT_OBJ_TAG) < 0)
+		goto on_error;
 
-	stream->write(stream, tagger_str, tagger_str_len);
-	free(tagger_str);
+	git_buf_free(&tag);
+	return 0;
 
-	stream->write(stream, "\n", 1);
-	stream->write(stream, message, message_len);
-
-
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
-
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to create tag");
-
-	if (!should_update_ref)
-		error = git_reference_create_oid(&new_ref, repo, ref_name, oid);
-	else
-		error = git_reference_set_oid(new_ref, oid);
-	
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create tag");
+on_error:
+	git_buf_free(&tag);
+	git_buf_free(&cleaned_message);
+	giterr_set(GITERR_OBJECT, "Failed to create tag annotation.");
+	return -1;
 }
 
-int git_tag_create_frombuffer(git_oid *oid, git_repository *repo, const char *buffer)
+static int git_tag_create__internal(
+		git_oid *oid,
+		git_repository *repo,
+		const char *tag_name,
+		const git_object *target,
+		const git_signature *tagger,
+		const char *message,
+		int allow_ref_overwrite,
+		int create_tag_annotation)
 {
-	git_tag tag;
+	git_reference *new_ref = NULL;
+	git_buf ref_name = GIT_BUF_INIT;
+
 	int error;
 
-	assert(oid && buffer);
+	assert(repo && tag_name && target);
+	assert(!create_tag_annotation || (tagger && message));
 
-	memset(&tag, 0, sizeof(tag));
+	if (git_object_owner(target) != repo) {
+		giterr_set(GITERR_INVALID, "The given target does not belong to this repository");
+		return -1;
+	}
 
-	if ((error = parse_tag_buffer(&tag, buffer, buffer + strlen(buffer))) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to create tag");
+	error = retrieve_tag_reference_oid(oid, &ref_name, repo, tag_name);
+	if (error < 0 && error != GIT_ENOTFOUND)
+		return -1;
 
-	error = git_tag_create(oid, repo, tag.tag_name, &tag.target, tag.type, tag.tagger, tag.message);
+	/** Ensure the tag name doesn't conflict with an already existing
+	 *	reference unless overwriting has explictly been requested **/
+	if (error == 0 && !allow_ref_overwrite) {
+		git_buf_free(&ref_name);
+		giterr_set(GITERR_TAG, "Tag already exists");
+		return GIT_EEXISTS;
+	}
 
-	git_signature_free(tag.tagger);
-	free(tag.tag_name);
-	free(tag.message);
+	if (create_tag_annotation) {
+		if (write_tag_annotation(oid, repo, tag_name, target, tagger, message) < 0)
+			return -1;
+	} else
+		git_oid_cpy(oid, git_object_id(target));
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to create tag");
-}
+	error = git_reference_create_oid(&new_ref, repo, ref_name.ptr, oid, allow_ref_overwrite);
 
-int git_tag_create_o(
-		git_oid *oid,
-		git_repository *repo,
-		const char *tag_name,
-		const git_object *target,
-		const git_signature *tagger,
-		const char *message)
-{
-	return tag_create(
-		oid, repo, tag_name, 
-		git_object_id(target),
-		git_object_type(target),
-		tagger, message, 0);
+	git_reference_free(new_ref);
+	git_buf_free(&ref_name);
+	return error;
 }
 
 int git_tag_create(
 		git_oid *oid,
 		git_repository *repo,
 		const char *tag_name,
-		const git_oid *target,
-		git_otype target_type,
+		const git_object *target,
 		const git_signature *tagger,
-		const char *message)
+		const char *message,
+		int allow_ref_overwrite)
 {
-	return tag_create(
-		oid, repo, tag_name, 
-		target,
-		target_type,
-		tagger, message, 0);
+	return git_tag_create__internal(oid, repo, tag_name, target, tagger, message, allow_ref_overwrite, 1);
 }
 
-int git_tag_create_fo(
+int git_tag_create_lightweight(
 		git_oid *oid,
 		git_repository *repo,
 		const char *tag_name,
 		const git_object *target,
-		const git_signature *tagger,
-		const char *message)
+		int allow_ref_overwrite)
 {
-	return tag_create(
-		oid, repo, tag_name, 
-		git_object_id(target),
-		git_object_type(target),
-		tagger, message, 1);
+	return git_tag_create__internal(oid, repo, tag_name, target, NULL, NULL, allow_ref_overwrite, 0);
 }
 
-int git_tag_create_f(
-		git_oid *oid,
-		git_repository *repo,
-		const char *tag_name,
-		const git_oid *target,
-		git_otype target_type,
-		const git_signature *tagger,
-		const char *message)
+int git_tag_create_frombuffer(git_oid *oid, git_repository *repo, const char *buffer, int allow_ref_overwrite)
 {
-	return tag_create(
-		oid, repo, tag_name, 
-		target,
-		target_type,
-		tagger, message, 1);
+	git_tag tag;
+	int error;
+	git_odb *odb;
+	git_odb_stream *stream;
+	git_odb_object *target_obj;
+
+	git_reference *new_ref = NULL;
+	git_buf ref_name = GIT_BUF_INIT;
+
+	assert(oid && buffer);
+
+	memset(&tag, 0, sizeof(tag));
+
+	if (git_repository_odb__weakptr(&odb, repo) < 0)
+		return -1;
+
+	/* validate the buffer */
+	if (git_tag__parse_buffer(&tag, buffer, strlen(buffer)) < 0)
+		return -1;
+
+	/* validate the target */
+	if (git_odb_read(&target_obj, odb, &tag.target) < 0)
+		goto on_error;
+
+	if (tag.type != target_obj->raw.type) {
+		giterr_set(GITERR_TAG, "The type for the given target is invalid");
+		goto on_error;
+	}
+
+	error = retrieve_tag_reference_oid(oid, &ref_name, repo, tag.tag_name);
+	if (error < 0 && error != GIT_ENOTFOUND)
+		goto on_error;
+
+	/* We don't need these objects after this */
+	git_signature_free(tag.tagger);
+	git__free(tag.tag_name);
+	git__free(tag.message);
+	git_odb_object_free(target_obj);
+
+	/** Ensure the tag name doesn't conflict with an already existing
+	 *	reference unless overwriting has explictly been requested **/
+	if (error == 0 && !allow_ref_overwrite) {
+		giterr_set(GITERR_TAG, "Tag already exists");
+		return GIT_EEXISTS;
+	}
+
+	/* write the buffer */
+	if (git_odb_open_wstream(&stream, odb, strlen(buffer), GIT_OBJ_TAG) < 0)
+		return -1;
+
+	stream->write(stream, buffer, strlen(buffer));
+
+	error = stream->finalize_write(oid, stream);
+	stream->free(stream);
+
+	if (error < 0) {
+		git_buf_free(&ref_name);
+		return -1;
+	}
+
+	error = git_reference_create_oid(&new_ref, repo, ref_name.ptr, oid, allow_ref_overwrite);
+
+	git_reference_free(new_ref);
+	git_buf_free(&ref_name);
+
+	return error;
+
+on_error:
+	git_signature_free(tag.tagger);
+	git__free(tag.tag_name);
+	git__free(tag.message);
+	git_odb_object_free(target_obj);
+	return -1;
 }
 
 int git_tag_delete(git_repository *repo, const char *tag_name)
 {
 	int error;
 	git_reference *tag_ref;
-	char ref_name[MAX_GITDIR_TREE_STRUCTURE_PATH_LENGTH];
+	git_buf ref_name = GIT_BUF_INIT;
 
-	error = retreive_tag_reference(&tag_ref, ref_name, repo, tag_name);
-	if (error < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to delete tag");
+	error = retrieve_tag_reference(&tag_ref, &ref_name, repo, tag_name);
+
+	git_buf_free(&ref_name);
+
+	if (error < 0)
+		return -1;
 
 	return git_reference_delete(tag_ref);
 }
@@ -378,32 +396,76 @@ int git_tag_delete(git_repository *repo, const char *tag_name)
 int git_tag__parse(git_tag *tag, git_odb_object *obj)
 {
 	assert(tag);
-	return parse_tag_buffer(tag, obj->raw.data, (char *)obj->raw.data + obj->raw.len);
+	return git_tag__parse_buffer(tag, obj->raw.data, obj->raw.len);
 }
 
+typedef struct {
+ git_vector *taglist;
+ const char *pattern;
+} tag_filter_data;
+
+#define GIT_REFS_TAGS_DIR_LEN strlen(GIT_REFS_TAGS_DIR)
+
 static int tag_list_cb(const char *tag_name, void *payload)
 {
-	if (git__prefixcmp(tag_name, GIT_REFS_TAGS_DIR) == 0)
-		return git_vector_insert((git_vector *)payload, git__strdup(tag_name));
+	tag_filter_data *filter;
 
-	return GIT_SUCCESS;
+	if (git__prefixcmp(tag_name, GIT_REFS_TAGS_DIR) != 0)
+		return 0;
+
+	filter = (tag_filter_data *)payload;
+	if (!*filter->pattern || p_fnmatch(filter->pattern, tag_name + GIT_REFS_TAGS_DIR_LEN, 0) == 0)
+		return git_vector_insert(filter->taglist, git__strdup(tag_name));
+
+	return 0;
 }
 
-int git_tag_list(git_strarray *tag_names, git_repository *repo)
+int git_tag_list_match(git_strarray *tag_names, const char *pattern, git_repository *repo)
 {
 	int error;
+	tag_filter_data filter;
 	git_vector taglist;
 
-	if (git_vector_init(&taglist, 8, NULL) < GIT_SUCCESS)
-		return GIT_ENOMEM;
+	assert(tag_names && repo && pattern);
 
-	error = git_reference_listcb(repo, GIT_REF_OID|GIT_REF_PACKED, &tag_list_cb, (void *)&taglist);
-	if (error < GIT_SUCCESS) {
+	if (git_vector_init(&taglist, 8, NULL) < 0)
+		return -1;
+
+	filter.taglist = &taglist;
+	filter.pattern = pattern;
+
+	error = git_reference_foreach(repo, GIT_REF_OID|GIT_REF_PACKED, &tag_list_cb, (void *)&filter);
+	if (error < 0) {
 		git_vector_free(&taglist);
-		return git__rethrow(error, "Failed to list tags");
+		return -1;
 	}
 
 	tag_names->strings = (char **)taglist.contents;
 	tag_names->count = taglist.length;
-	return GIT_SUCCESS;
+	return 0;
+}
+
+int git_tag_list(git_strarray *tag_names, git_repository *repo)
+{
+	return git_tag_list_match(tag_names, "", repo);
+}
+
+int git_tag_peel(git_object **tag_target, git_tag *tag)
+{
+	int error;
+	git_object *target;
+
+	assert(tag_target && tag);
+
+	if (git_tag_target(&target, tag) < 0)
+		return -1;
+
+	if (git_object_type(target) == GIT_OBJ_TAG) {
+		error = git_tag_peel(tag_target, (git_tag *)target);
+		git_object_free(target);
+		return error;
+	}
+
+	*tag_target = target;
+	return 0;
 }
diff --git a/src/tag.h b/src/tag.h
index eddf8fa3a..47f425509 100644
--- a/src/tag.h
+++ b/src/tag.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_tag_h__
 #define INCLUDE_tag_h__
 
@@ -18,5 +24,6 @@ struct git_tag {
 
 void git_tag__free(git_tag *tag);
 int git_tag__parse(git_tag *tag, git_odb_object *obj);
+int git_tag__parse_buffer(git_tag *tag, const char *data, size_t len);
 
 #endif
diff --git a/src/thread-utils.c b/src/thread-utils.c
index 5e8220f46..0ca01ef82 100644
--- a/src/thread-utils.c
+++ b/src/thread-utils.c
@@ -1,11 +1,17 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #include "common.h"
 #include "thread-utils.h"
 
 #ifdef _WIN32
-#  define WIN32_LEAN_AND_MEAN
-#  include <windows.h>
+#	define WIN32_LEAN_AND_MEAN
+#	include <windows.h>
 #elif defined(hpux) || defined(__hpux) || defined(_hpux)
-#  include <sys/pstat.h>
+#	include <sys/pstat.h>
 #endif
 
 /*
@@ -14,11 +20,11 @@
  * with this disgusting nest of #ifdefs.
  */
 #ifndef _SC_NPROCESSORS_ONLN
-#  ifdef _SC_NPROC_ONLN
-#    define _SC_NPROCESSORS_ONLN _SC_NPROC_ONLN
-#  elif defined _SC_CRAY_NCPU
-#    define _SC_NPROCESSORS_ONLN _SC_CRAY_NCPU
-#  endif
+#	ifdef _SC_NPROC_ONLN
+#		define _SC_NPROCESSORS_ONLN _SC_NPROC_ONLN
+#	elif defined _SC_CRAY_NCPU
+#		define _SC_NPROCESSORS_ONLN _SC_CRAY_NCPU
+#	endif
 #endif
 
 int git_online_cpus(void)
diff --git a/src/thread-utils.h b/src/thread-utils.h
index 20d6022ea..a309e93d1 100644
--- a/src/thread-utils.h
+++ b/src/thread-utils.h
@@ -1,11 +1,17 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_thread_utils_h__
 #define INCLUDE_thread_utils_h__
 
-
+#include "common.h"
 
 /* Common operations even if threading has been disabled */
 typedef struct {
-#if defined(_MSC_VER)
+#if defined(GIT_WIN32)
 	volatile long val;
 #else
 	volatile int val;
@@ -27,10 +33,10 @@ GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
 
 /* Pthreads Mutex */
 #define git_mutex pthread_mutex_t
-#define git_mutex_init(a)   pthread_mutex_init(a, NULL)
-#define git_mutex_lock(a)   pthread_mutex_lock(a)
+#define git_mutex_init(a)	pthread_mutex_init(a, NULL)
+#define git_mutex_lock(a)	pthread_mutex_lock(a)
 #define git_mutex_unlock(a) pthread_mutex_unlock(a)
-#define git_mutex_free(a)   pthread_mutex_destroy(a)
+#define git_mutex_free(a)	pthread_mutex_destroy(a)
 
 /* Pthreads condition vars -- disabled by now */
 #define git_cond unsigned int //pthread_cond_t
@@ -42,10 +48,10 @@ GIT_INLINE(void) git_atomic_set(git_atomic *a, int val)
 
 GIT_INLINE(int) git_atomic_inc(git_atomic *a)
 {
-#ifdef __GNUC__
-	return __sync_add_and_fetch(&a->val, 1);
-#elif defined(_MSC_VER)
+#if defined(GIT_WIN32)
 	return InterlockedIncrement(&a->val);
+#elif defined(__GNUC__)
+	return __sync_add_and_fetch(&a->val, 1);
 #else
 #	error "Unsupported architecture for atomic operations"
 #endif
@@ -53,10 +59,10 @@ GIT_INLINE(int) git_atomic_inc(git_atomic *a)
 
 GIT_INLINE(int) git_atomic_dec(git_atomic *a)
 {
-#ifdef __GNUC__
-	return __sync_sub_and_fetch(&a->val, 1);
-#elif defined(_MSC_VER)
+#if defined(GIT_WIN32)
 	return InterlockedDecrement(&a->val);
+#elif defined(__GNUC__)
+	return __sync_sub_and_fetch(&a->val, 1);
 #else
 #	error "Unsupported architecture for atomic operations"
 #endif
diff --git a/src/transport.c b/src/transport.c
new file mode 100644
index 000000000..fb2b94946
--- /dev/null
+++ b/src/transport.c
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "git2/types.h"
+#include "git2/remote.h"
+#include "git2/net.h"
+#include "transport.h"
+#include "path.h"
+
+static struct {
+	char *prefix;
+	git_transport_cb fn;
+} transports[] = {
+	{"git://", git_transport_git},
+	{"http://", git_transport_http},
+	{"https://", git_transport_https},
+	{"file://", git_transport_local},
+	{"git+ssh://", git_transport_dummy},
+	{"ssh+git://", git_transport_dummy},
+	{NULL, 0}
+};
+
+#define GIT_TRANSPORT_COUNT (sizeof(transports)/sizeof(transports[0])) - 1
+
+static git_transport_cb transport_find_fn(const char *url)
+{
+	size_t i = 0;
+
+	// First, check to see if it's an obvious URL, which a URL scheme
+	for (i = 0; i < GIT_TRANSPORT_COUNT; ++i) {
+		if (!strncasecmp(url, transports[i].prefix, strlen(transports[i].prefix)))
+			return transports[i].fn;
+	}
+
+	/* still here? Check to see if the path points to a file on the local file system */
+	if ((git_path_exists(url) == 0) && git_path_isdir(url))
+		return &git_transport_local;
+
+	/* It could be a SSH remote path. Check to see if there's a : */
+	if (strrchr(url, ':'))
+		return &git_transport_dummy;	/* SSH is an unsupported transport mechanism in this version of libgit2 */
+
+	return NULL;
+}
+
+/**************
+ * Public API *
+ **************/
+
+int git_transport_dummy(git_transport **transport)
+{
+	GIT_UNUSED(transport);
+	giterr_set(GITERR_NET, "This transport isn't implemented. Sorry");
+	return -1;
+}
+
+int git_transport_new(git_transport **out, const char *url)
+{
+	git_transport_cb fn;
+	git_transport *transport;
+	int error;
+
+	fn = transport_find_fn(url);
+
+	if (fn == NULL) {
+		giterr_set(GITERR_NET, "Unsupported URL protocol");
+		return -1;
+	}
+
+	error = fn(&transport);
+	if (error < 0)
+		return error;
+
+	transport->url = git__strdup(url);
+	GITERR_CHECK_ALLOC(transport->url);
+
+	*out = transport;
+
+	return 0;
+}
+
+/* from remote.h */
+int git_remote_valid_url(const char *url)
+{
+	return transport_find_fn(url) != NULL;
+}
+
+int git_remote_supported_url(const char* url)
+{
+	git_transport_cb transport_fn = transport_find_fn(url);
+
+	return ((transport_fn != NULL) && (transport_fn != &git_transport_dummy));
+}
diff --git a/src/transport.h b/src/transport.h
new file mode 100644
index 000000000..68b92f7a6
--- /dev/null
+++ b/src/transport.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_transport_h__
+#define INCLUDE_transport_h__
+
+#include "git2/net.h"
+#include "git2/indexer.h"
+#include "vector.h"
+#include "posix.h"
+#include "common.h"
+#ifdef GIT_SSL
+# include <openssl/ssl.h>
+# include <openssl/err.h>
+#endif
+
+
+#define GIT_CAP_OFS_DELTA "ofs-delta"
+
+typedef struct git_transport_caps {
+	int common:1,
+		ofs_delta:1;
+} git_transport_caps;
+
+#ifdef GIT_SSL
+typedef struct gitno_ssl {
+	SSL_CTX *ctx;
+	SSL *ssl;
+} gitno_ssl;
+#endif
+
+
+/*
+ * A day in the life of a network operation
+ * ========================================
+ *
+ * The library gets told to ls-remote/push/fetch on/to/from some
+ * remote. We look at the URL of the remote and fill the function
+ * table with whatever is appropriate (the remote may be git over git,
+ * ssh or http(s). It may even be an hg or svn repository, the library
+ * at this level doesn't care, it just calls the helpers.
+ *
+ * The first call is to ->connect() which connects to the remote,
+ * making use of the direction if necessary. This function must also
+ * store the remote heads and any other information it needs.
+ *
+ * The next useful step is to call ->ls() to get the list of
+ * references available to the remote. These references may have been
+ * collected on connect, or we may build them now. For ls-remote,
+ * nothing else is needed other than closing the connection.
+ * Otherwise, the higher leves decide which objects we want to
+ * have. ->send_have() is used to tell the other end what we have. If
+ * we do need to download a pack, ->download_pack() is called.
+ *
+ * When we're done, we call ->close() to close the
+ * connection. ->free() takes care of freeing all the resources.
+ */
+
+struct git_transport {
+	/**
+	 * Where the repo lives
+	 */
+	char *url;
+	/**
+	 * Whether we want to push or fetch
+	 */
+	int direction : 1, /* 0 fetch, 1 push */
+		connected : 1,
+		check_cert: 1,
+		encrypt : 1;
+#ifdef GIT_SSL
+	struct gitno_ssl ssl;
+#endif
+	GIT_SOCKET socket;
+	/**
+	 * Connect and store the remote heads
+	 */
+	int (*connect)(struct git_transport *transport, int dir);
+	/**
+	 * Give a list of references, useful for ls-remote
+	 */
+	int (*ls)(struct git_transport *transport, git_headlist_cb list_cb, void *opaque);
+	/**
+	 * Push the changes over
+	 */
+	int (*push)(struct git_transport *transport);
+	/**
+	 * Negotiate the minimal amount of objects that need to be
+	 * retrieved
+	 */
+	int (*negotiate_fetch)(struct git_transport *transport, git_repository *repo, const git_vector *wants);
+	/**
+	 * Download the packfile
+	 */
+	int (*download_pack)(struct git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats);
+	/**
+	 * Fetch the changes
+	 */
+	int (*fetch)(struct git_transport *transport);
+	/**
+	 * Close the connection
+	 */
+	int (*close)(struct git_transport *transport);
+	/**
+	 * Free the associated resources
+	 */
+	void (*free)(struct git_transport *transport);
+};
+
+
+int git_transport_new(struct git_transport **transport, const char *url);
+int git_transport_local(struct git_transport **transport);
+int git_transport_git(struct git_transport **transport);
+int git_transport_http(struct git_transport **transport);
+int git_transport_https(struct git_transport **transport);
+int git_transport_dummy(struct git_transport **transport);
+
+/**
+  Returns true if the passed URL is valid (a URL with a Git supported scheme,
+  or pointing to an existing path)
+*/
+int git_transport_valid_url(const char *url);
+
+typedef struct git_transport git_transport;
+typedef int (*git_transport_cb)(git_transport **transport);
+
+#endif
diff --git a/src/transports/git.c b/src/transports/git.c
new file mode 100644
index 000000000..45f571f20
--- /dev/null
+++ b/src/transports/git.c
@@ -0,0 +1,477 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "git2/net.h"
+#include "git2/common.h"
+#include "git2/types.h"
+#include "git2/errors.h"
+#include "git2/net.h"
+#include "git2/revwalk.h"
+
+#include "vector.h"
+#include "transport.h"
+#include "pkt.h"
+#include "common.h"
+#include "netops.h"
+#include "filebuf.h"
+#include "repository.h"
+#include "fetch.h"
+#include "protocol.h"
+
+typedef struct {
+	git_transport parent;
+	git_protocol proto;
+	git_vector refs;
+	git_remote_head **heads;
+	git_transport_caps caps;
+	char buff[1024];
+	gitno_buffer buf;
+#ifdef GIT_WIN32
+	WSADATA wsd;
+#endif
+} transport_git;
+
+/*
+ * Create a git procol request.
+ *
+ * For example: 0035git-upload-pack /libgit2/libgit2\0host=github.com\0
+ */
+static int gen_proto(git_buf *request, const char *cmd, const char *url)
+{
+	char *delim, *repo;
+	char default_command[] = "git-upload-pack";
+	char host[] = "host=";
+	size_t len;
+
+	delim = strchr(url, '/');
+	if (delim == NULL) {
+		giterr_set(GITERR_NET, "Malformed URL");
+		return -1;
+	}
+
+	repo = delim;
+
+	delim = strchr(url, ':');
+	if (delim == NULL)
+		delim = strchr(url, '/');
+
+	if (cmd == NULL)
+		cmd = default_command;
+
+	len = 4 + strlen(cmd) + 1 + strlen(repo) + 1 + strlen(host) + (delim - url) + 1;
+
+	git_buf_grow(request, len);
+	git_buf_printf(request, "%04x%s %s%c%s",
+		(unsigned int)(len & 0x0FFFF), cmd, repo, 0, host);
+	git_buf_put(request, url, delim - url);
+	git_buf_putc(request, '\0');
+
+	if (git_buf_oom(request))
+		return -1;
+
+	return 0;
+}
+
+static int send_request(git_transport *t, const char *cmd, const char *url)
+{
+	int error;
+	git_buf request = GIT_BUF_INIT;
+
+	error = gen_proto(&request, cmd, url);
+	if (error < 0)
+		goto cleanup;
+
+	error = gitno_send(t, request.ptr, request.size, 0);
+
+cleanup:
+	git_buf_free(&request);
+	return error;
+}
+
+/*
+ * Parse the URL and connect to a server, storing the socket in
+ * out. For convenience this also takes care of asking for the remote
+ * refs
+ */
+static int do_connect(transport_git *t, const char *url)
+{
+	char *host, *port;
+	const char prefix[] = "git://";
+
+	if (!git__prefixcmp(url, prefix))
+		url += strlen(prefix);
+
+	if (gitno_extract_host_and_port(&host, &port, url, GIT_DEFAULT_PORT) < 0)
+		return -1;
+
+	if (gitno_connect((git_transport *)t, host, port) < 0)
+		goto on_error;
+
+	if (send_request((git_transport *)t, NULL, url) < 0)
+		goto on_error;
+
+	git__free(host);
+	git__free(port);
+
+	return 0;
+
+on_error:
+	git__free(host);
+	git__free(port);
+	gitno_close(t->parent.socket);
+	return -1;
+}
+
+/*
+ * Read from the socket and store the references in the vector
+ */
+static int store_refs(transport_git *t)
+{
+	gitno_buffer *buf = &t->buf;
+	int ret = 0;
+
+	while (1) {
+		if ((ret = gitno_recv(buf)) < 0)
+			return -1;
+		if (ret == 0) /* Orderly shutdown, so exit */
+			return 0;
+
+		ret = git_protocol_store_refs(&t->proto, buf->data, buf->offset);
+		if (ret == GIT_EBUFS) {
+			gitno_consume_n(buf, buf->len);
+			continue;
+		}
+
+		if (ret < 0)
+			return ret;
+
+		gitno_consume_n(buf, buf->offset);
+
+		if (t->proto.flush) { /* No more refs */
+			t->proto.flush = 0;
+			return 0;
+		}
+	}
+}
+
+static int detect_caps(transport_git *t)
+{
+	git_vector *refs = &t->refs;
+	git_pkt_ref *pkt;
+	git_transport_caps *caps = &t->caps;
+	const char *ptr;
+
+	pkt = git_vector_get(refs, 0);
+	/* No refs or capabilites, odd but not a problem */
+	if (pkt == NULL || pkt->capabilities == NULL)
+		return 0;
+
+	ptr = pkt->capabilities;
+	while (ptr != NULL && *ptr != '\0') {
+		if (*ptr == ' ')
+			ptr++;
+
+		if(!git__prefixcmp(ptr, GIT_CAP_OFS_DELTA)) {
+			caps->common = caps->ofs_delta = 1;
+			ptr += strlen(GIT_CAP_OFS_DELTA);
+			continue;
+		}
+
+		/* We don't know this capability, so skip it */
+		ptr = strchr(ptr, ' ');
+	}
+
+	return 0;
+}
+
+/*
+ * Since this is a network connection, we need to parse and store the
+ * pkt-lines at this stage and keep them there.
+ */
+static int git_connect(git_transport *transport, int direction)
+{
+	transport_git *t = (transport_git *) transport;
+
+	if (direction == GIT_DIR_PUSH) {
+		giterr_set(GITERR_NET, "Pushing over git:// is not supported");
+		return -1;
+	}
+
+	t->parent.direction = direction;
+	if (git_vector_init(&t->refs, 16, NULL) < 0)
+		return -1;
+
+	/* Connect and ask for the refs */
+	if (do_connect(t, transport->url) < 0)
+		goto cleanup;
+
+	gitno_buffer_setup(transport, &t->buf, t->buff, sizeof(t->buff));
+
+	t->parent.connected = 1;
+	if (store_refs(t) < 0)
+		goto cleanup;
+
+	if (detect_caps(t) < 0)
+		goto cleanup;
+
+	return 0;
+cleanup:
+	git_vector_free(&t->refs);
+	return -1;
+}
+
+static int git_ls(git_transport *transport, git_headlist_cb list_cb, void *opaque)
+{
+	transport_git *t = (transport_git *) transport;
+	git_vector *refs = &t->refs;
+	unsigned int i;
+	git_pkt *p = NULL;
+
+	git_vector_foreach(refs, i, p) {
+		git_pkt_ref *pkt = NULL;
+
+		if (p->type != GIT_PKT_REF)
+			continue;
+
+		pkt = (git_pkt_ref *)p;
+
+		if (list_cb(&pkt->head, opaque) < 0) {
+			giterr_set(GITERR_NET, "User callback returned error");
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/* Wait until we get an ack from the */
+static int recv_pkt(gitno_buffer *buf)
+{
+	const char *ptr = buf->data, *line_end;
+	git_pkt *pkt;
+	int pkt_type, error;
+
+	do {
+		/* Wait for max. 1 second */
+		if ((error = gitno_select_in(buf, 1, 0)) < 0) {
+			return -1;
+		} else if (error == 0) {
+			/*
+			 * Some servers don't respond immediately, so if this
+			 * happens, we keep sending information until it
+			 * answers. Pretend we received a NAK to convince higher
+			 * layers to do so.
+			 */
+			return GIT_PKT_NAK;
+		}
+
+		if ((error = gitno_recv(buf)) < 0)
+			return -1;
+
+		error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->offset);
+		if (error == GIT_EBUFS)
+			continue;
+		if (error < 0)
+			return -1;
+	} while (error);
+
+	gitno_consume(buf, line_end);
+	pkt_type = pkt->type;
+	git__free(pkt);
+
+	return pkt_type;
+}
+
+static int git_negotiate_fetch(git_transport *transport, git_repository *repo, const git_vector *wants)
+{
+	transport_git *t = (transport_git *) transport;
+	git_revwalk *walk;
+	git_oid oid;
+	int error;
+	unsigned int i;
+	git_buf data = GIT_BUF_INIT;
+	gitno_buffer *buf = &t->buf;
+
+	if (git_pkt_buffer_wants(wants, &t->caps, &data) < 0)
+		return -1;
+
+	if (git_fetch_setup_walk(&walk, repo) < 0)
+		goto on_error;
+
+	if (gitno_send(transport, data.ptr, data.size, 0) < 0)
+		goto on_error;
+
+	git_buf_clear(&data);
+	/*
+	 * We don't support any kind of ACK extensions, so the negotiation
+	 * boils down to sending what we have and listening for an ACK
+	 * every once in a while.
+	 */
+	i = 0;
+	while ((error = git_revwalk_next(&oid, walk)) == 0) {
+		git_pkt_buffer_have(&oid, &data);
+		i++;
+		if (i % 20 == 0) {
+			int pkt_type;
+
+			git_pkt_buffer_flush(&data);
+			if (git_buf_oom(&data))
+				goto on_error;
+
+			if (gitno_send(transport, data.ptr, data.size, 0) < 0)
+				goto on_error;
+
+			pkt_type = recv_pkt(buf);
+
+			if (pkt_type == GIT_PKT_ACK) {
+				break;
+			} else if (pkt_type == GIT_PKT_NAK) {
+				continue;
+			} else {
+				giterr_set(GITERR_NET, "Unexpected pkt type");
+				goto on_error;
+			}
+
+		}
+	}
+	if (error < 0 && error != GIT_REVWALKOVER)
+		goto on_error;
+
+	/* Tell the other end that we're done negotiating */
+	git_buf_clear(&data);
+	git_pkt_buffer_flush(&data);
+	git_pkt_buffer_done(&data);
+	if (gitno_send(transport, data.ptr, data.size, 0) < 0)
+		goto on_error;
+
+	git_buf_free(&data);
+	git_revwalk_free(walk);
+	return 0;
+
+on_error:
+	git_buf_free(&data);
+	git_revwalk_free(walk);
+	return -1;
+}
+
+static int git_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
+{
+	transport_git *t = (transport_git *) transport;
+	int error = 0, read_bytes;
+	gitno_buffer *buf = &t->buf;
+	git_pkt *pkt;
+	const char *line_end, *ptr;
+
+	/*
+	 * For now, we ignore everything and wait for the pack
+	 */
+	do {
+		ptr = buf->data;
+		/* Whilst we're searching for the pack */
+		while (1) {
+			if (buf->offset == 0) {
+				break;
+			}
+
+			error = git_pkt_parse_line(&pkt, ptr, &line_end, buf->offset);
+			if (error == GIT_EBUFS)
+				break;
+
+			if (error < 0)
+				return error;
+
+			if (pkt->type == GIT_PKT_PACK) {
+				git__free(pkt);
+				return git_fetch__download_pack(buf->data, buf->offset, transport, repo, bytes, stats);
+			}
+
+			/* For now we don't care about anything */
+			git__free(pkt);
+			gitno_consume(buf, line_end);
+		}
+
+		read_bytes = gitno_recv(buf);
+	} while (read_bytes);
+
+	return read_bytes;
+}
+
+static int git_close(git_transport *t)
+{
+	git_buf buf = GIT_BUF_INIT;
+
+	if (git_pkt_buffer_flush(&buf) < 0)
+		return -1;
+	/* Can't do anything if there's an error, so don't bother checking  */
+	gitno_send(t, buf.ptr, buf.size, 0);
+
+	if (gitno_close(t->socket) < 0) {
+		giterr_set(GITERR_NET, "Failed to close socket");
+		return -1;
+	}
+
+	t->connected = 0;
+
+#ifdef GIT_WIN32
+	WSACleanup();
+#endif
+
+	return 0;
+}
+
+static void git_free(git_transport *transport)
+{
+	transport_git *t = (transport_git *) transport;
+	git_vector *refs = &t->refs;
+	unsigned int i;
+
+	for (i = 0; i < refs->length; ++i) {
+		git_pkt *p = git_vector_get(refs, i);
+		git_pkt_free(p);
+	}
+
+	git_vector_free(refs);
+	git__free(t->heads);
+	git_buf_free(&t->proto.buf);
+	git__free(t->parent.url);
+	git__free(t);
+}
+
+int git_transport_git(git_transport **out)
+{
+	transport_git *t;
+#ifdef GIT_WIN32
+	int ret;
+#endif
+
+	t = git__malloc(sizeof(transport_git));
+	GITERR_CHECK_ALLOC(t);
+
+	memset(t, 0x0, sizeof(transport_git));
+
+	t->parent.connect = git_connect;
+	t->parent.ls = git_ls;
+	t->parent.negotiate_fetch = git_negotiate_fetch;
+	t->parent.download_pack = git_download_pack;
+	t->parent.close = git_close;
+	t->parent.free = git_free;
+	t->proto.refs = &t->refs;
+	t->proto.transport = (git_transport *) t;
+
+	*out = (git_transport *) t;
+
+#ifdef GIT_WIN32
+	ret = WSAStartup(MAKEWORD(2,2), &t->wsd);
+	if (ret != 0) {
+		git_free(*out);
+		giterr_set(GITERR_NET, "Winsock init failed");
+		return -1;
+	}
+#endif
+
+	return 0;
+}
diff --git a/src/transports/http.c b/src/transports/http.c
new file mode 100644
index 000000000..4139a2fa6
--- /dev/null
+++ b/src/transports/http.c
@@ -0,0 +1,707 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include <stdlib.h>
+#include "git2.h"
+#include "http_parser.h"
+
+#include "transport.h"
+#include "common.h"
+#include "netops.h"
+#include "buffer.h"
+#include "pkt.h"
+#include "refs.h"
+#include "pack.h"
+#include "fetch.h"
+#include "filebuf.h"
+#include "repository.h"
+#include "protocol.h"
+
+enum last_cb {
+	NONE,
+	FIELD,
+	VALUE
+};
+
+typedef struct {
+	git_transport parent;
+	git_protocol proto;
+	git_vector refs;
+	git_vector common;
+	git_buf buf;
+	git_remote_head **heads;
+	int error;
+	int transfer_finished :1,
+		ct_found :1,
+		ct_finished :1,
+		pack_ready :1;
+	enum last_cb last_cb;
+	http_parser parser;
+	char *content_type;
+	char *path;
+	char *host;
+	char *port;
+	char *service;
+	git_transport_caps caps;
+#ifdef GIT_WIN32
+	WSADATA wsd;
+#endif
+} transport_http;
+
+static int gen_request(git_buf *buf, const char *path, const char *host, const char *op,
+                       const char *service, ssize_t content_length, int ls)
+{
+	if (path == NULL) /* Is 'git fetch http://host.com/' valid? */
+		path = "/";
+
+	if (ls) {
+		git_buf_printf(buf, "%s %s/info/refs?service=git-%s HTTP/1.1\r\n", op, path, service);
+	} else {
+		git_buf_printf(buf, "%s %s/git-%s HTTP/1.1\r\n", op, path, service);
+	}
+	git_buf_puts(buf, "User-Agent: git/1.0 (libgit2 " LIBGIT2_VERSION ")\r\n");
+	git_buf_printf(buf, "Host: %s\r\n", host);
+	if (content_length > 0) {
+		git_buf_printf(buf, "Accept: application/x-git-%s-result\r\n", service);
+		git_buf_printf(buf, "Content-Type: application/x-git-%s-request\r\n", service);
+		git_buf_printf(buf, "Content-Length: %"PRIuZ "\r\n", content_length);
+	} else {
+		git_buf_puts(buf, "Accept: */*\r\n");
+	}
+	git_buf_puts(buf, "\r\n");
+
+	if (git_buf_oom(buf))
+		return -1;
+
+	return 0;
+}
+
+static int do_connect(transport_http *t, const char *host, const char *port)
+{
+	if (t->parent.connected && http_should_keep_alive(&t->parser))
+		return 0;
+
+	if (gitno_connect((git_transport *) t, host, port) < 0)
+		return -1;
+
+	t->parent.connected = 1;
+
+	return 0;
+}
+
+/*
+ * The HTTP parser is streaming, so we need to wait until we're in the
+ * field handler before we can be sure that we can store the previous
+ * value. Right now, we only care about the
+ * Content-Type. on_header_{field,value} should be kept generic enough
+ * to work for any request.
+ */
+
+static const char *typestr = "Content-Type";
+
+static int on_header_field(http_parser *parser, const char *str, size_t len)
+{
+	transport_http *t = (transport_http *) parser->data;
+	git_buf *buf = &t->buf;
+
+	if (t->last_cb == VALUE && t->ct_found) {
+		t->ct_finished = 1;
+		t->ct_found = 0;
+		t->content_type = git__strdup(git_buf_cstr(buf));
+		GITERR_CHECK_ALLOC(t->content_type);
+		git_buf_clear(buf);
+	}
+
+	if (t->ct_found) {
+		t->last_cb = FIELD;
+		return 0;
+	}
+
+	if (t->last_cb != FIELD)
+		git_buf_clear(buf);
+
+	git_buf_put(buf, str, len);
+	t->last_cb = FIELD;
+
+	return git_buf_oom(buf);
+}
+
+static int on_header_value(http_parser *parser, const char *str, size_t len)
+{
+	transport_http *t = (transport_http *) parser->data;
+	git_buf *buf = &t->buf;
+
+	if (t->ct_finished) {
+		t->last_cb = VALUE;
+		return 0;
+	}
+
+	if (t->last_cb == VALUE)
+		git_buf_put(buf, str, len);
+
+	if (t->last_cb == FIELD && !strcmp(git_buf_cstr(buf), typestr)) {
+		t->ct_found = 1;
+		git_buf_clear(buf);
+		git_buf_put(buf, str, len);
+	}
+
+	t->last_cb = VALUE;
+
+	return git_buf_oom(buf);
+}
+
+static int on_headers_complete(http_parser *parser)
+{
+	transport_http *t = (transport_http *) parser->data;
+	git_buf *buf = &t->buf;
+
+	/* The content-type is text/plain for 404, so don't validate */
+	if (parser->status_code == 404) {
+		git_buf_clear(buf);
+		return 0;
+	}
+
+	if (t->content_type == NULL) {
+		t->content_type = git__strdup(git_buf_cstr(buf));
+		if (t->content_type == NULL)
+			return t->error = -1;
+	}
+
+	git_buf_clear(buf);
+	git_buf_printf(buf, "application/x-git-%s-advertisement", t->service);
+	if (git_buf_oom(buf))
+		return t->error = -1;
+
+	if (strcmp(t->content_type, git_buf_cstr(buf)))
+		return t->error = -1;
+
+	git_buf_clear(buf);
+	return 0;
+}
+
+static int on_body_store_refs(http_parser *parser, const char *str, size_t len)
+{
+	transport_http *t = (transport_http *) parser->data;
+
+	if (parser->status_code == 404) {
+		return git_buf_put(&t->buf, str, len);
+	}
+
+	return git_protocol_store_refs(&t->proto, str, len);
+}
+
+static int on_message_complete(http_parser *parser)
+{
+	transport_http *t = (transport_http *) parser->data;
+
+	t->transfer_finished = 1;
+
+	if (parser->status_code == 404) {
+		giterr_set(GITERR_NET, "Remote error: %s", git_buf_cstr(&t->buf));
+		t->error = -1;
+	}
+
+	return 0;
+}
+
+static int store_refs(transport_http *t)
+{
+	http_parser_settings settings;
+	char buffer[1024];
+	gitno_buffer buf;
+	git_pkt *pkt;
+	int ret;
+
+	http_parser_init(&t->parser, HTTP_RESPONSE);
+	t->parser.data = t;
+	memset(&settings, 0x0, sizeof(http_parser_settings));
+	settings.on_header_field = on_header_field;
+	settings.on_header_value = on_header_value;
+	settings.on_headers_complete = on_headers_complete;
+	settings.on_body = on_body_store_refs;
+	settings.on_message_complete = on_message_complete;
+
+	gitno_buffer_setup((git_transport *)t, &buf, buffer, sizeof(buffer));
+
+	while(1) {
+		size_t parsed;
+
+		if ((ret = gitno_recv(&buf)) < 0)
+			return -1;
+
+		parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset);
+		/* Both should happen at the same time */
+		if (parsed != buf.offset || t->error < 0)
+			return t->error;
+
+		gitno_consume_n(&buf, parsed);
+
+		if (ret == 0 || t->transfer_finished)
+			return 0;
+	}
+
+	pkt = git_vector_get(&t->refs, 0);
+	if (pkt == NULL || pkt->type != GIT_PKT_COMMENT) {
+		giterr_set(GITERR_NET, "Invalid HTTP response");
+		return t->error = -1;
+	} else {
+		git_vector_remove(&t->refs, 0);
+	}
+
+	return 0;
+}
+
+static int http_connect(git_transport *transport, int direction)
+{
+	transport_http *t = (transport_http *) transport;
+	int ret;
+	git_buf request = GIT_BUF_INIT;
+	const char *service = "upload-pack";
+	const char *url = t->parent.url, *prefix_http = "http://", *prefix_https = "https://";
+	const char *default_port;
+
+	if (direction == GIT_DIR_PUSH) {
+		giterr_set(GITERR_NET, "Pushing over HTTP is not implemented");
+		return -1;
+	}
+
+	t->parent.direction = direction;
+	if (git_vector_init(&t->refs, 16, NULL) < 0)
+		return -1;
+
+	if (!git__prefixcmp(url, prefix_http)) {
+		url = t->parent.url + strlen(prefix_http);
+		default_port = "80";
+	}
+
+	if (!git__prefixcmp(url, prefix_https)) {
+		url += strlen(prefix_https);
+		default_port = "443";
+	}
+
+	t->path = strchr(url, '/');
+
+	if ((ret = gitno_extract_host_and_port(&t->host, &t->port, url, default_port)) < 0)
+		goto cleanup;
+
+	t->service = git__strdup(service);
+	GITERR_CHECK_ALLOC(t->service);
+
+	if ((ret = do_connect(t, t->host, t->port)) < 0)
+		goto cleanup;
+
+	/* Generate and send the HTTP request */
+	if ((ret = gen_request(&request, t->path, t->host, "GET", service, 0, 1)) < 0) {
+		giterr_set(GITERR_NET, "Failed to generate request");
+		goto cleanup;
+	}
+
+
+	if (gitno_send(transport, request.ptr, request.size, 0) < 0)
+		goto cleanup;
+
+	ret = store_refs(t);
+
+cleanup:
+	git_buf_free(&request);
+	git_buf_clear(&t->buf);
+
+	return ret;
+}
+
+static int http_ls(git_transport *transport, git_headlist_cb list_cb, void *opaque)
+{
+	transport_http *t = (transport_http *) transport;
+	git_vector *refs = &t->refs;
+	unsigned int i;
+	git_pkt_ref *p;
+
+	git_vector_foreach(refs, i, p) {
+		if (p->type != GIT_PKT_REF)
+			continue;
+
+		if (list_cb(&p->head, opaque) < 0) {
+			giterr_set(GITERR_NET, "The user callback returned error");
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+static int on_body_parse_response(http_parser *parser, const char *str, size_t len)
+{
+	transport_http *t = (transport_http *) parser->data;
+	git_buf *buf = &t->buf;
+	git_vector *common = &t->common;
+	int error;
+	const char *line_end, *ptr;
+
+	if (len == 0) { /* EOF */
+		if (git_buf_len(buf) != 0) {
+			giterr_set(GITERR_NET, "Unexpected EOF");
+			return t->error = -1;
+		} else {
+			return 0;
+		}
+	}
+
+	git_buf_put(buf, str, len);
+	ptr = buf->ptr;
+	while (1) {
+		git_pkt *pkt;
+
+		if (git_buf_len(buf) == 0)
+			return 0;
+
+		error = git_pkt_parse_line(&pkt, ptr, &line_end, git_buf_len(buf));
+		if (error == GIT_EBUFS) {
+			return 0; /* Ask for more */
+		}
+		if (error < 0)
+			return t->error = -1;
+
+		git_buf_consume(buf, line_end);
+
+		if (pkt->type == GIT_PKT_PACK) {
+			git__free(pkt);
+			t->pack_ready = 1;
+			return 0;
+		}
+
+		if (pkt->type == GIT_PKT_NAK) {
+			git__free(pkt);
+			return 0;
+		}
+
+		if (pkt->type != GIT_PKT_ACK) {
+			git__free(pkt);
+			continue;
+		}
+
+		if (git_vector_insert(common, pkt) < 0)
+			return -1;
+	}
+
+	return error;
+
+}
+
+static int parse_response(transport_http *t)
+{
+	int ret = 0;
+	http_parser_settings settings;
+	char buffer[1024];
+	gitno_buffer buf;
+
+	http_parser_init(&t->parser, HTTP_RESPONSE);
+	t->parser.data = t;
+	t->transfer_finished = 0;
+	memset(&settings, 0x0, sizeof(http_parser_settings));
+	settings.on_header_field = on_header_field;
+	settings.on_header_value = on_header_value;
+	settings.on_headers_complete = on_headers_complete;
+	settings.on_body = on_body_parse_response;
+	settings.on_message_complete = on_message_complete;
+
+	gitno_buffer_setup((git_transport *)t, &buf, buffer, sizeof(buffer));
+
+	while(1) {
+		size_t parsed;
+
+		if ((ret = gitno_recv(&buf)) < 0)
+			return -1;
+
+		parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset);
+		/* Both should happen at the same time */
+		if (parsed != buf.offset || t->error < 0)
+			return t->error;
+
+		gitno_consume_n(&buf, parsed);
+
+		if (ret == 0 || t->transfer_finished || t->pack_ready) {
+			return 0;
+		}
+	}
+
+	return ret;
+}
+
+static int http_negotiate_fetch(git_transport *transport, git_repository *repo, const git_vector *wants)
+{
+	transport_http *t = (transport_http *) transport;
+	int ret;
+	unsigned int i;
+	char buff[128];
+	gitno_buffer buf;
+	git_revwalk *walk = NULL;
+	git_oid oid;
+	git_pkt_ack *pkt;
+	git_vector *common = &t->common;
+	git_buf request = GIT_BUF_INIT, data = GIT_BUF_INIT;
+
+	gitno_buffer_setup(transport, &buf, buff, sizeof(buff));
+
+	if (git_vector_init(common, 16, NULL) < 0)
+		return -1;
+
+	if (git_fetch_setup_walk(&walk, repo) < 0)
+		return -1;
+
+	do {
+		if ((ret = do_connect(t, t->host, t->port)) < 0)
+			goto cleanup;
+
+		if ((ret = git_pkt_buffer_wants(wants, &t->caps, &data)) < 0)
+			goto cleanup;
+
+		/* We need to send these on each connection */
+		git_vector_foreach (common, i, pkt) {
+			if ((ret = git_pkt_buffer_have(&pkt->oid, &data)) < 0)
+				goto cleanup;
+		}
+
+		i = 0;
+		while ((i < 20) && ((ret = git_revwalk_next(&oid, walk)) == 0)) {
+			if ((ret = git_pkt_buffer_have(&oid, &data)) < 0)
+				goto cleanup;
+
+			i++;
+		}
+
+		git_pkt_buffer_done(&data);
+
+		if ((ret = gen_request(&request, t->path, t->host, "POST", "upload-pack", data.size, 0)) < 0)
+			goto cleanup;
+
+		if ((ret = gitno_send(transport, request.ptr, request.size, 0)) < 0)
+			goto cleanup;
+
+		if ((ret = gitno_send(transport, data.ptr, data.size, 0)) < 0)
+			goto cleanup;
+
+		git_buf_clear(&request);
+		git_buf_clear(&data);
+
+		if (ret < 0 || i >= 256)
+			break;
+
+		if ((ret = parse_response(t)) < 0)
+			goto cleanup;
+
+		if (t->pack_ready) {
+			ret = 0;
+			goto cleanup;
+		}
+
+	} while(1);
+
+cleanup:
+	git_buf_free(&request);
+	git_buf_free(&data);
+	git_revwalk_free(walk);
+	return ret;
+}
+
+typedef struct {
+	git_indexer_stream *idx;
+	git_indexer_stats *stats;
+	transport_http *transport;
+} download_pack_cbdata;
+
+static int on_message_complete_download_pack(http_parser *parser)
+{
+	download_pack_cbdata *data = (download_pack_cbdata *) parser->data;
+
+	data->transport->transfer_finished = 1;
+
+	return 0;
+}
+static int on_body_download_pack(http_parser *parser, const char *str, size_t len)
+{
+	download_pack_cbdata *data = (download_pack_cbdata *) parser->data;
+	transport_http *t = data->transport;
+	git_indexer_stream *idx = data->idx;
+	git_indexer_stats *stats = data->stats;
+
+	return t->error = git_indexer_stream_add(idx, str, len, stats);
+}
+
+/*
+ * As the server is probably using Transfer-Encoding: chunked, we have
+ * to use the HTTP parser to download the pack instead of giving it to
+ * the simple downloader. Furthermore, we're using keep-alive
+ * connections, so the simple downloader would just hang.
+ */
+static int http_download_pack(git_transport *transport, git_repository *repo, git_off_t *bytes, git_indexer_stats *stats)
+{
+	transport_http *t = (transport_http *) transport;
+	git_buf *oldbuf = &t->buf;
+	int recvd;
+	http_parser_settings settings;
+	char buffer[1024];
+	gitno_buffer buf;
+	git_indexer_stream *idx = NULL;
+	download_pack_cbdata data;
+
+	gitno_buffer_setup(transport, &buf, buffer, sizeof(buffer));
+
+	if (memcmp(oldbuf->ptr, "PACK", strlen("PACK"))) {
+		giterr_set(GITERR_NET, "The pack doesn't start with a pack signature");
+		return -1;
+	}
+
+	if (git_indexer_stream_new(&idx, git_repository_path(repo)) < 0)
+		return -1;
+
+	/*
+	 * This is part of the previous response, so we don't want to
+	 * re-init the parser, just set these two callbacks.
+	 */
+	memset(stats, 0, sizeof(git_indexer_stats));
+	data.stats = stats;
+	data.idx = idx;
+	data.transport = t;
+	t->parser.data = &data;
+	t->transfer_finished = 0;
+	memset(&settings, 0x0, sizeof(settings));
+	settings.on_message_complete = on_message_complete_download_pack;
+	settings.on_body = on_body_download_pack;
+	*bytes = git_buf_len(oldbuf);
+
+	if (git_indexer_stream_add(idx, git_buf_cstr(oldbuf), git_buf_len(oldbuf), stats) < 0)
+		goto on_error;
+
+	gitno_buffer_setup(transport, &buf, buffer, sizeof(buffer));
+
+	do {
+		size_t parsed;
+
+		if ((recvd = gitno_recv(&buf)) < 0)
+			goto on_error;
+
+		parsed = http_parser_execute(&t->parser, &settings, buf.data, buf.offset);
+		if (parsed != buf.offset || t->error < 0)
+			goto on_error;
+
+		*bytes += recvd;
+		gitno_consume_n(&buf, parsed);
+	} while (recvd > 0 && !t->transfer_finished);
+
+	if (git_indexer_stream_finalize(idx, stats) < 0)
+		goto on_error;
+
+	git_indexer_stream_free(idx);
+	return 0;
+
+on_error:
+	git_indexer_stream_free(idx);
+	return -1;
+}
+
+static int http_close(git_transport *transport)
+{
+	if (gitno_ssl_teardown(transport) < 0)
+		return -1;
+
+	if (gitno_close(transport->socket) < 0) {
+		giterr_set(GITERR_OS, "Failed to close the socket: %s", strerror(errno));
+		return -1;
+	}
+
+	transport->connected = 0;
+
+	return 0;
+}
+
+
+static void http_free(git_transport *transport)
+{
+	transport_http *t = (transport_http *) transport;
+	git_vector *refs = &t->refs;
+	git_vector *common = &t->common;
+	unsigned int i;
+	git_pkt *p;
+
+#ifdef GIT_WIN32
+	/* cleanup the WSA context. note that this context
+	 * can be initialized more than once with WSAStartup(),
+	 * and needs to be cleaned one time for each init call
+	 */
+	WSACleanup();
+#endif
+
+	git_vector_foreach(refs, i, p) {
+		git_pkt_free(p);
+	}
+	git_vector_free(refs);
+	git_vector_foreach(common, i, p) {
+		git_pkt_free(p);
+	}
+	git_vector_free(common);
+	git_buf_free(&t->buf);
+	git_buf_free(&t->proto.buf);
+	git__free(t->heads);
+	git__free(t->content_type);
+	git__free(t->host);
+	git__free(t->port);
+	git__free(t->service);
+	git__free(t->parent.url);
+	git__free(t);
+}
+
+int git_transport_http(git_transport **out)
+{
+	transport_http *t;
+
+	t = git__malloc(sizeof(transport_http));
+	GITERR_CHECK_ALLOC(t);
+
+	memset(t, 0x0, sizeof(transport_http));
+
+	t->parent.connect = http_connect;
+	t->parent.ls = http_ls;
+	t->parent.negotiate_fetch = http_negotiate_fetch;
+	t->parent.download_pack = http_download_pack;
+	t->parent.close = http_close;
+	t->parent.free = http_free;
+	t->proto.refs = &t->refs;
+	t->proto.transport = (git_transport *) t;
+
+#ifdef GIT_WIN32
+	/* on win32, the WSA context needs to be initialized
+	 * before any socket calls can be performed */
+	if (WSAStartup(MAKEWORD(2,2), &t->wsd) != 0) {
+		http_free((git_transport *) t);
+		giterr_set(GITERR_OS, "Winsock init failed");
+		return -1;
+	}
+#endif
+
+	*out = (git_transport *) t;
+	return 0;
+}
+
+int git_transport_https(git_transport **out)
+{
+#ifdef GIT_SSL
+	transport_http *t;
+	if (git_transport_http((git_transport **)&t) < 0)
+		return -1;
+
+	t->parent.encrypt = 1;
+	t->parent.check_cert = 1;
+	*out = (git_transport *) t;
+
+	return 0;
+#else
+	GIT_UNUSED(out);
+
+	giterr_set(GITERR_NET, "HTTPS support not available");
+	return -1;
+#endif
+}
diff --git a/src/transports/local.c b/src/transports/local.c
new file mode 100644
index 000000000..0e1ae3752
--- /dev/null
+++ b/src/transports/local.c
@@ -0,0 +1,241 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "common.h"
+#include "git2/types.h"
+#include "git2/net.h"
+#include "git2/repository.h"
+#include "git2/object.h"
+#include "git2/tag.h"
+#include "refs.h"
+#include "transport.h"
+#include "posix.h"
+#include "path.h"
+#include "buffer.h"
+
+typedef struct {
+	git_transport parent;
+	git_repository *repo;
+	git_vector refs;
+} transport_local;
+
+static int add_ref(transport_local *t, const char *name)
+{
+	const char peeled[] = "^{}";
+	git_remote_head *head;
+	git_object *obj = NULL, *target = NULL;
+	git_buf buf = GIT_BUF_INIT;
+
+	head = git__malloc(sizeof(git_remote_head));
+	GITERR_CHECK_ALLOC(head);
+
+	head->name = git__strdup(name);
+	GITERR_CHECK_ALLOC(head->name);
+
+	if (git_reference_name_to_oid(&head->oid, t->repo, name) < 0 ||
+		git_vector_insert(&t->refs, head) < 0)
+	{
+		git__free(head->name);
+		git__free(head);
+		return -1;
+	}
+
+	/* If it's not a tag, we don't need to try to peel it */
+	if (git__prefixcmp(name, GIT_REFS_TAGS_DIR))
+		return 0;
+
+	if (git_object_lookup(&obj, t->repo, &head->oid, GIT_OBJ_ANY) < 0)
+		return -1;
+
+	head = NULL;
+
+	/* If it's not an annotated tag, just get out */
+	if (git_object_type(obj) != GIT_OBJ_TAG) {
+		git_object_free(obj);
+		return 0;
+	}
+
+	/* And if it's a tag, peel it, and add it to the list */
+	head = git__malloc(sizeof(git_remote_head));
+	GITERR_CHECK_ALLOC(head);
+	if (git_buf_join(&buf, 0, name, peeled) < 0)
+		return -1;
+
+	head->name = git_buf_detach(&buf);
+
+	if (git_tag_peel(&target, (git_tag *) obj) < 0)
+		goto on_error;
+
+	git_oid_cpy(&head->oid, git_object_id(target));
+	git_object_free(obj);
+	git_object_free(target);
+
+	if (git_vector_insert(&t->refs, head) < 0)
+		return -1;
+
+	return 0;
+
+on_error:
+	git_object_free(obj);
+	git_object_free(target);
+	return -1;
+}
+
+static int store_refs(transport_local *t)
+{
+	unsigned int i;
+	git_strarray ref_names = {0};
+
+	assert(t);
+
+	if (git_reference_list(&ref_names, t->repo, GIT_REF_LISTALL) < 0 ||
+		git_vector_init(&t->refs, (unsigned int)ref_names.count, NULL) < 0)
+		goto on_error;
+
+	/* Sort the references first */
+	git__tsort((void **)ref_names.strings, ref_names.count, &git__strcmp_cb);
+
+	/* Add HEAD */
+	if (add_ref(t, GIT_HEAD_FILE) < 0)
+		goto on_error;
+
+	for (i = 0; i < ref_names.count; ++i) {
+		if (add_ref(t, ref_names.strings[i]) < 0)
+			goto on_error;
+	}
+
+	git_strarray_free(&ref_names);
+	return 0;
+
+on_error:
+	git_vector_free(&t->refs);
+	git_strarray_free(&ref_names);
+	return -1;
+}
+
+static int local_ls(git_transport *transport, git_headlist_cb list_cb, void *payload)
+{
+	transport_local  *t = (transport_local *) transport;
+	git_vector *refs = &t->refs;
+	unsigned int i;
+	git_remote_head *h;
+
+	assert(transport && transport->connected);
+
+	git_vector_foreach(refs, i, h) {
+		if (list_cb(h, payload) < 0)
+			return -1;
+	}
+
+	return 0;
+}
+
+/*
+ * Try to open the url as a git directory. The direction doesn't
+ * matter in this case because we're calulating the heads ourselves.
+ */
+static int local_connect(git_transport *transport, int direction)
+{
+	git_repository *repo;
+	int error;
+	transport_local *t = (transport_local *) transport;
+	const char *path;
+	git_buf buf = GIT_BUF_INIT;
+
+	GIT_UNUSED(direction);
+
+	/* The repo layer doesn't want the prefix */
+	if (!git__prefixcmp(transport->url, "file://")) {
+		if (git_path_fromurl(&buf, transport->url) < 0) {
+			git_buf_free(&buf);
+			return -1;
+		}
+		path = git_buf_cstr(&buf);
+
+	} else { /* We assume transport->url is already a path */
+		path = transport->url;
+	}
+
+	error = git_repository_open(&repo, path);
+
+	git_buf_free(&buf);
+
+	if (error < 0)
+		return -1;
+
+	t->repo = repo;
+
+	if (store_refs(t) < 0)
+		return -1;
+
+	t->parent.connected = 1;
+
+	return 0;
+}
+
+static int local_negotiate_fetch(git_transport *transport, git_repository *repo, const git_vector *wants)
+{
+	GIT_UNUSED(transport);
+	GIT_UNUSED(repo);
+	GIT_UNUSED(wants);
+
+	giterr_set(GITERR_NET, "Fetch via local transport isn't implemented. Sorry");
+	return -1;
+}
+
+static int local_close(git_transport *transport)
+{
+	transport_local *t = (transport_local *)transport;
+
+	t->parent.connected = 0;
+	git_repository_free(t->repo);
+	t->repo = NULL;
+
+	return 0;
+}
+
+static void local_free(git_transport *transport)
+{
+	unsigned int i;
+	transport_local *t = (transport_local *) transport;
+	git_vector *vec = &t->refs;
+	git_remote_head *h;
+
+	assert(transport);
+
+	git_vector_foreach (vec, i, h) {
+		git__free(h->name);
+		git__free(h);
+	}
+	git_vector_free(vec);
+
+	git__free(t->parent.url);
+	git__free(t);
+}
+
+/**************
+ * Public API *
+ **************/
+
+int git_transport_local(git_transport **out)
+{
+	transport_local *t;
+
+	t = git__malloc(sizeof(transport_local));
+	GITERR_CHECK_ALLOC(t);
+
+	memset(t, 0x0, sizeof(transport_local));
+
+	t->parent.connect = local_connect;
+	t->parent.ls = local_ls;
+	t->parent.negotiate_fetch = local_negotiate_fetch;
+	t->parent.close = local_close;
+	t->parent.free = local_free;
+
+	*out = (git_transport *) t;
+
+	return 0;
+}
diff --git a/src/tree-cache.c b/src/tree-cache.c
new file mode 100644
index 000000000..ebc2c6807
--- /dev/null
+++ b/src/tree-cache.c
@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "tree-cache.h"
+
+static git_tree_cache *find_child(const git_tree_cache *tree, const char *path)
+{
+	size_t i, dirlen;
+	const char *end;
+
+	end = strchr(path, '/');
+	if (end == NULL) {
+		end = strrchr(path, '\0');
+	}
+
+	dirlen = end - path;
+
+	for (i = 0; i < tree->children_count; ++i) {
+		const char *childname = tree->children[i]->name;
+
+		if (strlen(childname) == dirlen && !memcmp(path, childname, dirlen))
+			return tree->children[i];
+	}
+
+	return NULL;
+}
+
+void git_tree_cache_invalidate_path(git_tree_cache *tree, const char *path)
+{
+	const char *ptr = path, *end;
+
+	if (tree == NULL)
+		return;
+
+	tree->entries = -1;
+
+	while (ptr != NULL) {
+		end = strchr(ptr, '/');
+
+		if (end == NULL) /* End of path */
+			break;
+
+		tree = find_child(tree, ptr);
+		if (tree == NULL) /* We don't have that tree */
+			return;
+
+		tree->entries = -1;
+		ptr = end + 1;
+	}
+}
+
+const git_tree_cache *git_tree_cache_get(const git_tree_cache *tree, const char *path)
+{
+	const char *ptr = path, *end;
+
+	if (tree == NULL) {
+		return NULL;
+	}
+
+	while (1) {
+		end = strchr(ptr, '/');
+
+		tree = find_child(tree, ptr);
+		if (tree == NULL) { /* Can't find it */
+			return NULL;
+		}
+
+		if (end == NULL || end + 1 == '\0')
+			return tree;
+
+		ptr = end + 1;
+	}
+}
+
+static int read_tree_internal(git_tree_cache **out,
+		const char **buffer_in, const char *buffer_end, git_tree_cache *parent)
+{
+	git_tree_cache *tree = NULL;
+	const char *name_start, *buffer;
+	int count;
+	size_t name_len;
+
+	buffer = name_start = *buffer_in;
+
+	if ((buffer = memchr(buffer, '\0', buffer_end - buffer)) == NULL)
+		goto corrupted;
+
+	if (++buffer >= buffer_end)
+		goto corrupted;
+
+	name_len = strlen(name_start);
+	tree = git__malloc(sizeof(git_tree_cache) + name_len + 1);
+	GITERR_CHECK_ALLOC(tree);
+
+	memset(tree, 0x0, sizeof(git_tree_cache));
+	tree->parent = parent;
+
+	/* NUL-terminated tree name */
+	memcpy(tree->name, name_start, name_len);
+	tree->name[name_len] = '\0';
+
+	/* Blank-terminated ASCII decimal number of entries in this tree */
+	if (git__strtol32(&count, buffer, &buffer, 10) < 0 || count < -1)
+		goto corrupted;
+
+	tree->entries = count;
+
+	if (*buffer != ' ' || ++buffer >= buffer_end)
+		goto corrupted;
+
+	 /* Number of children of the tree, newline-terminated */
+	if (git__strtol32(&count, buffer, &buffer, 10) < 0 || count < 0)
+		goto corrupted;
+
+	tree->children_count = count;
+
+	if (*buffer != '\n' || ++buffer > buffer_end)
+		goto corrupted;
+
+	/* The SHA1 is only there if it's not invalidated */
+	if (tree->entries >= 0) {
+		/* 160-bit SHA-1 for this tree and it's children */
+		if (buffer + GIT_OID_RAWSZ > buffer_end)
+			goto corrupted;
+
+		git_oid_fromraw(&tree->oid, (const unsigned char *)buffer);
+		buffer += GIT_OID_RAWSZ;
+	}
+
+	/* Parse children: */
+	if (tree->children_count > 0) {
+		unsigned int i;
+
+		tree->children = git__malloc(tree->children_count * sizeof(git_tree_cache *));
+		GITERR_CHECK_ALLOC(tree->children);
+
+		for (i = 0; i < tree->children_count; ++i) {
+			if (read_tree_internal(&tree->children[i], &buffer, buffer_end, tree) < 0)
+				return -1;
+		}
+	}
+
+	*buffer_in = buffer;
+	*out = tree;
+	return 0;
+
+ corrupted:
+	git_tree_cache_free(tree);
+	giterr_set(GITERR_INDEX, "Corruped TREE extension in index");
+	return -1;
+}
+
+int git_tree_cache_read(git_tree_cache **tree, const char *buffer, size_t buffer_size)
+{
+	const char *buffer_end = buffer + buffer_size;
+
+	if (read_tree_internal(tree, &buffer, buffer_end, NULL) < 0)
+		return -1;
+
+	if (buffer < buffer_end) {
+		giterr_set(GITERR_INDEX, "Corruped TREE extension in index (unexpected trailing data)");
+		return -1;
+	}
+
+	return 0;
+}
+
+void git_tree_cache_free(git_tree_cache *tree)
+{
+	unsigned int i;
+
+	if (tree == NULL)
+		return;
+
+	for (i = 0; i < tree->children_count; ++i)
+		git_tree_cache_free(tree->children[i]);
+
+	git__free(tree->children);
+	git__free(tree);
+}
diff --git a/src/tree-cache.h b/src/tree-cache.h
new file mode 100644
index 000000000..41fde997a
--- /dev/null
+++ b/src/tree-cache.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#ifndef INCLUDE_tree_cache_h__
+#define INCLUDE_tree_cache_h__
+
+#include "common.h"
+#include "git2/oid.h"
+
+struct git_tree_cache {
+	struct git_tree_cache *parent;
+	struct git_tree_cache **children;
+	size_t children_count;
+
+	ssize_t entries;
+	git_oid oid;
+	char name[GIT_FLEX_ARRAY];
+};
+
+typedef struct git_tree_cache git_tree_cache;
+
+int git_tree_cache_read(git_tree_cache **tree, const char *buffer, size_t buffer_size);
+void git_tree_cache_invalidate_path(git_tree_cache *tree, const char *path);
+const git_tree_cache *git_tree_cache_get(const git_tree_cache *tree, const char *path);
+void git_tree_cache_free(git_tree_cache *tree);
+
+#endif
diff --git a/src/tree.c b/src/tree.c
index 60413e276..92b1b1e39 100644
--- a/src/tree.c
+++ b/src/tree.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
@@ -33,33 +15,109 @@
 #define MAX_FILEMODE 0777777
 #define MAX_FILEMODE_BYTES 6
 
-static int valid_attributes(const int attributes) {
-	return attributes >= 0 && attributes <= MAX_FILEMODE; 
+static int valid_attributes(const int attributes)
+{
+	return attributes >= 0 && attributes <= MAX_FILEMODE;
+}
+
+static int valid_entry_name(const char *filename)
+{
+	return strlen(filename) > 0 && strchr(filename, '/') == NULL;
 }
 
-int entry_search_cmp(const void *key, const void *array_member)
+static int entry_sort_cmp(const void *a, const void *b)
 {
-	const char *filename = (const char *)key;
-	const git_tree_entry *entry = *(const git_tree_entry **)(array_member);
+	const git_tree_entry *entry_a = (const git_tree_entry *)(a);
+	const git_tree_entry *entry_b = (const git_tree_entry *)(b);
 
-	return strcmp(filename, entry->filename);
+	return git_path_cmp(
+		entry_a->filename, entry_a->filename_len, git_tree_entry__is_tree(entry_a),
+		entry_b->filename, entry_b->filename_len, git_tree_entry__is_tree(entry_b));
 }
 
-#if 0
-static int valid_attributes(const int attributes) {
-	return attributes >= 0 && attributes <= MAX_FILEMODE; 
+
+struct tree_key_search {
+	const char *filename;
+	size_t filename_len;
+};
+
+static int homing_search_cmp(const void *key, const void *array_member)
+{
+	const struct tree_key_search *ksearch = key;
+	const git_tree_entry *entry = array_member;
+
+	const size_t len1 = ksearch->filename_len;
+	const size_t len2 = entry->filename_len;
+
+	return memcmp(
+		ksearch->filename,
+		entry->filename,
+		len1 < len2 ? len1 : len2
+	);
 }
-#endif
 
-int entry_sort_cmp(const void *a, const void *b)
+/*
+ * Search for an entry in a given tree.
+ *
+ * Note that this search is performed in two steps because
+ * of the way tree entries are sorted internally in git:
+ *
+ * Entries in a tree are not sorted alphabetically; two entries
+ * with the same root prefix will have different positions
+ * depending on whether they are folders (subtrees) or normal files.
+ *
+ * Consequently, it is not possible to find an entry on the tree
+ * with a binary search if you don't know whether the filename
+ * you're looking for is a folder or a normal file.
+ *
+ * To work around this, we first perform a homing binary search
+ * on the tree, using the minimal length root prefix of our filename.
+ * Once the comparisons for this homing search start becoming
+ * ambiguous because of folder vs file sorting, we look linearly
+ * around the area for our target file.
+ */
+static int tree_key_search(git_vector *entries, const char *filename)
 {
-	const git_tree_entry *entry_a = *(const git_tree_entry **)(a);
-	const git_tree_entry *entry_b = *(const git_tree_entry **)(b);
+	struct tree_key_search ksearch;
+	const git_tree_entry *entry;
 
-	return gitfo_cmp_path(entry_a->filename, strlen(entry_a->filename),
-                                  entry_a->attr & 040000,
-                                  entry_b->filename, strlen(entry_b->filename),
-                                  entry_b->attr & 040000);
+	int homing, i;
+
+	ksearch.filename = filename;
+	ksearch.filename_len = strlen(filename);
+
+	/* Initial homing search; find an entry on the tree with
+	 * the same prefix as the filename we're looking for */
+	homing = git_vector_bsearch2(entries, &homing_search_cmp, &ksearch);
+	if (homing < 0)
+		return homing;
+
+	/* We found a common prefix. Look forward as long as
+	 * there are entries that share the common prefix */
+	for (i = homing; i < (int)entries->length; ++i) {
+		entry = entries->contents[i];
+
+		if (homing_search_cmp(&ksearch, entry) < 0)
+			break;
+
+		if (strcmp(filename, entry->filename) == 0)
+			return i;
+	}
+
+	/* If we haven't found our filename yet, look backwards
+	 * too as long as we have entries with the same prefix */
+	for (i = homing - 1; i >= 0; --i) {
+		entry = entries->contents[i];
+
+		if (homing_search_cmp(&ksearch, entry) > 0)
+			break;
+
+		if (strcmp(filename, entry->filename) == 0)
+			return i;
+	}
+
+	/* The filename doesn't exist at all */
+	return GIT_ENOTFOUND;
 }
 
 void git_tree__free(git_tree *tree)
@@ -70,12 +128,12 @@ void git_tree__free(git_tree *tree)
 		git_tree_entry *e;
 		e = git_vector_get(&tree->entries, i);
 
-		free(e->filename);
-		free(e);
+		git__free(e->filename);
+		git__free(e);
 	}
 
 	git_vector_free(&tree->entries);
-	free(tree);
+	git__free(tree);
 }
 
 const git_oid *git_tree_id(git_tree *c)
@@ -100,7 +158,22 @@ const git_oid *git_tree_entry_id(const git_tree_entry *entry)
 	return &entry->oid;
 }
 
-int git_tree_entry_2object(git_object **object_out, git_repository *repo, const git_tree_entry *entry)
+git_otype git_tree_entry_type(const git_tree_entry *entry)
+{
+	assert(entry);
+
+	if (S_ISGITLINK(entry->attr))
+		return GIT_OBJ_COMMIT;
+	else if (S_ISDIR(entry->attr))
+		return GIT_OBJ_TREE;
+	else
+		return GIT_OBJ_BLOB;
+}
+
+int git_tree_entry_to_object(
+	git_object **object_out,
+	git_repository *repo,
+	const git_tree_entry *entry)
 {
 	assert(entry && object_out);
 	return git_object_lookup(object_out, repo, &entry->oid, GIT_OBJ_ANY);
@@ -112,56 +185,84 @@ const git_tree_entry *git_tree_entry_byname(git_tree *tree, const char *filename
 
 	assert(tree && filename);
 
-	idx = git_vector_bsearch2(&tree->entries, entry_search_cmp, filename);
+	idx = tree_key_search(&tree->entries, filename);
 	if (idx == GIT_ENOTFOUND)
 		return NULL;
 
 	return git_vector_get(&tree->entries, idx);
 }
 
-const git_tree_entry *git_tree_entry_byindex(git_tree *tree, int idx)
+const git_tree_entry *git_tree_entry_byindex(git_tree *tree, unsigned int idx)
 {
 	assert(tree);
-	return git_vector_get(&tree->entries, (unsigned int)idx);
+	return git_vector_get(&tree->entries, idx);
 }
 
-size_t git_tree_entrycount(git_tree *tree)
+int git_tree__prefix_position(git_tree *tree, const char *path)
+{
+	git_vector *entries = &tree->entries;
+	struct tree_key_search ksearch;
+	unsigned int at_pos;
+
+	ksearch.filename = path;
+	ksearch.filename_len = strlen(path);
+
+	/* Find tree entry with appropriate prefix */
+	git_vector_bsearch3(&at_pos, entries, &homing_search_cmp, &ksearch);
+
+	for (; at_pos < entries->length; ++at_pos) {
+		const git_tree_entry *entry = entries->contents[at_pos];
+		if (homing_search_cmp(&ksearch, entry) < 0)
+			break;
+	}
+
+	for (; at_pos > 0; --at_pos) {
+		const git_tree_entry *entry = entries->contents[at_pos - 1];
+		if (homing_search_cmp(&ksearch, entry) > 0)
+			break;
+	}
+
+	return at_pos;
+}
+
+unsigned int git_tree_entrycount(git_tree *tree)
 {
 	assert(tree);
 	return tree->entries.length;
 }
 
-static int tree_parse_buffer(git_tree *tree, const char *buffer, const char *buffer_end)
+static int tree_error(const char *str)
 {
-	int error = GIT_SUCCESS;
+	giterr_set(GITERR_TREE, "%s", str);
+	return -1;
+}
 
-	if (git_vector_init(&tree->entries, DEFAULT_TREE_SIZE, entry_sort_cmp) < GIT_SUCCESS)
-		return GIT_ENOMEM;
+static int tree_parse_buffer(git_tree *tree, const char *buffer, const char *buffer_end)
+{
+	if (git_vector_init(&tree->entries, DEFAULT_TREE_SIZE, entry_sort_cmp) < 0)
+		return -1;
 
 	while (buffer < buffer_end) {
 		git_tree_entry *entry;
+		int tmp;
 
 		entry = git__calloc(1, sizeof(git_tree_entry));
-		if (entry == NULL) {
-			error = GIT_ENOMEM;
-			break;
-		}
+		GITERR_CHECK_ALLOC(entry);
 
-		if (git_vector_insert(&tree->entries, entry) < GIT_SUCCESS)
-			return GIT_ENOMEM;
+		if (git_vector_insert(&tree->entries, entry) < 0)
+			return -1;
 
-		if (git__strtol32((long *)&entry->attr, buffer, &buffer, 8) < GIT_SUCCESS)
-			return git__throw(GIT_EOBJCORRUPTED, "Failed to parse tree. Can't parse attributes");
+		if (git__strtol32(&tmp, buffer, &buffer, 8) < 0 ||
+			!buffer || !valid_attributes(tmp))
+			return tree_error("Failed to parse tree. Can't parse attributes");
 
-		if (*buffer++ != ' ') {
-			error = git__throw(GIT_EOBJCORRUPTED, "Failed to parse tree. Object it corrupted");
-			break;
-		}
+		entry->attr = tmp;
 
-		if (memchr(buffer, 0, buffer_end - buffer) == NULL) {
-			error = git__throw(GIT_EOBJCORRUPTED, "Failed to parse tree. Object it corrupted");
-			break;
-		}
+		if (*buffer++ != ' ')
+			return tree_error("Failed to parse tree. Object is corrupted");
+
+		if (memchr(buffer, 0, buffer_end - buffer) == NULL)
+			return tree_error("Failed to parse tree. Object is corrupted");
 
 		entry->filename = git__strdup(buffer);
 		entry->filename_len = strlen(buffer);
@@ -171,11 +272,11 @@ static int tree_parse_buffer(git_tree *tree, const char *buffer, const char *buf
 
 		buffer++;
 
-		git_oid_mkraw(&entry->oid, (const unsigned char *)buffer);
+		git_oid_fromraw(&entry->oid, (const unsigned char *)buffer);
 		buffer += GIT_OID_RAWSZ;
 	}
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to parse buffer");
+	return 0;
 }
 
 int git_tree__parse(git_tree *tree, git_odb_object *obj)
@@ -184,100 +285,169 @@ int git_tree__parse(git_tree *tree, git_odb_object *obj)
 	return tree_parse_buffer(tree, (char *)obj->raw.data, (char *)obj->raw.data + obj->raw.len);
 }
 
-static int write_index_entry(char *buffer, int mode, const char *path, size_t path_len, const git_oid *oid)
+static unsigned int find_next_dir(const char *dirname, git_index *index, unsigned int start)
 {
-	int written;
-	written = sprintf(buffer, "%o %.*s%c", mode, (int)path_len, path, 0);
-	memcpy(buffer + written, &oid->id, GIT_OID_RAWSZ);
-	return written + GIT_OID_RAWSZ;
+	unsigned int i, entries = git_index_entrycount(index);
+	size_t dirlen;
+
+	dirlen = strlen(dirname);
+	for (i = start; i < entries; ++i) {
+		git_index_entry *entry = git_index_get(index, i);
+		if (strlen(entry->path) < dirlen ||
+		    memcmp(entry->path, dirname, dirlen) ||
+			(dirlen > 0 && entry->path[dirlen] != '/')) {
+			break;
+		}
+	}
+
+	return i;
 }
 
-static int write_index(git_oid *oid, git_index *index, const char *base, int baselen, int entry_no, int maxentries)
+static int append_entry(git_treebuilder *bld, const char *filename, const git_oid *id, unsigned int attributes)
 {
-	size_t size, offset;
-	char *buffer;
-	int nr, error;
+	git_tree_entry *entry;
 
-	/* Guess at some random initial size */
-	size = maxentries * 40;
-	buffer = git__malloc(size);
-	if (buffer == NULL)
-		return GIT_ENOMEM;
-		
-	offset = 0;
-	
-	for (nr = entry_no; nr < maxentries; ++nr) {
-		git_index_entry *entry = git_index_get(index, nr);
+	entry = git__calloc(1, sizeof(git_tree_entry));
+	GITERR_CHECK_ALLOC(entry);
 
-		const char *pathname = entry->path, *filename, *dirname;
-		int pathlen = strlen(pathname), entrylen;
+	entry->filename = git__strdup(filename);
+	entry->filename_len = strlen(entry->filename);
 
-		unsigned int write_mode;
-		git_oid subtree_oid;
-		git_oid *write_oid;
-		
-		/* Did we hit the end of the directory? Return how many we wrote */
-		if (baselen >= pathlen || memcmp(base, pathname, baselen) != 0)
-			break;
-		
-		/* Do we have _further_ subdirectories? */
-		filename = pathname + baselen;
-		dirname = strchr(filename, '/');
+	git_oid_cpy(&entry->oid, id);
+	entry->attr = attributes;
 
-		write_oid = &entry->oid;
-		write_mode = entry->mode;
+	if (git_vector_insert(&bld->entries, entry) < 0)
+		return -1;
 
-		if (dirname) {
-			int subdir_written;
+	return 0;
+}
+
+static int write_tree(
+	git_oid *oid,
+	git_repository *repo,
+	git_index *index,
+	const char *dirname,
+	unsigned int start)
+{
+	git_treebuilder *bld = NULL;
 
-#if 0
-			if (entry->mode != S_IFDIR) {
-				free(buffer);
-				return GIT_EOBJCORRUPTED;
+	unsigned int i, entries = git_index_entrycount(index);
+	int error;
+	size_t dirname_len = strlen(dirname);
+	const git_tree_cache *cache;
+
+	cache = git_tree_cache_get(index->tree, dirname);
+	if (cache != NULL && cache->entries >= 0){
+		git_oid_cpy(oid, &cache->oid);
+		return find_next_dir(dirname, index, start);
+	}
+
+	error = git_treebuilder_create(&bld, NULL);
+	if (bld == NULL) {
+		return -1;
+	}
+
+	/*
+	 * This loop is unfortunate, but necessary. The index doesn't have
+	 * any directores, so we need to handle that manually, and we
+	 * need to keep track of the current position.
+	 */
+	for (i = start; i < entries; ++i) {
+		git_index_entry *entry = git_index_get(index, i);
+		char *filename, *next_slash;
+
+	/*
+	 * If we've left our (sub)tree, exit the loop and return. The
+	 * first check is an early out (and security for the
+	 * third). The second check is a simple prefix comparison. The
+	 * third check catches situations where there is a directory
+	 * win32/sys and a file win32mmap.c. Without it, the following
+	 * code believes there is a file win32/mmap.c
+	 */
+		if (strlen(entry->path) < dirname_len ||
+		    memcmp(entry->path, dirname, dirname_len) ||
+		    (dirname_len > 0 && entry->path[dirname_len] != '/')) {
+			break;
+		}
+
+		filename = entry->path + dirname_len;
+		if (*filename == '/')
+			filename++;
+		next_slash = strchr(filename, '/');
+		if (next_slash) {
+			git_oid sub_oid;
+			int written;
+			char *subdir, *last_comp;
+
+			subdir = git__strndup(entry->path, next_slash - entry->path);
+			GITERR_CHECK_ALLOC(subdir);
+
+			/* Write out the subtree */
+			written = write_tree(&sub_oid, repo, index, subdir, i);
+			if (written < 0) {
+				tree_error("Failed to write subtree");
+				goto on_error;
+			} else {
+				i = written - 1; /* -1 because of the loop increment */
 			}
-#endif
-			subdir_written = write_index(&subtree_oid, index, pathname, dirname - pathname + 1, nr, maxentries);
 
-			if (subdir_written < GIT_SUCCESS) {
-				free(buffer);
-				return subdir_written;
+			/*
+			 * We need to figure out what we want toinsert
+			 * into this tree. If we're traversing
+			 * deps/zlib/, then we only want to write
+			 * 'zlib' into the tree.
+			 */
+			last_comp = strrchr(subdir, '/');
+			if (last_comp) {
+				last_comp++; /* Get rid of the '/' */
+			} else {
+				last_comp = subdir;
+			}
+			error = append_entry(bld, last_comp, &sub_oid, S_IFDIR);
+			git__free(subdir);
+			if (error < 0) {
+				tree_error("Failed to insert dir");
+				goto on_error;
+			}
+		} else {
+			error = append_entry(bld, filename, &entry->oid, entry->mode);
+			if (error < 0) {
+				tree_error("Failed to insert file");
+				goto on_error;
 			}
-			
-			nr = subdir_written - 1;
-			
-			/* Now we need to write out the directory entry into this tree.. */
-			pathlen = dirname - pathname;
-			write_oid = &subtree_oid;
-			write_mode = S_IFDIR;
 		}
+	}
 
-		entrylen = pathlen - baselen;
-		if (offset + entrylen + 32 > size) {
-			size = alloc_nr(offset + entrylen + 32);
-			buffer = git__realloc(buffer, size);
-			
-			if (buffer == NULL)
-				return GIT_ENOMEM;
-		}
+	if (git_treebuilder_write(oid, repo, bld) < 0)
+		goto on_error;
 
-		offset += write_index_entry(buffer + offset, write_mode, filename, entrylen, write_oid);
-	}
-	
-	error = git_odb_write(oid, index->repository->db, buffer, offset, GIT_OBJ_TREE);
-	free(buffer);
+	git_treebuilder_free(bld);
+	return i;
 
-	return (error == GIT_SUCCESS) ? nr : git__rethrow(error, "Failed to write index");
+on_error:
+	git_treebuilder_free(bld);
+	return -1;
 }
 
 int git_tree_create_fromindex(git_oid *oid, git_index *index)
 {
-	int error;
+	int ret;
+	git_repository *repo;
 
-	if (index->repository == NULL)
-		return git__throw(GIT_EBAREINDEX, "Failed to create tree. The index file is not backed up by an existing repository");
+	repo = (git_repository *)GIT_REFCOUNT_OWNER(index);
 
-	error = write_index(oid, index, "", 0, 0, git_index_entrycount(index));
-	return (error < GIT_SUCCESS) ? git__rethrow(error, "Failed to create tree") : GIT_SUCCESS;
+	if (repo == NULL)
+		return tree_error("Failed to create tree. "
+		  "The index file is not backed up by an existing repository");
+
+	if (index->tree != NULL && index->tree->entries >= 0) {
+		git_oid_cpy(oid, &index->tree->oid);
+		return 0;
+	}
+
+	/* The tree cache didn't help us */
+	ret = write_tree(oid, repo, index, "", 0);
+	return ret < 0 ? ret : 0;
 }
 
 static void sort_entries(git_treebuilder *bld)
@@ -288,51 +458,34 @@ static void sort_entries(git_treebuilder *bld)
 int git_treebuilder_create(git_treebuilder **builder_p, const git_tree *source)
 {
 	git_treebuilder *bld;
-	size_t i, source_entries = DEFAULT_TREE_SIZE;
+	unsigned int i, source_entries = DEFAULT_TREE_SIZE;
 
 	assert(builder_p);
 
 	bld = git__calloc(1, sizeof(git_treebuilder));
-	if (bld == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(bld);
 
 	if (source != NULL)
 		source_entries = source->entries.length;
 
-	if (git_vector_init(&bld->entries, source_entries, entry_sort_cmp) < GIT_SUCCESS) {
-		free(bld);
-		return GIT_ENOMEM;
-	}
+	if (git_vector_init(&bld->entries, source_entries, entry_sort_cmp) < 0)
+		goto on_error;
 
 	if (source != NULL) {
-		bld->entry_count = source_entries;
 		for (i = 0; i < source->entries.length; ++i) {
 			git_tree_entry *entry_src = source->entries.contents[i];
-			git_tree_entry *entry = git__calloc(1, sizeof(git_tree_entry));
-
-			if (entry == NULL) {
-				git_treebuilder_free(bld);
-				return GIT_ENOMEM;
-			}
 
-			entry->filename = git__strdup(entry_src->filename);
-
-			if (entry->filename == NULL) {
-				free(entry);
-				git_treebuilder_free(bld);
-				return GIT_ENOMEM;
-			}
-
-			entry->filename_len = entry_src->filename_len;
-			git_oid_cpy(&entry->oid, &entry_src->oid);
-			entry->attr = entry_src->attr;
-
-			git_vector_insert(&bld->entries, entry);
+			if (append_entry(bld, entry_src->filename, &entry_src->oid, entry_src->attr) < 0)
+				goto on_error;
 		}
 	}
 
 	*builder_p = bld;
-	return GIT_SUCCESS;
+	return 0;
+
+on_error:
+	git_treebuilder_free(bld);
+	return -1;
 }
 
 int git_treebuilder_insert(git_tree_entry **entry_out, git_treebuilder *bld, const char *filename, const git_oid *id, unsigned int attributes)
@@ -343,23 +496,23 @@ int git_treebuilder_insert(git_tree_entry **entry_out, git_treebuilder *bld, con
 	assert(bld && id && filename);
 
 	if (!valid_attributes(attributes))
-		return git__throw(GIT_ERROR, "Failed to insert entry. Invalid atrributes");
+		return tree_error("Failed to insert entry. Invalid attributes");
 
-	if ((pos = git_vector_bsearch2(&bld->entries, entry_search_cmp, filename)) != GIT_ENOTFOUND) {
+	if (!valid_entry_name(filename))
+		return tree_error("Failed to insert entry. Invalid name for a tree entry");
+
+	pos = tree_key_search(&bld->entries, filename);
+
+	if (pos >= 0) {
 		entry = git_vector_get(&bld->entries, pos);
-		if (entry->removed) {
+		if (entry->removed)
 			entry->removed = 0;
-			bld->entry_count++;
-		}
 	} else {
-		if ((entry = git__malloc(sizeof(git_tree_entry))) == NULL)
-			return GIT_ENOMEM;
+		entry = git__calloc(1, sizeof(git_tree_entry));
+		GITERR_CHECK_ALLOC(entry);
 
-		memset(entry, 0x0, sizeof(git_tree_entry));
 		entry->filename = git__strdup(filename);
 		entry->filename_len = strlen(entry->filename);
-
-		bld->entry_count++;
 	}
 
 	git_oid_cpy(&entry->oid, id);
@@ -367,25 +520,24 @@ int git_treebuilder_insert(git_tree_entry **entry_out, git_treebuilder *bld, con
 
 	if (pos == GIT_ENOTFOUND) {
 		if (git_vector_insert(&bld->entries, entry) < 0)
-			return GIT_ENOMEM;
+			return -1;
 	}
 
 	if (entry_out != NULL)
 		*entry_out = entry;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-const git_tree_entry *git_treebuilder_get(git_treebuilder *bld, const char *filename)
+static git_tree_entry *treebuilder_get(git_treebuilder *bld, const char *filename)
 {
 	int idx;
 	git_tree_entry *entry;
 
 	assert(bld && filename);
 
-	sort_entries(bld);
-	idx = git_vector_bsearch2(&bld->entries, entry_search_cmp, filename);
-	if (idx == GIT_ENOTFOUND)
+	idx = tree_key_search(&bld->entries, filename);
+	if (idx < 0)
 		return NULL;
 
 	entry = git_vector_get(&bld->entries, idx);
@@ -395,65 +547,67 @@ const git_tree_entry *git_treebuilder_get(git_treebuilder *bld, const char *file
 	return entry;
 }
 
+const git_tree_entry *git_treebuilder_get(git_treebuilder *bld, const char *filename)
+{
+	return treebuilder_get(bld, filename);
+}
+
 int git_treebuilder_remove(git_treebuilder *bld, const char *filename)
 {
-	git_tree_entry *remove_ptr = (git_tree_entry *)git_treebuilder_get(bld, filename);
+	git_tree_entry *remove_ptr = treebuilder_get(bld, filename);
 
 	if (remove_ptr == NULL || remove_ptr->removed)
-		return git__throw(GIT_ENOTFOUND, "Failed to remove entry. File isn't in the tree");
+		return tree_error("Failed to remove entry. File isn't in the tree");
 
 	remove_ptr->removed = 1;
-	bld->entry_count--;
-	return GIT_SUCCESS;
+	return 0;
 }
 
 int git_treebuilder_write(git_oid *oid, git_repository *repo, git_treebuilder *bld)
 {
-	size_t i, size = 0;
-	char filemode[MAX_FILEMODE_BYTES + 1 + 1];
-	git_odb_stream *stream;
-	int error;
+	unsigned int i;
+	git_buf tree = GIT_BUF_INIT;
+	git_odb *odb;
 
 	assert(bld);
 
 	sort_entries(bld);
 
+	/* Grow the buffer beforehand to an estimated size */
+	git_buf_grow(&tree, bld->entries.length * 72);
+
 	for (i = 0; i < bld->entries.length; ++i) {
 		git_tree_entry *entry = bld->entries.contents[i];
 
 		if (entry->removed)
 			continue;
 
-		snprintf(filemode, sizeof(filemode), "%o ", entry->attr);
-		size += strlen(filemode);
-		size += entry->filename_len + 1;
-		size += GIT_OID_RAWSZ;
+		git_buf_printf(&tree, "%o ", entry->attr);
+		git_buf_put(&tree, entry->filename, entry->filename_len + 1);
+		git_buf_put(&tree, (char *)entry->oid.id, GIT_OID_RAWSZ);
 	}
 
-	if ((error = git_odb_open_wstream(&stream, git_repository_database(repo), size, GIT_OBJ_TREE)) < GIT_SUCCESS)
-		return git__rethrow(error, "Failed to write tree. Can't open write stream");
+	if (git_buf_oom(&tree))
+		goto on_error;
 
-	for (i = 0; i < bld->entries.length; ++i) {
-		git_tree_entry *entry = bld->entries.contents[i];
+	if (git_repository_odb__weakptr(&odb, repo) < 0)
+		goto on_error;
 
-		if (entry->removed)
-			continue;
 
-		snprintf(filemode, sizeof(filemode), "%o ", entry->attr);
-		stream->write(stream, filemode, strlen(filemode));
-		stream->write(stream, entry->filename, entry->filename_len + 1);
-		stream->write(stream, (char *)entry->oid.id, GIT_OID_RAWSZ);
-	} 
+	if (git_odb_write(oid, odb, tree.ptr, tree.size, GIT_OBJ_TREE) < 0)
+		goto on_error;
 
-	error = stream->finalize_write(oid, stream);
-	stream->free(stream);
+	git_buf_free(&tree);
+	return 0;
 
-	return error == GIT_SUCCESS ? GIT_SUCCESS : git__rethrow(error, "Failed to write tree");
+on_error:
+	git_buf_free(&tree);
+	return -1;
 }
 
 void git_treebuilder_filter(git_treebuilder *bld, int (*filter)(const git_tree_entry *, void *), void *payload)
 {
-	size_t i;
+	unsigned int i;
 
 	assert(bld && filter);
 
@@ -466,13 +620,13 @@ void git_treebuilder_filter(git_treebuilder *bld, int (*filter)(const git_tree_e
 
 void git_treebuilder_clear(git_treebuilder *bld)
 {
-	size_t i;
+	unsigned int i;
 	assert(bld);
 
 	for (i = 0; i < bld->entries.length; ++i) {
 		git_tree_entry *e = bld->entries.contents[i];
-		free(e->filename);
-		free(e);
+		git__free(e->filename);
+		git__free(e);
 	}
 
 	git_vector_clear(&bld->entries);
@@ -482,7 +636,150 @@ void git_treebuilder_free(git_treebuilder *bld)
 {
 	git_treebuilder_clear(bld);
 	git_vector_free(&bld->entries);
-	free(bld);
+	git__free(bld);
+}
+
+static int tree_frompath(
+	git_tree **parent_out,
+	git_tree *root,
+	git_buf *treeentry_path,
+	size_t offset)
+{
+	char *slash_pos = NULL;
+	const git_tree_entry* entry;
+	int error = 0;
+	git_tree *subtree;
+
+	if (!*(treeentry_path->ptr + offset)) {
+		giterr_set(GITERR_INVALID,
+			"Invalid relative path to a tree entry '%s'.", treeentry_path->ptr);
+		return -1;
+	}
+
+	slash_pos = (char *)strchr(treeentry_path->ptr + offset, '/');
+
+	if (slash_pos == NULL)
+		return git_tree_lookup(
+			parent_out,
+			root->object.repo,
+			git_object_id((const git_object *)root)
+		);
+
+	if (slash_pos == treeentry_path->ptr + offset) {
+		giterr_set(GITERR_INVALID,
+			"Invalid relative path to a tree entry '%s'.", treeentry_path->ptr);
+		return -1;
+	}
+
+	*slash_pos = '\0';
+
+	entry = git_tree_entry_byname(root, treeentry_path->ptr + offset);
+
+	if (slash_pos != NULL)
+		*slash_pos = '/';
+
+	if (entry == NULL) {
+		giterr_set(GITERR_TREE,
+			"No tree entry can be found from "
+			"the given tree and relative path '%s'.", treeentry_path->ptr);
+		return GIT_ENOTFOUND;
+	}
+
+
+	if (git_tree_lookup(&subtree, root->object.repo, &entry->oid) < 0)
+		return error;
+
+	error = tree_frompath(
+		parent_out,
+		subtree,
+		treeentry_path,
+		(slash_pos - treeentry_path->ptr) + 1
+	);
+
+	git_tree_free(subtree);
+	return error;
+}
+
+int git_tree_get_subtree(
+	git_tree **subtree,
+	git_tree *root,
+	const char *subtree_path)
+{
+	int error;
+	git_buf buffer = GIT_BUF_INIT;
+
+	assert(subtree && root && subtree_path);
+
+	if ((error = git_buf_sets(&buffer, subtree_path)) == 0)
+		error = tree_frompath(subtree, root, &buffer, 0);
+
+	git_buf_free(&buffer);
+
+	return error;
+}
+
+static int tree_walk_post(
+	git_tree *tree,
+	git_treewalk_cb callback,
+	git_buf *path,
+	void *payload)
+{
+	int error = 0;
+	unsigned int i;
+
+	for (i = 0; i < tree->entries.length; ++i) {
+		git_tree_entry *entry = tree->entries.contents[i];
+
+		if (callback(path->ptr, entry, payload) < 0)
+			continue;
+
+		if (git_tree_entry__is_tree(entry)) {
+			git_tree *subtree;
+			size_t path_len = git_buf_len(path);
+
+			if ((error = git_tree_lookup(
+				&subtree, tree->object.repo, &entry->oid)) < 0)
+				break;
+
+			/* append the next entry to the path */
+			git_buf_puts(path, entry->filename);
+			git_buf_putc(path, '/');
+
+			if (git_buf_oom(path))
+				return -1;
+
+			if (tree_walk_post(subtree, callback, path, payload) < 0)
+				return -1;
+
+			git_buf_truncate(path, path_len);
+			git_tree_free(subtree);
+		}
+	}
+
+	return 0;
 }
 
+int git_tree_walk(git_tree *tree, git_treewalk_cb callback, int mode, void *payload)
+{
+	int error = 0;
+	git_buf root_path = GIT_BUF_INIT;
+
+	switch (mode) {
+		case GIT_TREEWALK_POST:
+			error = tree_walk_post(tree, callback, &root_path, payload);
+			break;
+
+		case GIT_TREEWALK_PRE:
+			tree_error("Preorder tree walking is still not implemented");
+			return -1;
+
+		default:
+			giterr_set(GITERR_INVALID, "Invalid walking mode for tree walk");
+			return -1;
+	}
+
+	git_buf_free(&root_path);
+
+	return error;
+}
 
diff --git a/src/tree.h b/src/tree.h
index bff3f8edb..498a90d66 100644
--- a/src/tree.h
+++ b/src/tree.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_tree_h__
 #define INCLUDE_tree_h__
 
@@ -21,11 +27,25 @@ struct git_tree {
 
 struct git_treebuilder {
 	git_vector entries;
-	size_t entry_count;
 };
 
 
+GIT_INLINE(bool) git_tree_entry__is_tree(const struct git_tree_entry *e)
+{
+	return (S_ISDIR(e->attr) && !S_ISGITLINK(e->attr));
+}
+
 void git_tree__free(git_tree *tree);
 int git_tree__parse(git_tree *tree, git_odb_object *obj);
 
+/**
+ * Lookup the first position in the tree with a given prefix.
+ *
+ * @param tree a previously loaded tree.
+ * @param prefix the beginning of a path to find in the tree.
+ * @return index of the first item at or after the given prefix.
+ */
+int git_tree__prefix_position(git_tree *tree, const char *prefix);
+
+
 #endif
diff --git a/src/tsort.c b/src/tsort.c
new file mode 100644
index 000000000..f54c21e50
--- /dev/null
+++ b/src/tsort.c
@@ -0,0 +1,367 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+
+/**
+ * An array-of-pointers implementation of Python's Timsort
+ * Based on code by Christopher Swenson under the MIT license
+ *
+ * Copyright (c) 2010 Christopher Swenson
+ * Copyright (c) 2011 Vicent Marti
+ */
+
+#ifndef MAX
+#	define MAX(x,y) (((x) > (y) ? (x) : (y)))
+#endif
+
+#ifndef MIN
+#	define MIN(x,y) (((x) < (y) ? (x) : (y)))
+#endif
+
+typedef int (*cmp_ptr_t)(const void *, const void *);
+
+static int binsearch(void **dst, const void *x, size_t size, cmp_ptr_t cmp)
+{
+	int l, c, r;
+	void *lx, *cx;
+
+	assert(size > 0);
+
+	l = 0;
+	r = (int)size - 1;
+	c = r >> 1;
+	lx = dst[l];
+
+	/* check for beginning conditions */
+	if (cmp(x, lx) < 0)
+		return 0;
+
+	else if (cmp(x, lx) == 0) {
+		int i = 1;
+		while (cmp(x, dst[i]) == 0)
+			i++;
+		return i;
+	}
+
+	/* guaranteed not to be >= rx */
+	cx = dst[c];
+	while (1) {
+		const int val = cmp(x, cx);
+		if (val < 0) {
+			if (c - l <= 1) return c;
+			r = c;
+		} else if (val > 0) {
+			if (r - c <= 1) return c + 1;
+			l = c;
+			lx = cx;
+		} else {
+			do {
+				cx = dst[++c];
+			} while (cmp(x, cx) == 0);
+			return c;
+		}
+		c = l + ((r - l) >> 1);
+		cx = dst[c];
+	}
+}
+
+/* Binary insertion sort, but knowing that the first "start" entries are sorted. Used in timsort. */
+static void bisort(void **dst, size_t start, size_t size, cmp_ptr_t cmp)
+{
+	size_t i;
+	void *x;
+	int location;
+
+	for (i = start; i < size; i++) {
+		int j;
+		/* If this entry is already correct, just move along */
+		if (cmp(dst[i - 1], dst[i]) <= 0)
+			continue;
+
+		/* Else we need to find the right place, shift everything over, and squeeze in */
+		x = dst[i];
+		location = binsearch(dst, x, i, cmp);
+		for (j = (int)i - 1; j >= location; j--) {
+			dst[j + 1] = dst[j];
+		}
+		dst[location] = x;
+	}
+}
+
+
+/* timsort implementation, based on timsort.txt */
+struct tsort_run {
+	ssize_t start;
+	ssize_t length;
+};
+
+struct tsort_store {
+	size_t alloc;
+	cmp_ptr_t cmp;
+	void **storage;
+};
+
+static void reverse_elements(void **dst, ssize_t start, ssize_t end)
+{
+	while (start < end) {
+		void *tmp = dst[start];
+		dst[start] = dst[end];
+		dst[end] = tmp;
+
+		start++;
+		end--;
+	}
+}
+
+static ssize_t count_run(void **dst, ssize_t start, ssize_t size, struct tsort_store *store)
+{
+	ssize_t curr = start + 2;
+
+	if (size - start == 1)
+		return 1;
+
+	if (start >= size - 2) {
+		if (store->cmp(dst[size - 2], dst[size - 1]) > 0) {
+			void *tmp = dst[size - 1];
+			dst[size - 1] = dst[size - 2];
+			dst[size - 2] = tmp;
+		}
+
+		return 2;
+	}
+
+	if (store->cmp(dst[start], dst[start + 1]) <= 0) {
+		while (curr < size - 1 && store->cmp(dst[curr - 1], dst[curr]) <= 0)
+			curr++;
+
+		return curr - start;
+	} else {
+		while (curr < size - 1 && store->cmp(dst[curr - 1], dst[curr]) > 0)
+			curr++;
+
+		/* reverse in-place */
+		reverse_elements(dst, start, curr - 1);
+		return curr - start;
+	}
+}
+
+static size_t compute_minrun(size_t n)
+{
+	int r = 0;
+	while (n >= 64) {
+		r |= n & 1;
+		n >>= 1;
+	}
+	return n + r;
+}
+
+static int check_invariant(struct tsort_run *stack, ssize_t stack_curr)
+{
+	if (stack_curr < 2)
+		return 1;
+
+	else if (stack_curr == 2) {
+		const ssize_t A = stack[stack_curr - 2].length;
+		const ssize_t B = stack[stack_curr - 1].length;
+		return (A > B);
+	} else {
+		const ssize_t A = stack[stack_curr - 3].length;
+		const ssize_t B = stack[stack_curr - 2].length;
+		const ssize_t C = stack[stack_curr - 1].length;
+		return !((A <= B + C) || (B <= C));
+	}
+}
+
+static int resize(struct tsort_store *store, size_t new_size)
+{
+	if (store->alloc < new_size) {
+		void **tempstore = git__realloc(store->storage, new_size * sizeof(void *));
+
+		/**
+		 * Do not propagate on OOM; this will abort the sort and
+		 * leave the array unsorted, but no error code will be
+		 * raised
+		 */
+		if (tempstore == NULL)
+			return -1;
+
+		store->storage = tempstore;
+		store->alloc = new_size;
+	}
+
+	return 0;
+}
+
+static void merge(void **dst, const struct tsort_run *stack, ssize_t stack_curr, struct tsort_store *store)
+{
+	const ssize_t A = stack[stack_curr - 2].length;
+	const ssize_t B = stack[stack_curr - 1].length;
+	const ssize_t curr = stack[stack_curr - 2].start;
+
+	void **storage;
+	ssize_t i, j, k;
+
+	if (resize(store, MIN(A, B)) < 0)
+		return;
+
+	storage = store->storage;
+
+	/* left merge */
+	if (A < B) {
+		memcpy(storage, &dst[curr], A * sizeof(void *));
+		i = 0;
+		j = curr + A;
+
+		for (k = curr; k < curr + A + B; k++) {
+			if ((i < A) && (j < curr + A + B)) {
+				if (store->cmp(storage[i], dst[j]) <= 0)
+					dst[k] = storage[i++];
+				else
+					dst[k] = dst[j++];
+			} else if (i < A) {
+				dst[k] = storage[i++];
+			} else
+				dst[k] = dst[j++];
+		}
+	} else {
+		memcpy(storage, &dst[curr + A], B * sizeof(void *));
+		i = B - 1;
+		j = curr + A - 1;
+
+		for (k = curr + A + B - 1; k >= curr; k--) {
+			if ((i >= 0) && (j >= curr)) {
+				if (store->cmp(dst[j], storage[i]) > 0)
+					dst[k] = dst[j--];
+				else
+					dst[k] = storage[i--];
+			} else if (i >= 0)
+				dst[k] = storage[i--];
+			else
+				dst[k] = dst[j--];
+		}
+	}
+}
+
+static ssize_t collapse(void **dst, struct tsort_run *stack, ssize_t stack_curr, struct tsort_store *store, ssize_t size)
+{
+	ssize_t A, B, C;
+
+	while (1) {
+		/* if the stack only has one thing on it, we are done with the collapse */
+		if (stack_curr <= 1)
+			break;
+
+		/* if this is the last merge, just do it */
+		if ((stack_curr == 2) && (stack[0].length + stack[1].length == size)) {
+			merge(dst, stack, stack_curr, store);
+			stack[0].length += stack[1].length;
+			stack_curr--;
+			break;
+		}
+
+		/* check if the invariant is off for a stack of 2 elements */
+		else if ((stack_curr == 2) && (stack[0].length <= stack[1].length)) {
+			merge(dst, stack, stack_curr, store);
+			stack[0].length += stack[1].length;
+			stack_curr--;
+			break;
+		}
+		else if (stack_curr == 2)
+			break;
+
+		A = stack[stack_curr - 3].length;
+		B = stack[stack_curr - 2].length;
+		C = stack[stack_curr - 1].length;
+
+		/* check first invariant */
+		if (A <= B + C) {
+			if (A < C) {
+				merge(dst, stack, stack_curr - 1, store);
+				stack[stack_curr - 3].length += stack[stack_curr - 2].length;
+				stack[stack_curr - 2] = stack[stack_curr - 1];
+				stack_curr--;
+			} else {
+				merge(dst, stack, stack_curr, store);
+				stack[stack_curr - 2].length += stack[stack_curr - 1].length;
+				stack_curr--;
+			}
+		} else if (B <= C) {
+			merge(dst, stack, stack_curr, store);
+			stack[stack_curr - 2].length += stack[stack_curr - 1].length;
+			stack_curr--;
+		} else
+			break;
+	}
+
+	return stack_curr;
+}
+
+#define PUSH_NEXT() do {\
+	len = count_run(dst, curr, size, store);\
+	run = minrun;\
+	if (run < minrun) run = minrun;\
+	if (run > (ssize_t)size - curr) run = size - curr;\
+	if (run > len) {\
+		bisort(&dst[curr], len, run, cmp);\
+		len = run;\
+	}\
+	run_stack[stack_curr].start = curr;\
+	run_stack[stack_curr++].length = len;\
+	curr += len;\
+	if (curr == (ssize_t)size) {\
+		/* finish up */ \
+		while (stack_curr > 1) { \
+			merge(dst, run_stack, stack_curr, store); \
+			run_stack[stack_curr - 2].length += run_stack[stack_curr - 1].length; \
+			stack_curr--; \
+		} \
+		if (store->storage != NULL) {\
+			git__free(store->storage);\
+			store->storage = NULL;\
+		}\
+		return;\
+	}\
+}\
+while (0)
+
+void git__tsort(void **dst, size_t size, cmp_ptr_t cmp)
+{
+	struct tsort_store _store, *store = &_store;
+	struct tsort_run run_stack[128];
+
+	ssize_t stack_curr = 0;
+	ssize_t len, run;
+	ssize_t curr = 0;
+	ssize_t minrun;
+
+	if (size < 64) {
+		bisort(dst, 1, size, cmp);
+		return;
+	}
+
+	/* compute the minimum run length */
+	minrun = (ssize_t)compute_minrun(size);
+
+	/* temporary storage for merges */
+	store->alloc = 0;
+	store->storage = NULL;
+	store->cmp = cmp;
+
+	PUSH_NEXT();
+	PUSH_NEXT();
+	PUSH_NEXT();
+
+	while (1) {
+		if (!check_invariant(run_stack, stack_curr)) {
+			stack_curr = collapse(dst, run_stack, stack_curr, store, size);
+			continue;
+		}
+
+		PUSH_NEXT();
+	}
+}
diff --git a/src/unix/map.c b/src/unix/map.c
index 1613152a0..65f4ac91c 100644
--- a/src/unix/map.c
+++ b/src/unix/map.c
@@ -1,3 +1,12 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include <git2/common.h>
+
+#ifndef GIT_WIN32
 
 #include "map.h"
 #ifndef __amigaos4__
@@ -5,19 +14,13 @@
 #endif
 #include <errno.h>
 
-
-int git__mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset)
+int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset)
 {
 #ifndef __amigaos4__
 	int mprot = 0;
 	int mflag = 0;
 
-	assert((out != NULL) && (len > 0));
-
-	if ((out == NULL) || (len == 0)) {
-		errno = EINVAL;
-		return git__throw(GIT_ERROR, "Failed to mmap. No map or zero length");
-	}
+	GIT_MMAP_VALIDATE(out, len, prot, flags);
 
 	out->data = NULL;
 	out->len = 0;
@@ -26,40 +29,31 @@ int git__mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t o
 		mprot = PROT_WRITE;
 	else if (prot & GIT_PROT_READ)
 		mprot = PROT_READ;
-	else {
-		errno = EINVAL;
-		return git__throw(GIT_ERROR, "Failed to mmap. Invalid protection parameters");
-	}
 
 	if ((flags & GIT_MAP_TYPE) == GIT_MAP_SHARED)
 		mflag = MAP_SHARED;
 	else if ((flags & GIT_MAP_TYPE) == GIT_MAP_PRIVATE)
 		mflag = MAP_PRIVATE;
 
-	if (flags & GIT_MAP_FIXED) {
-		errno = EINVAL;
-		return git__throw(GIT_ERROR, "Failed to mmap. FIXED not set");
+	out->data = mmap(NULL, len, mprot, mflag, fd, offset);
+	if (!out->data || out->data == MAP_FAILED) {
+		giterr_set(GITERR_OS, "Failed to mmap. Could not write data");
+		return -1;
 	}
 
-	out->data = mmap(NULL, len, mprot, mflag, fd, offset);
-	if (!out->data || out->data == MAP_FAILED)
-		return git__throw(GIT_EOSERR, "Failed to mmap. Could not write data");
 	out->len = len;
 #endif
-	return GIT_SUCCESS;
+	return 0;
 }
 
-int git__munmap(git_map *map)
+int p_munmap(git_map *map)
 {
 #ifndef __amigaos4__
 	assert(map != NULL);
-
-	if (!map)
-		return git__throw(GIT_ERROR, "Failed to munmap. Map does not exist");
-
 	munmap(map->data, map->len);
 #endif
-	return GIT_SUCCESS;
+	return 0;
 }
 
+#endif
 
diff --git a/src/unix/posix.h b/src/unix/posix.h
new file mode 100644
index 000000000..48b492941
--- /dev/null
+++ b/src/unix/posix.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_posix__w32_h__
+#define INCLUDE_posix__w32_h__
+
+#ifndef __sun
+#	include <fnmatch.h>
+#	define p_fnmatch(p, s, f) fnmatch(p, s, f)
+#else
+#	include "compat/fnmatch.h"
+#endif
+
+#include <stdio.h>
+
+#define p_lstat(p,b) lstat(p,b)
+#define p_readlink(a, b, c) readlink(a, b, c)
+#define p_link(o,n) link(o, n)
+#define p_unlink(p) unlink(p)
+#define p_mkdir(p,m) mkdir(p, m)
+#define p_fsync(fd) fsync(fd)
+#define p_realpath(p, po) realpath(p, po)
+#define p_vsnprintf(b, c, f, a) vsnprintf(b, c, f, a)
+#define p_snprintf(b, c, f, ...) snprintf(b, c, f, __VA_ARGS__)
+#define p_mkstemp(p) mkstemp(p)
+#define p_setenv(n,v,o) setenv(n,v,o)
+
+#endif
diff --git a/src/util.c b/src/util.c
index 560c40dbb..3093cd767 100644
--- a/src/util.c
+++ b/src/util.c
@@ -1,22 +1,69 @@
-#define GIT__NO_HIDE_MALLOC
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include <git2.h>
 #include "common.h"
 #include <stdarg.h>
 #include <stdio.h>
 #include <ctype.h>
+#include "posix.h"
+
+#ifdef _MSC_VER
+# include <Shlwapi.h>
+#endif
+
+void git_libgit2_version(int *major, int *minor, int *rev)
+{
+	*major = LIBGIT2_VER_MAJOR;
+	*minor = LIBGIT2_VER_MINOR;
+	*rev = LIBGIT2_VER_REVISION;
+}
 
 void git_strarray_free(git_strarray *array)
 {
 	size_t i;
 	for (i = 0; i < array->count; ++i)
-		free(array->strings[i]);
+		git__free(array->strings[i]);
+
+	git__free(array->strings);
+}
+
+int git_strarray_copy(git_strarray *tgt, const git_strarray *src)
+{
+	size_t i;
+
+	assert(tgt && src);
+
+	memset(tgt, 0, sizeof(*tgt));
+
+	if (!src->count)
+		return 0;
 
-	free(array->strings);
+	tgt->strings = git__calloc(src->count, sizeof(char *));
+	GITERR_CHECK_ALLOC(tgt->strings);
+
+	for (i = 0; i < src->count; ++i) {
+		tgt->strings[tgt->count] = git__strdup(src->strings[i]);
+
+		if (!tgt->strings[tgt->count]) {
+			git_strarray_free(tgt);
+			memset(tgt, 0, sizeof(*tgt));
+			return -1;
+		}
+
+		tgt->count++;
+	}
+
+	return 0;
 }
 
-int git__strtol32(long *result, const char *nptr, const char **endptr, int base)
+int git__strtol64(int64_t *result, const char *nptr, const char **endptr, int base)
 {
 	const char *p;
-	long n, nn;
+	int64_t n, nn;
 	int c, ovfl, v, neg, ndig;
 
 	p = nptr;
@@ -28,7 +75,7 @@ int git__strtol32(long *result, const char *nptr, const char **endptr, int base)
 	/*
 	 * White space
 	 */
-	while (isspace(*p))
+	while (git__isspace(*p))
 		p++;
 
 	/*
@@ -78,35 +125,46 @@ int git__strtol32(long *result, const char *nptr, const char **endptr, int base)
 	}
 
 Return:
-	if (ndig == 0)
-		return git__throw(GIT_ENOTNUM, "Failed to convert string to long. Not a number");
+	if (ndig == 0) {
+		giterr_set(GITERR_INVALID, "Failed to convert string to long. Not a number");
+		return -1;
+	}
 
 	if (endptr)
 		*endptr = p;
 
-	if (ovfl)
-		return git__throw(GIT_EOVERFLOW, "Failed to convert string to long. Overflow error");
+	if (ovfl) {
+		giterr_set(GITERR_INVALID, "Failed to convert string to long. Overflow error");
+		return -1;
+	}
 
 	*result = neg ? -n : n;
-	return GIT_SUCCESS;
+	return 0;
 }
 
-int git__fmt(char *buf, size_t buf_sz, const char *fmt, ...)
+int git__strtol32(int32_t *result, const char *nptr, const char **endptr, int base)
 {
-	va_list va;
-	int r;
-
-	va_start(va, fmt);
-	r = vsnprintf(buf, buf_sz, fmt, va);
-	va_end(va);
-	if (r < 0 || ((size_t) r) >= buf_sz)
-		return git__throw(GIT_ERROR, "Failed to format string");
-	return r;
+	int error;
+	int32_t tmp_int;
+	int64_t tmp_long;
+
+	if ((error = git__strtol64(&tmp_long, nptr, endptr, base)) < 0)
+		return error;
+
+	tmp_int = tmp_long & 0xFFFFFFFF;
+	if (tmp_int != tmp_long) {
+		giterr_set(GITERR_INVALID, "Failed to convert. '%s' is too large", nptr);
+		return -1;
+	}
+
+	*result = tmp_int;
+
+	return error;
 }
 
-void git__strntolower(char *str, int len)
+void git__strntolower(char *str, size_t len)
 {
-	int i;
+	size_t i;
 
 	for (i = 0; i < len; ++i) {
 		str[i] = (char) tolower(str[i]);
@@ -121,7 +179,7 @@ void git__strtolower(char *str)
 int git__prefixcmp(const char *str, const char *prefix)
 {
 	for (;;) {
-		char p = *(prefix++), s;
+		unsigned char p = *(prefix++), s;
 		if (!p)
 			return 0;
 		if ((s = *(str++)) != p)
@@ -138,205 +196,6 @@ int git__suffixcmp(const char *str, const char *suffix)
 	return strcmp(str + (a - b), suffix);
 }
 
-/*
- * Based on the Android implementation, BSD licensed.
- * Check http://android.git.kernel.org/
- */
-int git__basename_r(char *buffer, size_t bufflen, const char *path)
-{
-	const char *endp, *startp;
-	int len, result;
-
-	/* Empty or NULL string gets treated as "." */
-	if (path == NULL || *path == '\0') {
-		startp  = ".";
-		len     = 1;
-		goto Exit;
-	}
-
-	/* Strip trailing slashes */
-	endp = path + strlen(path) - 1;
-	while (endp > path && *endp == '/')
-		endp--;
-
-	/* All slashes becomes "/" */
-	if (endp == path && *endp == '/') {
-		startp = "/";
-		len    = 1;
-		goto Exit;
-	}
-
-	/* Find the start of the base */
-	startp = endp;
-	while (startp > path && *(startp - 1) != '/')
-		startp--;
-
-	len = endp - startp +1;
-
-Exit:
-	result = len;
-	if (buffer == NULL) {
-		return result;
-	}
-	if (len > (int)bufflen-1) {
-		len    = (int)bufflen-1;
-		result = GIT_ENOMEM;
-	}
-
-	if (len >= 0) {
-		memmove(buffer, startp, len);
-		buffer[len] = 0;
-	}
-	return result;
-}
-
-/*
- * Based on the Android implementation, BSD licensed.
- * Check http://android.git.kernel.org/
- */
-int git__dirname_r(char *buffer, size_t bufflen, const char *path)
-{
-    const char *endp;
-    int result, len;
-
-    /* Empty or NULL string gets treated as "." */
-    if (path == NULL || *path == '\0') {
-        path = ".";
-        len  = 1;
-        goto Exit;
-    }
-
-    /* Strip trailing slashes */
-    endp = path + strlen(path) - 1;
-    while (endp > path && *endp == '/')
-        endp--;
-
-    /* Find the start of the dir */
-    while (endp > path && *endp != '/')
-        endp--;
-
-    /* Either the dir is "/" or there are no slashes */
-    if (endp == path) {
-        path = (*endp == '/') ? "/" : ".";
-        len  = 1;
-        goto Exit;
-    }
-
-    do {
-        endp--;
-    } while (endp > path && *endp == '/');
-
-    len = endp - path +1;
-
-Exit:
-    result = len;
-    if (len+1 > GIT_PATH_MAX) {
-        return GIT_ENOMEM;
-    }
-    if (buffer == NULL)
-        return result;
-
-    if (len > (int)bufflen-1) {
-        len    = (int)bufflen-1;
-        result = GIT_ENOMEM;
-    }
-
-    if (len >= 0) {
-        memmove(buffer, path, len);
-        buffer[len] = 0;
-    }
-    return result;
-}
-
-
-char *git__dirname(const char *path)
-{
-    char *dname = NULL;
-    int len;
-
-	len = (path ? strlen(path) : 0) + 2;
-	dname = (char *)git__malloc(len);
-	if (dname == NULL)
-		return NULL;
-
-    if (git__dirname_r(dname, len, path) < GIT_SUCCESS) {
-		free(dname);
-		return NULL;
-	}
-
-    return dname;
-}
-
-char *git__basename(const char *path)
-{
-    char *bname = NULL;
-    int len;
-
-	len = (path ? strlen(path) : 0) + 2;
-	bname = (char *)git__malloc(len);
-	if (bname == NULL)
-		return NULL;
-
-    if (git__basename_r(bname, len, path) < GIT_SUCCESS) {
-		free(bname);
-		return NULL;
-	}
-
-    return bname;
-}
-
-
-const char *git__topdir(const char *path)
-{
-	size_t len;
-	int i;
-
-	assert(path);
-	len = strlen(path);
-
-	if (!len || path[len - 1] != '/')
-		return NULL;
-
-	for (i = len - 2; i >= 0; --i)
-		if (path[i] == '/')
-			break;
-
-	return &path[i + 1];
-}
-
-void git__joinpath_n(char *buffer_out, int count, ...)
-{
-	va_list ap;
-	int i;
-	char *buffer_start = buffer_out;
-
-	va_start(ap, count);
-	for (i = 0; i < count; ++i) {
-		const char *path;
-		int len;
-
-		path = va_arg(ap, const char *);
-
-		assert((i == 0) || path != buffer_start);
-
-		if (i > 0 && *path == '/' && buffer_out > buffer_start && buffer_out[-1] == '/')
-			path++;
-
-		if (!*path)
-			continue;
-
-		len = strlen(path);
-		memmove(buffer_out, path, len);
-		buffer_out = buffer_out + len;
-
-		if (i < count - 1 && buffer_out[-1] != '/')
-			*buffer_out++ = '/';
-	}
-	va_end(ap);
-
-	*buffer_out = '\0';
-}
-
 char *git__strtok(char **end, const char *sep)
 {
 	char *ptr = *end;
@@ -393,7 +252,7 @@ void git__hexdump(const char *buffer, size_t len)
 			printf("%02X ", (unsigned char)*line & 0xFF);
 
 		for (j = 0; j < (LINE_WIDTH - last_line); ++j)
-			printf("   ");
+			printf("	");
 
 		printf("| ");
 
@@ -419,22 +278,22 @@ uint32_t git__hash(const void *key, int len, unsigned int seed)
 	while(len >= 4) {
 		uint32_t k = *(uint32_t *)data;
 
-		k *= m; 
-		k ^= k >> r; 
-		k *= m; 
-		
-		h *= m; 
+		k *= m;
+		k ^= k >> r;
+		k *= m;
+
+		h *= m;
 		h ^= k;
 
 		data += 4;
 		len -= 4;
 	}
-	
+
 	switch(len) {
 	case 3: h ^= data[2] << 16;
 	case 2: h ^= data[1] << 8;
 	case 1: h ^= data[0];
-	        h *= m;
+			h *= m;
 	};
 
 	h ^= h >> 13;
@@ -442,7 +301,7 @@ uint32_t git__hash(const void *key, int len, unsigned int seed)
 	h ^= h >> 15;
 
 	return h;
-} 
+}
 #else
 /*
 	Cross-platform version of Murmurhash3
@@ -455,13 +314,13 @@ uint32_t git__hash(const void *key, int len, uint32_t seed)
 {
 
 #define MURMUR_BLOCK() {\
-    k1 *= c1; \
-    k1  = git__rotl(k1,11);\
-    k1 *= c2;\
-    h1 ^= k1;\
-    h1 = h1*3 + 0x52dce729;\
-    c1 = c1*5 + 0x7b7d159c;\
-    c2 = c2*5 + 0x6bce6396;\
+	k1 *= c1; \
+	k1 = git__rotl(k1,11);\
+	k1 *= c2;\
+	h1 ^= k1;\
+	h1 = h1*3 + 0x52dce729;\
+	c1 = c1*5 + 0x7b7d159c;\
+	c2 = c2*5 + 0x6bce6396;\
 }
 
 	const uint8_t *data = (const uint8_t*)key;
@@ -500,5 +359,79 @@ uint32_t git__hash(const void *key, int len, uint32_t seed)
 	h1 ^= h1 >> 16;
 
 	return h1;
-} 
+}
 #endif
+
+/**
+ * A modified `bsearch` from the BSD glibc.
+ *
+ * Copyright (c) 1990 Regents of the University of California.
+ * All rights reserved.
+ */
+int git__bsearch(
+	void **array,
+	size_t array_len,
+	const void *key,
+	int (*compare)(const void *, const void *),
+	size_t *position)
+{
+	unsigned int lim;
+	int cmp = -1;
+	void **part, **base = array;
+
+	for (lim = (unsigned int)array_len; lim != 0; lim >>= 1) {
+		part = base + (lim >> 1);
+		cmp = (*compare)(key, *part);
+		if (cmp == 0) {
+			base = part;
+			break;
+		}
+		if (cmp > 0) { /* key > p; take right partition */
+			base = part + 1;
+			lim--;
+		} /* else take left partition */
+	}
+
+	if (position)
+		*position = (base - array);
+
+	return (cmp == 0) ? 0 : -1;
+}
+
+/**
+ * A strcmp wrapper
+ * 
+ * We don't want direct pointers to the CRT on Windows, we may
+ * get stdcall conflicts.
+ */
+int git__strcmp_cb(const void *a, const void *b)
+{
+	const char *stra = (const char *)a;
+	const char *strb = (const char *)b;
+
+	return strcmp(stra, strb);
+}
+
+int git__parse_bool(int *out, const char *value)
+{
+	/* A missing value means true */
+	if (value == NULL) {
+		*out = 1;
+		return 0;
+	}
+
+	if (!strcasecmp(value, "true") ||
+		!strcasecmp(value, "yes") ||
+		!strcasecmp(value, "on")) {
+		*out = 1;
+		return 0;
+	}
+	if (!strcasecmp(value, "false") ||
+		!strcasecmp(value, "no") ||
+		!strcasecmp(value, "off")) {
+		*out = 0;
+		return 0;
+	}
+
+	return -1;
+}
diff --git a/src/util.h b/src/util.h
index 72e3a9c68..c6851ac7e 100644
--- a/src/util.h
+++ b/src/util.h
@@ -1,11 +1,20 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_util_h__
 #define INCLUDE_util_h__
 
 #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0]))
-#define bitsizeof(x)  (CHAR_BIT * sizeof(x))
+#define bitsizeof(x) (CHAR_BIT * sizeof(x))
 #define MSB(x, bits) ((x) & (~0ULL << (bitsizeof(x) - (bits))))
+#ifndef min
+# define min(a,b) ((a) < (b) ? (a) : (b))
+#endif
 
-/* 
+/*
  * Custom memory allocation wrappers
  * that set error code and error message
  * on allocation failure
@@ -13,24 +22,21 @@
 GIT_INLINE(void *) git__malloc(size_t len)
 {
 	void *ptr = malloc(len);
-	if (!ptr)
-		git__throw(GIT_ENOMEM, "Out of memory. Failed to allocate %d bytes.", (int)len);
+	if (!ptr) giterr_set_oom();
 	return ptr;
 }
 
 GIT_INLINE(void *) git__calloc(size_t nelem, size_t elsize)
 {
 	void *ptr = calloc(nelem, elsize);
-	if (!ptr)
-		git__throw(GIT_ENOMEM, "Out of memory. Failed to allocate %d bytes.", (int)elsize);
+	if (!ptr) giterr_set_oom();
 	return ptr;
 }
 
 GIT_INLINE(char *) git__strdup(const char *str)
 {
 	char *ptr = strdup(str);
-	if (!ptr)
-		git__throw(GIT_ENOMEM, "Out of memory. Failed to duplicate string");
+	if (!ptr) giterr_set_oom();
 	return ptr;
 }
 
@@ -43,12 +49,14 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n)
 	if (n < length)
 		length = n;
 
-	ptr = malloc(length + 1);
-	if (!ptr)
-		git__throw(GIT_ENOMEM, "Out of memory. Failed to duplicate string");
+	ptr = (char*)malloc(length + 1);
+	if (!ptr) {
+		giterr_set_oom();
+		return NULL;
+	}
 
 	memcpy(ptr, str, length);
-	ptr[length] = 0;
+	ptr[length] = '\0';
 
 	return ptr;
 }
@@ -56,75 +64,21 @@ GIT_INLINE(char *) git__strndup(const char *str, size_t n)
 GIT_INLINE(void *) git__realloc(void *ptr, size_t size)
 {
 	void *new_ptr = realloc(ptr, size);
-	if (!new_ptr)
-		git__throw(GIT_ENOMEM, "Out of memory. Failed to allocate %d bytes.", (int)size);
+	if (!new_ptr) giterr_set_oom();
 	return new_ptr;
 }
 
-extern int git__fmt(char *, size_t, const char *, ...)
-	GIT_FORMAT_PRINTF(3, 4);
+#define git__free(ptr) free(ptr)
+
 extern int git__prefixcmp(const char *str, const char *prefix);
 extern int git__suffixcmp(const char *str, const char *suffix);
 
-extern int git__strtol32(long *n, const char *buff, const char **end_buf, int base);
-
-/*
- * The dirname() function shall take a pointer to a character string
- * that contains a pathname, and return a pointer to a string that is a
- * pathname of the parent directory of that file. Trailing '/' characters
- * in the path are not counted as part of the path.
- *
- * If path does not contain a '/', then dirname() shall return a pointer to
- * the string ".". If path is a null pointer or points to an empty string,
- * dirname() shall return a pointer to the string "." .
- *
- * The `git__dirname` implementation is thread safe. The returned 
- * string must be manually free'd.
- *
- * The `git__dirname_r` implementation expects a string allocated
- * by the user with big enough size.
- */
-extern char *git__dirname(const char *path);
-extern int git__dirname_r(char *buffer, size_t bufflen, const char *path);
-
-/*
- * This function returns the basename of the file, which is the last
- * part of its full name given by fname, with the drive letter and
- * leading directories stripped off. For example, the basename of
- * c:/foo/bar/file.ext is file.ext, and the basename of a:foo is foo.
- *
- * Trailing slashes and backslashes are significant: the basename of
- * c:/foo/bar/ is an empty string after the rightmost slash.
- *
- * The `git__basename` implementation is thread safe. The returned 
- * string must be manually free'd.
- *
- * The `git__basename_r` implementation expects a string allocated
- * by the user with big enough size.
- */
-extern char *git__basename(const char *path);
-extern int git__basename_r(char *buffer, size_t bufflen, const char *path);
-
-extern const char *git__topdir(const char *path);
-
-/**
- * Join two paths together. Takes care of properly fixing the
- * middle slashes and everything
- *
- * The paths are joined together into buffer_out; this is expected
- * to be an user allocated buffer of `GIT_PATH_MAX` size 
- */
-extern void git__joinpath_n(char *buffer_out, int npath, ...);
-
-GIT_INLINE(void) git__joinpath(char *buffer_out, const char *path_a, const char *path_b)
-{
-	git__joinpath_n(buffer_out, 2, path_a, path_b);
-}
+extern int git__strtol32(int32_t *n, const char *buff, const char **end_buf, int base);
+extern int git__strtol64(int64_t *n, const char *buff, const char **end_buf, int base);
 
 extern void git__hexdump(const char *buffer, size_t n);
 extern uint32_t git__hash(const void *key, int len, uint32_t seed);
 
-
 /** @return true if p fits into the range of a size_t */
 GIT_INLINE(int) git__is_sizet(git_off_t p)
 {
@@ -141,30 +95,132 @@ GIT_INLINE(int) git__is_sizet(git_off_t p)
 
 extern char *git__strtok(char **end, const char *sep);
 
-extern void git__strntolower(char *str, int len);
+extern void git__strntolower(char *str, size_t len);
 extern void git__strtolower(char *str);
 
-#define STRLEN(str) (sizeof(str) - 1)
+GIT_INLINE(const char *) git__next_line(const char *s)
+{
+	while (*s && *s != '\n') s++;
+	while (*s == '\n' || *s == '\r') s++;
+	return s;
+}
+
+extern void git__tsort(void **dst, size_t size, int (*cmp)(const void *, const void *));
+
+/**
+ * @param position If non-NULL, this will be set to the position where the
+ * 		element is or would be inserted if not found.
+ * @return pos (>=0) if found or -1 if not found
+ */
+extern int git__bsearch(
+	void **array,
+	size_t array_len,
+	const void *key,
+	int (*compare)(const void *, const void *),
+	size_t *position);
 
-#define GIT_OID_LINE_LENGTH(header) (STRLEN(header) + 1 + GIT_OID_HEXSZ + 1)
+extern int git__strcmp_cb(const void *a, const void *b);
+
+typedef struct {
+	short refcount;
+	void *owner;
+} git_refcount;
+
+typedef void (*git_refcount_freeptr)(void *r);
+
+#define GIT_REFCOUNT_INC(r) { \
+	((git_refcount *)(r))->refcount++; \
+}
+
+#define GIT_REFCOUNT_DEC(_r, do_free) { \
+	git_refcount *r = (git_refcount *)(_r); \
+	r->refcount--; \
+	if (r->refcount <= 0 && r->owner == NULL) { do_free(_r); } \
+}
+
+#define GIT_REFCOUNT_OWN(r, o) { \
+	((git_refcount *)(r))->owner = o; \
+}
+
+#define GIT_REFCOUNT_OWNER(r) (((git_refcount *)(r))->owner)
+
+static signed char from_hex[] = {
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 00 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 10 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 20 */
+ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, /* 30 */
+-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 40 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 50 */
+-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 60 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 70 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 80 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* 90 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* a0 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* b0 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* c0 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* d0 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* e0 */
+-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* f0 */
+};
+
+GIT_INLINE(int) git__fromhex(char h)
+{
+	return from_hex[(unsigned char) h];
+}
+
+GIT_INLINE(int) git__ishex(const char *str)
+{
+	unsigned i;
+	for (i=0; i<strlen(str); i++)
+		if (git__fromhex(str[i]) < 0)
+			return 0;
+	return 1;
+}
+
+GIT_INLINE(size_t) git__size_t_bitmask(size_t v)
+{
+	v--;
+	v |= v >> 1;
+	v |= v >> 2;
+	v |= v >> 4;
+	v |= v >> 8;
+	v |= v >> 16;
+
+	return v;
+}
+
+GIT_INLINE(size_t) git__size_t_powerof2(size_t v)
+{
+	return git__size_t_bitmask(v) + 1;
+}
+
+GIT_INLINE(bool) git__isupper(int c)
+{
+    return (c >= 'A' && c <= 'Z');
+}
+
+GIT_INLINE(bool) git__isalpha(int c)
+{
+    return ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'));
+}
+
+GIT_INLINE(bool) git__isspace(int c)
+{
+    return (c == ' ' || c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == '\v');
+}
+
+GIT_INLINE(bool) git__iswildcard(int c)
+{
+	return (c == '*' || c == '?' || c == '[');
+}
 
 /*
- * Realloc the buffer pointed at by variable 'x' so that it can hold
- * at least 'nr' entries; the number of entries currently allocated
- * is 'alloc', using the standard growing factor alloc_nr() macro.
+ * Parse a string value as a boolean, just like Core Git
+ * does.
  *
- * DO NOT USE any expression with side-effect for 'x' or 'alloc'.
+ * Valid values for true are: 'true', 'yes', 'on'
+ * Valid values for false are: 'false', 'no', 'off'
  */
-#define alloc_nr(x) (((x)+16)*3/2)
-#define ALLOC_GROW(x, nr, alloc) \
-	do { \
-		if ((nr) > alloc) { \
-			if (alloc_nr(alloc) < (nr)) \
-				alloc = (nr); \
-			else \
-				alloc = alloc_nr(alloc); \
-			x = xrealloc((x), alloc * sizeof(*(x))); \
-		} \
-	} while (0)
+extern int git__parse_bool(int *out, const char *value);
 
 #endif /* INCLUDE_util_h__ */
diff --git a/src/vector.c b/src/vector.c
index 1ddc26e3e..6f9aacccf 100644
--- a/src/vector.c
+++ b/src/vector.c
@@ -1,26 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "common.h"
@@ -28,7 +10,7 @@
 #include "vector.h"
 
 static const double resize_factor = 1.75;
-static const size_t minimum_size = 8;
+static const unsigned int minimum_size = 8;
 
 static int resize_vector(git_vector *v)
 {
@@ -36,18 +18,21 @@ static int resize_vector(git_vector *v)
 	if (v->_alloc_size < minimum_size)
 		v->_alloc_size = minimum_size;
 
-	v->contents = realloc(v->contents, v->_alloc_size * sizeof(void *));
-	if (v->contents == NULL)
-		return GIT_ENOMEM;
+	v->contents = git__realloc(v->contents, v->_alloc_size * sizeof(void *));
+	GITERR_CHECK_ALLOC(v->contents);
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-
 void git_vector_free(git_vector *v)
 {
 	assert(v);
-	free(v->contents);
+
+	git__free(v->contents);
+	v->contents = NULL;
+
+	v->length = 0;
+	v->_alloc_size = 0;
 }
 
 int git_vector_init(git_vector *v, unsigned int initial_size, git_vector_cmp cmp)
@@ -61,30 +46,63 @@ int git_vector_init(git_vector *v, unsigned int initial_size, git_vector_cmp cmp
 
 	v->_alloc_size = initial_size;
 	v->_cmp = cmp;
-	
+
 	v->length = 0;
 	v->sorted = 1;
 
 	v->contents = git__malloc(v->_alloc_size * sizeof(void *));
-	if (v->contents == NULL)
-		return GIT_ENOMEM;
+	GITERR_CHECK_ALLOC(v->contents);
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
 int git_vector_insert(git_vector *v, void *element)
 {
 	assert(v);
 
-	if (v->length >= v->_alloc_size) {
-		if (resize_vector(v) < 0)
-			return GIT_ENOMEM;
-	}
+	if (v->length >= v->_alloc_size &&
+		resize_vector(v) < 0)
+		return -1;
 
 	v->contents[v->length++] = element;
 	v->sorted = 0;
 
-	return GIT_SUCCESS;
+	return 0;
+}
+
+int git_vector_insert_sorted(
+	git_vector *v, void *element, int (*on_dup)(void **old, void *new))
+{
+	int result;
+	size_t pos;
+
+	assert(v && v->_cmp);
+
+	if (!v->sorted)
+		git_vector_sort(v);
+
+	if (v->length >= v->_alloc_size &&
+		resize_vector(v) < 0)
+		return -1;
+
+	/* If we find the element and have a duplicate handler callback,
+	 * invoke it.  If it returns non-zero, then cancel insert, otherwise
+	 * proceed with normal insert.
+	 */
+	if (git__bsearch(v->contents, v->length, element, v->_cmp, &pos) >= 0 &&
+		on_dup != NULL &&
+		(result = on_dup(&v->contents[pos], element)) < 0)
+		return result;
+
+	/* shift elements to the right */
+	if (pos < v->length) {
+		memmove(v->contents + pos + 1, v->contents + pos,
+		        (v->length - pos) * sizeof(void *));
+	}
+
+	v->contents[pos] = element;
+	v->length++;
+	return 0;
 }
 
 void git_vector_sort(git_vector *v)
@@ -94,30 +112,36 @@ void git_vector_sort(git_vector *v)
 	if (v->sorted || v->_cmp == NULL)
 		return;
 
-	qsort(v->contents, v->length, sizeof(void *), v->_cmp);
+	git__tsort(v->contents, v->length, v->_cmp);
 	v->sorted = 1;
 }
 
-int git_vector_bsearch2(git_vector *v, git_vector_cmp key_lookup, const void *key)
+int git_vector_bsearch3(
+	unsigned int *at_pos,
+	git_vector *v,
+	git_vector_cmp key_lookup,
+	const void *key)
 {
-	void **find;
+	int rval;
+	size_t pos;
 
 	assert(v && key && key_lookup);
 
 	/* need comparison function to sort the vector */
-	if (v->_cmp == NULL)
-		return git__throw(GIT_ENOTFOUND, "Can't sort vector. No comparison function set");
+	assert(v->_cmp != NULL);
 
 	git_vector_sort(v);
 
-	find = bsearch(key, v->contents, v->length, sizeof(void *), key_lookup);
-	if (find != NULL)
-		return (int)(find - v->contents);
+	rval = git__bsearch(v->contents, v->length, key, key_lookup, &pos);
 
-	return git__throw(GIT_ENOTFOUND, "Can't find element");
+	if (at_pos != NULL)
+		*at_pos = (unsigned int)pos;
+
+	return (rval >= 0) ? (int)pos : GIT_ENOTFOUND;
 }
 
-int git_vector_search2(git_vector *v, git_vector_cmp key_lookup, const void *key)
+int git_vector_search2(
+	git_vector *v, git_vector_cmp key_lookup, const void *key)
 {
 	unsigned int i;
 
@@ -128,7 +152,7 @@ int git_vector_search2(git_vector *v, git_vector_cmp key_lookup, const void *key
 			return i;
 	}
 
-	return git__throw(GIT_ENOTFOUND, "Can't find element");
+	return GIT_ENOTFOUND;
 }
 
 static int strict_comparison(const void *a, const void *b)
@@ -141,11 +165,6 @@ int git_vector_search(git_vector *v, const void *entry)
 	return git_vector_search2(v, v->_cmp ? v->_cmp : strict_comparison, entry);
 }
 
-int git_vector_bsearch(git_vector *v, const void *key)
-{
-	return git_vector_bsearch2(v, v->_cmp, key);
-}
-
 int git_vector_remove(git_vector *v, unsigned int idx)
 {
 	unsigned int i;
@@ -153,13 +172,39 @@ int git_vector_remove(git_vector *v, unsigned int idx)
 	assert(v);
 
 	if (idx >= v->length || v->length == 0)
-		return git__throw(GIT_ENOTFOUND, "Can't remove element. Index out of bounds");
+		return GIT_ENOTFOUND;
 
 	for (i = idx; i < v->length - 1; ++i)
 		v->contents[i] = v->contents[i + 1];
 
 	v->length--;
-	return GIT_SUCCESS;
+	return 0;
+}
+
+void git_vector_pop(git_vector *v)
+{
+	if (v->length > 0)
+		v->length--;
+}
+
+void git_vector_uniq(git_vector *v)
+{
+	git_vector_cmp cmp;
+	unsigned int i, j;
+
+	if (v->length <= 1)
+		return;
+
+	git_vector_sort(v);
+	cmp = v->_cmp ? v->_cmp : strict_comparison;
+
+	for (i = 0, j = 1 ; j < v->length; ++j)
+		if (!cmp(v->contents[i], v->contents[j]))
+			v->contents[i] = v->contents[j];
+		else
+			v->contents[++i] = v->contents[j];
+
+	v->length -= j - i - 1;
 }
 
 void git_vector_clear(git_vector *v)
@@ -169,4 +214,14 @@ void git_vector_clear(git_vector *v)
 	v->sorted = 1;
 }
 
+void git_vector_swap(git_vector *a, git_vector *b)
+{
+	git_vector t;
+
+	if (!a || !b || a == b)
+		return;
 
+	memcpy(&t, a, sizeof(t));
+	memcpy(a, b, sizeof(t));
+	memcpy(b, &t, sizeof(t));
+}
diff --git a/src/vector.h b/src/vector.h
index 256452ee5..9139db345 100644
--- a/src/vector.h
+++ b/src/vector.h
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #ifndef INCLUDE_vector_h__
 #define INCLUDE_vector_h__
 
@@ -13,24 +19,60 @@ typedef struct git_vector {
 	int sorted;
 } git_vector;
 
+#define GIT_VECTOR_INIT {0}
+
 int git_vector_init(git_vector *v, unsigned int initial_size, git_vector_cmp cmp);
 void git_vector_free(git_vector *v);
 void git_vector_clear(git_vector *v);
+void git_vector_swap(git_vector *a, git_vector *b);
+
+void git_vector_sort(git_vector *v);
 
 int git_vector_search(git_vector *v, const void *entry);
 int git_vector_search2(git_vector *v, git_vector_cmp cmp, const void *key);
 
-int git_vector_bsearch(git_vector *v, const void *entry);
-int git_vector_bsearch2(git_vector *v, git_vector_cmp cmp, const void *key);
+int git_vector_bsearch3(
+	unsigned int *at_pos, git_vector *v, git_vector_cmp cmp, const void *key);
 
-void git_vector_sort(git_vector *v);
+GIT_INLINE(int) git_vector_bsearch(git_vector *v, const void *key)
+{
+	return git_vector_bsearch3(NULL, v, v->_cmp, key);
+}
+
+GIT_INLINE(int) git_vector_bsearch2(
+	git_vector *v, git_vector_cmp cmp, const void *key)
+{
+	return git_vector_bsearch3(NULL, v, cmp, key);
+}
 
 GIT_INLINE(void *) git_vector_get(git_vector *v, unsigned int position)
 {
 	return (position < v->length) ? v->contents[position] : NULL;
 }
 
+GIT_INLINE(const void *) git_vector_get_const(const git_vector *v, unsigned int position)
+{
+	return (position < v->length) ? v->contents[position] : NULL;
+}
+
+#define GIT_VECTOR_GET(V,I) ((I) < (V)->length ? (V)->contents[(I)] : NULL)
+
+GIT_INLINE(void *) git_vector_last(git_vector *v)
+{
+	return (v->length > 0) ? git_vector_get(v, v->length - 1) : NULL;
+}
+
+#define git_vector_foreach(v, iter, elem)	\
+	for ((iter) = 0; (iter) < (v)->length && ((elem) = (v)->contents[(iter)], 1); (iter)++ )
+
+#define git_vector_rforeach(v, iter, elem)	\
+	for ((iter) = (v)->length; (iter) > 0 && ((elem) = (v)->contents[(iter)-1], 1); (iter)-- )
+
 int git_vector_insert(git_vector *v, void *element);
+int git_vector_insert_sorted(git_vector *v, void *element,
+	int (*on_dup)(void **old, void *new));
 int git_vector_remove(git_vector *v, unsigned int idx);
+void git_vector_pop(git_vector *v);
+void git_vector_uniq(git_vector *v);
 
 #endif
diff --git a/src/win32/dir.c b/src/win32/dir.c
index 069a41c3a..bc3d40fa5 100644
--- a/src/win32/dir.c
+++ b/src/win32/dir.c
@@ -1,5 +1,13 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 #define GIT__WIN32_NO_WRAP_DIR
 #include "dir.h"
+#include "utf-conv.h"
+#include "git2/windows.h"
 
 static int init_filter(char *filter, size_t n, const char *dir)
 {
@@ -19,7 +27,8 @@ static int init_filter(char *filter, size_t n, const char *dir)
 git__DIR *git__opendir(const char *dir)
 {
 	char filter[4096];
-	git__DIR *new;
+	wchar_t* filter_w = NULL;
+	git__DIR *new = NULL;
 
 	if (!dir || !init_filter(filter, sizeof(filter), dir))
 		return NULL;
@@ -28,71 +37,119 @@ git__DIR *git__opendir(const char *dir)
 	if (!new)
 		return NULL;
 
-	new->dir = git__malloc(strlen(dir)+1);
-	if (!new->dir) {
-		free(new);
-		return NULL;
-	}
-	strcpy(new->dir, dir);
+	new->dir = git__strdup(dir);
+	if (!new->dir)
+		goto fail;
+
+	filter_w = gitwin_to_utf16(filter);
+	if (!filter_w)
+		goto fail;
+
+	new->h = FindFirstFileW(filter_w, &new->f);
+	git__free(filter_w);
 
-	new->h = FindFirstFile(filter, &new->f);
 	if (new->h == INVALID_HANDLE_VALUE) {
-		free(new->dir);
-		free(new);
-		return NULL;
+		giterr_set(GITERR_OS, "Could not open directory '%s'", dir);
+		goto fail;
 	}
-	new->first = 1;
 
+	new->first = 1;
 	return new;
+
+fail:
+	git__free(new->dir);
+	git__free(new);
+	return NULL;
 }
 
-struct git__dirent *git__readdir(git__DIR *d)
+int git__readdir_ext(
+	git__DIR *d,
+	struct git__dirent *entry,
+	struct git__dirent **result,
+	int *is_dir)
 {
-	if (!d || d->h == INVALID_HANDLE_VALUE)
-		return NULL;
+	if (!d || !entry || !result || d->h == INVALID_HANDLE_VALUE)
+		return -1;
+
+	*result = NULL;
 
 	if (d->first)
 		d->first = 0;
-	else {
-		if (!FindNextFile(d->h, &d->f))
-			return NULL;
+	else if (!FindNextFileW(d->h, &d->f)) {
+		if (GetLastError() == ERROR_NO_MORE_FILES)
+			return 0;
+		giterr_set(GITERR_OS, "Could not read from directory '%s'", d->dir);
+		return -1;
 	}
 
-	if (strlen(d->f.cFileName) >= sizeof(d->entry.d_name))
-		return NULL;
+	if (wcslen(d->f.cFileName) >= sizeof(entry->d_name))
+		return -1;
+
+	entry->d_ino = 0;
+
+	if (WideCharToMultiByte(
+		gitwin_get_codepage(), 0, d->f.cFileName, -1,
+		entry->d_name, GIT_PATH_MAX, NULL, NULL) == 0)
+	{
+		giterr_set(GITERR_OS, "Could not convert filename to UTF-8");
+		return -1;
+	}
 
-	d->entry.d_ino = 0;
-	strcpy(d->entry.d_name, d->f.cFileName);
+	*result = entry;
 
-	return &d->entry;
+	if (is_dir != NULL)
+		*is_dir = ((d->f.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0);
+
+	return 0;
+}
+
+struct git__dirent *git__readdir(git__DIR *d)
+{
+	struct git__dirent *result;
+	if (git__readdir_ext(d, &d->entry, &result, NULL) < 0)
+		return NULL;
+	return result;
 }
 
 void git__rewinddir(git__DIR *d)
 {
 	char filter[4096];
+	wchar_t* filter_w;
 
-	if (d) {
-		if (d->h != INVALID_HANDLE_VALUE)
-			FindClose(d->h);
+	if (!d)
+		return;
+
+	if (d->h != INVALID_HANDLE_VALUE) {
+		FindClose(d->h);
 		d->h = INVALID_HANDLE_VALUE;
 		d->first = 0;
-		if (init_filter(filter, sizeof(filter), d->dir)) {
-			d->h = FindFirstFile(filter, &d->f);
-			if (d->h != INVALID_HANDLE_VALUE)
-				d->first = 1;
-		}
 	}
+
+	if (!init_filter(filter, sizeof(filter), d->dir) ||
+		(filter_w = gitwin_to_utf16(filter)) == NULL)
+		return;
+
+	d->h = FindFirstFileW(filter_w, &d->f);
+	git__free(filter_w);
+
+	if (d->h == INVALID_HANDLE_VALUE)
+		giterr_set(GITERR_OS, "Could not open directory '%s'", d->dir);
+	else
+		d->first = 1;
 }
 
 int git__closedir(git__DIR *d)
 {
-	if (d) {
-		if (d->h != INVALID_HANDLE_VALUE)
-			FindClose(d->h);
-		if (d->dir)
-			free(d->dir);
-		free(d);
+	if (!d)
+		return 0;
+
+	if (d->h != INVALID_HANDLE_VALUE) {
+		FindClose(d->h);
+		d->h = INVALID_HANDLE_VALUE;
 	}
+	git__free(d->dir);
+	d->dir = NULL;
+	git__free(d);
 	return 0;
 }
 
diff --git a/src/win32/dir.h b/src/win32/dir.h
new file mode 100644
index 000000000..c816d79bb
--- /dev/null
+++ b/src/win32/dir.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_dir_h__
+#define INCLUDE_dir_h__
+
+#include "common.h"
+
+struct git__dirent {
+	int d_ino;
+	char d_name[261];
+};
+
+typedef struct {
+	HANDLE h;
+	WIN32_FIND_DATAW f;
+	struct git__dirent entry;
+	char *dir;
+	int first;
+} git__DIR;
+
+extern git__DIR *git__opendir(const char *);
+extern struct git__dirent *git__readdir(git__DIR *);
+extern int git__readdir_ext(
+	git__DIR *, struct git__dirent *, struct git__dirent **, int *);
+extern void git__rewinddir(git__DIR *);
+extern int git__closedir(git__DIR *);
+
+# ifndef GIT__WIN32_NO_WRAP_DIR
+#	define dirent git__dirent
+#	define DIR git__DIR
+#	define opendir	git__opendir
+#	define readdir	git__readdir
+#   define readdir_r(d,e,r) git__readdir_ext((d),(e),(r),NULL)
+#	define rewinddir git__rewinddir
+#	define closedir git__closedir
+# endif
+
+#endif /* INCLUDE_dir_h__ */
diff --git a/src/win32/fileops.c b/src/win32/fileops.c
deleted file mode 100644
index d435e706e..000000000
--- a/src/win32/fileops.c
+++ /dev/null
@@ -1,41 +0,0 @@
-#define GIT__WIN32_NO_HIDE_FILEOPS
-#include "fileops.h"
-#include <errno.h>
-
-int git__unlink(const char *path)
-{
-	chmod(path, 0666);
-	return unlink(path);
-}
-
-int git__mkstemp(char *template)
-{
-	char *file = mktemp(template);
-	if (file == NULL)
-		return -1;
-	return open(file, O_RDWR | O_CREAT | O_BINARY, 0600);
-}
-
-int git__fsync(int fd)
-{
-	HANDLE fh = (HANDLE)_get_osfhandle(fd);
-
-	if (fh == INVALID_HANDLE_VALUE) {
-		errno = EBADF;
-		return -1;
-	}
-
-	if (!FlushFileBuffers(fh)) {
-		DWORD code = GetLastError();
-
-		if (code == ERROR_INVALID_HANDLE)
-			errno = EINVAL;
-		else
-			errno = EIO;
-
-		return -1;
-	}
-
-	return 0;
-}
-
diff --git a/src/win32/git2.rc b/src/win32/git2.rc
new file mode 100644
index 000000000..3a65c0a0f
--- /dev/null
+++ b/src/win32/git2.rc
@@ -0,0 +1,42 @@
+#include <winver.h>
+#include "../../include/git2/version.h"
+
+#ifndef INCLUDE_LIB
+#define LIBGIT2_FILENAME "git2.dll"
+#else
+#define LIBGIT2_FILENAME "libgit2.dll"
+#endif
+
+VS_VERSION_INFO		VERSIONINFO	MOVEABLE IMPURE LOADONCALL DISCARDABLE
+  FILEVERSION		LIBGIT2_VER_MAJOR,LIBGIT2_VER_MINOR,LIBGIT2_VER_REVISION,0
+  PRODUCTVERSION	LIBGIT2_VER_MAJOR,LIBGIT2_VER_MINOR,LIBGIT2_VER_REVISION,0
+  FILEFLAGSMASK		VS_FFI_FILEFLAGSMASK
+#ifdef _DEBUG
+  FILEFLAGS		1
+#else
+  FILEFLAGS		0
+#endif
+  FILEOS		VOS__WINDOWS32
+  FILETYPE		VFT_DLL
+  FILESUBTYPE		0	// not used
+BEGIN
+  BLOCK "StringFileInfo"
+  BEGIN
+    BLOCK "040904E4"
+    //language ID = U.S. English, char set = Windows, Multilingual
+    BEGIN
+      VALUE "FileDescription",	"libgit2 - the Git linkable library\0"
+      VALUE "FileVersion",	LIBGIT2_VERSION "\0"
+      VALUE "InternalName",	LIBGIT2_FILENAME "\0"
+      VALUE "LegalCopyright",	"Copyright (C) 2009-2012 the libgit2 contributors\0"
+      VALUE "OriginalFilename",	LIBGIT2_FILENAME "\0"
+      VALUE "ProductName",	"libgit2\0"
+      VALUE "ProductVersion",	LIBGIT2_VERSION "\0"
+      VALUE "Comments",		"For more information visit http://libgit2.github.com/\0"
+    END
+  END
+  BLOCK "VarFileInfo"
+  BEGIN
+    VALUE "Translation", 0x0409, 1252
+  END
+END
diff --git a/src/win32/map.c b/src/win32/map.c
index c7a39fcf6..f730120cc 100644
--- a/src/win32/map.c
+++ b/src/win32/map.c
@@ -1,3 +1,9 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
 
 #include "map.h"
 #include <errno.h>
@@ -16,23 +22,18 @@ static DWORD get_page_size(void)
 	return page_size;
 }
 
-int git__mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset)
+int p_mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t offset)
 {
 	HANDLE fh = (HANDLE)_get_osfhandle(fd);
 	DWORD page_size = get_page_size();
 	DWORD fmap_prot = 0;
 	DWORD view_prot = 0;
 	DWORD off_low = 0;
-	DWORD off_hi  = 0;
+	DWORD off_hi = 0;
 	git_off_t page_start;
 	git_off_t page_offset;
 
-	assert((out != NULL) && (len > 0));
-
-	if ((out == NULL) || (len == 0)) {
-		errno = EINVAL;
-		return git__throw(GIT_ERROR, "Failed to mmap. No map or zero length");
-	}
+	GIT_MMAP_VALIDATE(out, len, prot, flags);
 
 	out->data = NULL;
 	out->len = 0;
@@ -40,86 +41,75 @@ int git__mmap(git_map *out, size_t len, int prot, int flags, int fd, git_off_t o
 
 	if (fh == INVALID_HANDLE_VALUE) {
 		errno = EBADF;
-		return git__throw(GIT_ERROR, "Failed to mmap. Invalid handle value");
+		giterr_set(GITERR_OS, "Failed to mmap. Invalid handle value");
+		return -1;
 	}
 
 	if (prot & GIT_PROT_WRITE)
 		fmap_prot |= PAGE_READWRITE;
 	else if (prot & GIT_PROT_READ)
 		fmap_prot |= PAGE_READONLY;
-	else {
-		errno = EINVAL;
-		return git__throw(GIT_ERROR, "Failed to mmap. Invalid protection parameters");
-	}
 
 	if (prot & GIT_PROT_WRITE)
 		view_prot |= FILE_MAP_WRITE;
 	if (prot & GIT_PROT_READ)
 		view_prot |= FILE_MAP_READ;
 
-	if (flags & GIT_MAP_FIXED) {
-		errno = EINVAL;
-		return git__throw(GIT_ERROR, "Failed to mmap. FIXED not set");
-	}
-
 	page_start = (offset / page_size) * page_size;
 	page_offset = offset - page_start;
 
-	if (page_offset != 0) {  /* offset must be multiple of page size */
+	if (page_offset != 0) { /* offset must be multiple of page size */
 		errno = EINVAL;
-		return git__throw(GIT_ERROR, "Failed to mmap. Offset must be multiple of page size");
+		giterr_set(GITERR_OS, "Failed to mmap. Offset must be multiple of page size");
+		return -1;
 	}
 
 	out->fmh = CreateFileMapping(fh, NULL, fmap_prot, 0, 0, NULL);
 	if (!out->fmh || out->fmh == INVALID_HANDLE_VALUE) {
-		/* errno = ? */
+		giterr_set(GITERR_OS, "Failed to mmap. Invalid handle value");
 		out->fmh = NULL;
-		return git__throw(GIT_ERROR, "Failed to mmap. Invalid handle value");
+		return -1;
 	}
 
 	assert(sizeof(git_off_t) == 8);
+
 	off_low = (DWORD)(page_start);
 	off_hi = (DWORD)(page_start >> 32);
 	out->data = MapViewOfFile(out->fmh, view_prot, off_hi, off_low, len);
 	if (!out->data) {
-		/* errno = ? */
+		giterr_set(GITERR_OS, "Failed to mmap. No data written");
 		CloseHandle(out->fmh);
 		out->fmh = NULL;
-		return git__throw(GIT_ERROR, "Failed to mmap. No data written");
+		return -1;
 	}
 	out->len = len;
 
-	return GIT_SUCCESS;
+	return 0;
 }
 
-int git__munmap(git_map *map)
+int p_munmap(git_map *map)
 {
-	assert(map != NULL);
+	int error = 0;
 
-	if (!map)
-		return git__throw(GIT_ERROR, "Failed to munmap. Map does not exist");
+	assert(map != NULL);
 
 	if (map->data) {
 		if (!UnmapViewOfFile(map->data)) {
-			/* errno = ? */
-			CloseHandle(map->fmh);
-			map->data = NULL;
-			map->fmh = NULL;
-			return git__throw(GIT_ERROR, "Failed to munmap. Could not unmap view of file");
+			giterr_set(GITERR_OS, "Failed to munmap. Could not unmap view of file");
+			error = -1;
 		}
 		map->data = NULL;
 	}
 
 	if (map->fmh) {
 		if (!CloseHandle(map->fmh)) {
-			/* errno = ? */
-			map->fmh = NULL;
-			return git__throw(GIT_ERROR, "Failed to munmap. Could not close handle");
+			giterr_set(GITERR_OS, "Failed to munmap. Could not close handle");
+			error = -1;
 		}
 		map->fmh = NULL;
 	}
 
-	return GIT_SUCCESS;
+	return error;
 }
 
 
diff --git a/src/win32/mingw-compat.h b/src/win32/mingw-compat.h
new file mode 100644
index 000000000..6200dc094
--- /dev/null
+++ b/src/win32/mingw-compat.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_mingw_compat__
+#define INCLUDE_mingw_compat__
+
+#if defined(__MINGW32__)
+
+/* use a 64-bit file offset type */
+# define lseek _lseeki64
+# define stat _stati64
+# define fstat _fstati64
+
+/* stat: file mode type testing macros */
+# define _S_IFLNK 0120000
+# define S_IFLNK _S_IFLNK
+# define S_ISLNK(m) (((m) & _S_IFMT) == _S_IFLNK)
+
+#endif
+
+#endif /* INCLUDE_mingw_compat__ */
diff --git a/src/win32/msvc-compat.h b/src/win32/msvc-compat.h
new file mode 100644
index 000000000..ccc091cd0
--- /dev/null
+++ b/src/win32/msvc-compat.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_msvc_compat__
+#define INCLUDE_msvc_compat__
+
+#if defined(_MSC_VER)
+
+/* access() mode parameter #defines	*/
+#	define F_OK 0 /* existence check */
+#	define W_OK 2 /* write mode check */
+#	define R_OK 4 /* read mode check */
+
+#	define lseek _lseeki64
+#	define stat _stat64
+#	define fstat _fstat64
+
+/* stat: file mode type testing macros */
+#	define _S_IFLNK 0120000
+#	define S_IFLNK _S_IFLNK
+#	define S_IXUSR 00100
+
+#	define S_ISDIR(m)	(((m) & _S_IFMT) == _S_IFDIR)
+#	define S_ISREG(m)	(((m) & _S_IFMT) == _S_IFREG)
+#	define S_ISFIFO(m) (((m) & _S_IFMT) == _S_IFIFO)
+#	define S_ISLNK(m) (((m) & _S_IFMT) == _S_IFLNK)
+
+#	define mode_t unsigned short
+
+/* case-insensitive string comparison */
+#	define strcasecmp	_stricmp
+#	define strncasecmp _strnicmp
+
+/* MSVC doesn't define ssize_t at all */
+typedef SSIZE_T ssize_t;
+
+#endif
+
+#endif /* INCLUDE_msvc_compat__ */
diff --git a/src/win32/posix.h b/src/win32/posix.h
new file mode 100644
index 000000000..baa4a3b4e
--- /dev/null
+++ b/src/win32/posix.h
@@ -0,0 +1,55 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_posix__w32_h__
+#define INCLUDE_posix__w32_h__
+
+#include "common.h"
+#include "compat/fnmatch.h"
+#include "utf-conv.h"
+
+GIT_INLINE(int) p_link(const char *old, const char *new)
+{
+	GIT_UNUSED(old);
+	GIT_UNUSED(new);
+	errno = ENOSYS;
+	return -1;
+}
+
+GIT_INLINE(int) p_mkdir(const char *path, mode_t mode)
+{
+	wchar_t* buf = gitwin_to_utf16(path);
+	int ret = _wmkdir(buf);
+
+	GIT_UNUSED(mode);
+
+	git__free(buf);
+	return ret;
+}
+
+extern int p_unlink(const char *path);
+extern int p_lstat(const char *file_name, struct stat *buf);
+extern int p_readlink(const char *link, char *target, size_t target_len);
+extern int p_hide_directory__w32(const char *path);
+extern char *p_realpath(const char *orig_path, char *buffer);
+extern int p_vsnprintf(char *buffer, size_t count, const char *format, va_list argptr);
+extern int p_snprintf(char *buffer, size_t count, const char *format, ...) GIT_FORMAT_PRINTF(3, 4);
+extern int p_mkstemp(char *tmp_path);
+extern int p_setenv(const char* name, const char* value, int overwrite);
+extern int p_stat(const char* path, struct stat* buf);
+extern int p_chdir(const char* path);
+extern int p_chmod(const char* path, mode_t mode);
+extern int p_rmdir(const char* path);
+extern int p_access(const char* path, mode_t mode);
+extern int p_fsync(int fd);
+extern int p_open(const char *path, int flags, ...);
+extern int p_creat(const char *path, mode_t mode);
+extern int p_getcwd(char *buffer_out, size_t size);
+extern int p_rename(const char *from, const char *to);
+extern int p_recv(GIT_SOCKET socket, void *buffer, size_t length, int flags);
+extern int p_send(GIT_SOCKET socket, const void *buffer, size_t length, int flags);
+
+#endif
diff --git a/src/win32/posix_w32.c b/src/win32/posix_w32.c
new file mode 100644
index 000000000..10de70da8
--- /dev/null
+++ b/src/win32/posix_w32.c
@@ -0,0 +1,472 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#include "../posix.h"
+#include "path.h"
+#include "utf-conv.h"
+#include <errno.h>
+#include <io.h>
+#include <fcntl.h>
+
+
+int p_unlink(const char *path)
+{
+	int ret = 0;
+	wchar_t* buf;
+
+	if ((buf = gitwin_to_utf16(path)) != NULL) {
+		_wchmod(buf, 0666);
+		ret = _wunlink(buf);
+		git__free(buf);
+	}
+
+	return ret;
+}
+
+int p_fsync(int fd)
+{
+	HANDLE fh = (HANDLE)_get_osfhandle(fd);
+
+	if (fh == INVALID_HANDLE_VALUE) {
+		errno = EBADF;
+		return -1;
+	}
+
+	if (!FlushFileBuffers(fh)) {
+		DWORD code = GetLastError();
+
+		if (code == ERROR_INVALID_HANDLE)
+			errno = EINVAL;
+		else
+			errno = EIO;
+
+		return -1;
+	}
+
+	return 0;
+}
+
+GIT_INLINE(time_t) filetime_to_time_t(const FILETIME *ft)
+{
+	long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime;
+	winTime -= 116444736000000000LL; /* Windows to Unix Epoch conversion */
+	winTime /= 10000000;		 /* Nano to seconds resolution */
+	return (time_t)winTime;
+}
+
+static int do_lstat(const char *file_name, struct stat *buf)
+{
+	WIN32_FILE_ATTRIBUTE_DATA fdata;
+	wchar_t* fbuf = gitwin_to_utf16(file_name);
+	if (!fbuf)
+		return -1;
+
+	if (GetFileAttributesExW(fbuf, GetFileExInfoStandard, &fdata)) {
+		int fMode = S_IREAD;
+
+		if (fdata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+			fMode |= S_IFDIR;
+		else
+			fMode |= S_IFREG;
+
+		if (!(fdata.dwFileAttributes & FILE_ATTRIBUTE_READONLY))
+			fMode |= S_IWRITE;
+
+		if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)
+			fMode |= S_IFLNK;
+
+		buf->st_ino = 0;
+		buf->st_gid = 0;
+		buf->st_uid = 0;
+		buf->st_nlink = 1;
+		buf->st_mode = (mode_t)fMode;
+		buf->st_size = ((git_off_t)fdata.nFileSizeHigh << 32) + fdata.nFileSizeLow;
+		buf->st_dev = buf->st_rdev = (_getdrive() - 1);
+		buf->st_atime = filetime_to_time_t(&(fdata.ftLastAccessTime));
+		buf->st_mtime = filetime_to_time_t(&(fdata.ftLastWriteTime));
+		buf->st_ctime = filetime_to_time_t(&(fdata.ftCreationTime));
+
+		git__free(fbuf);
+		return 0;
+	}
+
+	git__free(fbuf);
+	return -1;
+}
+
+int p_lstat(const char *file_name, struct stat *buf)
+{
+	int error;
+	size_t namelen;
+	char *alt_name;
+
+	if (do_lstat(file_name, buf) == 0)
+		return 0;
+
+	/* if file_name ended in a '/', Windows returned ENOENT;
+	 * try again without trailing slashes
+	 */
+	namelen = strlen(file_name);
+	if (namelen && file_name[namelen-1] != '/')
+		return -1;
+
+	while (namelen && file_name[namelen-1] == '/')
+		--namelen;
+
+	if (!namelen)
+		return -1;
+
+	alt_name = git__strndup(file_name, namelen);
+	if (!alt_name)
+		return -1;
+
+	error = do_lstat(alt_name, buf);
+
+	git__free(alt_name);
+	return error;
+}
+
+int p_readlink(const char *link, char *target, size_t target_len)
+{
+	typedef DWORD (WINAPI *fpath_func)(HANDLE, LPWSTR, DWORD, DWORD);
+	static fpath_func pGetFinalPath = NULL;
+	HANDLE hFile;
+	DWORD dwRet;
+	wchar_t* link_w;
+	wchar_t* target_w;
+	int error = 0;
+
+	assert(link && target && target_len > 0);
+
+	/*
+	 * Try to load the pointer to pGetFinalPath dynamically, because
+	 * it is not available in platforms older than Vista
+	 */
+	if (pGetFinalPath == NULL) {
+		HINSTANCE library = LoadLibrary("kernel32");
+
+		if (library != NULL)
+			pGetFinalPath = (fpath_func)GetProcAddress(library, "GetFinalPathNameByHandleW");
+
+		if (pGetFinalPath == NULL) {
+			giterr_set(GITERR_OS,
+				"'GetFinalPathNameByHandleW' is not available in this platform");
+			return -1;
+		}
+	}
+
+	link_w = gitwin_to_utf16(link);
+	GITERR_CHECK_ALLOC(link_w);
+
+	hFile = CreateFileW(link_w,			// file to open
+			GENERIC_READ,			// open for reading
+			FILE_SHARE_READ,		// share for reading
+			NULL,					// default security
+			OPEN_EXISTING,			// existing file only
+			FILE_FLAG_BACKUP_SEMANTICS, // normal file
+			NULL);					// no attr. template
+
+	git__free(link_w);
+
+	if (hFile == INVALID_HANDLE_VALUE) {
+		giterr_set(GITERR_OS, "Cannot open '%s' for reading", link);
+		return -1;
+	}
+
+	target_w = (wchar_t*)git__malloc(target_len * sizeof(wchar_t));
+	GITERR_CHECK_ALLOC(target_w);
+
+	dwRet = pGetFinalPath(hFile, target_w, (DWORD)target_len, 0x0);
+	if (dwRet == 0 ||
+		dwRet >= target_len ||
+		!WideCharToMultiByte(CP_UTF8, 0, target_w, -1, target,
+			(int)(target_len * sizeof(char)), NULL, NULL))
+		error = -1;
+
+	git__free(target_w);
+	CloseHandle(hFile);
+
+	if (error)
+		return error;
+
+	/* Skip first 4 characters if they are "\\?\" */
+	if (dwRet > 4 &&
+		target[0] == '\\' && target[1] == '\\' &&
+		target[2] == '?' && target[3] == '\\')
+	{
+		unsigned int offset = 4;
+		dwRet -= 4;
+
+		/* \??\UNC\ */
+		if (dwRet > 7 &&
+			target[4] == 'U' && target[5] == 'N' && target[6] == 'C')
+		{
+			offset += 2;
+			dwRet -= 2;
+			target[offset] = '\\';
+		}
+
+		memmove(target, target + offset, dwRet);
+	}
+
+	target[dwRet] = '\0';
+
+	return dwRet;
+}
+
+int p_open(const char *path, int flags, ...)
+{
+	int fd;
+	wchar_t* buf;
+	mode_t mode = 0;
+
+	buf = gitwin_to_utf16(path);
+	if (!buf)
+		return -1;
+
+	if (flags & O_CREAT)
+	{
+		va_list arg_list;
+
+		va_start(arg_list, flags);
+		mode = va_arg(arg_list, mode_t);
+		va_end(arg_list);
+	}
+
+	fd = _wopen(buf, flags | _O_BINARY, mode);
+
+	git__free(buf);
+	return fd;
+}
+
+int p_creat(const char *path, mode_t mode)
+{
+	int fd;
+	wchar_t* buf = gitwin_to_utf16(path);
+	if (!buf)
+		return -1;
+	fd = _wopen(buf, _O_WRONLY | _O_CREAT | _O_TRUNC | _O_BINARY, mode);
+	git__free(buf);
+	return fd;
+}
+
+int p_getcwd(char *buffer_out, size_t size)
+{
+	int ret;
+	wchar_t* buf;
+
+	if ((size_t)((int)size) != size)
+		return -1;
+
+	buf = (wchar_t*)git__malloc(sizeof(wchar_t) * (int)size);
+	GITERR_CHECK_ALLOC(buf);
+
+	_wgetcwd(buf, (int)size);
+
+	ret = WideCharToMultiByte(
+		CP_UTF8, 0, buf, -1, buffer_out, (int)size, NULL, NULL);
+
+	git__free(buf);
+	return !ret ? -1 : 0;
+}
+
+int p_stat(const char* path, struct stat* buf)
+{
+	return do_lstat(path, buf);
+}
+
+int p_chdir(const char* path)
+{
+	wchar_t* buf = gitwin_to_utf16(path);
+	int ret;
+	if (!buf)
+		return -1;
+	ret = _wchdir(buf);
+	git__free(buf);
+	return ret;
+}
+
+int p_chmod(const char* path, mode_t mode)
+{
+	wchar_t* buf = gitwin_to_utf16(path);
+	int ret;
+	if (!buf)
+		return -1;
+	ret = _wchmod(buf, mode);
+	git__free(buf);
+	return ret;
+}
+
+int p_rmdir(const char* path)
+{
+	wchar_t* buf = gitwin_to_utf16(path);
+	int ret;
+	if (!buf)
+		return -1;
+	ret = _wrmdir(buf);
+	git__free(buf);
+	return ret;
+}
+
+int p_hide_directory__w32(const char *path)
+{
+	int res;
+	wchar_t* buf = gitwin_to_utf16(path);
+	if (!buf)
+		return -1;
+
+	res = SetFileAttributesW(buf, FILE_ATTRIBUTE_HIDDEN);
+	git__free(buf);
+
+	return (res != 0) ? 0 : -1; /* MSDN states a "non zero" value indicates a success */
+}
+
+char *p_realpath(const char *orig_path, char *buffer)
+{
+	int ret, buffer_sz = 0;
+	wchar_t* orig_path_w = gitwin_to_utf16(orig_path);
+	wchar_t* buffer_w = (wchar_t*)git__malloc(GIT_PATH_MAX * sizeof(wchar_t));
+
+	if (!orig_path_w || !buffer_w)
+		return NULL;
+
+	ret = GetFullPathNameW(orig_path_w, GIT_PATH_MAX, buffer_w, NULL);
+	git__free(orig_path_w);
+
+	/* According to MSDN, a return value equals to zero means a failure. */
+	if (ret == 0 || ret > GIT_PATH_MAX) {
+		buffer = NULL;
+		goto done;
+	}
+
+	if (buffer == NULL) {
+		buffer_sz = WideCharToMultiByte(CP_UTF8, 0, buffer_w, -1, NULL, 0, NULL, NULL);
+
+		if (!buffer_sz ||
+			!(buffer = (char *)git__malloc(buffer_sz)) ||
+			!WideCharToMultiByte(CP_UTF8, 0, buffer_w, -1, buffer, buffer_sz, NULL, NULL))
+		{
+			git__free(buffer);
+			buffer = NULL;
+			goto done;
+		}
+	} else {
+		if (!WideCharToMultiByte(CP_UTF8, 0, buffer_w, -1, buffer, GIT_PATH_MAX, NULL, NULL)) {
+			buffer = NULL;
+			goto done;
+		}
+	}
+
+	if (!git_path_exists(buffer))
+	{
+		if (buffer_sz > 0)
+			git__free(buffer);
+
+		buffer = NULL;
+		errno = ENOENT;
+	}
+
+done:
+	git__free(buffer_w);
+	if (buffer)
+		git_path_mkposix(buffer);
+	return buffer;
+}
+
+int p_vsnprintf(char *buffer, size_t count, const char *format, va_list argptr)
+{
+#ifdef _MSC_VER
+	int len;
+
+	if (count == 0 || (len = _vsnprintf(buffer, count, format, argptr)) < 0)
+		return _vscprintf(format, argptr);
+
+	return len;
+#else /* MinGW */
+	return vsnprintf(buffer, count, format, argptr);
+#endif
+}
+
+int p_snprintf(char *buffer, size_t count, const char *format, ...)
+{
+	va_list va;
+	int r;
+
+	va_start(va, format);
+	r = p_vsnprintf(buffer, count, format, va);
+	va_end(va);
+
+	return r;
+}
+
+extern int p_creat(const char *path, mode_t mode);
+
+int p_mkstemp(char *tmp_path)
+{
+#if defined(_MSC_VER)
+	if (_mktemp_s(tmp_path, strlen(tmp_path) + 1) != 0)
+		return -1;
+#else
+	if (_mktemp(tmp_path) == NULL)
+		return -1;
+#endif
+
+	return p_creat(tmp_path, 0744);
+}
+
+int p_setenv(const char* name, const char* value, int overwrite)
+{
+	if (overwrite != 1)
+		return -1;
+
+	return (SetEnvironmentVariableA(name, value) == 0 ? -1 : 0);
+}
+
+int p_access(const char* path, mode_t mode)
+{
+	wchar_t *buf = gitwin_to_utf16(path);
+	int ret;
+	if (!buf)
+		return -1;
+
+	ret = _waccess(buf, mode);
+	git__free(buf);
+
+	return ret;
+}
+
+int p_rename(const char *from, const char *to)
+{
+	wchar_t *wfrom = gitwin_to_utf16(from);
+	wchar_t *wto = gitwin_to_utf16(to);
+	int ret;
+
+	if (!wfrom || !wto)
+		return -1;
+
+	ret = MoveFileExW(wfrom, wto, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) ? 0 : -1;
+
+	git__free(wfrom);
+	git__free(wto);
+
+	return ret;
+}
+
+int p_recv(GIT_SOCKET socket, void *buffer, size_t length, int flags)
+{
+	if ((size_t)((int)length) != length)
+		return -1; /* giterr_set will be done by caller */
+
+	return recv(socket, buffer, (int)length, flags);
+}
+
+int p_send(GIT_SOCKET socket, const void *buffer, size_t length, int flags)
+{
+	if ((size_t)((int)length) != length)
+		return -1; /* giterr_set will be done by caller */
+
+	return send(socket, buffer, (int)length, flags);
+}
diff --git a/src/win32/pthread.c b/src/win32/pthread.c
index 41cf5b35b..3a186c8d9 100644
--- a/src/win32/pthread.c
+++ b/src/win32/pthread.c
@@ -1,85 +1,68 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- *  Original code by Ramiro Polla (Public Domain)
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #include "pthread.h"
 
-int pthread_create(pthread_t *GIT_RESTRICT thread,
-                   const pthread_attr_t *GIT_RESTRICT GIT_UNUSED(attr),
-                   void *(*start_routine)(void*), void *GIT_RESTRICT arg)
+int pthread_create(
+	pthread_t *GIT_RESTRICT thread,
+	const pthread_attr_t *GIT_RESTRICT attr,
+	void *(*start_routine)(void*),
+	void *GIT_RESTRICT arg)
 {
-	GIT_UNUSED_ARG(attr);
-	*thread = (pthread_t) CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE)start_routine, arg, 0, NULL);
-	return *thread ? GIT_SUCCESS : git__throw(GIT_EOSERR, "Failed to create pthread");
+	GIT_UNUSED(attr);
+	*thread = (pthread_t) CreateThread(
+		NULL, 0, (LPTHREAD_START_ROUTINE)start_routine, arg, 0, NULL);
+	return *thread ? 0 : -1;
 }
 
 int pthread_join(pthread_t thread, void **value_ptr)
 {
-    int ret;
-    ret = WaitForSingleObject(thread, INFINITE);
-    if (ret && value_ptr)
-        GetExitCodeThread(thread, (void*) value_ptr);
-    return -(!!ret);
+	int ret;
+	ret = WaitForSingleObject(thread, INFINITE);
+	if (ret && value_ptr)
+		GetExitCodeThread(thread, (void*) value_ptr);
+	return -(!!ret);
 }
 
 int pthread_mutex_init(pthread_mutex_t *GIT_RESTRICT mutex,
-                       const pthread_mutexattr_t *GIT_RESTRICT GIT_UNUSED(mutexattr))
+						const pthread_mutexattr_t *GIT_RESTRICT mutexattr)
 {
-    GIT_UNUSED_ARG(mutexattr);
-    InitializeCriticalSection(mutex);
-    return 0;
+	GIT_UNUSED(mutexattr);
+	InitializeCriticalSection(mutex);
+	return 0;
 }
 
 int pthread_mutex_destroy(pthread_mutex_t *mutex)
 {
-    DeleteCriticalSection(mutex);
-    return 0;
+	DeleteCriticalSection(mutex);
+	return 0;
 }
 
 int pthread_mutex_lock(pthread_mutex_t *mutex)
 {
-    EnterCriticalSection(mutex);
-    return 0;
+	EnterCriticalSection(mutex);
+	return 0;
 }
 
 int pthread_mutex_unlock(pthread_mutex_t *mutex)
 {
-    LeaveCriticalSection(mutex);
-    return 0;
+	LeaveCriticalSection(mutex);
+	return 0;
 }
 
 int pthread_num_processors_np(void)
 {
-    DWORD_PTR p, s;
-    int n = 0;
+	DWORD_PTR p, s;
+	int n = 0;
 
-    if (GetProcessAffinityMask(GetCurrentProcess(), &p, &s))
-        for (; p; p >>= 1)
-            n += p&1;
+	if (GetProcessAffinityMask(GetCurrentProcess(), &p, &s))
+		for (; p; p >>= 1)
+			n += p&1;
 
-    return n ? n : 1;
+	return n ? n : 1;
 }
 
diff --git a/src/win32/pthread.h b/src/win32/pthread.h
index 10949f1eb..b194cbfa7 100644
--- a/src/win32/pthread.h
+++ b/src/win32/pthread.h
@@ -1,28 +1,8 @@
 /*
- * This file is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License, version 2,
- * as published by the Free Software Foundation.
+ * Copyright (C) 2009-2012 the libgit2 contributors
  *
- * In addition to the permissions in the GNU General Public License,
- * the authors give you unlimited permission to link the compiled
- * version of this file into combinations with other programs,
- * and to distribute those combinations without any restriction
- * coming from the use of this file.  (The General Public License
- * restrictions do apply in other respects; for example, they cover
- * modification of the file, and distribution when not linked into
- * a combined executable.)
- *
- * This file is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; see the file COPYING.  If not, write to
- * the Free Software Foundation, 51 Franklin Street, Fifth Floor,
- * Boston, MA 02110-1301, USA.
- *
- *  Original code by Ramiro Polla (Public Domain)
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
  */
 
 #ifndef GIT_PTHREAD_H
@@ -45,8 +25,8 @@ typedef HANDLE pthread_t;
 #define PTHREAD_MUTEX_INITIALIZER {(void*)-1};
 
 int pthread_create(pthread_t *GIT_RESTRICT,
-                   const pthread_attr_t *GIT_RESTRICT,
-                   void *(*start_routine)(void*), void *__restrict);
+					const pthread_attr_t *GIT_RESTRICT,
+					void *(*start_routine)(void*), void *__restrict);
 
 int pthread_join(pthread_t, void **);
 
diff --git a/src/win32/utf-conv.c b/src/win32/utf-conv.c
new file mode 100644
index 000000000..0a705c0ad
--- /dev/null
+++ b/src/win32/utf-conv.c
@@ -0,0 +1,92 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include "common.h"
+#include "utf-conv.h"
+#include "git2/windows.h"
+
+/*
+ * Default codepage value
+ */
+static int _active_codepage = CP_UTF8;
+
+void gitwin_set_codepage(unsigned int codepage)
+{
+	_active_codepage = codepage;
+}
+
+unsigned int gitwin_get_codepage(void)
+{
+	return _active_codepage;
+}
+
+void gitwin_set_utf8(void)
+{
+	_active_codepage = CP_UTF8;
+}
+
+wchar_t* gitwin_to_utf16(const char* str)
+{
+	wchar_t* ret;
+	int cb;
+
+	if (!str)
+		return NULL;
+
+	cb = MultiByteToWideChar(_active_codepage, 0, str, -1, NULL, 0);
+	if (cb == 0)
+		return (wchar_t *)git__calloc(1, sizeof(wchar_t));
+
+	ret = (wchar_t *)git__malloc(cb * sizeof(wchar_t));
+	if (!ret)
+		return NULL;
+
+	if (MultiByteToWideChar(_active_codepage, 0, str, -1, ret, (int)cb) == 0) {
+		giterr_set(GITERR_OS, "Could not convert string to UTF-16");
+		git__free(ret);
+		ret = NULL;
+	}
+
+	return ret;
+}
+
+int gitwin_append_utf16(wchar_t *buffer, const char *str, size_t len)
+{
+	int result = MultiByteToWideChar(
+		_active_codepage, 0, str, -1, buffer, (int)len);
+	if (result == 0)
+		giterr_set(GITERR_OS, "Could not convert string to UTF-16");
+	return result;
+}
+
+char* gitwin_from_utf16(const wchar_t* str)
+{
+	char* ret;
+	int cb;
+
+	if (!str)
+		return NULL;
+
+	cb = WideCharToMultiByte(_active_codepage, 0, str, -1, NULL, 0, NULL, NULL);
+	if (cb == 0)
+		return (char *)git__calloc(1, sizeof(char));
+
+	ret = (char*)git__malloc(cb);
+	if (!ret)
+		return NULL;
+
+	if (WideCharToMultiByte(
+		_active_codepage, 0, str, -1, ret, (int)cb, NULL, NULL) == 0)
+	{
+		giterr_set(GITERR_OS, "Could not convert string to UTF-8");
+		git__free(ret);
+		ret = NULL;
+	}
+
+	return ret;
+
+}
diff --git a/src/win32/utf-conv.h b/src/win32/utf-conv.h
new file mode 100644
index 000000000..ae9f29f6c
--- /dev/null
+++ b/src/win32/utf-conv.h
@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2009-2012 the libgit2 contributors
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+
+#include <wchar.h>
+
+#ifndef INCLUDE_git_utfconv_h__
+#define INCLUDE_git_utfconv_h__
+
+wchar_t* gitwin_to_utf16(const char* str);
+int gitwin_append_utf16(wchar_t *buffer, const char *str, size_t len);
+char* gitwin_from_utf16(const wchar_t* str);
+
+#endif
+
diff --git a/src/xdiff/xdiff.h b/src/xdiff/xdiff.h
new file mode 100644
index 000000000..cb8b235b5
--- /dev/null
+++ b/src/xdiff/xdiff.h
@@ -0,0 +1,135 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#if !defined(XDIFF_H)
+#define XDIFF_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* #ifdef __cplusplus */
+
+
+#define XDF_NEED_MINIMAL (1 << 1)
+#define XDF_IGNORE_WHITESPACE (1 << 2)
+#define XDF_IGNORE_WHITESPACE_CHANGE (1 << 3)
+#define XDF_IGNORE_WHITESPACE_AT_EOL (1 << 4)
+#define XDF_PATIENCE_DIFF (1 << 5)
+#define XDF_HISTOGRAM_DIFF (1 << 6)
+#define XDF_WHITESPACE_FLAGS (XDF_IGNORE_WHITESPACE | XDF_IGNORE_WHITESPACE_CHANGE | XDF_IGNORE_WHITESPACE_AT_EOL)
+
+#define XDL_PATCH_NORMAL '-'
+#define XDL_PATCH_REVERSE '+'
+#define XDL_PATCH_MODEMASK ((1 << 8) - 1)
+#define XDL_PATCH_IGNOREBSPACE (1 << 8)
+
+#define XDL_EMIT_FUNCNAMES (1 << 0)
+#define XDL_EMIT_COMMON (1 << 1)
+#define XDL_EMIT_FUNCCONTEXT (1 << 2)
+
+#define XDL_MMB_READONLY (1 << 0)
+
+#define XDL_MMF_ATOMIC (1 << 0)
+
+#define XDL_BDOP_INS 1
+#define XDL_BDOP_CPY 2
+#define XDL_BDOP_INSB 3
+
+/* merge simplification levels */
+#define XDL_MERGE_MINIMAL 0
+#define XDL_MERGE_EAGER 1
+#define XDL_MERGE_ZEALOUS 2
+#define XDL_MERGE_ZEALOUS_ALNUM 3
+
+/* merge favor modes */
+#define XDL_MERGE_FAVOR_OURS 1
+#define XDL_MERGE_FAVOR_THEIRS 2
+#define XDL_MERGE_FAVOR_UNION 3
+
+/* merge output styles */
+#define XDL_MERGE_DIFF3 1
+
+typedef struct s_mmfile {
+	char *ptr;
+	size_t size;
+} mmfile_t;
+
+typedef struct s_mmbuffer {
+	char *ptr;
+	size_t size;
+} mmbuffer_t;
+
+typedef struct s_xpparam {
+	unsigned long flags;
+} xpparam_t;
+
+typedef struct s_xdemitcb {
+	void *priv;
+	int (*outf)(void *, mmbuffer_t *, int);
+} xdemitcb_t;
+
+typedef long (*find_func_t)(const char *line, long line_len, char *buffer, long buffer_size, void *priv);
+
+typedef struct s_xdemitconf {
+	long ctxlen;
+	long interhunkctxlen;
+	unsigned long flags;
+	find_func_t find_func;
+	void *find_func_priv;
+	void (*emit_func)(void);
+} xdemitconf_t;
+
+typedef struct s_bdiffparam {
+	long bsize;
+} bdiffparam_t;
+
+
+#define xdl_malloc(x) malloc(x)
+#define xdl_free(ptr) free(ptr)
+#define xdl_realloc(ptr,x) realloc(ptr,x)
+
+void *xdl_mmfile_first(mmfile_t *mmf, long *size);
+long xdl_mmfile_size(mmfile_t *mmf);
+
+int xdl_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+	     xdemitconf_t const *xecfg, xdemitcb_t *ecb);
+
+typedef struct s_xmparam {
+	xpparam_t xpp;
+	int marker_size;
+	int level;
+	int favor;
+	int style;
+	const char *ancestor;	/* label for orig */
+	const char *file1;	/* label for mf1 */
+	const char *file2;	/* label for mf2 */
+} xmparam_t;
+
+#define DEFAULT_CONFLICT_MARKER_SIZE 7
+
+int xdl_merge(mmfile_t *orig, mmfile_t *mf1, mmfile_t *mf2,
+		xmparam_t const *xmp, mmbuffer_t *result);
+
+#ifdef __cplusplus
+}
+#endif /* #ifdef __cplusplus */
+
+#endif /* #if !defined(XDIFF_H) */
diff --git a/src/xdiff/xdiffi.c b/src/xdiff/xdiffi.c
new file mode 100644
index 000000000..75a392275
--- /dev/null
+++ b/src/xdiff/xdiffi.c
@@ -0,0 +1,572 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003	Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#include "xinclude.h"
+
+
+
+#define XDL_MAX_COST_MIN 256
+#define XDL_HEUR_MIN_COST 256
+#define XDL_LINE_MAX (long)((1UL << (CHAR_BIT * sizeof(long) - 1)) - 1)
+#define XDL_SNAKE_CNT 20
+#define XDL_K_HEUR 4
+
+
+
+typedef struct s_xdpsplit {
+	long i1, i2;
+	int min_lo, min_hi;
+} xdpsplit_t;
+
+
+
+
+static long xdl_split(unsigned long const *ha1, long off1, long lim1,
+		      unsigned long const *ha2, long off2, long lim2,
+		      long *kvdf, long *kvdb, int need_min, xdpsplit_t *spl,
+		      xdalgoenv_t *xenv);
+static xdchange_t *xdl_add_change(xdchange_t *xscr, long i1, long i2, long chg1, long chg2);
+
+
+
+
+
+/*
+ * See "An O(ND) Difference Algorithm and its Variations", by Eugene Myers.
+ * Basically considers a "box" (off1, off2, lim1, lim2) and scan from both
+ * the forward diagonal starting from (off1, off2) and the backward diagonal
+ * starting from (lim1, lim2). If the K values on the same diagonal crosses
+ * returns the furthest point of reach. We might end up having to expensive
+ * cases using this algorithm is full, so a little bit of heuristic is needed
+ * to cut the search and to return a suboptimal point.
+ */
+static long xdl_split(unsigned long const *ha1, long off1, long lim1,
+		      unsigned long const *ha2, long off2, long lim2,
+		      long *kvdf, long *kvdb, int need_min, xdpsplit_t *spl,
+		      xdalgoenv_t *xenv) {
+	long dmin = off1 - lim2, dmax = lim1 - off2;
+	long fmid = off1 - off2, bmid = lim1 - lim2;
+	long odd = (fmid - bmid) & 1;
+	long fmin = fmid, fmax = fmid;
+	long bmin = bmid, bmax = bmid;
+	long ec, d, i1, i2, prev1, best, dd, v, k;
+
+	/*
+	 * Set initial diagonal values for both forward and backward path.
+	 */
+	kvdf[fmid] = off1;
+	kvdb[bmid] = lim1;
+
+	for (ec = 1;; ec++) {
+		int got_snake = 0;
+
+		/*
+		 * We need to extent the diagonal "domain" by one. If the next
+		 * values exits the box boundaries we need to change it in the
+		 * opposite direction because (max - min) must be a power of two.
+		 * Also we initialize the external K value to -1 so that we can
+		 * avoid extra conditions check inside the core loop.
+		 */
+		if (fmin > dmin)
+			kvdf[--fmin - 1] = -1;
+		else
+			++fmin;
+		if (fmax < dmax)
+			kvdf[++fmax + 1] = -1;
+		else
+			--fmax;
+
+		for (d = fmax; d >= fmin; d -= 2) {
+			if (kvdf[d - 1] >= kvdf[d + 1])
+				i1 = kvdf[d - 1] + 1;
+			else
+				i1 = kvdf[d + 1];
+			prev1 = i1;
+			i2 = i1 - d;
+			for (; i1 < lim1 && i2 < lim2 && ha1[i1] == ha2[i2]; i1++, i2++);
+			if (i1 - prev1 > xenv->snake_cnt)
+				got_snake = 1;
+			kvdf[d] = i1;
+			if (odd && bmin <= d && d <= bmax && kvdb[d] <= i1) {
+				spl->i1 = i1;
+				spl->i2 = i2;
+				spl->min_lo = spl->min_hi = 1;
+				return ec;
+			}
+		}
+
+		/*
+		 * We need to extent the diagonal "domain" by one. If the next
+		 * values exits the box boundaries we need to change it in the
+		 * opposite direction because (max - min) must be a power of two.
+		 * Also we initialize the external K value to -1 so that we can
+		 * avoid extra conditions check inside the core loop.
+		 */
+		if (bmin > dmin)
+			kvdb[--bmin - 1] = XDL_LINE_MAX;
+		else
+			++bmin;
+		if (bmax < dmax)
+			kvdb[++bmax + 1] = XDL_LINE_MAX;
+		else
+			--bmax;
+
+		for (d = bmax; d >= bmin; d -= 2) {
+			if (kvdb[d - 1] < kvdb[d + 1])
+				i1 = kvdb[d - 1];
+			else
+				i1 = kvdb[d + 1] - 1;
+			prev1 = i1;
+			i2 = i1 - d;
+			for (; i1 > off1 && i2 > off2 && ha1[i1 - 1] == ha2[i2 - 1]; i1--, i2--);
+			if (prev1 - i1 > xenv->snake_cnt)
+				got_snake = 1;
+			kvdb[d] = i1;
+			if (!odd && fmin <= d && d <= fmax && i1 <= kvdf[d]) {
+				spl->i1 = i1;
+				spl->i2 = i2;
+				spl->min_lo = spl->min_hi = 1;
+				return ec;
+			}
+		}
+
+		if (need_min)
+			continue;
+
+		/*
+		 * If the edit cost is above the heuristic trigger and if
+		 * we got a good snake, we sample current diagonals to see
+		 * if some of the, have reached an "interesting" path. Our
+		 * measure is a function of the distance from the diagonal
+		 * corner (i1 + i2) penalized with the distance from the
+		 * mid diagonal itself. If this value is above the current
+		 * edit cost times a magic factor (XDL_K_HEUR) we consider
+		 * it interesting.
+		 */
+		if (got_snake && ec > xenv->heur_min) {
+			for (best = 0, d = fmax; d >= fmin; d -= 2) {
+				dd = d > fmid ? d - fmid: fmid - d;
+				i1 = kvdf[d];
+				i2 = i1 - d;
+				v = (i1 - off1) + (i2 - off2) - dd;
+
+				if (v > XDL_K_HEUR * ec && v > best &&
+				    off1 + xenv->snake_cnt <= i1 && i1 < lim1 &&
+				    off2 + xenv->snake_cnt <= i2 && i2 < lim2) {
+					for (k = 1; ha1[i1 - k] == ha2[i2 - k]; k++)
+						if (k == xenv->snake_cnt) {
+							best = v;
+							spl->i1 = i1;
+							spl->i2 = i2;
+							break;
+						}
+				}
+			}
+			if (best > 0) {
+				spl->min_lo = 1;
+				spl->min_hi = 0;
+				return ec;
+			}
+
+			for (best = 0, d = bmax; d >= bmin; d -= 2) {
+				dd = d > bmid ? d - bmid: bmid - d;
+				i1 = kvdb[d];
+				i2 = i1 - d;
+				v = (lim1 - i1) + (lim2 - i2) - dd;
+
+				if (v > XDL_K_HEUR * ec && v > best &&
+				    off1 < i1 && i1 <= lim1 - xenv->snake_cnt &&
+				    off2 < i2 && i2 <= lim2 - xenv->snake_cnt) {
+					for (k = 0; ha1[i1 + k] == ha2[i2 + k]; k++)
+						if (k == xenv->snake_cnt - 1) {
+							best = v;
+							spl->i1 = i1;
+							spl->i2 = i2;
+							break;
+						}
+				}
+			}
+			if (best > 0) {
+				spl->min_lo = 0;
+				spl->min_hi = 1;
+				return ec;
+			}
+		}
+
+		/*
+		 * Enough is enough. We spent too much time here and now we collect
+		 * the furthest reaching path using the (i1 + i2) measure.
+		 */
+		if (ec >= xenv->mxcost) {
+			long fbest, fbest1, bbest, bbest1;
+
+			fbest = fbest1 = -1;
+			for (d = fmax; d >= fmin; d -= 2) {
+				i1 = XDL_MIN(kvdf[d], lim1);
+				i2 = i1 - d;
+				if (lim2 < i2)
+					i1 = lim2 + d, i2 = lim2;
+				if (fbest < i1 + i2) {
+					fbest = i1 + i2;
+					fbest1 = i1;
+				}
+			}
+
+			bbest = bbest1 = XDL_LINE_MAX;
+			for (d = bmax; d >= bmin; d -= 2) {
+				i1 = XDL_MAX(off1, kvdb[d]);
+				i2 = i1 - d;
+				if (i2 < off2)
+					i1 = off2 + d, i2 = off2;
+				if (i1 + i2 < bbest) {
+					bbest = i1 + i2;
+					bbest1 = i1;
+				}
+			}
+
+			if ((lim1 + lim2) - bbest < fbest - (off1 + off2)) {
+				spl->i1 = fbest1;
+				spl->i2 = fbest - fbest1;
+				spl->min_lo = 1;
+				spl->min_hi = 0;
+			} else {
+				spl->i1 = bbest1;
+				spl->i2 = bbest - bbest1;
+				spl->min_lo = 0;
+				spl->min_hi = 1;
+			}
+			return ec;
+		}
+	}
+}
+
+
+/*
+ * Rule: "Divide et Impera". Recursively split the box in sub-boxes by calling
+ * the box splitting function. Note that the real job (marking changed lines)
+ * is done in the two boundary reaching checks.
+ */
+int xdl_recs_cmp(diffdata_t *dd1, long off1, long lim1,
+		 diffdata_t *dd2, long off2, long lim2,
+		 long *kvdf, long *kvdb, int need_min, xdalgoenv_t *xenv) {
+	unsigned long const *ha1 = dd1->ha, *ha2 = dd2->ha;
+
+	/*
+	 * Shrink the box by walking through each diagonal snake (SW and NE).
+	 */
+	for (; off1 < lim1 && off2 < lim2 && ha1[off1] == ha2[off2]; off1++, off2++);
+	for (; off1 < lim1 && off2 < lim2 && ha1[lim1 - 1] == ha2[lim2 - 1]; lim1--, lim2--);
+
+	/*
+	 * If one dimension is empty, then all records on the other one must
+	 * be obviously changed.
+	 */
+	if (off1 == lim1) {
+		char *rchg2 = dd2->rchg;
+		long *rindex2 = dd2->rindex;
+
+		for (; off2 < lim2; off2++)
+			rchg2[rindex2[off2]] = 1;
+	} else if (off2 == lim2) {
+		char *rchg1 = dd1->rchg;
+		long *rindex1 = dd1->rindex;
+
+		for (; off1 < lim1; off1++)
+			rchg1[rindex1[off1]] = 1;
+	} else {
+		xdpsplit_t spl;
+		spl.i1 = spl.i2 = 0;
+
+		/*
+		 * Divide ...
+		 */
+		if (xdl_split(ha1, off1, lim1, ha2, off2, lim2, kvdf, kvdb,
+			      need_min, &spl, xenv) < 0) {
+
+			return -1;
+		}
+
+		/*
+		 * ... et Impera.
+		 */
+		if (xdl_recs_cmp(dd1, off1, spl.i1, dd2, off2, spl.i2,
+				 kvdf, kvdb, spl.min_lo, xenv) < 0 ||
+		    xdl_recs_cmp(dd1, spl.i1, lim1, dd2, spl.i2, lim2,
+				 kvdf, kvdb, spl.min_hi, xenv) < 0) {
+
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+
+int xdl_do_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+		xdfenv_t *xe) {
+	long ndiags;
+	long *kvd, *kvdf, *kvdb;
+	xdalgoenv_t xenv;
+	diffdata_t dd1, dd2;
+
+	if (xpp->flags & XDF_PATIENCE_DIFF)
+		return xdl_do_patience_diff(mf1, mf2, xpp, xe);
+
+	if (xpp->flags & XDF_HISTOGRAM_DIFF)
+		return xdl_do_histogram_diff(mf1, mf2, xpp, xe);
+
+	if (xdl_prepare_env(mf1, mf2, xpp, xe) < 0) {
+
+		return -1;
+	}
+
+	/*
+	 * Allocate and setup K vectors to be used by the differential algorithm.
+	 * One is to store the forward path and one to store the backward path.
+	 */
+	ndiags = xe->xdf1.nreff + xe->xdf2.nreff + 3;
+	if (!(kvd = (long *) xdl_malloc((2 * ndiags + 2) * sizeof(long)))) {
+
+		xdl_free_env(xe);
+		return -1;
+	}
+	kvdf = kvd;
+	kvdb = kvdf + ndiags;
+	kvdf += xe->xdf2.nreff + 1;
+	kvdb += xe->xdf2.nreff + 1;
+
+	xenv.mxcost = xdl_bogosqrt(ndiags);
+	if (xenv.mxcost < XDL_MAX_COST_MIN)
+		xenv.mxcost = XDL_MAX_COST_MIN;
+	xenv.snake_cnt = XDL_SNAKE_CNT;
+	xenv.heur_min = XDL_HEUR_MIN_COST;
+
+	dd1.nrec = xe->xdf1.nreff;
+	dd1.ha = xe->xdf1.ha;
+	dd1.rchg = xe->xdf1.rchg;
+	dd1.rindex = xe->xdf1.rindex;
+	dd2.nrec = xe->xdf2.nreff;
+	dd2.ha = xe->xdf2.ha;
+	dd2.rchg = xe->xdf2.rchg;
+	dd2.rindex = xe->xdf2.rindex;
+
+	if (xdl_recs_cmp(&dd1, 0, dd1.nrec, &dd2, 0, dd2.nrec,
+			 kvdf, kvdb, (xpp->flags & XDF_NEED_MINIMAL) != 0, &xenv) < 0) {
+
+		xdl_free(kvd);
+		xdl_free_env(xe);
+		return -1;
+	}
+
+	xdl_free(kvd);
+
+	return 0;
+}
+
+
+static xdchange_t *xdl_add_change(xdchange_t *xscr, long i1, long i2, long chg1, long chg2) {
+	xdchange_t *xch;
+
+	if (!(xch = (xdchange_t *) xdl_malloc(sizeof(xdchange_t))))
+		return NULL;
+
+	xch->next = xscr;
+	xch->i1 = i1;
+	xch->i2 = i2;
+	xch->chg1 = chg1;
+	xch->chg2 = chg2;
+
+	return xch;
+}
+
+
+int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags) {
+	long ix, ixo, ixs, ixref, grpsiz, nrec = xdf->nrec;
+	char *rchg = xdf->rchg, *rchgo = xdfo->rchg;
+	xrecord_t **recs = xdf->recs;
+
+	/*
+	 * This is the same of what GNU diff does. Move back and forward
+	 * change groups for a consistent and pretty diff output. This also
+	 * helps in finding joinable change groups and reduce the diff size.
+	 */
+	for (ix = ixo = 0;;) {
+		/*
+		 * Find the first changed line in the to-be-compacted file.
+		 * We need to keep track of both indexes, so if we find a
+		 * changed lines group on the other file, while scanning the
+		 * to-be-compacted file, we need to skip it properly. Note
+		 * that loops that are testing for changed lines on rchg* do
+		 * not need index bounding since the array is prepared with
+		 * a zero at position -1 and N.
+		 */
+		for (; ix < nrec && !rchg[ix]; ix++)
+			while (rchgo[ixo++]);
+		if (ix == nrec)
+			break;
+
+		/*
+		 * Record the start of a changed-group in the to-be-compacted file
+		 * and find the end of it, on both to-be-compacted and other file
+		 * indexes (ix and ixo).
+		 */
+		ixs = ix;
+		for (ix++; rchg[ix]; ix++);
+		for (; rchgo[ixo]; ixo++);
+
+		do {
+			grpsiz = ix - ixs;
+
+			/*
+			 * If the line before the current change group, is equal to
+			 * the last line of the current change group, shift backward
+			 * the group.
+			 */
+			while (ixs > 0 && recs[ixs - 1]->ha == recs[ix - 1]->ha &&
+			       xdl_recmatch(recs[ixs - 1]->ptr, recs[ixs - 1]->size, recs[ix - 1]->ptr, recs[ix - 1]->size, flags)) {
+				rchg[--ixs] = 1;
+				rchg[--ix] = 0;
+
+				/*
+				 * This change might have joined two change groups,
+				 * so we try to take this scenario in account by moving
+				 * the start index accordingly (and so the other-file
+				 * end-of-group index).
+				 */
+				for (; rchg[ixs - 1]; ixs--);
+				while (rchgo[--ixo]);
+			}
+
+			/*
+			 * Record the end-of-group position in case we are matched
+			 * with a group of changes in the other file (that is, the
+			 * change record before the end-of-group index in the other
+			 * file is set).
+			 */
+			ixref = rchgo[ixo - 1] ? ix: nrec;
+
+			/*
+			 * If the first line of the current change group, is equal to
+			 * the line next of the current change group, shift forward
+			 * the group.
+			 */
+			while (ix < nrec && recs[ixs]->ha == recs[ix]->ha &&
+			       xdl_recmatch(recs[ixs]->ptr, recs[ixs]->size, recs[ix]->ptr, recs[ix]->size, flags)) {
+				rchg[ixs++] = 0;
+				rchg[ix++] = 1;
+
+				/*
+				 * This change might have joined two change groups,
+				 * so we try to take this scenario in account by moving
+				 * the start index accordingly (and so the other-file
+				 * end-of-group index). Keep tracking the reference
+				 * index in case we are shifting together with a
+				 * corresponding group of changes in the other file.
+				 */
+				for (; rchg[ix]; ix++);
+				while (rchgo[++ixo])
+					ixref = ix;
+			}
+		} while (grpsiz != ix - ixs);
+
+		/*
+		 * Try to move back the possibly merged group of changes, to match
+		 * the recorded postion in the other file.
+		 */
+		while (ixref < ix) {
+			rchg[--ixs] = 1;
+			rchg[--ix] = 0;
+			while (rchgo[--ixo]);
+		}
+	}
+
+	return 0;
+}
+
+
+int xdl_build_script(xdfenv_t *xe, xdchange_t **xscr) {
+	xdchange_t *cscr = NULL, *xch;
+	char *rchg1 = xe->xdf1.rchg, *rchg2 = xe->xdf2.rchg;
+	long i1, i2, l1, l2;
+
+	/*
+	 * Trivial. Collects "groups" of changes and creates an edit script.
+	 */
+	for (i1 = xe->xdf1.nrec, i2 = xe->xdf2.nrec; i1 >= 0 || i2 >= 0; i1--, i2--)
+		if (rchg1[i1 - 1] || rchg2[i2 - 1]) {
+			for (l1 = i1; rchg1[i1 - 1]; i1--);
+			for (l2 = i2; rchg2[i2 - 1]; i2--);
+
+			if (!(xch = xdl_add_change(cscr, i1, i2, l1 - i1, l2 - i2))) {
+				xdl_free_script(cscr);
+				return -1;
+			}
+			cscr = xch;
+		}
+
+	*xscr = cscr;
+
+	return 0;
+}
+
+
+void xdl_free_script(xdchange_t *xscr) {
+	xdchange_t *xch;
+
+	while ((xch = xscr) != NULL) {
+		xscr = xscr->next;
+		xdl_free(xch);
+	}
+}
+
+
+int xdl_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+	     xdemitconf_t const *xecfg, xdemitcb_t *ecb) {
+	xdchange_t *xscr;
+	xdfenv_t xe;
+	emit_func_t ef = xecfg->emit_func ?
+		(emit_func_t)xecfg->emit_func : xdl_emit_diff;
+
+	if (xdl_do_diff(mf1, mf2, xpp, &xe) < 0) {
+
+		return -1;
+	}
+	if (xdl_change_compact(&xe.xdf1, &xe.xdf2, xpp->flags) < 0 ||
+	    xdl_change_compact(&xe.xdf2, &xe.xdf1, xpp->flags) < 0 ||
+	    xdl_build_script(&xe, &xscr) < 0) {
+
+		xdl_free_env(&xe);
+		return -1;
+	}
+	if (xscr) {
+		if (ef(&xe, xscr, ecb, xecfg) < 0) {
+
+			xdl_free_script(xscr);
+			xdl_free_env(&xe);
+			return -1;
+		}
+		xdl_free_script(xscr);
+	}
+	xdl_free_env(&xe);
+
+	return 0;
+}
diff --git a/src/xdiff/xdiffi.h b/src/xdiff/xdiffi.h
new file mode 100644
index 000000000..7a92ea9c4
--- /dev/null
+++ b/src/xdiff/xdiffi.h
@@ -0,0 +1,63 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#if !defined(XDIFFI_H)
+#define XDIFFI_H
+
+
+typedef struct s_diffdata {
+	long nrec;
+	unsigned long const *ha;
+	long *rindex;
+	char *rchg;
+} diffdata_t;
+
+typedef struct s_xdalgoenv {
+	long mxcost;
+	long snake_cnt;
+	long heur_min;
+} xdalgoenv_t;
+
+typedef struct s_xdchange {
+	struct s_xdchange *next;
+	long i1, i2;
+	long chg1, chg2;
+} xdchange_t;
+
+
+
+int xdl_recs_cmp(diffdata_t *dd1, long off1, long lim1,
+		 diffdata_t *dd2, long off2, long lim2,
+		 long *kvdf, long *kvdb, int need_min, xdalgoenv_t *xenv);
+int xdl_do_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+		xdfenv_t *xe);
+int xdl_change_compact(xdfile_t *xdf, xdfile_t *xdfo, long flags);
+int xdl_build_script(xdfenv_t *xe, xdchange_t **xscr);
+void xdl_free_script(xdchange_t *xscr);
+int xdl_emit_diff(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb,
+		  xdemitconf_t const *xecfg);
+int xdl_do_patience_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+		xdfenv_t *env);
+int xdl_do_histogram_diff(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+		xdfenv_t *env);
+
+#endif /* #if !defined(XDIFFI_H) */
diff --git a/src/xdiff/xemit.c b/src/xdiff/xemit.c
new file mode 100644
index 000000000..e3e63d902
--- /dev/null
+++ b/src/xdiff/xemit.c
@@ -0,0 +1,253 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003	Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#include "xinclude.h"
+
+
+
+
+static long xdl_get_rec(xdfile_t *xdf, long ri, char const **rec);
+static int xdl_emit_record(xdfile_t *xdf, long ri, char const *pre, xdemitcb_t *ecb);
+
+
+
+
+static long xdl_get_rec(xdfile_t *xdf, long ri, char const **rec) {
+
+	*rec = xdf->recs[ri]->ptr;
+
+	return xdf->recs[ri]->size;
+}
+
+
+static int xdl_emit_record(xdfile_t *xdf, long ri, char const *pre, xdemitcb_t *ecb) {
+	long size, psize = (long)strlen(pre);
+	char const *rec;
+
+	size = xdl_get_rec(xdf, ri, &rec);
+	if (xdl_emit_diffrec(rec, size, pre, psize, ecb) < 0) {
+
+		return -1;
+	}
+
+	return 0;
+}
+
+
+/*
+ * Starting at the passed change atom, find the latest change atom to be included
+ * inside the differential hunk according to the specified configuration.
+ */
+xdchange_t *xdl_get_hunk(xdchange_t *xscr, xdemitconf_t const *xecfg) {
+	xdchange_t *xch, *xchp;
+	long max_common = 2 * xecfg->ctxlen + xecfg->interhunkctxlen;
+
+	for (xchp = xscr, xch = xscr->next; xch; xchp = xch, xch = xch->next)
+		if (xch->i1 - (xchp->i1 + xchp->chg1) > max_common)
+			break;
+
+	return xchp;
+}
+
+
+static long def_ff(const char *rec, long len, char *buf, long sz, void *priv)
+{
+	(void)priv;
+
+	if (len > 0 &&
+			(isalpha((unsigned char)*rec) || /* identifier? */
+			 *rec == '_' ||	/* also identifier? */
+			 *rec == '$')) { /* identifiers from VMS and other esoterico */
+		if (len > sz)
+			len = sz;
+		while (0 < len && isspace((unsigned char)rec[len - 1]))
+			len--;
+		memcpy(buf, rec, len);
+		return len;
+	}
+	return -1;
+}
+
+static int xdl_emit_common(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb,
+                           xdemitconf_t const *xecfg) {
+	xdfile_t *xdf = &xe->xdf2;
+	const char *rchg = xdf->rchg;
+	long ix;
+
+	(void)xscr;
+	(void)xecfg;
+
+	for (ix = 0; ix < xdf->nrec; ix++) {
+		if (rchg[ix])
+			continue;
+		if (xdl_emit_record(xdf, ix, "", ecb))
+			return -1;
+	}
+	return 0;
+}
+
+struct func_line {
+	long len;
+	char buf[80];
+};
+
+static long get_func_line(xdfenv_t *xe, xdemitconf_t const *xecfg,
+			  struct func_line *func_line, long start, long limit)
+{
+	find_func_t ff = xecfg->find_func ? xecfg->find_func : def_ff;
+	long l, size, step = (start > limit) ? -1 : 1;
+	char *buf, dummy[1];
+
+	buf = func_line ? func_line->buf : dummy;
+	size = func_line ? sizeof(func_line->buf) : sizeof(dummy);
+
+	for (l = start; l != limit && 0 <= l && l < xe->xdf1.nrec; l += step) {
+		const char *rec;
+		long reclen = xdl_get_rec(&xe->xdf1, l, &rec);
+		long len = ff(rec, reclen, buf, size, xecfg->find_func_priv);
+		if (len >= 0) {
+			if (func_line)
+				func_line->len = len;
+			return l;
+		}
+	}
+	return -1;
+}
+
+int xdl_emit_diff(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb,
+		  xdemitconf_t const *xecfg) {
+	long s1, s2, e1, e2, lctx;
+	xdchange_t *xch, *xche;
+	long funclineprev = -1;
+	struct func_line func_line = { 0 };
+
+	if (xecfg->flags & XDL_EMIT_COMMON)
+		return xdl_emit_common(xe, xscr, ecb, xecfg);
+
+	for (xch = xscr; xch; xch = xche->next) {
+		xche = xdl_get_hunk(xch, xecfg);
+
+		s1 = XDL_MAX(xch->i1 - xecfg->ctxlen, 0);
+		s2 = XDL_MAX(xch->i2 - xecfg->ctxlen, 0);
+
+		if (xecfg->flags & XDL_EMIT_FUNCCONTEXT) {
+			long fs1 = get_func_line(xe, xecfg, NULL, xch->i1, -1);
+			if (fs1 < 0)
+				fs1 = 0;
+			if (fs1 < s1) {
+				s2 -= s1 - fs1;
+				s1 = fs1;
+			}
+		}
+
+ again:
+		lctx = xecfg->ctxlen;
+		lctx = XDL_MIN(lctx, xe->xdf1.nrec - (xche->i1 + xche->chg1));
+		lctx = XDL_MIN(lctx, xe->xdf2.nrec - (xche->i2 + xche->chg2));
+
+		e1 = xche->i1 + xche->chg1 + lctx;
+		e2 = xche->i2 + xche->chg2 + lctx;
+
+		if (xecfg->flags & XDL_EMIT_FUNCCONTEXT) {
+			long fe1 = get_func_line(xe, xecfg, NULL,
+						 xche->i1 + xche->chg1,
+						 xe->xdf1.nrec);
+			if (fe1 < 0)
+				fe1 = xe->xdf1.nrec;
+			if (fe1 > e1) {
+				e2 += fe1 - e1;
+				e1 = fe1;
+			}
+
+			/*
+			 * Overlap with next change?  Then include it
+			 * in the current hunk and start over to find
+			 * its new end.
+			 */
+			if (xche->next) {
+				long l = xche->next->i1;
+				if (l <= e1 ||
+				    get_func_line(xe, xecfg, NULL, l, e1) < 0) {
+					xche = xche->next;
+					goto again;
+				}
+			}
+		}
+
+		/*
+		 * Emit current hunk header.
+		 */
+
+		if (xecfg->flags & XDL_EMIT_FUNCNAMES) {
+			get_func_line(xe, xecfg, &func_line,
+				      s1 - 1, funclineprev);
+			funclineprev = s1 - 1;
+		}
+		if (xdl_emit_hunk_hdr(s1 + 1, e1 - s1, s2 + 1, e2 - s2,
+				      func_line.buf, func_line.len, ecb) < 0)
+			return -1;
+
+		/*
+		 * Emit pre-context.
+		 */
+		for (; s2 < xch->i2; s2++)
+			if (xdl_emit_record(&xe->xdf2, s2, " ", ecb) < 0)
+				return -1;
+
+		for (s1 = xch->i1, s2 = xch->i2;; xch = xch->next) {
+			/*
+			 * Merge previous with current change atom.
+			 */
+			for (; s1 < xch->i1 && s2 < xch->i2; s1++, s2++)
+				if (xdl_emit_record(&xe->xdf2, s2, " ", ecb) < 0)
+					return -1;
+
+			/*
+			 * Removes lines from the first file.
+			 */
+			for (s1 = xch->i1; s1 < xch->i1 + xch->chg1; s1++)
+				if (xdl_emit_record(&xe->xdf1, s1, "-", ecb) < 0)
+					return -1;
+
+			/*
+			 * Adds lines from the second file.
+			 */
+			for (s2 = xch->i2; s2 < xch->i2 + xch->chg2; s2++)
+				if (xdl_emit_record(&xe->xdf2, s2, "+", ecb) < 0)
+					return -1;
+
+			if (xch == xche)
+				break;
+			s1 = xch->i1 + xch->chg1;
+			s2 = xch->i2 + xch->chg2;
+		}
+
+		/*
+		 * Emit post-context.
+		 */
+		for (s2 = xche->i2 + xche->chg2; s2 < e2; s2++)
+			if (xdl_emit_record(&xe->xdf2, s2, " ", ecb) < 0)
+				return -1;
+	}
+
+	return 0;
+}
diff --git a/src/xdiff/xemit.h b/src/xdiff/xemit.h
new file mode 100644
index 000000000..c2e2e8302
--- /dev/null
+++ b/src/xdiff/xemit.h
@@ -0,0 +1,36 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#if !defined(XEMIT_H)
+#define XEMIT_H
+
+
+typedef int (*emit_func_t)(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb,
+			   xdemitconf_t const *xecfg);
+
+xdchange_t *xdl_get_hunk(xdchange_t *xscr, xdemitconf_t const *xecfg);
+int xdl_emit_diff(xdfenv_t *xe, xdchange_t *xscr, xdemitcb_t *ecb,
+		  xdemitconf_t const *xecfg);
+
+
+
+#endif /* #if !defined(XEMIT_H) */
diff --git a/src/xdiff/xhistogram.c b/src/xdiff/xhistogram.c
new file mode 100644
index 000000000..5d101754d
--- /dev/null
+++ b/src/xdiff/xhistogram.c
@@ -0,0 +1,371 @@
+/*
+ * Copyright (C) 2010, Google Inc.
+ * and other copyright owners as documented in JGit's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "xinclude.h"
+#include "xtypes.h"
+#include "xdiff.h"
+
+#define MAX_PTR	UINT_MAX
+#define MAX_CNT	UINT_MAX
+
+#define LINE_END(n) (line##n + count##n - 1)
+#define LINE_END_PTR(n) (*line##n + *count##n - 1)
+
+struct histindex {
+	struct record {
+		unsigned int ptr, cnt;
+		struct record *next;
+	} **records, /* an ocurrence */
+	  **line_map; /* map of line to record chain */
+	chastore_t rcha;
+	unsigned int *next_ptrs;
+	unsigned int table_bits,
+		     records_size,
+		     line_map_size;
+
+	unsigned int max_chain_length,
+		     key_shift,
+		     ptr_shift;
+
+	unsigned int cnt,
+		     has_common;
+
+	xdfenv_t *env;
+	xpparam_t const *xpp;
+};
+
+struct region {
+	unsigned int begin1, end1;
+	unsigned int begin2, end2;
+};
+
+#define LINE_MAP(i, a) (i->line_map[(a) - i->ptr_shift])
+
+#define NEXT_PTR(index, ptr) \
+	(index->next_ptrs[(ptr) - index->ptr_shift])
+
+#define CNT(index, ptr) \
+	((LINE_MAP(index, ptr))->cnt)
+
+#define REC(env, s, l) \
+	(env->xdf##s.recs[l - 1])
+
+static int cmp_recs(xpparam_t const *xpp,
+	xrecord_t *r1, xrecord_t *r2)
+{
+	return r1->ha == r2->ha &&
+		xdl_recmatch(r1->ptr, r1->size, r2->ptr, r2->size,
+			    xpp->flags);
+}
+
+#define CMP_ENV(xpp, env, s1, l1, s2, l2) \
+	(cmp_recs(xpp, REC(env, s1, l1), REC(env, s2, l2)))
+
+#define CMP(i, s1, l1, s2, l2) \
+	(cmp_recs(i->xpp, REC(i->env, s1, l1), REC(i->env, s2, l2)))
+
+#define TABLE_HASH(index, side, line) \
+	XDL_HASHLONG((REC(index->env, side, line))->ha, index->table_bits)
+
+static int scanA(struct histindex *index, unsigned int line1, unsigned int count1)
+{
+	unsigned int ptr;
+	unsigned int tbl_idx;
+	unsigned int chain_len;
+	struct record **rec_chain, *rec;
+
+	for (ptr = LINE_END(1); line1 <= ptr; ptr--) {
+		tbl_idx = TABLE_HASH(index, 1, ptr);
+		rec_chain = index->records + tbl_idx;
+		rec = *rec_chain;
+
+		chain_len = 0;
+		while (rec) {
+			if (CMP(index, 1, rec->ptr, 1, ptr)) {
+				/*
+				 * ptr is identical to another element. Insert
+				 * it onto the front of the existing element
+				 * chain.
+				 */
+				NEXT_PTR(index, ptr) = rec->ptr;
+				rec->ptr = ptr;
+				/* cap rec->cnt at MAX_CNT */
+				rec->cnt = XDL_MIN(MAX_CNT, rec->cnt + 1);
+				LINE_MAP(index, ptr) = rec;
+				goto continue_scan;
+			}
+
+			rec = rec->next;
+			chain_len++;
+		}
+
+		if (chain_len == index->max_chain_length)
+			return -1;
+
+		/*
+		 * This is the first time we have ever seen this particular
+		 * element in the sequence. Construct a new chain for it.
+		 */
+		if (!(rec = xdl_cha_alloc(&index->rcha)))
+			return -1;
+		rec->ptr = ptr;
+		rec->cnt = 1;
+		rec->next = *rec_chain;
+		*rec_chain = rec;
+		LINE_MAP(index, ptr) = rec;
+
+continue_scan:
+		; /* no op */
+	}
+
+	return 0;
+}
+
+static int try_lcs(
+	struct histindex *index, struct region *lcs, unsigned int b_ptr,
+	unsigned int line1, unsigned int count1,
+	unsigned int line2, unsigned int count2)
+{
+	unsigned int b_next = b_ptr + 1;
+	struct record *rec = index->records[TABLE_HASH(index, 2, b_ptr)];
+	unsigned int as, ae, bs, be, np, rc;
+	int should_break;
+
+	for (; rec; rec = rec->next) {
+		if (rec->cnt > index->cnt) {
+			if (!index->has_common)
+				index->has_common = CMP(index, 1, rec->ptr, 2, b_ptr);
+			continue;
+		}
+
+		as = rec->ptr;
+		if (!CMP(index, 1, as, 2, b_ptr))
+			continue;
+
+		index->has_common = 1;
+		for (;;) {
+			should_break = 0;
+			np = NEXT_PTR(index, as);
+			bs = b_ptr;
+			ae = as;
+			be = bs;
+			rc = rec->cnt;
+
+			while (line1 < as && line2 < bs
+				&& CMP(index, 1, as - 1, 2, bs - 1)) {
+				as--;
+				bs--;
+				if (1 < rc)
+					rc = XDL_MIN(rc, CNT(index, as));
+			}
+			while (ae < LINE_END(1) && be < LINE_END(2)
+				&& CMP(index, 1, ae + 1, 2, be + 1)) {
+				ae++;
+				be++;
+				if (1 < rc)
+					rc = XDL_MIN(rc, CNT(index, ae));
+			}
+
+			if (b_next <= be)
+				b_next = be + 1;
+			if (lcs->end1 - lcs->begin1 < ae - as || rc < index->cnt) {
+				lcs->begin1 = as;
+				lcs->begin2 = bs;
+				lcs->end1 = ae;
+				lcs->end2 = be;
+				index->cnt = rc;
+			}
+
+			if (np == 0)
+				break;
+
+			while (np <= ae) {
+				np = NEXT_PTR(index, np);
+				if (np == 0) {
+					should_break = 1;
+					break;
+				}
+			}
+
+			if (should_break)
+				break;
+
+			as = np;
+		}
+	}
+	return b_next;
+}
+
+static int find_lcs(
+	struct histindex *index, struct region *lcs,
+	unsigned int line1, unsigned int count1,
+	unsigned int line2, unsigned int count2)
+{
+	unsigned int b_ptr;
+
+	if (scanA(index, line1, count1))
+		return -1;
+
+	index->cnt = index->max_chain_length + 1;
+
+	for (b_ptr = line2; b_ptr <= LINE_END(2); )
+		b_ptr = try_lcs(index, lcs, b_ptr, line1, count1, line2, count2);
+
+	return index->has_common && index->max_chain_length < index->cnt;
+}
+
+static int fall_back_to_classic_diff(struct histindex *index,
+		int line1, int count1, int line2, int count2)
+{
+	xpparam_t xpp;
+	xpp.flags = index->xpp->flags & ~XDF_HISTOGRAM_DIFF;
+
+	return xdl_fall_back_diff(index->env, &xpp,
+				  line1, count1, line2, count2);
+}
+
+static int histogram_diff(
+	xpparam_t const *xpp, xdfenv_t *env,
+	unsigned int line1, unsigned int count1,
+	unsigned int line2, unsigned int count2)
+{
+	struct histindex index;
+	struct region lcs;
+	unsigned int sz;
+	int result = -1;
+
+	if (count1 <= 0 && count2 <= 0)
+		return 0;
+
+	if (LINE_END(1) >= MAX_PTR)
+		return -1;
+
+	if (!count1) {
+		while(count2--)
+			env->xdf2.rchg[line2++ - 1] = 1;
+		return 0;
+	} else if (!count2) {
+		while(count1--)
+			env->xdf1.rchg[line1++ - 1] = 1;
+		return 0;
+	}
+
+	memset(&index, 0, sizeof(index));
+
+	index.env = env;
+	index.xpp = xpp;
+
+	index.records = NULL;
+	index.line_map = NULL;
+	/* in case of early xdl_cha_free() */
+	index.rcha.head = NULL;
+
+	index.table_bits = xdl_hashbits(count1);
+	sz = index.records_size = 1 << index.table_bits;
+	sz *= sizeof(struct record *);
+	if (!(index.records = (struct record **) xdl_malloc(sz)))
+		goto cleanup;
+	memset(index.records, 0, sz);
+
+	sz = index.line_map_size = count1;
+	sz *= sizeof(struct record *);
+	if (!(index.line_map = (struct record **) xdl_malloc(sz)))
+		goto cleanup;
+	memset(index.line_map, 0, sz);
+
+	sz = index.line_map_size;
+	sz *= sizeof(unsigned int);
+	if (!(index.next_ptrs = (unsigned int *) xdl_malloc(sz)))
+		goto cleanup;
+	memset(index.next_ptrs, 0, sz);
+
+	/* lines / 4 + 1 comes from xprepare.c:xdl_prepare_ctx() */
+	if (xdl_cha_init(&index.rcha, sizeof(struct record), count1 / 4 + 1) < 0)
+		goto cleanup;
+
+	index.ptr_shift = line1;
+	index.max_chain_length = 64;
+
+	memset(&lcs, 0, sizeof(lcs));
+	if (find_lcs(&index, &lcs, line1, count1, line2, count2))
+		result = fall_back_to_classic_diff(&index, line1, count1, line2, count2);
+	else {
+		if (lcs.begin1 == 0 && lcs.begin2 == 0) {
+			while (count1--)
+				env->xdf1.rchg[line1++ - 1] = 1;
+			while (count2--)
+				env->xdf2.rchg[line2++ - 1] = 1;
+			result = 0;
+		} else {
+			result = histogram_diff(xpp, env,
+						line1, lcs.begin1 - line1,
+						line2, lcs.begin2 - line2);
+			if (result)
+				goto cleanup;
+			result = histogram_diff(xpp, env,
+						lcs.end1 + 1, LINE_END(1) - lcs.end1,
+						lcs.end2 + 1, LINE_END(2) - lcs.end2);
+			if (result)
+				goto cleanup;
+		}
+	}
+
+cleanup:
+	xdl_free(index.records);
+	xdl_free(index.line_map);
+	xdl_free(index.next_ptrs);
+	xdl_cha_free(&index.rcha);
+
+	return result;
+}
+
+int xdl_do_histogram_diff(mmfile_t *file1, mmfile_t *file2,
+	xpparam_t const *xpp, xdfenv_t *env)
+{
+	if (xdl_prepare_env(file1, file2, xpp, env) < 0)
+		return -1;
+
+	return histogram_diff(xpp, env,
+		env->xdf1.dstart + 1, env->xdf1.dend - env->xdf1.dstart + 1,
+		env->xdf2.dstart + 1, env->xdf2.dend - env->xdf2.dstart + 1);
+}
diff --git a/src/xdiff/xinclude.h b/src/xdiff/xinclude.h
new file mode 100644
index 000000000..4a1cde909
--- /dev/null
+++ b/src/xdiff/xinclude.h
@@ -0,0 +1,46 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#if !defined(XINCLUDE_H)
+#define XINCLUDE_H
+
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+
+#ifdef _WIN32
+#else
+#include <unistd.h>
+#endif
+
+#include "xmacros.h"
+#include "xdiff.h"
+#include "xtypes.h"
+#include "xutils.h"
+#include "xprepare.h"
+#include "xdiffi.h"
+#include "xemit.h"
+
+
+#endif /* #if !defined(XINCLUDE_H) */
diff --git a/src/xdiff/xmacros.h b/src/xdiff/xmacros.h
new file mode 100644
index 000000000..165a895a9
--- /dev/null
+++ b/src/xdiff/xmacros.h
@@ -0,0 +1,54 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#if !defined(XMACROS_H)
+#define XMACROS_H
+
+
+
+
+#define XDL_MIN(a, b) ((a) < (b) ? (a): (b))
+#define XDL_MAX(a, b) ((a) > (b) ? (a): (b))
+#define XDL_ABS(v) ((v) >= 0 ? (v): -(v))
+#define XDL_ISDIGIT(c) ((c) >= '0' && (c) <= '9')
+#define XDL_ISSPACE(c) (isspace((unsigned char)(c)))
+#define XDL_ADDBITS(v,b)	((v) + ((v) >> (b)))
+#define XDL_MASKBITS(b)		((1UL << (b)) - 1)
+#define XDL_HASHLONG(v,b)	(XDL_ADDBITS((unsigned long)(v), b) & XDL_MASKBITS(b))
+#define XDL_PTRFREE(p) do { if (p) { xdl_free(p); (p) = NULL; } } while (0)
+#define XDL_LE32_PUT(p, v) \
+do { \
+	unsigned char *__p = (unsigned char *) (p); \
+	*__p++ = (unsigned char) (v); \
+	*__p++ = (unsigned char) ((v) >> 8); \
+	*__p++ = (unsigned char) ((v) >> 16); \
+	*__p = (unsigned char) ((v) >> 24); \
+} while (0)
+#define XDL_LE32_GET(p, v) \
+do { \
+	unsigned char const *__p = (unsigned char const *) (p); \
+	(v) = (unsigned long) __p[0] | ((unsigned long) __p[1]) << 8 | \
+		((unsigned long) __p[2]) << 16 | ((unsigned long) __p[3]) << 24; \
+} while (0)
+
+
+#endif /* #if !defined(XMACROS_H) */
diff --git a/src/xdiff/xmerge.c b/src/xdiff/xmerge.c
new file mode 100644
index 000000000..84e424672
--- /dev/null
+++ b/src/xdiff/xmerge.c
@@ -0,0 +1,619 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003-2006 Davide Libenzi, Johannes E. Schindelin
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#include "xinclude.h"
+
+typedef struct s_xdmerge {
+	struct s_xdmerge *next;
+	/*
+	 * 0 = conflict,
+	 * 1 = no conflict, take first,
+	 * 2 = no conflict, take second.
+	 * 3 = no conflict, take both.
+	 */
+	int mode;
+	/*
+	 * These point at the respective postimages.  E.g. <i1,chg1> is
+	 * how side #1 wants to change the common ancestor; if there is no
+	 * overlap, lines before i1 in the postimage of side #1 appear
+	 * in the merge result as a region touched by neither side.
+	 */
+	long i1, i2;
+	long chg1, chg2;
+	/*
+	 * These point at the preimage; of course there is just one
+	 * preimage, that is from the shared common ancestor.
+	 */
+	long i0;
+	long chg0;
+} xdmerge_t;
+
+static int xdl_append_merge(xdmerge_t **merge, int mode,
+			    long i0, long chg0,
+			    long i1, long chg1,
+			    long i2, long chg2)
+{
+	xdmerge_t *m = *merge;
+	if (m && (i1 <= m->i1 + m->chg1 || i2 <= m->i2 + m->chg2)) {
+		if (mode != m->mode)
+			m->mode = 0;
+		m->chg0 = i0 + chg0 - m->i0;
+		m->chg1 = i1 + chg1 - m->i1;
+		m->chg2 = i2 + chg2 - m->i2;
+	} else {
+		m = xdl_malloc(sizeof(xdmerge_t));
+		if (!m)
+			return -1;
+		m->next = NULL;
+		m->mode = mode;
+		m->i0 = i0;
+		m->chg0 = chg0;
+		m->i1 = i1;
+		m->chg1 = chg1;
+		m->i2 = i2;
+		m->chg2 = chg2;
+		if (*merge)
+			(*merge)->next = m;
+		*merge = m;
+	}
+	return 0;
+}
+
+static int xdl_cleanup_merge(xdmerge_t *c)
+{
+	int count = 0;
+	xdmerge_t *next_c;
+
+	/* were there conflicts? */
+	for (; c; c = next_c) {
+		if (c->mode == 0)
+			count++;
+		next_c = c->next;
+		free(c);
+	}
+	return count;
+}
+
+static int xdl_merge_cmp_lines(xdfenv_t *xe1, int i1, xdfenv_t *xe2, int i2,
+		int line_count, long flags)
+{
+	int i;
+	xrecord_t **rec1 = xe1->xdf2.recs + i1;
+	xrecord_t **rec2 = xe2->xdf2.recs + i2;
+
+	for (i = 0; i < line_count; i++) {
+		int result = xdl_recmatch(rec1[i]->ptr, rec1[i]->size,
+			rec2[i]->ptr, rec2[i]->size, flags);
+		if (!result)
+			return -1;
+	}
+	return 0;
+}
+
+static int xdl_recs_copy_0(int use_orig, xdfenv_t *xe, int i, int count, int add_nl, char *dest)
+{
+	xrecord_t **recs;
+	int size = 0;
+
+	recs = (use_orig ? xe->xdf1.recs : xe->xdf2.recs) + i;
+
+	if (count < 1)
+		return 0;
+
+	for (i = 0; i < count; size += recs[i++]->size)
+		if (dest)
+			memcpy(dest + size, recs[i]->ptr, recs[i]->size);
+	if (add_nl) {
+		i = recs[count - 1]->size;
+		if (i == 0 || recs[count - 1]->ptr[i - 1] != '\n') {
+			if (dest)
+				dest[size] = '\n';
+			size++;
+		}
+	}
+	return size;
+}
+
+static int xdl_recs_copy(xdfenv_t *xe, int i, int count, int add_nl, char *dest)
+{
+	return xdl_recs_copy_0(0, xe, i, count, add_nl, dest);
+}
+
+static int xdl_orig_copy(xdfenv_t *xe, int i, int count, int add_nl, char *dest)
+{
+	return xdl_recs_copy_0(1, xe, i, count, add_nl, dest);
+}
+
+static int fill_conflict_hunk(xdfenv_t *xe1, const char *name1,
+			      xdfenv_t *xe2, const char *name2,
+			      const char *name3,
+			      int size, int i, int style,
+			      xdmerge_t *m, char *dest, int marker_size)
+{
+	int marker1_size = (name1 ? (int)strlen(name1) + 1 : 0);
+	int marker2_size = (name2 ? (int)strlen(name2) + 1 : 0);
+	int marker3_size = (name3 ? (int)strlen(name3) + 1 : 0);
+
+	if (marker_size <= 0)
+		marker_size = DEFAULT_CONFLICT_MARKER_SIZE;
+
+	/* Before conflicting part */
+	size += xdl_recs_copy(xe1, i, m->i1 - i, 0,
+			      dest ? dest + size : NULL);
+
+	if (!dest) {
+		size += marker_size + 1 + marker1_size;
+	} else {
+		memset(dest + size, '<', marker_size);
+		size += marker_size;
+		if (marker1_size) {
+			dest[size] = ' ';
+			memcpy(dest + size + 1, name1, marker1_size - 1);
+			size += marker1_size;
+		}
+		dest[size++] = '\n';
+	}
+
+	/* Postimage from side #1 */
+	size += xdl_recs_copy(xe1, m->i1, m->chg1, 1,
+			      dest ? dest + size : NULL);
+
+	if (style == XDL_MERGE_DIFF3) {
+		/* Shared preimage */
+		if (!dest) {
+			size += marker_size + 1 + marker3_size;
+		} else {
+			memset(dest + size, '|', marker_size);
+			size += marker_size;
+			if (marker3_size) {
+				dest[size] = ' ';
+				memcpy(dest + size + 1, name3, marker3_size - 1);
+				size += marker3_size;
+			}
+			dest[size++] = '\n';
+		}
+		size += xdl_orig_copy(xe1, m->i0, m->chg0, 1,
+				      dest ? dest + size : NULL);
+	}
+
+	if (!dest) {
+		size += marker_size + 1;
+	} else {
+		memset(dest + size, '=', marker_size);
+		size += marker_size;
+		dest[size++] = '\n';
+	}
+
+	/* Postimage from side #2 */
+	size += xdl_recs_copy(xe2, m->i2, m->chg2, 1,
+			      dest ? dest + size : NULL);
+	if (!dest) {
+		size += marker_size + 1 + marker2_size;
+	} else {
+		memset(dest + size, '>', marker_size);
+		size += marker_size;
+		if (marker2_size) {
+			dest[size] = ' ';
+			memcpy(dest + size + 1, name2, marker2_size - 1);
+			size += marker2_size;
+		}
+		dest[size++] = '\n';
+	}
+	return size;
+}
+
+static int xdl_fill_merge_buffer(xdfenv_t *xe1, const char *name1,
+				 xdfenv_t *xe2, const char *name2,
+				 const char *ancestor_name,
+				 int favor,
+				 xdmerge_t *m, char *dest, int style,
+				 int marker_size)
+{
+	int size, i;
+
+	for (size = i = 0; m; m = m->next) {
+		if (favor && !m->mode)
+			m->mode = favor;
+
+		if (m->mode == 0)
+			size = fill_conflict_hunk(xe1, name1, xe2, name2,
+						  ancestor_name,
+						  size, i, style, m, dest,
+						  marker_size);
+		else if (m->mode & 3) {
+			/* Before conflicting part */
+			size += xdl_recs_copy(xe1, i, m->i1 - i, 0,
+					      dest ? dest + size : NULL);
+			/* Postimage from side #1 */
+			if (m->mode & 1)
+				size += xdl_recs_copy(xe1, m->i1, m->chg1, 1,
+						      dest ? dest + size : NULL);
+			/* Postimage from side #2 */
+			if (m->mode & 2)
+				size += xdl_recs_copy(xe2, m->i2, m->chg2, 1,
+						      dest ? dest + size : NULL);
+		} else
+			continue;
+		i = m->i1 + m->chg1;
+	}
+	size += xdl_recs_copy(xe1, i, xe1->xdf2.nrec - i, 0,
+			      dest ? dest + size : NULL);
+	return size;
+}
+
+/*
+ * Sometimes, changes are not quite identical, but differ in only a few
+ * lines. Try hard to show only these few lines as conflicting.
+ */
+static int xdl_refine_conflicts(xdfenv_t *xe1, xdfenv_t *xe2, xdmerge_t *m,
+		xpparam_t const *xpp)
+{
+	for (; m; m = m->next) {
+		mmfile_t t1, t2;
+		xdfenv_t xe;
+		xdchange_t *xscr, *x;
+		int i1 = m->i1, i2 = m->i2;
+
+		/* let's handle just the conflicts */
+		if (m->mode)
+			continue;
+
+		/* no sense refining a conflict when one side is empty */
+		if (m->chg1 == 0 || m->chg2 == 0)
+			continue;
+
+		/*
+		 * This probably does not work outside git, since
+		 * we have a very simple mmfile structure.
+		 */
+		t1.ptr = (char *)xe1->xdf2.recs[m->i1]->ptr;
+		t1.size = xe1->xdf2.recs[m->i1 + m->chg1 - 1]->ptr
+			+ xe1->xdf2.recs[m->i1 + m->chg1 - 1]->size - t1.ptr;
+		t2.ptr = (char *)xe2->xdf2.recs[m->i2]->ptr;
+		t2.size = xe2->xdf2.recs[m->i2 + m->chg2 - 1]->ptr
+			+ xe2->xdf2.recs[m->i2 + m->chg2 - 1]->size - t2.ptr;
+		if (xdl_do_diff(&t1, &t2, xpp, &xe) < 0)
+			return -1;
+		if (xdl_change_compact(&xe.xdf1, &xe.xdf2, xpp->flags) < 0 ||
+		    xdl_change_compact(&xe.xdf2, &xe.xdf1, xpp->flags) < 0 ||
+		    xdl_build_script(&xe, &xscr) < 0) {
+			xdl_free_env(&xe);
+			return -1;
+		}
+		if (!xscr) {
+			/* If this happens, the changes are identical. */
+			xdl_free_env(&xe);
+			m->mode = 4;
+			continue;
+		}
+		x = xscr;
+		m->i1 = xscr->i1 + i1;
+		m->chg1 = xscr->chg1;
+		m->i2 = xscr->i2 + i2;
+		m->chg2 = xscr->chg2;
+		while (xscr->next) {
+			xdmerge_t *m2 = xdl_malloc(sizeof(xdmerge_t));
+			if (!m2) {
+				xdl_free_env(&xe);
+				xdl_free_script(x);
+				return -1;
+			}
+			xscr = xscr->next;
+			m2->next = m->next;
+			m->next = m2;
+			m = m2;
+			m->mode = 0;
+			m->i1 = xscr->i1 + i1;
+			m->chg1 = xscr->chg1;
+			m->i2 = xscr->i2 + i2;
+			m->chg2 = xscr->chg2;
+		}
+		xdl_free_env(&xe);
+		xdl_free_script(x);
+	}
+	return 0;
+}
+
+static int line_contains_alnum(const char *ptr, long size)
+{
+	while (size--)
+		if (isalnum((unsigned char)*(ptr++)))
+			return 1;
+	return 0;
+}
+
+static int lines_contain_alnum(xdfenv_t *xe, int i, int chg)
+{
+	for (; chg; chg--, i++)
+		if (line_contains_alnum(xe->xdf2.recs[i]->ptr,
+				xe->xdf2.recs[i]->size))
+			return 1;
+	return 0;
+}
+
+/*
+ * This function merges m and m->next, marking everything between those hunks
+ * as conflicting, too.
+ */
+static void xdl_merge_two_conflicts(xdmerge_t *m)
+{
+	xdmerge_t *next_m = m->next;
+	m->chg1 = next_m->i1 + next_m->chg1 - m->i1;
+	m->chg2 = next_m->i2 + next_m->chg2 - m->i2;
+	m->next = next_m->next;
+	free(next_m);
+}
+
+/*
+ * If there are less than 3 non-conflicting lines between conflicts,
+ * it appears simpler -- because it takes up less (or as many) lines --
+ * if the lines are moved into the conflicts.
+ */
+static int xdl_simplify_non_conflicts(xdfenv_t *xe1, xdmerge_t *m,
+				      int simplify_if_no_alnum)
+{
+	int result = 0;
+
+	if (!m)
+		return result;
+	for (;;) {
+		xdmerge_t *next_m = m->next;
+		int begin, end;
+
+		if (!next_m)
+			return result;
+
+		begin = m->i1 + m->chg1;
+		end = next_m->i1;
+
+		if (m->mode != 0 || next_m->mode != 0 ||
+		    (end - begin > 3 &&
+		     (!simplify_if_no_alnum ||
+		      lines_contain_alnum(xe1, begin, end - begin)))) {
+			m = next_m;
+		} else {
+			result++;
+			xdl_merge_two_conflicts(m);
+		}
+	}
+}
+
+/*
+ * level == 0: mark all overlapping changes as conflict
+ * level == 1: mark overlapping changes as conflict only if not identical
+ * level == 2: analyze non-identical changes for minimal conflict set
+ * level == 3: analyze non-identical changes for minimal conflict set, but
+ *             treat hunks not containing any letter or number as conflicting
+ *
+ * returns < 0 on error, == 0 for no conflicts, else number of conflicts
+ */
+static int xdl_do_merge(xdfenv_t *xe1, xdchange_t *xscr1,
+		xdfenv_t *xe2, xdchange_t *xscr2,
+		xmparam_t const *xmp, mmbuffer_t *result)
+{
+	xdmerge_t *changes, *c;
+	xpparam_t const *xpp = &xmp->xpp;
+	const char *const ancestor_name = xmp->ancestor;
+	const char *const name1 = xmp->file1;
+	const char *const name2 = xmp->file2;
+	int i0, i1, i2, chg0, chg1, chg2;
+	int level = xmp->level;
+	int style = xmp->style;
+	int favor = xmp->favor;
+
+	if (style == XDL_MERGE_DIFF3) {
+		/*
+		 * "diff3 -m" output does not make sense for anything
+		 * more aggressive than XDL_MERGE_EAGER.
+		 */
+		if (XDL_MERGE_EAGER < level)
+			level = XDL_MERGE_EAGER;
+	}
+
+	c = changes = NULL;
+
+	while (xscr1 && xscr2) {
+		if (!changes)
+			changes = c;
+		if (xscr1->i1 + xscr1->chg1 < xscr2->i1) {
+			i0 = xscr1->i1;
+			i1 = xscr1->i2;
+			i2 = xscr2->i2 - xscr2->i1 + xscr1->i1;
+			chg0 = xscr1->chg1;
+			chg1 = xscr1->chg2;
+			chg2 = xscr1->chg1;
+			if (xdl_append_merge(&c, 1,
+					     i0, chg0, i1, chg1, i2, chg2)) {
+				xdl_cleanup_merge(changes);
+				return -1;
+			}
+			xscr1 = xscr1->next;
+			continue;
+		}
+		if (xscr2->i1 + xscr2->chg1 < xscr1->i1) {
+			i0 = xscr2->i1;
+			i1 = xscr1->i2 - xscr1->i1 + xscr2->i1;
+			i2 = xscr2->i2;
+			chg0 = xscr2->chg1;
+			chg1 = xscr2->chg1;
+			chg2 = xscr2->chg2;
+			if (xdl_append_merge(&c, 2,
+					     i0, chg0, i1, chg1, i2, chg2)) {
+				xdl_cleanup_merge(changes);
+				return -1;
+			}
+			xscr2 = xscr2->next;
+			continue;
+		}
+		if (level == XDL_MERGE_MINIMAL || xscr1->i1 != xscr2->i1 ||
+				xscr1->chg1 != xscr2->chg1 ||
+				xscr1->chg2 != xscr2->chg2 ||
+				xdl_merge_cmp_lines(xe1, xscr1->i2,
+					xe2, xscr2->i2,
+					xscr1->chg2, xpp->flags)) {
+			/* conflict */
+			int off = xscr1->i1 - xscr2->i1;
+			int ffo = off + xscr1->chg1 - xscr2->chg1;
+
+			i0 = xscr1->i1;
+			i1 = xscr1->i2;
+			i2 = xscr2->i2;
+			if (off > 0) {
+				i0 -= off;
+				i1 -= off;
+			}
+			else
+				i2 += off;
+			chg0 = xscr1->i1 + xscr1->chg1 - i0;
+			chg1 = xscr1->i2 + xscr1->chg2 - i1;
+			chg2 = xscr2->i2 + xscr2->chg2 - i2;
+			if (ffo < 0) {
+				chg0 -= ffo;
+				chg1 -= ffo;
+			} else
+				chg2 += ffo;
+			if (xdl_append_merge(&c, 0,
+					     i0, chg0, i1, chg1, i2, chg2)) {
+				xdl_cleanup_merge(changes);
+				return -1;
+			}
+		}
+
+		i1 = xscr1->i1 + xscr1->chg1;
+		i2 = xscr2->i1 + xscr2->chg1;
+
+		if (i1 >= i2)
+			xscr2 = xscr2->next;
+		if (i2 >= i1)
+			xscr1 = xscr1->next;
+	}
+	while (xscr1) {
+		if (!changes)
+			changes = c;
+		i0 = xscr1->i1;
+		i1 = xscr1->i2;
+		i2 = xscr1->i1 + xe2->xdf2.nrec - xe2->xdf1.nrec;
+		chg0 = xscr1->chg1;
+		chg1 = xscr1->chg2;
+		chg2 = xscr1->chg1;
+		if (xdl_append_merge(&c, 1,
+				     i0, chg0, i1, chg1, i2, chg2)) {
+			xdl_cleanup_merge(changes);
+			return -1;
+		}
+		xscr1 = xscr1->next;
+	}
+	while (xscr2) {
+		if (!changes)
+			changes = c;
+		i0 = xscr2->i1;
+		i1 = xscr2->i1 + xe1->xdf2.nrec - xe1->xdf1.nrec;
+		i2 = xscr2->i2;
+		chg0 = xscr2->chg1;
+		chg1 = xscr2->chg1;
+		chg2 = xscr2->chg2;
+		if (xdl_append_merge(&c, 2,
+				     i0, chg0, i1, chg1, i2, chg2)) {
+			xdl_cleanup_merge(changes);
+			return -1;
+		}
+		xscr2 = xscr2->next;
+	}
+	if (!changes)
+		changes = c;
+	/* refine conflicts */
+	if (XDL_MERGE_ZEALOUS <= level &&
+	    (xdl_refine_conflicts(xe1, xe2, changes, xpp) < 0 ||
+	     xdl_simplify_non_conflicts(xe1, changes,
+					XDL_MERGE_ZEALOUS < level) < 0)) {
+		xdl_cleanup_merge(changes);
+		return -1;
+	}
+	/* output */
+	if (result) {
+		int marker_size = xmp->marker_size;
+		int size = xdl_fill_merge_buffer(xe1, name1, xe2, name2,
+						 ancestor_name,
+						 favor, changes, NULL, style,
+						 marker_size);
+		result->ptr = xdl_malloc(size);
+		if (!result->ptr) {
+			xdl_cleanup_merge(changes);
+			return -1;
+		}
+		result->size = size;
+		xdl_fill_merge_buffer(xe1, name1, xe2, name2,
+				      ancestor_name, favor, changes,
+				      result->ptr, style, marker_size);
+	}
+	return xdl_cleanup_merge(changes);
+}
+
+int xdl_merge(mmfile_t *orig, mmfile_t *mf1, mmfile_t *mf2,
+		xmparam_t const *xmp, mmbuffer_t *result)
+{
+	xdchange_t *xscr1, *xscr2;
+	xdfenv_t xe1, xe2;
+	int status;
+	xpparam_t const *xpp = &xmp->xpp;
+
+	result->ptr = NULL;
+	result->size = 0;
+
+	if (xdl_do_diff(orig, mf1, xpp, &xe1) < 0 ||
+			xdl_do_diff(orig, mf2, xpp, &xe2) < 0) {
+		return -1;
+	}
+	if (xdl_change_compact(&xe1.xdf1, &xe1.xdf2, xpp->flags) < 0 ||
+	    xdl_change_compact(&xe1.xdf2, &xe1.xdf1, xpp->flags) < 0 ||
+	    xdl_build_script(&xe1, &xscr1) < 0) {
+		xdl_free_env(&xe1);
+		return -1;
+	}
+	if (xdl_change_compact(&xe2.xdf1, &xe2.xdf2, xpp->flags) < 0 ||
+	    xdl_change_compact(&xe2.xdf2, &xe2.xdf1, xpp->flags) < 0 ||
+	    xdl_build_script(&xe2, &xscr2) < 0) {
+		xdl_free_env(&xe2);
+		return -1;
+	}
+	status = 0;
+	if (!xscr1) {
+		result->ptr = xdl_malloc(mf2->size);
+		memcpy(result->ptr, mf2->ptr, mf2->size);
+		result->size = mf2->size;
+	} else if (!xscr2) {
+		result->ptr = xdl_malloc(mf1->size);
+		memcpy(result->ptr, mf1->ptr, mf1->size);
+		result->size = mf1->size;
+	} else {
+		status = xdl_do_merge(&xe1, xscr1,
+				      &xe2, xscr2,
+				      xmp, result);
+	}
+	xdl_free_script(xscr1);
+	xdl_free_script(xscr2);
+
+	xdl_free_env(&xe1);
+	xdl_free_env(&xe2);
+
+	return status;
+}
diff --git a/src/xdiff/xpatience.c b/src/xdiff/xpatience.c
new file mode 100644
index 000000000..fdd7d0263
--- /dev/null
+++ b/src/xdiff/xpatience.c
@@ -0,0 +1,358 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003-2009 Davide Libenzi, Johannes E. Schindelin
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+#include "xinclude.h"
+#include "xtypes.h"
+#include "xdiff.h"
+
+/*
+ * The basic idea of patience diff is to find lines that are unique in
+ * both files.  These are intuitively the ones that we want to see as
+ * common lines.
+ *
+ * The maximal ordered sequence of such line pairs (where ordered means
+ * that the order in the sequence agrees with the order of the lines in
+ * both files) naturally defines an initial set of common lines.
+ *
+ * Now, the algorithm tries to extend the set of common lines by growing
+ * the line ranges where the files have identical lines.
+ *
+ * Between those common lines, the patience diff algorithm is applied
+ * recursively, until no unique line pairs can be found; these line ranges
+ * are handled by the well-known Myers algorithm.
+ */
+
+#define NON_UNIQUE ULONG_MAX
+
+/*
+ * This is a hash mapping from line hash to line numbers in the first and
+ * second file.
+ */
+struct hashmap {
+	int nr, alloc;
+	struct entry {
+		unsigned long hash;
+		/*
+		 * 0 = unused entry, 1 = first line, 2 = second, etc.
+		 * line2 is NON_UNIQUE if the line is not unique
+		 * in either the first or the second file.
+		 */
+		unsigned long line1, line2;
+		/*
+		 * "next" & "previous" are used for the longest common
+		 * sequence;
+		 * initially, "next" reflects only the order in file1.
+		 */
+		struct entry *next, *previous;
+	} *entries, *first, *last;
+	/* were common records found? */
+	unsigned long has_matches;
+	mmfile_t *file1, *file2;
+	xdfenv_t *env;
+	xpparam_t const *xpp;
+};
+
+/* The argument "pass" is 1 for the first file, 2 for the second. */
+static void insert_record(int line, struct hashmap *map, int pass)
+{
+	xrecord_t **records = pass == 1 ?
+		map->env->xdf1.recs : map->env->xdf2.recs;
+	xrecord_t *record = records[line - 1], *other;
+	/*
+	 * After xdl_prepare_env() (or more precisely, due to
+	 * xdl_classify_record()), the "ha" member of the records (AKA lines)
+	 * is _not_ the hash anymore, but a linearized version of it.  In
+	 * other words, the "ha" member is guaranteed to start with 0 and
+	 * the second record's ha can only be 0 or 1, etc.
+	 *
+	 * So we multiply ha by 2 in the hope that the hashing was
+	 * "unique enough".
+	 */
+	int index = (int)((record->ha << 1) % map->alloc);
+
+	while (map->entries[index].line1) {
+		other = map->env->xdf1.recs[map->entries[index].line1 - 1];
+		if (map->entries[index].hash != record->ha ||
+				!xdl_recmatch(record->ptr, record->size,
+					other->ptr, other->size,
+					map->xpp->flags)) {
+			if (++index >= map->alloc)
+				index = 0;
+			continue;
+		}
+		if (pass == 2)
+			map->has_matches = 1;
+		if (pass == 1 || map->entries[index].line2)
+			map->entries[index].line2 = NON_UNIQUE;
+		else
+			map->entries[index].line2 = line;
+		return;
+	}
+	if (pass == 2)
+		return;
+	map->entries[index].line1 = line;
+	map->entries[index].hash = record->ha;
+	if (!map->first)
+		map->first = map->entries + index;
+	if (map->last) {
+		map->last->next = map->entries + index;
+		map->entries[index].previous = map->last;
+	}
+	map->last = map->entries + index;
+	map->nr++;
+}
+
+/*
+ * This function has to be called for each recursion into the inter-hunk
+ * parts, as previously non-unique lines can become unique when being
+ * restricted to a smaller part of the files.
+ *
+ * It is assumed that env has been prepared using xdl_prepare().
+ */
+static int fill_hashmap(mmfile_t *file1, mmfile_t *file2,
+		xpparam_t const *xpp, xdfenv_t *env,
+		struct hashmap *result,
+		int line1, int count1, int line2, int count2)
+{
+	result->file1 = file1;
+	result->file2 = file2;
+	result->xpp = xpp;
+	result->env = env;
+
+	/* We know exactly how large we want the hash map */
+	result->alloc = count1 * 2;
+	result->entries = (struct entry *)
+		xdl_malloc(result->alloc * sizeof(struct entry));
+	if (!result->entries)
+		return -1;
+	memset(result->entries, 0, result->alloc * sizeof(struct entry));
+
+	/* First, fill with entries from the first file */
+	while (count1--)
+		insert_record(line1++, result, 1);
+
+	/* Then search for matches in the second file */
+	while (count2--)
+		insert_record(line2++, result, 2);
+
+	return 0;
+}
+
+/*
+ * Find the longest sequence with a smaller last element (meaning a smaller
+ * line2, as we construct the sequence with entries ordered by line1).
+ */
+static int binary_search(struct entry **sequence, int longest,
+		struct entry *entry)
+{
+	int left = -1, right = longest;
+
+	while (left + 1 < right) {
+		int middle = (left + right) / 2;
+		/* by construction, no two entries can be equal */
+		if (sequence[middle]->line2 > entry->line2)
+			right = middle;
+		else
+			left = middle;
+	}
+	/* return the index in "sequence", _not_ the sequence length */
+	return left;
+}
+
+/*
+ * The idea is to start with the list of common unique lines sorted by
+ * the order in file1.  For each of these pairs, the longest (partial)
+ * sequence whose last element's line2 is smaller is determined.
+ *
+ * For efficiency, the sequences are kept in a list containing exactly one
+ * item per sequence length: the sequence with the smallest last
+ * element (in terms of line2).
+ */
+static struct entry *find_longest_common_sequence(struct hashmap *map)
+{
+	struct entry **sequence = xdl_malloc(map->nr * sizeof(struct entry *));
+	int longest = 0, i;
+	struct entry *entry;
+
+	for (entry = map->first; entry; entry = entry->next) {
+		if (!entry->line2 || entry->line2 == NON_UNIQUE)
+			continue;
+		i = binary_search(sequence, longest, entry);
+		entry->previous = i < 0 ? NULL : sequence[i];
+		sequence[++i] = entry;
+		if (i == longest)
+			longest++;
+	}
+
+	/* No common unique lines were found */
+	if (!longest) {
+		xdl_free(sequence);
+		return NULL;
+	}
+
+	/* Iterate starting at the last element, adjusting the "next" members */
+	entry = sequence[longest - 1];
+	entry->next = NULL;
+	while (entry->previous) {
+		entry->previous->next = entry;
+		entry = entry->previous;
+	}
+	xdl_free(sequence);
+	return entry;
+}
+
+static int match(struct hashmap *map, int line1, int line2)
+{
+	xrecord_t *record1 = map->env->xdf1.recs[line1 - 1];
+	xrecord_t *record2 = map->env->xdf2.recs[line2 - 1];
+	return xdl_recmatch(record1->ptr, record1->size,
+		record2->ptr, record2->size, map->xpp->flags);
+}
+
+static int patience_diff(mmfile_t *file1, mmfile_t *file2,
+		xpparam_t const *xpp, xdfenv_t *env,
+		int line1, int count1, int line2, int count2);
+
+static int walk_common_sequence(struct hashmap *map, struct entry *first,
+		int line1, int count1, int line2, int count2)
+{
+	int end1 = line1 + count1, end2 = line2 + count2;
+	int next1, next2;
+
+	for (;;) {
+		/* Try to grow the line ranges of common lines */
+		if (first) {
+			next1 = first->line1;
+			next2 = first->line2;
+			while (next1 > line1 && next2 > line2 &&
+					match(map, next1 - 1, next2 - 1)) {
+				next1--;
+				next2--;
+			}
+		} else {
+			next1 = end1;
+			next2 = end2;
+		}
+		while (line1 < next1 && line2 < next2 &&
+				match(map, line1, line2)) {
+			line1++;
+			line2++;
+		}
+
+		/* Recurse */
+		if (next1 > line1 || next2 > line2) {
+			struct hashmap submap;
+
+			memset(&submap, 0, sizeof(submap));
+			if (patience_diff(map->file1, map->file2,
+					map->xpp, map->env,
+					line1, next1 - line1,
+					line2, next2 - line2))
+				return -1;
+		}
+
+		if (!first)
+			return 0;
+
+		while (first->next &&
+				first->next->line1 == first->line1 + 1 &&
+				first->next->line2 == first->line2 + 1)
+			first = first->next;
+
+		line1 = first->line1 + 1;
+		line2 = first->line2 + 1;
+
+		first = first->next;
+	}
+}
+
+static int fall_back_to_classic_diff(struct hashmap *map,
+		int line1, int count1, int line2, int count2)
+{
+	xpparam_t xpp;
+	xpp.flags = map->xpp->flags & ~XDF_PATIENCE_DIFF;
+
+	return xdl_fall_back_diff(map->env, &xpp,
+				  line1, count1, line2, count2);
+}
+
+/*
+ * Recursively find the longest common sequence of unique lines,
+ * and if none was found, ask xdl_do_diff() to do the job.
+ *
+ * This function assumes that env was prepared with xdl_prepare_env().
+ */
+static int patience_diff(mmfile_t *file1, mmfile_t *file2,
+		xpparam_t const *xpp, xdfenv_t *env,
+		int line1, int count1, int line2, int count2)
+{
+	struct hashmap map;
+	struct entry *first;
+	int result = 0;
+
+	/* trivial case: one side is empty */
+	if (!count1) {
+		while(count2--)
+			env->xdf2.rchg[line2++ - 1] = 1;
+		return 0;
+	} else if (!count2) {
+		while(count1--)
+			env->xdf1.rchg[line1++ - 1] = 1;
+		return 0;
+	}
+
+	memset(&map, 0, sizeof(map));
+	if (fill_hashmap(file1, file2, xpp, env, &map,
+			line1, count1, line2, count2))
+		return -1;
+
+	/* are there any matching lines at all? */
+	if (!map.has_matches) {
+		while(count1--)
+			env->xdf1.rchg[line1++ - 1] = 1;
+		while(count2--)
+			env->xdf2.rchg[line2++ - 1] = 1;
+		xdl_free(map.entries);
+		return 0;
+	}
+
+	first = find_longest_common_sequence(&map);
+	if (first)
+		result = walk_common_sequence(&map, first,
+			line1, count1, line2, count2);
+	else
+		result = fall_back_to_classic_diff(&map,
+			line1, count1, line2, count2);
+
+	xdl_free(map.entries);
+	return result;
+}
+
+int xdl_do_patience_diff(mmfile_t *file1, mmfile_t *file2,
+		xpparam_t const *xpp, xdfenv_t *env)
+{
+	if (xdl_prepare_env(file1, file2, xpp, env) < 0)
+		return -1;
+
+	/* environment is cleaned up in xdl_diff() */
+	return patience_diff(file1, file2, xpp, env,
+			1, env->xdf1.nrec, 1, env->xdf2.nrec);
+}
diff --git a/src/xdiff/xprepare.c b/src/xdiff/xprepare.c
new file mode 100644
index 000000000..e419f4f72
--- /dev/null
+++ b/src/xdiff/xprepare.c
@@ -0,0 +1,483 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#include "xinclude.h"
+
+
+#define XDL_KPDIS_RUN 4
+#define XDL_MAX_EQLIMIT 1024
+#define XDL_SIMSCAN_WINDOW 100
+#define XDL_GUESS_NLINES1 256
+#define XDL_GUESS_NLINES2 20
+
+
+typedef struct s_xdlclass {
+	struct s_xdlclass *next;
+	unsigned long ha;
+	char const *line;
+	long size;
+	long idx;
+	long len1, len2;
+} xdlclass_t;
+
+typedef struct s_xdlclassifier {
+	unsigned int hbits;
+	long hsize;
+	xdlclass_t **rchash;
+	chastore_t ncha;
+	xdlclass_t **rcrecs;
+	long alloc;
+	long count;
+	long flags;
+} xdlclassifier_t;
+
+
+
+
+static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags);
+static void xdl_free_classifier(xdlclassifier_t *cf);
+static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t **rhash,
+			       unsigned int hbits, xrecord_t *rec);
+static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp,
+			   xdlclassifier_t *cf, xdfile_t *xdf);
+static void xdl_free_ctx(xdfile_t *xdf);
+static int xdl_clean_mmatch(char const *dis, long i, long s, long e);
+static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2);
+static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2);
+static int xdl_optimize_ctxs(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2);
+
+
+
+
+static int xdl_init_classifier(xdlclassifier_t *cf, long size, long flags) {
+	cf->flags = flags;
+
+	cf->hbits = xdl_hashbits((unsigned int) size);
+	cf->hsize = 1 << cf->hbits;
+
+	if (xdl_cha_init(&cf->ncha, sizeof(xdlclass_t), size / 4 + 1) < 0) {
+
+		return -1;
+	}
+	if (!(cf->rchash = (xdlclass_t **) xdl_malloc(cf->hsize * sizeof(xdlclass_t *)))) {
+
+		xdl_cha_free(&cf->ncha);
+		return -1;
+	}
+	memset(cf->rchash, 0, cf->hsize * sizeof(xdlclass_t *));
+
+	cf->alloc = size;
+	if (!(cf->rcrecs = (xdlclass_t **) xdl_malloc(cf->alloc * sizeof(xdlclass_t *)))) {
+
+		xdl_free(cf->rchash);
+		xdl_cha_free(&cf->ncha);
+		return -1;
+	}
+
+	cf->count = 0;
+
+	return 0;
+}
+
+
+static void xdl_free_classifier(xdlclassifier_t *cf) {
+
+	xdl_free(cf->rcrecs);
+	xdl_free(cf->rchash);
+	xdl_cha_free(&cf->ncha);
+}
+
+
+static int xdl_classify_record(unsigned int pass, xdlclassifier_t *cf, xrecord_t **rhash,
+			       unsigned int hbits, xrecord_t *rec) {
+	long hi;
+	char const *line;
+	xdlclass_t *rcrec;
+	xdlclass_t **rcrecs;
+
+	line = rec->ptr;
+	hi = (long) XDL_HASHLONG(rec->ha, cf->hbits);
+	for (rcrec = cf->rchash[hi]; rcrec; rcrec = rcrec->next)
+		if (rcrec->ha == rec->ha &&
+				xdl_recmatch(rcrec->line, rcrec->size,
+					rec->ptr, rec->size, cf->flags))
+			break;
+
+	if (!rcrec) {
+		if (!(rcrec = xdl_cha_alloc(&cf->ncha))) {
+
+			return -1;
+		}
+		rcrec->idx = cf->count++;
+		if (cf->count > cf->alloc) {
+			cf->alloc *= 2;
+			if (!(rcrecs = (xdlclass_t **) xdl_realloc(cf->rcrecs, cf->alloc * sizeof(xdlclass_t *)))) {
+
+				return -1;
+			}
+			cf->rcrecs = rcrecs;
+		}
+		cf->rcrecs[rcrec->idx] = rcrec;
+		rcrec->line = line;
+		rcrec->size = rec->size;
+		rcrec->ha = rec->ha;
+		rcrec->len1 = rcrec->len2 = 0;
+		rcrec->next = cf->rchash[hi];
+		cf->rchash[hi] = rcrec;
+	}
+
+	(pass == 1) ? rcrec->len1++ : rcrec->len2++;
+
+	rec->ha = (unsigned long) rcrec->idx;
+
+	hi = (long) XDL_HASHLONG(rec->ha, hbits);
+	rec->next = rhash[hi];
+	rhash[hi] = rec;
+
+	return 0;
+}
+
+
+static int xdl_prepare_ctx(unsigned int pass, mmfile_t *mf, long narec, xpparam_t const *xpp,
+			   xdlclassifier_t *cf, xdfile_t *xdf) {
+	unsigned int hbits;
+	long nrec, hsize, bsize;
+	unsigned long hav;
+	char const *blk, *cur, *top, *prev;
+	xrecord_t *crec;
+	xrecord_t **recs, **rrecs;
+	xrecord_t **rhash;
+	unsigned long *ha;
+	char *rchg;
+	long *rindex;
+
+	ha = NULL;
+	rindex = NULL;
+	rchg = NULL;
+	rhash = NULL;
+	recs = NULL;
+
+	if (xdl_cha_init(&xdf->rcha, sizeof(xrecord_t), narec / 4 + 1) < 0)
+		goto abort;
+	if (!(recs = (xrecord_t **) xdl_malloc(narec * sizeof(xrecord_t *))))
+		goto abort;
+
+	if (xpp->flags & XDF_HISTOGRAM_DIFF)
+		hbits = hsize = 0;
+	else {
+		hbits = xdl_hashbits((unsigned int) narec);
+		hsize = 1 << hbits;
+		if (!(rhash = (xrecord_t **) xdl_malloc(hsize * sizeof(xrecord_t *))))
+			goto abort;
+		memset(rhash, 0, hsize * sizeof(xrecord_t *));
+	}
+
+	nrec = 0;
+	if ((cur = blk = xdl_mmfile_first(mf, &bsize)) != NULL) {
+		for (top = blk + bsize; cur < top; ) {
+			prev = cur;
+			hav = xdl_hash_record(&cur, top, xpp->flags);
+			if (nrec >= narec) {
+				narec *= 2;
+				if (!(rrecs = (xrecord_t **) xdl_realloc(recs, narec * sizeof(xrecord_t *))))
+					goto abort;
+				recs = rrecs;
+			}
+			if (!(crec = xdl_cha_alloc(&xdf->rcha)))
+				goto abort;
+			crec->ptr = prev;
+			crec->size = (long) (cur - prev);
+			crec->ha = hav;
+			recs[nrec++] = crec;
+
+			if (!(xpp->flags & XDF_HISTOGRAM_DIFF) &&
+				xdl_classify_record(pass, cf, rhash, hbits, crec) < 0)
+				goto abort;
+		}
+	}
+
+	if (!(rchg = (char *) xdl_malloc((nrec + 2) * sizeof(char))))
+		goto abort;
+	memset(rchg, 0, (nrec + 2) * sizeof(char));
+
+	if (!(rindex = (long *) xdl_malloc((nrec + 1) * sizeof(long))))
+		goto abort;
+	if (!(ha = (unsigned long *) xdl_malloc((nrec + 1) * sizeof(unsigned long))))
+		goto abort;
+
+	xdf->nrec = nrec;
+	xdf->recs = recs;
+	xdf->hbits = hbits;
+	xdf->rhash = rhash;
+	xdf->rchg = rchg + 1;
+	xdf->rindex = rindex;
+	xdf->nreff = 0;
+	xdf->ha = ha;
+	xdf->dstart = 0;
+	xdf->dend = nrec - 1;
+
+	return 0;
+
+abort:
+	xdl_free(ha);
+	xdl_free(rindex);
+	xdl_free(rchg);
+	xdl_free(rhash);
+	xdl_free(recs);
+	xdl_cha_free(&xdf->rcha);
+	return -1;
+}
+
+
+static void xdl_free_ctx(xdfile_t *xdf) {
+
+	xdl_free(xdf->rhash);
+	xdl_free(xdf->rindex);
+	xdl_free(xdf->rchg - 1);
+	xdl_free(xdf->ha);
+	xdl_free(xdf->recs);
+	xdl_cha_free(&xdf->rcha);
+}
+
+
+int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+		    xdfenv_t *xe) {
+	long enl1, enl2, sample;
+	xdlclassifier_t cf;
+
+	memset(&cf, 0, sizeof(cf));
+
+	/*
+	 * For histogram diff, we can afford a smaller sample size and
+	 * thus a poorer estimate of the number of lines, as the hash
+	 * table (rhash) won't be filled up/grown. The number of lines
+	 * (nrecs) will be updated correctly anyway by
+	 * xdl_prepare_ctx().
+	 */
+	sample = xpp->flags & XDF_HISTOGRAM_DIFF ? XDL_GUESS_NLINES2 : XDL_GUESS_NLINES1;
+
+	enl1 = xdl_guess_lines(mf1, sample) + 1;
+	enl2 = xdl_guess_lines(mf2, sample) + 1;
+
+	if (!(xpp->flags & XDF_HISTOGRAM_DIFF) &&
+		xdl_init_classifier(&cf, enl1 + enl2 + 1, xpp->flags) < 0) {
+
+		return -1;
+	}
+
+	if (xdl_prepare_ctx(1, mf1, enl1, xpp, &cf, &xe->xdf1) < 0) {
+
+		xdl_free_classifier(&cf);
+		return -1;
+	}
+	if (xdl_prepare_ctx(2, mf2, enl2, xpp, &cf, &xe->xdf2) < 0) {
+
+		xdl_free_ctx(&xe->xdf1);
+		xdl_free_classifier(&cf);
+		return -1;
+	}
+
+	if (!(xpp->flags & XDF_PATIENCE_DIFF) &&
+			!(xpp->flags & XDF_HISTOGRAM_DIFF) &&
+			xdl_optimize_ctxs(&cf, &xe->xdf1, &xe->xdf2) < 0) {
+
+		xdl_free_ctx(&xe->xdf2);
+		xdl_free_ctx(&xe->xdf1);
+		return -1;
+	}
+
+	if (!(xpp->flags & XDF_HISTOGRAM_DIFF))
+		xdl_free_classifier(&cf);
+
+	return 0;
+}
+
+
+void xdl_free_env(xdfenv_t *xe) {
+
+	xdl_free_ctx(&xe->xdf2);
+	xdl_free_ctx(&xe->xdf1);
+}
+
+
+static int xdl_clean_mmatch(char const *dis, long i, long s, long e) {
+	long r, rdis0, rpdis0, rdis1, rpdis1;
+
+	/*
+	 * Limits the window the is examined during the similar-lines
+	 * scan. The loops below stops when dis[i - r] == 1 (line that
+	 * has no match), but there are corner cases where the loop
+	 * proceed all the way to the extremities by causing huge
+	 * performance penalties in case of big files.
+	 */
+	if (i - s > XDL_SIMSCAN_WINDOW)
+		s = i - XDL_SIMSCAN_WINDOW;
+	if (e - i > XDL_SIMSCAN_WINDOW)
+		e = i + XDL_SIMSCAN_WINDOW;
+
+	/*
+	 * Scans the lines before 'i' to find a run of lines that either
+	 * have no match (dis[j] == 0) or have multiple matches (dis[j] > 1).
+	 * Note that we always call this function with dis[i] > 1, so the
+	 * current line (i) is already a multimatch line.
+	 */
+	for (r = 1, rdis0 = 0, rpdis0 = 1; (i - r) >= s; r++) {
+		if (!dis[i - r])
+			rdis0++;
+		else if (dis[i - r] == 2)
+			rpdis0++;
+		else
+			break;
+	}
+	/*
+	 * If the run before the line 'i' found only multimatch lines, we
+	 * return 0 and hence we don't make the current line (i) discarded.
+	 * We want to discard multimatch lines only when they appear in the
+	 * middle of runs with nomatch lines (dis[j] == 0).
+	 */
+	if (rdis0 == 0)
+		return 0;
+	for (r = 1, rdis1 = 0, rpdis1 = 1; (i + r) <= e; r++) {
+		if (!dis[i + r])
+			rdis1++;
+		else if (dis[i + r] == 2)
+			rpdis1++;
+		else
+			break;
+	}
+	/*
+	 * If the run after the line 'i' found only multimatch lines, we
+	 * return 0 and hence we don't make the current line (i) discarded.
+	 */
+	if (rdis1 == 0)
+		return 0;
+	rdis1 += rdis0;
+	rpdis1 += rpdis0;
+
+	return rpdis1 * XDL_KPDIS_RUN < (rpdis1 + rdis1);
+}
+
+
+/*
+ * Try to reduce the problem complexity, discard records that have no
+ * matches on the other file. Also, lines that have multiple matches
+ * might be potentially discarded if they happear in a run of discardable.
+ */
+static int xdl_cleanup_records(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2) {
+	long i, nm, nreff, mlim;
+	xrecord_t **recs;
+	xdlclass_t *rcrec;
+	char *dis, *dis1, *dis2;
+
+	if (!(dis = (char *) xdl_malloc(xdf1->nrec + xdf2->nrec + 2))) {
+
+		return -1;
+	}
+	memset(dis, 0, xdf1->nrec + xdf2->nrec + 2);
+	dis1 = dis;
+	dis2 = dis1 + xdf1->nrec + 1;
+
+	if ((mlim = xdl_bogosqrt(xdf1->nrec)) > XDL_MAX_EQLIMIT)
+		mlim = XDL_MAX_EQLIMIT;
+	for (i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart]; i <= xdf1->dend; i++, recs++) {
+		rcrec = cf->rcrecs[(*recs)->ha];
+		nm = rcrec ? rcrec->len2 : 0;
+		dis1[i] = (nm == 0) ? 0: (nm >= mlim) ? 2: 1;
+	}
+
+	if ((mlim = xdl_bogosqrt(xdf2->nrec)) > XDL_MAX_EQLIMIT)
+		mlim = XDL_MAX_EQLIMIT;
+	for (i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart]; i <= xdf2->dend; i++, recs++) {
+		rcrec = cf->rcrecs[(*recs)->ha];
+		nm = rcrec ? rcrec->len1 : 0;
+		dis2[i] = (nm == 0) ? 0: (nm >= mlim) ? 2: 1;
+	}
+
+	for (nreff = 0, i = xdf1->dstart, recs = &xdf1->recs[xdf1->dstart];
+	     i <= xdf1->dend; i++, recs++) {
+		if (dis1[i] == 1 ||
+		    (dis1[i] == 2 && !xdl_clean_mmatch(dis1, i, xdf1->dstart, xdf1->dend))) {
+			xdf1->rindex[nreff] = i;
+			xdf1->ha[nreff] = (*recs)->ha;
+			nreff++;
+		} else
+			xdf1->rchg[i] = 1;
+	}
+	xdf1->nreff = nreff;
+
+	for (nreff = 0, i = xdf2->dstart, recs = &xdf2->recs[xdf2->dstart];
+	     i <= xdf2->dend; i++, recs++) {
+		if (dis2[i] == 1 ||
+		    (dis2[i] == 2 && !xdl_clean_mmatch(dis2, i, xdf2->dstart, xdf2->dend))) {
+			xdf2->rindex[nreff] = i;
+			xdf2->ha[nreff] = (*recs)->ha;
+			nreff++;
+		} else
+			xdf2->rchg[i] = 1;
+	}
+	xdf2->nreff = nreff;
+
+	xdl_free(dis);
+
+	return 0;
+}
+
+
+/*
+ * Early trim initial and terminal matching records.
+ */
+static int xdl_trim_ends(xdfile_t *xdf1, xdfile_t *xdf2) {
+	long i, lim;
+	xrecord_t **recs1, **recs2;
+
+	recs1 = xdf1->recs;
+	recs2 = xdf2->recs;
+	for (i = 0, lim = XDL_MIN(xdf1->nrec, xdf2->nrec); i < lim;
+	     i++, recs1++, recs2++)
+		if ((*recs1)->ha != (*recs2)->ha)
+			break;
+
+	xdf1->dstart = xdf2->dstart = i;
+
+	recs1 = xdf1->recs + xdf1->nrec - 1;
+	recs2 = xdf2->recs + xdf2->nrec - 1;
+	for (lim -= i, i = 0; i < lim; i++, recs1--, recs2--)
+		if ((*recs1)->ha != (*recs2)->ha)
+			break;
+
+	xdf1->dend = xdf1->nrec - i - 1;
+	xdf2->dend = xdf2->nrec - i - 1;
+
+	return 0;
+}
+
+
+static int xdl_optimize_ctxs(xdlclassifier_t *cf, xdfile_t *xdf1, xdfile_t *xdf2) {
+
+	if (xdl_trim_ends(xdf1, xdf2) < 0 ||
+	    xdl_cleanup_records(cf, xdf1, xdf2) < 0) {
+
+		return -1;
+	}
+
+	return 0;
+}
diff --git a/src/xdiff/xprepare.h b/src/xdiff/xprepare.h
new file mode 100644
index 000000000..8fb06a537
--- /dev/null
+++ b/src/xdiff/xprepare.h
@@ -0,0 +1,34 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#if !defined(XPREPARE_H)
+#define XPREPARE_H
+
+
+
+int xdl_prepare_env(mmfile_t *mf1, mmfile_t *mf2, xpparam_t const *xpp,
+		    xdfenv_t *xe);
+void xdl_free_env(xdfenv_t *xe);
+
+
+
+#endif /* #if !defined(XPREPARE_H) */
diff --git a/src/xdiff/xtypes.h b/src/xdiff/xtypes.h
new file mode 100644
index 000000000..2511aef8d
--- /dev/null
+++ b/src/xdiff/xtypes.h
@@ -0,0 +1,67 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#if !defined(XTYPES_H)
+#define XTYPES_H
+
+
+
+typedef struct s_chanode {
+	struct s_chanode *next;
+	long icurr;
+} chanode_t;
+
+typedef struct s_chastore {
+	chanode_t *head, *tail;
+	long isize, nsize;
+	chanode_t *ancur;
+	chanode_t *sncur;
+	long scurr;
+} chastore_t;
+
+typedef struct s_xrecord {
+	struct s_xrecord *next;
+	char const *ptr;
+	long size;
+	unsigned long ha;
+} xrecord_t;
+
+typedef struct s_xdfile {
+	chastore_t rcha;
+	long nrec;
+	unsigned int hbits;
+	xrecord_t **rhash;
+	long dstart, dend;
+	xrecord_t **recs;
+	char *rchg;
+	long *rindex;
+	long nreff;
+	unsigned long *ha;
+} xdfile_t;
+
+typedef struct s_xdfenv {
+	xdfile_t xdf1, xdf2;
+} xdfenv_t;
+
+
+
+#endif /* #if !defined(XTYPES_H) */
diff --git a/src/xdiff/xutils.c b/src/xdiff/xutils.c
new file mode 100644
index 000000000..bb7bdee49
--- /dev/null
+++ b/src/xdiff/xutils.c
@@ -0,0 +1,419 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003	Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#include "xinclude.h"
+
+
+
+
+long xdl_bogosqrt(long n) {
+	long i;
+
+	/*
+	 * Classical integer square root approximation using shifts.
+	 */
+	for (i = 1; n > 0; n >>= 2)
+		i <<= 1;
+
+	return i;
+}
+
+
+int xdl_emit_diffrec(char const *rec, long size, char const *pre, long psize,
+		     xdemitcb_t *ecb) {
+	int i = 2;
+	mmbuffer_t mb[3];
+
+	mb[0].ptr = (char *) pre;
+	mb[0].size = psize;
+	mb[1].ptr = (char *) rec;
+	mb[1].size = size;
+	if (size > 0 && rec[size - 1] != '\n') {
+		mb[2].ptr = (char *) "\n\\ No newline at end of file\n";
+		mb[2].size = strlen(mb[2].ptr);
+		i++;
+	}
+	if (ecb->outf(ecb->priv, mb, i) < 0) {
+
+		return -1;
+	}
+
+	return 0;
+}
+
+void *xdl_mmfile_first(mmfile_t *mmf, long *size)
+{
+	*size = (long)mmf->size;
+	return mmf->ptr;
+}
+
+
+long xdl_mmfile_size(mmfile_t *mmf)
+{
+	return (long)mmf->size;
+}
+
+
+int xdl_cha_init(chastore_t *cha, long isize, long icount) {
+
+	cha->head = cha->tail = NULL;
+	cha->isize = isize;
+	cha->nsize = icount * isize;
+	cha->ancur = cha->sncur = NULL;
+	cha->scurr = 0;
+
+	return 0;
+}
+
+
+void xdl_cha_free(chastore_t *cha) {
+	chanode_t *cur, *tmp;
+
+	for (cur = cha->head; (tmp = cur) != NULL;) {
+		cur = cur->next;
+		xdl_free(tmp);
+	}
+}
+
+
+void *xdl_cha_alloc(chastore_t *cha) {
+	chanode_t *ancur;
+	void *data;
+
+	if (!(ancur = cha->ancur) || ancur->icurr == cha->nsize) {
+		if (!(ancur = (chanode_t *) xdl_malloc(sizeof(chanode_t) + cha->nsize))) {
+
+			return NULL;
+		}
+		ancur->icurr = 0;
+		ancur->next = NULL;
+		if (cha->tail)
+			cha->tail->next = ancur;
+		if (!cha->head)
+			cha->head = ancur;
+		cha->tail = ancur;
+		cha->ancur = ancur;
+	}
+
+	data = (char *) ancur + sizeof(chanode_t) + ancur->icurr;
+	ancur->icurr += cha->isize;
+
+	return data;
+}
+
+
+void *xdl_cha_first(chastore_t *cha) {
+	chanode_t *sncur;
+
+	if (!(cha->sncur = sncur = cha->head))
+		return NULL;
+
+	cha->scurr = 0;
+
+	return (char *) sncur + sizeof(chanode_t) + cha->scurr;
+}
+
+
+void *xdl_cha_next(chastore_t *cha) {
+	chanode_t *sncur;
+
+	if (!(sncur = cha->sncur))
+		return NULL;
+	cha->scurr += cha->isize;
+	if (cha->scurr == sncur->icurr) {
+		if (!(sncur = cha->sncur = sncur->next))
+			return NULL;
+		cha->scurr = 0;
+	}
+
+	return (char *) sncur + sizeof(chanode_t) + cha->scurr;
+}
+
+
+long xdl_guess_lines(mmfile_t *mf, long sample) {
+	long nl = 0, size, tsize = 0;
+	char const *data, *cur, *top;
+
+	if ((cur = data = xdl_mmfile_first(mf, &size)) != NULL) {
+		for (top = data + size; nl < sample && cur < top; ) {
+			nl++;
+			if (!(cur = memchr(cur, '\n', top - cur)))
+				cur = top;
+			else
+				cur++;
+		}
+		tsize += (long) (cur - data);
+	}
+
+	if (nl && tsize)
+		nl = xdl_mmfile_size(mf) / (tsize / nl);
+
+	return nl + 1;
+}
+
+int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags)
+{
+	int i1, i2;
+
+	if (s1 == s2 && !memcmp(l1, l2, s1))
+		return 1;
+	if (!(flags & XDF_WHITESPACE_FLAGS))
+		return 0;
+
+	i1 = 0;
+	i2 = 0;
+
+	/*
+	 * -w matches everything that matches with -b, and -b in turn
+	 * matches everything that matches with --ignore-space-at-eol.
+	 *
+	 * Each flavor of ignoring needs different logic to skip whitespaces
+	 * while we have both sides to compare.
+	 */
+	if (flags & XDF_IGNORE_WHITESPACE) {
+		goto skip_ws;
+		while (i1 < s1 && i2 < s2) {
+			if (l1[i1++] != l2[i2++])
+				return 0;
+		skip_ws:
+			while (i1 < s1 && XDL_ISSPACE(l1[i1]))
+				i1++;
+			while (i2 < s2 && XDL_ISSPACE(l2[i2]))
+				i2++;
+		}
+	} else if (flags & XDF_IGNORE_WHITESPACE_CHANGE) {
+		while (i1 < s1 && i2 < s2) {
+			if (XDL_ISSPACE(l1[i1]) && XDL_ISSPACE(l2[i2])) {
+				/* Skip matching spaces and try again */
+				while (i1 < s1 && XDL_ISSPACE(l1[i1]))
+					i1++;
+				while (i2 < s2 && XDL_ISSPACE(l2[i2]))
+					i2++;
+				continue;
+			}
+			if (l1[i1++] != l2[i2++])
+				return 0;
+		}
+	} else if (flags & XDF_IGNORE_WHITESPACE_AT_EOL) {
+		while (i1 < s1 && i2 < s2 && l1[i1++] == l2[i2++])
+			; /* keep going */
+	}
+
+	/*
+	 * After running out of one side, the remaining side must have
+	 * nothing but whitespace for the lines to match.  Note that
+	 * ignore-whitespace-at-eol case may break out of the loop
+	 * while there still are characters remaining on both lines.
+	 */
+	if (i1 < s1) {
+		while (i1 < s1 && XDL_ISSPACE(l1[i1]))
+			i1++;
+		if (s1 != i1)
+			return 0;
+	}
+	if (i2 < s2) {
+		while (i2 < s2 && XDL_ISSPACE(l2[i2]))
+			i2++;
+		return (s2 == i2);
+	}
+	return 1;
+}
+
+static unsigned long xdl_hash_record_with_whitespace(char const **data,
+		char const *top, long flags) {
+	unsigned long ha = 5381;
+	char const *ptr = *data;
+
+	for (; ptr < top && *ptr != '\n'; ptr++) {
+		if (XDL_ISSPACE(*ptr)) {
+			const char *ptr2 = ptr;
+			int at_eol;
+			while (ptr + 1 < top && XDL_ISSPACE(ptr[1])
+					&& ptr[1] != '\n')
+				ptr++;
+			at_eol = (top <= ptr + 1 || ptr[1] == '\n');
+			if (flags & XDF_IGNORE_WHITESPACE)
+				; /* already handled */
+			else if (flags & XDF_IGNORE_WHITESPACE_CHANGE
+				 && !at_eol) {
+				ha += (ha << 5);
+				ha ^= (unsigned long) ' ';
+			}
+			else if (flags & XDF_IGNORE_WHITESPACE_AT_EOL
+				 && !at_eol) {
+				while (ptr2 != ptr + 1) {
+					ha += (ha << 5);
+					ha ^= (unsigned long) *ptr2;
+					ptr2++;
+				}
+			}
+			continue;
+		}
+		ha += (ha << 5);
+		ha ^= (unsigned long) *ptr;
+	}
+	*data = ptr < top ? ptr + 1: ptr;
+
+	return ha;
+}
+
+
+unsigned long xdl_hash_record(char const **data, char const *top, long flags) {
+	unsigned long ha = 5381;
+	char const *ptr = *data;
+
+	if (flags & XDF_WHITESPACE_FLAGS)
+		return xdl_hash_record_with_whitespace(data, top, flags);
+
+	for (; ptr < top && *ptr != '\n'; ptr++) {
+		ha += (ha << 5);
+		ha ^= (unsigned long) *ptr;
+	}
+	*data = ptr < top ? ptr + 1: ptr;
+
+	return ha;
+}
+
+
+unsigned int xdl_hashbits(unsigned int size) {
+	unsigned int val = 1, bits = 0;
+
+	for (; val < size && bits < CHAR_BIT * sizeof(unsigned int); val <<= 1, bits++);
+	return bits ? bits: 1;
+}
+
+
+int xdl_num_out(char *out, long val) {
+	char *ptr, *str = out;
+	char buf[32];
+
+	ptr = buf + sizeof(buf) - 1;
+	*ptr = '\0';
+	if (val < 0) {
+		*--ptr = '-';
+		val = -val;
+	}
+	for (; val && ptr > buf; val /= 10)
+		*--ptr = "0123456789"[val % 10];
+	if (*ptr)
+		for (; *ptr; ptr++, str++)
+			*str = *ptr;
+	else
+		*str++ = '0';
+	*str = '\0';
+
+	return (int)(str - out);
+}
+
+
+long xdl_atol(char const *str, char const **next) {
+	long val, base;
+	char const *top;
+
+	for (top = str; XDL_ISDIGIT(*top); top++);
+	if (next)
+		*next = top;
+	for (val = 0, base = 1, top--; top >= str; top--, base *= 10)
+		val += base * (long)(*top - '0');
+	return val;
+}
+
+
+int xdl_emit_hunk_hdr(long s1, long c1, long s2, long c2,
+		      const char *func, long funclen, xdemitcb_t *ecb) {
+	int nb = 0;
+	mmbuffer_t mb;
+	char buf[128];
+
+	memcpy(buf, "@@ -", 4);
+	nb += 4;
+
+	nb += xdl_num_out(buf + nb, c1 ? s1: s1 - 1);
+
+	if (c1 != 1) {
+		memcpy(buf + nb, ",", 1);
+		nb += 1;
+
+		nb += xdl_num_out(buf + nb, c1);
+	}
+
+	memcpy(buf + nb, " +", 2);
+	nb += 2;
+
+	nb += xdl_num_out(buf + nb, c2 ? s2: s2 - 1);
+
+	if (c2 != 1) {
+		memcpy(buf + nb, ",", 1);
+		nb += 1;
+
+		nb += xdl_num_out(buf + nb, c2);
+	}
+
+	memcpy(buf + nb, " @@", 3);
+	nb += 3;
+	if (func && funclen) {
+		buf[nb++] = ' ';
+		if (funclen > (long)sizeof(buf) - nb - 1)
+			funclen = (long)sizeof(buf) - nb - 1;
+		memcpy(buf + nb, func, funclen);
+		nb += funclen;
+	}
+	buf[nb++] = '\n';
+
+	mb.ptr = buf;
+	mb.size = nb;
+	if (ecb->outf(ecb->priv, &mb, 1) < 0)
+		return -1;
+
+	return 0;
+}
+
+int xdl_fall_back_diff(xdfenv_t *diff_env, xpparam_t const *xpp,
+		int line1, int count1, int line2, int count2)
+{
+	/*
+	 * This probably does not work outside Git, since
+	 * we have a very simple mmfile structure.
+	 *
+	 * Note: ideally, we would reuse the prepared environment, but
+	 * the libxdiff interface does not (yet) allow for diffing only
+	 * ranges of lines instead of the whole files.
+	 */
+	mmfile_t subfile1, subfile2;
+	xdfenv_t env;
+
+	subfile1.ptr = (char *)diff_env->xdf1.recs[line1 - 1]->ptr;
+	subfile1.size = diff_env->xdf1.recs[line1 + count1 - 2]->ptr +
+		diff_env->xdf1.recs[line1 + count1 - 2]->size - subfile1.ptr;
+	subfile2.ptr = (char *)diff_env->xdf2.recs[line2 - 1]->ptr;
+	subfile2.size = diff_env->xdf2.recs[line2 + count2 - 2]->ptr +
+		diff_env->xdf2.recs[line2 + count2 - 2]->size - subfile2.ptr;
+	if (xdl_do_diff(&subfile1, &subfile2, xpp, &env) < 0)
+		return -1;
+
+	memcpy(diff_env->xdf1.rchg + line1 - 1, env.xdf1.rchg, count1);
+	memcpy(diff_env->xdf2.rchg + line2 - 1, env.xdf2.rchg, count2);
+
+	xdl_free_env(&env);
+
+	return 0;
+}
diff --git a/src/xdiff/xutils.h b/src/xdiff/xutils.h
new file mode 100644
index 000000000..714719a89
--- /dev/null
+++ b/src/xdiff/xutils.h
@@ -0,0 +1,49 @@
+/*
+ *  LibXDiff by Davide Libenzi ( File Differential Library )
+ *  Copyright (C) 2003  Davide Libenzi
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public
+ *  License along with this library; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ *  Davide Libenzi <davidel@xmailserver.org>
+ *
+ */
+
+#if !defined(XUTILS_H)
+#define XUTILS_H
+
+
+
+long xdl_bogosqrt(long n);
+int xdl_emit_diffrec(char const *rec, long size, char const *pre, long psize,
+		     xdemitcb_t *ecb);
+int xdl_cha_init(chastore_t *cha, long isize, long icount);
+void xdl_cha_free(chastore_t *cha);
+void *xdl_cha_alloc(chastore_t *cha);
+void *xdl_cha_first(chastore_t *cha);
+void *xdl_cha_next(chastore_t *cha);
+long xdl_guess_lines(mmfile_t *mf, long sample);
+int xdl_recmatch(const char *l1, long s1, const char *l2, long s2, long flags);
+unsigned long xdl_hash_record(char const **data, char const *top, long flags);
+unsigned int xdl_hashbits(unsigned int size);
+int xdl_num_out(char *out, long val);
+long xdl_atol(char const *str, char const **next);
+int xdl_emit_hunk_hdr(long s1, long c1, long s2, long c2,
+		      const char *func, long funclen, xdemitcb_t *ecb);
+int xdl_fall_back_diff(xdfenv_t *diff_env, xpparam_t const *xpp,
+		       int line1, int count1, int line2, int count2);
+
+
+
+#endif /* #if !defined(XUTILS_H) */