Honor `core.fsyncObjectFiles`ethomson/fsync

13 files changed, 173 insertions, 55 deletions

diff --git a/src/config_cache.c b/src/config_cache.c
index dbea871b9..840722274 100644
--- a/src/config_cache.c
+++ b/src/config_cache.c
@@ -78,6 +78,7 @@ static struct map_data _cvar_maps[] = {
 	{"core.logallrefupdates", NULL, 0, GIT_LOGALLREFUPDATES_DEFAULT },
 	{"core.protecthfs", NULL, 0, GIT_PROTECTHFS_DEFAULT },
 	{"core.protectntfs", NULL, 0, GIT_PROTECTNTFS_DEFAULT },
+	{"core.fsyncobjectfiles", NULL, 0, GIT_FSYNCOBJECTFILES_DEFAULT },
 };
 
 int git_config__cvar(int *out, git_config *config, git_cvar_cached cvar)
diff --git a/src/indexer.c b/src/indexer.c
index 3fd7223e5..ce67240ce 100644
--- a/src/indexer.c
+++ b/src/indexer.c
@@ -34,7 +34,8 @@ struct git_indexer {
 	unsigned int parsed_header :1,
 		pack_committed :1,
 		have_stream :1,
-		have_delta :1;
+		have_delta :1,
+		do_fsync :1;
 	struct git_pack_header hdr;
 	struct git_pack_file *pack;
 	unsigned int mode;
@@ -124,6 +125,9 @@ int git_indexer_new(
 	git_hash_ctx_init(&idx->hash_ctx);
 	git_hash_ctx_init(&idx->trailer);
 
+	if (git_object__synchronous_writing)
+		idx->do_fsync = 1;
+
 	error = git_buf_joinpath(&path, prefix, suff);
 	if (error < 0)
 		goto cleanup;
@@ -162,6 +166,11 @@ cleanup:
 	return -1;
 }
 
+void git_indexer__set_fsync(git_indexer *idx, int do_fsync)
+{
+	idx->do_fsync = !!do_fsync;
+}
+
 /* Try to store the delta so we can try to resolve it later */
 static int store_delta(git_indexer *idx)
 {
@@ -991,7 +1000,7 @@ int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 
 	if (git_filebuf_open(&index_file, filename.ptr,
 		GIT_FILEBUF_HASH_CONTENTS |
-		(git_object__synchronous_writing ? GIT_FILEBUF_FSYNC : 0),
+		(idx->do_fsync ? GIT_FILEBUF_FSYNC : 0),
 		idx->mode) < 0)
 		goto on_error;
 
@@ -1069,7 +1078,7 @@ int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 		return -1;
 	}
 
-	if (git_object__synchronous_writing && p_fsync(idx->pack->mwf.fd) < 0) {
+	if (idx->do_fsync && p_fsync(idx->pack->mwf.fd) < 0) {
 		giterr_set(GITERR_OS, "failed to fsync packfile");
 		goto on_error;
 	}
@@ -1090,7 +1099,7 @@ int git_indexer_commit(git_indexer *idx, git_transfer_progress *stats)
 		goto on_error;
 
 	/* And fsync the parent directory if we're asked to. */
-	if (git_object__synchronous_writing &&
+	if (idx->do_fsync &&
 		git_futils_fsync_parent(git_buf_cstr(&filename)) < 0)
 		goto on_error;
 
diff --git a/src/indexer.h b/src/indexer.h
new file mode 100644
index 000000000..702694bbf
--- /dev/null
+++ b/src/indexer.h
@@ -0,0 +1,12 @@
+/*
+ * Copyright (C) the libgit2 contributors. All rights reserved.
+ *
+ * This file is part of libgit2, distributed under the GNU GPL v2 with
+ * a Linking Exception. For full terms see the included COPYING file.
+ */
+#ifndef INCLUDE_indexer_h__
+#define INCLUDE_indexer_h__
+
+extern int git_indexer__set_fsync(git_indexer *idx, int do_fsync);
+
+#endif
diff --git a/src/odb.c b/src/odb.c
index dc98a6ff7..0e0dc5256 100644
--- a/src/odb.c
+++ b/src/odb.c
@@ -496,7 +496,7 @@ int git_odb_get_backend(git_odb_backend **out, git_odb *odb, size_t pos)
 	return GIT_ENOTFOUND;
 }
 
-static int add_default_backends(
+int git_odb__add_default_backends(
 	git_odb *db, const char *objects_dir,
 	bool as_alternates, int alternate_depth)
 {
@@ -531,7 +531,7 @@ static int add_default_backends(
 #endif
 
 	/* add the loose object backend */
-	if (git_odb_backend_loose(&loose, objects_dir, -1, 0, 0, 0) < 0 ||
+	if (git_odb_backend_loose(&loose, objects_dir, -1, db->do_fsync, 0, 0) < 0 ||
 		add_backend_internal(db, loose, GIT_LOOSE_PRIORITY, as_alternates, inode) < 0)
 		return -1;
 
@@ -586,7 +586,7 @@ static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_
 			alternate = git_buf_cstr(&alternates_path);
 		}
 
-		if ((result = add_default_backends(odb, alternate, true, alternate_depth + 1)) < 0)
+		if ((result = git_odb__add_default_backends(odb, alternate, true, alternate_depth + 1)) < 0)
 			break;
 	}
 
@@ -598,7 +598,7 @@ static int load_alternates(git_odb *odb, const char *objects_dir, int alternate_
 
 int git_odb_add_disk_alternate(git_odb *odb, const char *path)
 {
-	return add_default_backends(odb, path, true, 0);
+	return git_odb__add_default_backends(odb, path, true, 0);
 }
 
 int git_odb_open(git_odb **out, const char *objects_dir)
@@ -612,7 +612,7 @@ int git_odb_open(git_odb **out, const char *objects_dir)
 	if (git_odb_new(&db) < 0)
 		return -1;
 
-	if (add_default_backends(db, objects_dir, 0, 0) < 0) {
+	if (git_odb__add_default_backends(db, objects_dir, 0, 0) < 0) {
 		git_odb_free(db);
 		return -1;
 	}
@@ -621,6 +621,24 @@ int git_odb_open(git_odb **out, const char *objects_dir)
 	return 0;
 }
 
+int git_odb__set_caps(git_odb *odb, int caps)
+{
+	if (caps == GIT_ODB_CAP_FROM_OWNER) {
+		git_repository *repo = odb->rc.owner;
+		int val;
+
+		if (!repo) {
+			giterr_set(GITERR_ODB, "cannot access repository to set odb caps");
+			return -1;
+		}
+
+		if (!git_repository__cvar(&val, repo, GIT_CVAR_FSYNCOBJECTFILES))
+			odb->do_fsync = !!val;
+	}
+
+	return 0;
+}
+
 static void odb_free(git_odb *db)
 {
 	size_t i;
diff --git a/src/odb.h b/src/odb.h
index 31a9fd1b9..8ae0643ae 100644
--- a/src/odb.h
+++ b/src/odb.h
@@ -38,8 +38,25 @@ struct git_odb {
 	git_refcount rc;
 	git_vector backends;
 	git_cache own_cache;
+	unsigned int do_fsync :1;
 };
 
+typedef enum {
+	GIT_ODB_CAP_FROM_OWNER = -1,
+} git_odb_cap_t;
+
+/*
+ * Set the capabilities for the object database.
+ */
+int git_odb__set_caps(git_odb *odb, int caps);
+
+/*
+ * Add the default loose and packed backends for a database.
+ */
+int git_odb__add_default_backends(
+	git_odb *db, const char *objects_dir,
+	bool as_alternates, int alternate_depth);
+
 /*
  * Hash a git_rawobj internally.
  * The `git_rawobj` is supposed to be previously initialized
diff --git a/src/pack-objects.c b/src/pack-objects.c
index 58b7b94b3..ef272e8f5 100644
--- a/src/pack-objects.c
+++ b/src/pack-objects.c
@@ -1385,6 +1385,7 @@ int git_packbuilder_write(
 	git_indexer *indexer;
 	git_transfer_progress stats;
 	struct pack_write_context ctx;
+	int t;
 
 	PREPARE_PACK;
 
@@ -1392,6 +1393,9 @@ int git_packbuilder_write(
 		&indexer, path, mode, pb->odb, progress_cb, progress_cb_payload) < 0)
 		return -1;
 
+	if (!git_repository__cvar(&t, pb->repo, GIT_CVAR_FSYNCOBJECTFILES) && t)
+		git_indexer__set_fsync(indexer, 1);
+
 	ctx.indexer = indexer;
 	ctx.stats = &stats;
 
diff --git a/src/pack-objects.h b/src/pack-objects.h
index 5a84f4158..e1e0ee3c8 100644
--- a/src/pack-objects.h
+++ b/src/pack-objects.h
@@ -16,6 +16,7 @@
 #include "netops.h"
 #include "zstream.h"
 #include "pool.h"
+#include "indexer.h"
 
 #include "git2/oid.h"
 #include "git2/pack.h"
diff --git a/src/refdb_fs.c b/src/refdb_fs.c
index d7a458a87..ade50f7c3 100644
--- a/src/refdb_fs.c
+++ b/src/refdb_fs.c
@@ -62,6 +62,7 @@ typedef struct refdb_fs_backend {
 	int peeling_mode;
 	git_iterator_flag_t iterator_flags;
 	uint32_t direach_flags;
+	int fsync;
 } refdb_fs_backend;
 
 static int refdb_reflog_fs__delete(git_refdb_backend *_backend, const char *name);
@@ -756,7 +757,7 @@ static int loose_lock(git_filebuf *file, refdb_fs_backend *backend, const char *
 		return -1;
 
 	filebuf_flags = GIT_FILEBUF_FORCE;
-	if (git_object__synchronous_writing)
+	if (backend->fsync)
 		filebuf_flags |= GIT_FILEBUF_FSYNC;
 
 	error = git_filebuf_open(file, ref_path.ptr, filebuf_flags, GIT_REFS_FILE_MODE);
@@ -1001,7 +1002,7 @@ static int packed_write(refdb_fs_backend *backend)
 	if ((error = git_sortedcache_wlock(refcache)) < 0)
 		return error;
 
-	if (git_object__synchronous_writing)
+	if (backend->fsync)
 		open_flags = GIT_FILEBUF_FSYNC;
 
 	/* Open the file! */
@@ -1861,7 +1862,7 @@ static int reflog_append(refdb_fs_backend *backend, const git_reference *ref, co
 
 	open_flags = O_WRONLY | O_CREAT | O_APPEND;
 
-	if (git_object__synchronous_writing)
+	if (backend->fsync)
 		open_flags |= O_FSYNC;
 
 	error = git_futils_writebuffer(&buf, git_buf_cstr(&path), open_flags, GIT_REFLOG_FILE_MODE);
@@ -2014,6 +2015,9 @@ int git_refdb_backend_fs(
 		backend->iterator_flags |= GIT_ITERATOR_PRECOMPOSE_UNICODE;
 		backend->direach_flags  |= GIT_PATH_DIR_PRECOMPOSE_UNICODE;
 	}
+	if ((!git_repository__cvar(&t, backend->repo, GIT_CVAR_FSYNCOBJECTFILES) && t) ||
+		git_object__synchronous_writing)
+		backend->fsync = 1;
 
 	backend->parent.exists = &refdb_fs_backend__exists;
 	backend->parent.lookup = &refdb_fs_backend__lookup;
diff --git a/src/repository.c b/src/repository.c
index 0db481638..425ef796f 100644
--- a/src/repository.c
+++ b/src/repository.c
@@ -1055,18 +1055,22 @@ int git_repository_odb__weakptr(git_odb **out, git_repository *repo)
 		git_odb *odb;
 
 		if ((error = git_repository_item_path(&odb_path, repo,
-				GIT_REPOSITORY_ITEM_OBJECTS)) < 0)
+				GIT_REPOSITORY_ITEM_OBJECTS)) < 0 ||
+			(error = git_odb_new(&odb)) < 0)
 			return error;
 
-		error = git_odb_open(&odb, odb_path.ptr);
-		if (!error) {
-			GIT_REFCOUNT_OWN(odb, repo);
+		GIT_REFCOUNT_OWN(odb, repo);
 
-			odb = git__compare_and_swap(&repo->_odb, NULL, odb);
-			if (odb != NULL) {
-				GIT_REFCOUNT_OWN(odb, NULL);
-				git_odb_free(odb);
-			}
+		if ((error = git_odb__set_caps(odb, GIT_ODB_CAP_FROM_OWNER)) < 0 ||
+			(error = git_odb__add_default_backends(odb, odb_path.ptr, 0, 0)) < 0) {
+			git_odb_free(odb);
+			return error;
+		}
+
+		odb = git__compare_and_swap(&repo->_odb, NULL, odb);
+		if (odb != NULL) {
+			GIT_REFCOUNT_OWN(odb, NULL);
+			git_odb_free(odb);
 		}
 
 		git_buf_free(&odb_path);
diff --git a/src/repository.h b/src/repository.h
index c328ecd21..33adfa60a 100644
--- a/src/repository.h
+++ b/src/repository.h
@@ -46,6 +46,7 @@ typedef enum {
 	GIT_CVAR_LOGALLREFUPDATES, /* core.logallrefupdates */
 	GIT_CVAR_PROTECTHFS,    /* core.protectHFS */
 	GIT_CVAR_PROTECTNTFS,   /* core.protectNTFS */
+	GIT_CVAR_FSYNCOBJECTFILES, /* core.fsyncObjectFiles */
 	GIT_CVAR_CACHE_MAX
 } git_cvar_cached;
 
@@ -106,6 +107,8 @@ typedef enum {
 	GIT_PROTECTHFS_DEFAULT = GIT_CVAR_FALSE,
 	/* core.protectNTFS */
 	GIT_PROTECTNTFS_DEFAULT = GIT_CVAR_FALSE,
+	/* core.fsyncObjectFiles */
+	GIT_FSYNCOBJECTFILES_DEFAULT = GIT_CVAR_FALSE,
 } git_cvar_value;
 
 /* internal repository init flags */
diff --git a/tests/odb/loose.c b/tests/odb/loose.c
index fd4a53837..dd686aa01 100644
--- a/tests/odb/loose.c
+++ b/tests/odb/loose.c
@@ -3,6 +3,7 @@
 #include "git2/odb_backend.h"
 #include "posix.h"
 #include "loose_data.h"
+#include "repository.h"
 
 #ifdef __ANDROID_API__
 # define S_IREAD        S_IRUSR
@@ -184,3 +185,23 @@ void test_odb_loose__fsync_obeys_global_setting(void)
 	write_object_to_loose_odb(0);
 	cl_assert(p_fsync__cnt > 0);
 }
+
+void test_odb_loose__fsync_obeys_repo_setting(void)
+{
+	git_repository *repo;
+	git_odb *odb;
+	git_oid oid;
+
+	cl_git_pass(git_repository_init(&repo, "test-objects", 1));
+	cl_git_pass(git_repository_odb__weakptr(&odb, repo));
+	cl_git_pass(git_odb_write(&oid, odb, "No fsync here\n", 14, GIT_OBJ_BLOB));
+	cl_assert(p_fsync__cnt == 0);
+	git_repository_free(repo);
+
+	cl_git_pass(git_repository_open(&repo, "test-objects"));
+	cl_repo_set_bool(repo, "core.fsyncObjectFiles", true);
+	cl_git_pass(git_repository_odb__weakptr(&odb, repo));
+	cl_git_pass(git_odb_write(&oid, odb, "Now fsync\n", 10, GIT_OBJ_BLOB));
+	cl_assert(p_fsync__cnt > 0);
+	git_repository_free(repo);
+}
diff --git a/tests/pack/packbuilder.c b/tests/pack/packbuilder.c
index 5cdd9a8d2..1d7becef7 100644
--- a/tests/pack/packbuilder.c
+++ b/tests/pack/packbuilder.c
@@ -198,22 +198,31 @@ void test_pack_packbuilder__does_not_fsync_by_default(void)
 	cl_assert_equal_sz(0, p_fsync__cnt);
 }
 
-void test_pack_packbuilder__fsync_when_asked(void)
-{
-	/* We fsync the packfile and index.  On non-Windows, we also fsync
-	 * the parent directories.
-	 */
+/* We fsync the packfile and index.  On non-Windows, we also fsync
+ * the parent directories.
+ */
 #ifdef GIT_WIN32
-	int expected = 2;
+static int expected_fsyncs = 2;
 #else
-	int expected = 4;
+static int expected_fsyncs = 4;
 #endif
 
+void test_pack_packbuilder__fsync_global_setting(void)
+{
 	cl_git_pass(git_libgit2_opts(GIT_OPT_ENABLE_SYNCHRONOUS_OBJECT_CREATION, 1));
 	p_fsync__cnt = 0;
 	seed_packbuilder();
 	git_packbuilder_write(_packbuilder, ".", 0666, NULL, NULL);
-	cl_assert_equal_sz(expected, p_fsync__cnt);
+	cl_assert_equal_sz(expected_fsyncs, p_fsync__cnt);
+}
+
+void test_pack_packbuilder__fsync_repo_setting(void)
+{
+	cl_repo_set_bool(_repo, "core.fsyncObjectFiles", true);
+	p_fsync__cnt = 0;
+	seed_packbuilder();
+	git_packbuilder_write(_packbuilder, ".", 0666, NULL, NULL);
+	cl_assert_equal_sz(expected_fsyncs, p_fsync__cnt);
 }
 
 static int foreach_cb(void *buf, size_t len, void *payload)
diff --git a/tests/refs/create.c b/tests/refs/create.c
index 6265aee47..7b582d708 100644
--- a/tests/refs/create.c
+++ b/tests/refs/create.c
@@ -300,53 +300,68 @@ void test_refs_create__creating_a_loose_ref_with_invalid_windows_name(void)
 	test_win32_name("refs/heads/com1");
 }
 
-void test_refs_create__does_not_fsync_by_default(void)
+/* Creating a loose ref involves fsync'ing the reference, the
+ * reflog and (on non-Windows) the containing directories.
+ * Creating a packed ref involves fsync'ing the packed ref file
+ * and (on non-Windows) the containing directory.
+ */
+#ifdef GIT_WIN32
+static int expected_fsyncs_create = 2, expected_fsyncs_compress = 1;
+#else
+static int expected_fsyncs_create = 4, expected_fsyncs_compress = 2;
+#endif
+
+static void count_fsyncs(size_t *create_count, size_t *compress_count)
 {
 	git_reference *ref = NULL;
 	git_refdb *refdb;
 	git_oid id;
 
+	p_fsync__cnt = 0;
+
 	git_oid_fromstr(&id, current_master_tip);
 	cl_git_pass(git_reference_create(&ref, g_repo, "refs/heads/fsync_test", &id, 0, "log message"));
 	git_reference_free(ref);
 
+	*create_count = p_fsync__cnt;
+	p_fsync__cnt = 0;
+
 	cl_git_pass(git_repository_refdb(&refdb, g_repo));
 	cl_git_pass(git_refdb_compress(refdb));
 	git_refdb_free(refdb);
 
-	cl_assert_equal_i(0, p_fsync__cnt);
+	*compress_count = p_fsync__cnt;
+	p_fsync__cnt = 0;
 }
 
-void test_refs_create__fsyncs_when_requested(void)
+void test_refs_create__does_not_fsync_by_default(void)
 {
-	git_reference *ref = NULL;
-	git_refdb *refdb;
-	git_oid id;
+	size_t create_count, compress_count;
+	count_fsyncs(&create_count, &compress_count);
 
-	/* Creating a loose ref involves fsync'ing the reference, the
-	 * reflog and (on non-Windows) the containing directories.
-	 * Creating a packed ref involves fsync'ing the packed ref file
-	 * and (on non-Windows) the containing directory.
-	 */
-#ifdef GIT_WIN32
-	int expected_create = 2, expected_compress = 1;
-#else
-	int expected_create = 4, expected_compress = 2;
-#endif
+	cl_assert_equal_i(0, create_count);
+	cl_assert_equal_i(0, compress_count);
+}
+
+void test_refs_create__fsyncs_when_global_opt_set(void)
+{
+	size_t create_count, compress_count;
 
 	cl_git_pass(git_libgit2_opts(GIT_OPT_ENABLE_SYNCHRONOUS_OBJECT_CREATION, 1));
-	p_fsync__cnt = 0;
+	count_fsyncs(&create_count, &compress_count);
 
-	git_oid_fromstr(&id, current_master_tip);
-	cl_git_pass(git_reference_create(&ref, g_repo, "refs/heads/fsync_test", &id, 0, "log message"));
-	git_reference_free(ref);
-	cl_assert_equal_i(expected_create, p_fsync__cnt);
+	cl_assert_equal_i(expected_fsyncs_create, create_count);
+	cl_assert_equal_i(expected_fsyncs_compress, compress_count);
+}
 
-	p_fsync__cnt = 0;
+void test_refs_create__fsyncs_when_repo_config_set(void)
+{
+	size_t create_count, compress_count;
 
-	cl_git_pass(git_repository_refdb(&refdb, g_repo));
-	cl_git_pass(git_refdb_compress(refdb));
-	git_refdb_free(refdb);
+	cl_repo_set_bool(g_repo, "core.fsyncObjectFiles", true);
+
+	count_fsyncs(&create_count, &compress_count);
 
-	cl_assert_equal_i(expected_compress, p_fsync__cnt);
+	cl_assert_equal_i(expected_fsyncs_create, create_count);
+	cl_assert_equal_i(expected_fsyncs_compress, compress_count);
 }