More filemode cleanups for FAT on MacOS

This cleans up some additional issues.  The main change is that
on a filesystem that doesn't support mode bits, libgit2 will now
create new blobs with GIT_FILEMODE_BLOB always instead of being
at the mercy to the filesystem driver to report executable or not.
This means that if "core.filemode" lies and claims that filemode
is not supported, then we will ignore the executable bit from the
filesystem.  Previously we would have allowed it.

This adds an option to the new git_repository_reset_filesystem to
recurse through submodules if desired.  There may be other types
of APIs that would like a "recurse submodules" option, but this
one is particularly useful.

This also has a number of cleanups, etc., for related things
including trying to give better error messages when problems come
up from the filesystem.  For example, the FAT filesystem driver on
MacOS appears to return errno EINVAL if you attempt to write a
filename with invalid UTF-8 in it.  We try to capture that with a
better error message now.

9 files changed, 41 insertions, 62 deletions

diff --git a/tests-clar/clar_libgit2.c b/tests-clar/clar_libgit2.c
index c3abc1f95..82ec5c065 100644
--- a/tests-clar/clar_libgit2.c
+++ b/tests-clar/clar_libgit2.c
@@ -191,7 +191,7 @@ git_repository *cl_git_sandbox_init(const char *sandbox)
 	cl_git_pass(git_repository_open(&_cl_repo, sandbox));
 
 	/* Adjust configs after copying to new filesystem */
-	cl_git_pass(git_repository_reset_filesystem(_cl_repo));
+	cl_git_pass(git_repository_reset_filesystem(_cl_repo, 0));
 
 	return _cl_repo;
 }
diff --git a/tests-clar/diff/patch.c b/tests-clar/diff/patch.c
index 7aab8f409..1dbfc9ac5 100644
--- a/tests-clar/diff/patch.c
+++ b/tests-clar/diff/patch.c
@@ -240,7 +240,7 @@ void test_diff_patch__hunks_have_correct_line_numbers(void)
 	git_repository_set_config(g_repo, cfg);
 	git_config_free(cfg);
 
-	git_repository_reset_filesystem(g_repo);
+	git_repository_reset_filesystem(g_repo, false);
 
 	cl_git_pass(
 		git_futils_readbuffer(&old_content, "renames/songof7cities.txt"));
@@ -524,7 +524,7 @@ void test_diff_patch__line_counts_with_eofnl(void)
 	git_repository_set_config(g_repo, cfg);
 	git_config_free(cfg);
 
-	git_repository_reset_filesystem(g_repo);
+	git_repository_reset_filesystem(g_repo, false);
 
 	cl_git_pass(git_futils_readbuffer(&content, "renames/songof7cities.txt"));
 
diff --git a/tests-clar/diff/workdir.c b/tests-clar/diff/workdir.c
index 6c17b41c6..aeef7b963 100644
--- a/tests-clar/diff/workdir.c
+++ b/tests-clar/diff/workdir.c
@@ -761,16 +761,7 @@ void test_diff_workdir__submodules(void)
 	git_diff_list *diff = NULL;
 	diff_expects exp;
 
-	g_repo = cl_git_sandbox_init("submod2");
-
-	cl_fixture_sandbox("submod2_target");
-	p_rename("submod2_target/.gitted", "submod2_target/.git");
-
-	rewrite_gitmodules(git_repository_workdir(g_repo));
-	p_rename("submod2/not-submodule/.gitted", "submod2/not-submodule/.git");
-	p_rename("submod2/not/.gitted", "submod2/not/.git");
-
-	cl_fixture_cleanup("submod2_target");
+	g_repo = setup_fixture_submod2();
 
 	a = resolve_commit_oid_to_tree(g_repo, a_commit);
 
diff --git a/tests-clar/index/addall.c b/tests-clar/index/addall.c
index f46a1e16c..3b5f5f22d 100644
--- a/tests-clar/index/addall.c
+++ b/tests-clar/index/addall.c
@@ -111,8 +111,9 @@ static void check_stat_data(git_index *index, const char *path, bool match)
 		cl_assert(st.st_gid  == entry->gid);
 		cl_assert_equal_i_fmt(
 			GIT_MODE_TYPE(st.st_mode), GIT_MODE_TYPE(entry->mode), "%07o");
-		cl_assert_equal_b(
-			GIT_PERMS_IS_EXEC(st.st_mode), GIT_PERMS_IS_EXEC(entry->mode));
+		if (cl_is_chmod_supported())
+			cl_assert_equal_b(
+				GIT_PERMS_IS_EXEC(st.st_mode), GIT_PERMS_IS_EXEC(entry->mode));
 	} else {
 		/* most things will still match */
 		cl_assert(st.st_size != entry->file_size);
diff --git a/tests-clar/index/filemodes.c b/tests-clar/index/filemodes.c
index 8cbd6ff3a..013932696 100644
--- a/tests-clar/index/filemodes.c
+++ b/tests-clar/index/filemodes.c
@@ -74,7 +74,6 @@ static void add_and_check_mode_(
 void test_index_filemodes__untrusted(void)
 {
 	git_index *index;
-	bool can_filemode = cl_is_chmod_supported();
 
 	cl_repo_set_bool(g_repo, "core.filemode", false);
 
@@ -97,20 +96,15 @@ void test_index_filemodes__untrusted(void)
 	replace_file_with_mode("exec_on", "filemodes/exec_on.1", 0755);
 	add_and_check_mode(index, "exec_on", GIT_FILEMODE_BLOB_EXECUTABLE);
 
-	/* these tests of newly added files won't give predictable results on
-	 * filesystems without actual filemode support, so skip them.
-	 */
-	if (can_filemode) {
-		/*  5 - add new 0644 -> expect 0644 */
-		cl_git_write2file("filemodes/new_off", "blah", 0,
-			O_WRONLY | O_CREAT | O_TRUNC, 0644);
-		add_and_check_mode(index, "new_off", GIT_FILEMODE_BLOB);
-
-		/* 6 - add new 0755 -> expect 0755 */
-		cl_git_write2file("filemodes/new_on", "blah", 0,
-			O_WRONLY | O_CREAT | O_TRUNC, 0755);
-		add_and_check_mode(index, "new_on", GIT_FILEMODE_BLOB_EXECUTABLE);
-	}
+	/*  5 - add new 0644 -> expect 0644 */
+	cl_git_write2file("filemodes/new_off", "blah", 0,
+		O_WRONLY | O_CREAT | O_TRUNC, 0644);
+	add_and_check_mode(index, "new_off", GIT_FILEMODE_BLOB);
+
+	/* 6 - add new 0755 -> expect 0644 if core.filemode == false */
+	cl_git_write2file("filemodes/new_on", "blah", 0,
+		O_WRONLY | O_CREAT | O_TRUNC, 0755);
+	add_and_check_mode(index, "new_on", GIT_FILEMODE_BLOB);
 
 	git_index_free(index);
 }
diff --git a/tests-clar/refs/unicode.c b/tests-clar/refs/unicode.c
index 2ec103275..b29c63e2b 100644
--- a/tests-clar/refs/unicode.c
+++ b/tests-clar/refs/unicode.c
@@ -22,23 +22,38 @@ void test_refs_unicode__create_and_lookup(void)
 	git_reference *ref0, *ref1, *ref2;
 	git_repository *repo2;
 
-	const char *REFNAME = "refs/heads/" "\305" "ngstr" "\366" "m";
+	const char *REFNAME = "refs/heads/" "\303\205" "ngstr" "\303\266" "m";
+	const char *REFNAME_DECOMPOSED =
+		"refs/heads/" "A" "\314\212" "ngstro" "\314\210" "m";
 	const char *master = "refs/heads/master";
 
 	/* Create the reference */
 	cl_git_pass(git_reference_lookup(&ref0, repo, master));
 	cl_git_pass(git_reference_create(&ref1, repo, REFNAME, git_reference_target(ref0), 0));
 	cl_assert_equal_s(REFNAME, git_reference_name(ref1));
+	git_reference_free(ref0);
 
 	/* Lookup the reference in a different instance of the repository */
 	cl_git_pass(git_repository_open(&repo2, "testrepo.git"));
+
 	cl_git_pass(git_reference_lookup(&ref2, repo2, REFNAME));
+	cl_assert_equal_i(
+		0, git_oid_cmp(git_reference_target(ref1), git_reference_target(ref2)));
+	cl_assert_equal_s(REFNAME, git_reference_name(ref2));
+	git_reference_free(ref2);
+
+#if GIT_USE_ICONV
+	/* Lookup reference by decomposed unicode name */
 
-	cl_assert(git_oid_cmp(git_reference_target(ref1), git_reference_target(ref2)) == 0);
+	cl_git_pass(git_reference_lookup(&ref2, repo2, REFNAME_DECOMPOSED));
+	cl_assert_equal_i(
+		0, git_oid_cmp(git_reference_target(ref1), git_reference_target(ref2)));
 	cl_assert_equal_s(REFNAME, git_reference_name(ref2));
+	git_reference_free(ref2);
+#endif
+
+	/* Cleanup */
 
-	git_reference_free(ref0);
 	git_reference_free(ref1);
-	git_reference_free(ref2);
 	git_repository_free(repo2);
 }
diff --git a/tests-clar/submodule/lookup.c b/tests-clar/submodule/lookup.c
index b626cdf04..5f320e702 100644
--- a/tests-clar/submodule/lookup.c
+++ b/tests-clar/submodule/lookup.c
@@ -7,20 +7,7 @@ static git_repository *g_repo = NULL;
 
 void test_submodule_lookup__initialize(void)
 {
-	g_repo = cl_git_sandbox_init("submod2");
-
-	cl_fixture_sandbox("submod2_target");
-	p_rename("submod2_target/.gitted", "submod2_target/.git");
-
-	/* must create submod2_target before rewrite so prettify will work */
-	rewrite_gitmodules(git_repository_workdir(g_repo));
-	p_rename("submod2/not-submodule/.gitted", "submod2/not-submodule/.git");
-}
-
-void test_submodule_lookup__cleanup(void)
-{
-	cl_git_sandbox_cleanup();
-	cl_fixture_cleanup("submod2_target");
+	g_repo = setup_fixture_submod2();
 }
 
 void test_submodule_lookup__simple_lookup(void)
diff --git a/tests-clar/submodule/modify.c b/tests-clar/submodule/modify.c
index c0498ce6e..e326287a6 100644
--- a/tests-clar/submodule/modify.c
+++ b/tests-clar/submodule/modify.c
@@ -11,20 +11,7 @@ static git_repository *g_repo = NULL;
 
 void test_submodule_modify__initialize(void)
 {
-	g_repo = cl_git_sandbox_init("submod2");
-
-	cl_fixture_sandbox("submod2_target");
-	p_rename("submod2_target/.gitted", "submod2_target/.git");
-
-	/* must create submod2_target before rewrite so prettify will work */
-	rewrite_gitmodules(git_repository_workdir(g_repo));
-	p_rename("submod2/not-submodule/.gitted", "submod2/not-submodule/.git");
-}
-
-void test_submodule_modify__cleanup(void)
-{
-	cl_git_sandbox_cleanup();
-	cl_fixture_cleanup("submod2_target");
+	g_repo = setup_fixture_submod2();
 }
 
 void test_submodule_modify__add(void)
diff --git a/tests-clar/submodule/submodule_helpers.c b/tests-clar/submodule/submodule_helpers.c
index 3e79c77fd..cc3b7a2e3 100644
--- a/tests-clar/submodule/submodule_helpers.c
+++ b/tests-clar/submodule/submodule_helpers.c
@@ -102,6 +102,8 @@ git_repository *setup_fixture_submodules(void)
 
 	cl_set_cleanup(cleanup_fixture_submodules, "testrepo.git");
 
+	cl_git_pass(git_repository_reset_filesystem(repo, 1));
+
 	return repo;
 }
 
@@ -118,5 +120,7 @@ git_repository *setup_fixture_submod2(void)
 
 	cl_set_cleanup(cleanup_fixture_submodules, "submod2_target");
 
+	cl_git_pass(git_repository_reset_filesystem(repo, 1));
+
 	return repo;
 }