From a5436b57941d99302a4cf68373da6288c11f0340 Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Mon, 24 Oct 2016 20:02:59 +0200 Subject: sha1_file: rename git_open_noatime() to git_open() This function is meant to be used when reading from files in the object store, and the original objective was to avoid smudging atime of loose object files too often, hence its name. Because we'll be extending its role in the next commit to also arrange the file descriptors they return auto-closed in the child processes, rename it to lose "noatime" part that is too specific. Signed-off-by: Lars Schneider Signed-off-by: Junio C Hamano --- builtin/pack-objects.c | 2 +- cache.h | 2 +- pack-bitmap.c | 2 +- sha1_file.c | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 1e7c2a98a5..0fd52bd6b4 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -720,7 +720,7 @@ static off_t write_reused_pack(struct sha1file *f) if (!is_pack_valid(reuse_packfile)) die("packfile is invalid: %s", reuse_packfile->pack_name); - fd = git_open_noatime(reuse_packfile->pack_name); + fd = git_open(reuse_packfile->pack_name); if (fd < 0) die_errno("unable to open packfile for reuse: %s", reuse_packfile->pack_name); diff --git a/cache.h b/cache.h index 0dc39a998c..a902ca1f8e 100644 --- a/cache.h +++ b/cache.h @@ -1122,7 +1122,7 @@ extern int write_sha1_file(const void *buf, unsigned long len, const char *type, extern int hash_sha1_file_literally(const void *buf, unsigned long len, const char *type, unsigned char *sha1, unsigned flags); extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *); extern int force_object_loose(const unsigned char *sha1, time_t mtime); -extern int git_open_noatime(const char *name); +extern int git_open(const char *name); extern void *map_sha1_file(const unsigned char *sha1, unsigned long *size); extern int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz); extern int parse_sha1_header(const char *hdr, unsigned long *sizep); diff --git a/pack-bitmap.c b/pack-bitmap.c index b949e51716..39bcc16846 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -266,7 +266,7 @@ static int open_pack_bitmap_1(struct packed_git *packfile) return -1; idx_name = pack_bitmap_filename(packfile); - fd = git_open_noatime(idx_name); + fd = git_open(idx_name); free(idx_name); if (fd < 0) diff --git a/sha1_file.c b/sha1_file.c index 266152de36..5d2bcd3ed1 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -370,7 +370,7 @@ void read_info_alternates(const char * relative_base, int depth) int fd; path = xstrfmt("%s/info/alternates", relative_base); - fd = git_open_noatime(path); + fd = git_open(path); free(path); if (fd < 0) return; @@ -663,7 +663,7 @@ static int check_packed_git_idx(const char *path, struct packed_git *p) struct pack_idx_header *hdr; size_t idx_size; uint32_t version, nr, i, *index; - int fd = git_open_noatime(path); + int fd = git_open(path); struct stat st; if (fd < 0) @@ -1069,7 +1069,7 @@ static int open_packed_git_1(struct packed_git *p) while (pack_max_fds <= pack_open_fds && close_one_pack()) ; /* nothing */ - p->pack_fd = git_open_noatime(p->pack_name); + p->pack_fd = git_open(p->pack_name); if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) return -1; pack_open_fds++; @@ -1559,7 +1559,7 @@ int check_sha1_signature(const unsigned char *sha1, void *map, return hashcmp(sha1, real_sha1) ? -1 : 0; } -int git_open_noatime(const char *name) +int git_open(const char *name) { static int sha1_file_open_flag = O_NOATIME; @@ -1605,7 +1605,7 @@ static int open_sha1_file(const unsigned char *sha1) struct alternate_object_database *alt; int most_interesting_errno; - fd = git_open_noatime(sha1_file_name(sha1)); + fd = git_open(sha1_file_name(sha1)); if (fd >= 0) return fd; most_interesting_errno = errno; @@ -1613,7 +1613,7 @@ static int open_sha1_file(const unsigned char *sha1) prepare_alt_odb(); for (alt = alt_odb_list; alt; alt = alt->next) { const char *path = alt_sha1_path(alt, sha1); - fd = git_open_noatime(path); + fd = git_open(path); if (fd >= 0) return fd; if (most_interesting_errno == ENOENT) -- cgit v1.2.1 From cd66ada06588f797a424dd1f6da1c6bb51de1660 Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Mon, 24 Oct 2016 20:02:59 +0200 Subject: sha1_file: open window into packfiles with O_CLOEXEC All processes that the Git main process spawns inherit the open file descriptors of the main process. These leaked file descriptors can cause problems. Use the O_CLOEXEC flag similar to 05d1ed61 to fix the leaked file descriptors. Signed-off-by: Lars Schneider Signed-off-by: Junio C Hamano --- sha1_file.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sha1_file.c b/sha1_file.c index 5d2bcd3ed1..09045df1dc 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -1561,7 +1561,7 @@ int check_sha1_signature(const unsigned char *sha1, void *map, int git_open(const char *name) { - static int sha1_file_open_flag = O_NOATIME; + static int sha1_file_open_flag = O_NOATIME | O_CLOEXEC; for (;;) { int fd; @@ -1571,12 +1571,17 @@ int git_open(const char *name) if (fd >= 0) return fd; - /* Might the failure be due to O_NOATIME? */ - if (errno != ENOENT && sha1_file_open_flag) { - sha1_file_open_flag = 0; + /* Try again w/o O_CLOEXEC: the kernel might not support it */ + if ((sha1_file_open_flag & O_CLOEXEC) && errno == EINVAL) { + sha1_file_open_flag &= ~O_CLOEXEC; continue; } + /* Might the failure be due to O_NOATIME? */ + if (errno != ENOENT && (sha1_file_open_flag & O_NOATIME)) { + sha1_file_open_flag &= ~O_NOATIME; + continue; + } return -1; } } -- cgit v1.2.1 From a0a6cb96625cebe8590841c469bfbb461a132ae3 Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Mon, 24 Oct 2016 20:03:00 +0200 Subject: read-cache: make sure file handles are not inherited by child processes This fixes "convert: add filter..process option" (edcc8581) on Windows. Consider the case of a file that requires filtering and is present in branch A but not in branch B. If A is the current HEAD and we checkout B then the following happens: 1. ce_compare_data() opens the file 2. index_fd() detects that the file requires to run a clean filter and calls index_stream_convert_blob() 4. index_stream_convert_blob() calls convert_to_git_filter_fd() 5. convert_to_git_filter_fd() calls apply_filter() which creates a new long running filter process (in case it is the first file of this kind to be filtered) 6. The new filter process inherits all file handles. This is the default on Linux/OSX and is explicitly defined in the `CreateProcessW` call in `mingw.c` on Windows. 7. ce_compare_data() closes the file 8. Git unlinks the file as it is not present in B The unlink operation does not work on Windows because the filter process has still an open handle to the file. On Linux/OSX the unlink operation succeeds but the file descriptors still leak into the child process. Fix this problem by opening files in read-cache with the O_CLOEXEC flag to ensure that the file descriptor does not remain open in a newly spawned process similar to 05d1ed6148 ("mingw: ensure temporary file handles are not inherited by child processes", 2016-08-22). Signed-off-by: Lars Schneider Signed-off-by: Junio C Hamano --- read-cache.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/read-cache.c b/read-cache.c index 38d67faf70..db5d910642 100644 --- a/read-cache.c +++ b/read-cache.c @@ -156,7 +156,14 @@ void fill_stat_cache_info(struct cache_entry *ce, struct stat *st) static int ce_compare_data(const struct cache_entry *ce, struct stat *st) { int match = -1; - int fd = open(ce->name, O_RDONLY); + static int cloexec = O_CLOEXEC; + int fd = open(ce->name, O_RDONLY | cloexec); + + if ((cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) { + /* Try again w/o O_CLOEXEC: the kernel might not support it */ + cloexec &= ~O_CLOEXEC; + fd = open(ce->name, O_RDONLY | cloexec); + } if (fd >= 0) { unsigned char sha1[20]; -- cgit v1.2.1