diff options
| -rw-r--r-- | cache.h | 3 | ||||
| -rw-r--r-- | compat/mingw.h | 16 | ||||
| -rw-r--r-- | fast-import.c | 1 | ||||
| -rw-r--r-- | sha1_file.c | 84 |
4 files changed, 85 insertions, 19 deletions
@@ -913,7 +913,8 @@ extern struct packed_git { time_t mtime; int pack_fd; unsigned pack_local:1, - pack_keep:1; + pack_keep:1, + do_not_close:1; unsigned char sha1[20]; /* something like ".git/objects/pack/xxxxx.pack" */ char pack_name[FLEX_ARRAY]; /* more */ diff --git a/compat/mingw.h b/compat/mingw.h index fe6ba34043..62eccd3391 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -231,6 +231,22 @@ int mingw_getpagesize(void); #define getpagesize mingw_getpagesize #endif +struct rlimit { + unsigned int rlim_cur; +}; +#define RLIMIT_NOFILE 0 + +static inline int getrlimit(int resource, struct rlimit *rlp) +{ + if (resource != RLIMIT_NOFILE) { + errno = EINVAL; + return -1; + } + + rlp->rlim_cur = 2048; + return 0; +} + /* Use mingw_lstat() instead of lstat()/stat() and * mingw_fstat() instead of fstat() on Windows. */ diff --git a/fast-import.c b/fast-import.c index e1268b8cb4..e24ee2c63d 100644 --- a/fast-import.c +++ b/fast-import.c @@ -877,6 +877,7 @@ static void start_packfile(void) p = xcalloc(1, sizeof(*p) + strlen(tmpfile) + 2); strcpy(p->pack_name, tmpfile); p->pack_fd = pack_fd; + p->do_not_close = 1; pack_file = sha1fd(pack_fd, p->pack_name); hdr.hdr_signature = htonl(PACK_SIGNATURE); diff --git a/sha1_file.c b/sha1_file.c index d949b35c33..b4fcca8ffd 100644 --- a/sha1_file.c +++ b/sha1_file.c @@ -418,6 +418,8 @@ static unsigned int pack_used_ctr; static unsigned int pack_mmap_calls; static unsigned int peak_pack_open_windows; static unsigned int pack_open_windows; +static unsigned int pack_open_fds; +static unsigned int pack_max_fds; static size_t peak_pack_mapped; static size_t pack_mapped; struct packed_git *packed_git; @@ -595,8 +597,10 @@ static int unuse_one_window(struct packed_git *current, int keep_fd) lru_l->next = lru_w->next; else { lru_p->windows = lru_w->next; - if (!lru_p->windows && lru_p->pack_fd != keep_fd) { + if (!lru_p->windows && lru_p->pack_fd != -1 + && lru_p->pack_fd != keep_fd) { close(lru_p->pack_fd); + pack_open_fds--; lru_p->pack_fd = -1; } } @@ -681,8 +685,10 @@ void free_pack_by_name(const char *pack_name) if (strcmp(pack_name, p->pack_name) == 0) { clear_delta_base_cache(); close_pack_windows(p); - if (p->pack_fd != -1) + if (p->pack_fd != -1) { close(p->pack_fd); + pack_open_fds--; + } close_pack_index(p); free(p->bad_object_sha1); *pp = p->next; @@ -708,9 +714,29 @@ static int open_packed_git_1(struct packed_git *p) if (!p->index_data && open_pack_index(p)) return error("packfile %s index unavailable", p->pack_name); + if (!pack_max_fds) { + struct rlimit lim; + unsigned int max_fds; + + if (getrlimit(RLIMIT_NOFILE, &lim)) + die_errno("cannot get RLIMIT_NOFILE"); + + max_fds = lim.rlim_cur; + + /* Save 3 for stdin/stdout/stderr, 22 for work */ + if (25 < max_fds) + pack_max_fds = max_fds - 25; + else + pack_max_fds = 1; + } + + while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1)) + ; /* nothing */ + p->pack_fd = git_open_noatime(p->pack_name, p); if (p->pack_fd < 0 || fstat(p->pack_fd, &st)) return -1; + pack_open_fds++; /* If we created the struct before we had the pack we lack size. */ if (!p->pack_size) { @@ -762,6 +788,7 @@ static int open_packed_git(struct packed_git *p) return 0; if (p->pack_fd != -1) { close(p->pack_fd); + pack_open_fds--; p->pack_fd = -1; } return -1; @@ -787,14 +814,13 @@ unsigned char *use_pack(struct packed_git *p, { struct pack_window *win = *w_cursor; - if (p->pack_fd == -1 && open_packed_git(p)) - die("packfile %s cannot be accessed", p->pack_name); - /* Since packfiles end in a hash of their content and it's * pointless to ask for an offset into the middle of that * hash, and the in_window function above wouldn't match * don't allow an offset too close to the end of the file. */ + if (!p->pack_size && p->pack_fd == -1 && open_packed_git(p)) + die("packfile %s cannot be accessed", p->pack_name); if (offset > (p->pack_size - 20)) die("offset beyond end of packfile (truncated pack?)"); @@ -808,6 +834,10 @@ unsigned char *use_pack(struct packed_git *p, if (!win) { size_t window_align = packed_git_window_size / 2; off_t len; + + if (p->pack_fd == -1 && open_packed_git(p)) + die("packfile %s cannot be accessed", p->pack_name); + win = xcalloc(1, sizeof(*win)); win->offset = (offset / window_align) * window_align; len = p->pack_size - win->offset; @@ -825,6 +855,12 @@ unsigned char *use_pack(struct packed_git *p, die("packfile %s cannot be mapped: %s", p->pack_name, strerror(errno)); + if (!win->offset && win->len == p->pack_size + && !p->do_not_close) { + close(p->pack_fd); + pack_open_fds--; + p->pack_fd = -1; + } pack_mmap_calls++; pack_open_windows++; if (pack_mapped > peak_pack_mapped) @@ -919,6 +955,9 @@ struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path) void install_packed_git(struct packed_git *pack) { + if (pack->pack_fd != -1) + pack_open_fds++; + pack->next = packed_git; packed_git = pack; } @@ -936,8 +975,6 @@ static void prepare_packed_git_one(char *objdir, int local) sprintf(path, "%s/pack", objdir); len = strlen(path); dir = opendir(path); - while (!dir && errno == EMFILE && unuse_one_window(NULL, -1)) - dir = opendir(path); if (!dir) { if (errno != ENOENT) error("unable to open object pack directory: %s: %s", @@ -1093,14 +1130,6 @@ static int git_open_noatime(const char *name, struct packed_git *p) if (fd >= 0) return fd; - /* Might the failure be insufficient file descriptors? */ - if (errno == EMFILE) { - if (unuse_one_window(p, -1)) - continue; - else - return -1; - } - /* Might the failure be due to O_NOATIME? */ if (errno != ENOENT && sha1_file_open_flag) { sha1_file_open_flag = 0; @@ -1932,6 +1961,27 @@ off_t find_pack_entry_one(const unsigned char *sha1, return 0; } +static int is_pack_valid(struct packed_git *p) +{ + /* An already open pack is known to be valid. */ + if (p->pack_fd != -1) + return 1; + + /* If the pack has one window completely covering the + * file size, the pack is known to be valid even if + * the descriptor is not currently open. + */ + if (p->windows) { + struct pack_window *w = p->windows; + + if (!w->offset && w->len == p->pack_size) + return 1; + } + + /* Force the pack to open to prove its valid. */ + return !open_packed_git(p); +} + static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e) { static struct packed_git *last_found = (void *)1; @@ -1961,7 +2011,7 @@ static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e) * it may have been deleted since the index * was loaded! */ - if (p->pack_fd == -1 && open_packed_git(p)) { + if (!is_pack_valid(p)) { error("packfile %s cannot be accessed", p->pack_name); goto next; } @@ -2360,8 +2410,6 @@ static int write_loose_object(const unsigned char *sha1, char *hdr, int hdrlen, filename = sha1_file_name(sha1); fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename); - while (fd < 0 && errno == EMFILE && unuse_one_window(NULL, -1)) - fd = create_tmpfile(tmpfile, sizeof(tmpfile), filename); if (fd < 0) { if (errno == EACCES) return error("insufficient permission for adding an object to repository database %s\n", get_object_directory()); |