diff options
author | lhchavez <lhchavez@lhchavez.com> | 2020-02-16 02:00:56 +0000 |
---|---|---|
committer | Edward Thomson <ethomson@edwardthomson.com> | 2020-11-27 11:40:02 +0000 |
commit | f847fa7b34abdc7850b8739747e1d25eefafa5f2 (patch) | |
tree | ac8b28e09c7563a05196f4d354e065b5faaafc0a | |
parent | fa618a595bff675fa1d001dc9d57f8fd6c1b052e (diff) | |
download | libgit2-f847fa7b34abdc7850b8739747e1d25eefafa5f2.tar.gz |
midx: Support multi-pack-index files in odb_pack.c
This change adds support for reading multi-pack-index files from the
packfile odb backend. This also makes git_pack_file objects open their
backing failes lazily in more scenarios, since the multi-pack-index can
avoid having to open them in some cases (yay!).
This change also refreshes the documentation found in src/odb_pack.c to
match the updated code.
Part of: #5399
-rw-r--r-- | src/midx.c | 73 | ||||
-rw-r--r-- | src/midx.h | 11 | ||||
-rw-r--r-- | src/odb_pack.c | 405 | ||||
-rw-r--r-- | src/pack.c | 6 | ||||
-rw-r--r-- | tests/pack/midx.c | 16 |
5 files changed, 399 insertions, 112 deletions
diff --git a/src/midx.c b/src/midx.c index 00d73153d..ee93b03c1 100644 --- a/src/midx.c +++ b/src/midx.c @@ -13,8 +13,6 @@ #include "odb.h" #include "pack.h" -#define GIT_MIDX_FILE_MODE 0444 - #define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */ #define MIDX_VERSION 1 #define MIDX_OBJECT_ID_VERSION 1 @@ -116,7 +114,7 @@ static int midx_parse_oid_lookup( return midx_error("missing OID Lookup chunk"); if (chunk_oid_lookup->length == 0) return midx_error("empty OID Lookup chunk"); - if (chunk_oid_lookup->length != idx->num_objects * 20) + if (chunk_oid_lookup->length != idx->num_objects * GIT_OID_RAWSZ) return midx_error("OID Lookup chunk has wrong length"); idx->oid_lookup = oid = (git_oid *)(data + chunk_oid_lookup->offset); @@ -183,7 +181,7 @@ int git_midx_parse( GIT_ASSERT_ARG(idx); - if (size < sizeof(struct git_midx_header) + 20) + if (size < sizeof(struct git_midx_header) + GIT_OID_RAWSZ) return midx_error("multi-pack index is too short"); hdr = ((struct git_midx_header *)data); @@ -203,7 +201,7 @@ int git_midx_parse( last_chunk_offset = sizeof(struct git_midx_header) + (1 + hdr->chunks) * 12; - trailer_offset = size - 20; + trailer_offset = size - GIT_OID_RAWSZ; if (trailer_offset < last_chunk_offset) return midx_error("wrong index size"); git_oid_cpy(&idx->checksum, (git_oid *)(data + trailer_offset)); @@ -309,6 +307,10 @@ int git_midx_open( idx = git__calloc(1, sizeof(git_midx_file)); GIT_ERROR_CHECK_ALLOC(idx); + error = git_buf_sets(&idx->filename, path); + if (error < 0) + return error; + error = git_futils_mmap_ro(&idx->index_map, fd, 0, idx_size); p_close(fd); if (error < 0) { @@ -325,6 +327,46 @@ int git_midx_open( return 0; } +bool git_midx_needs_refresh( + const git_midx_file *idx, + const char *path) +{ + git_file fd = -1; + struct stat st; + ssize_t bytes_read; + git_oid idx_checksum = {{0}}; + + /* TODO: properly open the file without access time using O_NOATIME */ + fd = git_futils_open_ro(path); + if (fd < 0) + return true; + + if (p_fstat(fd, &st) < 0) { + p_close(fd); + return true; + } + + if (!S_ISREG(st.st_mode) || + !git__is_sizet(st.st_size) || + (size_t)st.st_size != idx->index_map.len) { + p_close(fd); + return true; + } + + if (p_lseek(fd, -GIT_OID_RAWSZ, SEEK_END) < 0) { + p_close(fd); + return true; + } + + bytes_read = p_read(fd, &idx_checksum, GIT_OID_RAWSZ); + p_close(fd); + + if (bytes_read != GIT_OID_RAWSZ) + return true; + + return git_oid_cmp(&idx_checksum, &idx->checksum) == 0; +} + int git_midx_entry_find( git_midx_entry *e, git_midx_file *idx, @@ -343,7 +385,7 @@ int git_midx_entry_find( hi = ntohl(idx->oid_fanout[(int)short_oid->id[0]]); lo = ((short_oid->id[0] == 0x0) ? 0 : ntohl(idx->oid_fanout[(int)short_oid->id[0] - 1])); - pos = git_pack__lookup_sha1(idx->oid_lookup, 20, lo, hi, short_oid->id); + pos = git_pack__lookup_sha1(idx->oid_lookup, GIT_OID_RAWSZ, lo, hi, short_oid->id); if (pos >= 0) { /* An object matching exactly the oid was found */ @@ -399,6 +441,24 @@ int git_midx_entry_find( return 0; } +int git_midx_foreach_entry( + git_midx_file *idx, + git_odb_foreach_cb cb, + void *data) +{ + size_t i; + int error; + + GIT_ASSERT_ARG(idx); + + for (i = 0; i < idx->num_objects; ++i) { + if ((error = cb(&idx->oid_lookup[i], data)) != 0) + return git_error_set_after_callback(error); + } + + return error; +} + int git_midx_close(git_midx_file *idx) { GIT_ASSERT_ARG(idx); @@ -416,6 +476,7 @@ void git_midx_free(git_midx_file *idx) if (!idx) return; + git_buf_dispose(&idx->filename); git_midx_close(idx); git__free(idx); } diff --git a/src/midx.h b/src/midx.h index 3b802952c..543ff2178 100644 --- a/src/midx.h +++ b/src/midx.h @@ -14,6 +14,7 @@ #include "map.h" #include "mwindow.h" +#include "odb.h" /* * A multi-pack-index file. @@ -49,6 +50,9 @@ typedef struct git_midx_file { /* The trailer of the file. Contains the SHA1-checksum of the whole file. */ git_oid checksum; + + /* something like ".git/objects/pack/multi-pack-index". */ + git_buf filename; } git_midx_file; /* @@ -66,11 +70,18 @@ typedef struct git_midx_entry { int git_midx_open( git_midx_file **idx_out, const char *path); +bool git_midx_needs_refresh( + const git_midx_file *idx, + const char *path); int git_midx_entry_find( git_midx_entry *e, git_midx_file *idx, const git_oid *short_oid, size_t len); +int git_midx_foreach_entry( + git_midx_file *idx, + git_odb_foreach_cb cb, + void *data); int git_midx_close(git_midx_file *idx); void git_midx_free(git_midx_file *idx); diff --git a/src/odb_pack.c b/src/odb_pack.c index e4ad0f4b7..3df8a4267 100644 --- a/src/odb_pack.c +++ b/src/odb_pack.c @@ -11,11 +11,12 @@ #include "git2/repository.h" #include "git2/indexer.h" #include "git2/sys/odb_backend.h" +#include "delta.h" #include "futils.h" #include "hash.h" -#include "odb.h" -#include "delta.h" +#include "midx.h" #include "mwindow.h" +#include "odb.h" #include "pack.h" #include "git2/odb_backend.h" @@ -25,6 +26,8 @@ struct pack_backend { git_odb_backend parent; + git_midx_file *midx; + git_vector midx_packs; git_vector packs; struct git_pack_file *last_found; char *pack_folder; @@ -47,36 +50,43 @@ struct pack_writepack { * Initialization of the Pack Backend * -------------------------------------------------- * - * # git_odb_backend_pack - * | Creates the pack backend structure, initializes the - * | callback pointers to our default read() and exist() methods, - * | and tries to preload all the known packfiles in the ODB. + * # git_odb_backend_pack + * | Creates the pack backend structure, initializes the + * | callback pointers to our default read() and exist() methods, + * | and tries to find the `pack` folder, if it exists. ODBs without a `pack` + * | folder are ignored altogether. If there is a `pack` folder, it tries to + * | preload all the known packfiles in the ODB. * | - * |-# packfile_load_all - * | Tries to find the `pack` folder, if it exists. ODBs without - * | a pack folder are ignored altogether. If there's a `pack` folder - * | we run a `dirent` callback through every file in the pack folder - * | to find our packfiles. The packfiles are then sorted according - * | to a sorting callback. - * | - * |-# packfile_load__cb - * | | This callback is called from `dirent` with every single file - * | | inside the pack folder. We find the packs by actually locating - * | | their index (ends in ".idx"). From that index, we verify that - * | | the corresponding packfile exists and is valid, and if so, we - * | | add it to the pack list. - * | | - * | |-# packfile_check - * | Make sure that there's a packfile to back this index, and store - * | some very basic information regarding the packfile itself, - * | such as the full path, the size, and the modification time. - * | We don't actually open the packfile to check for internal consistency. - * | - * |-# packfile_sort__cb - * Sort all the preloaded packs according to some specific criteria: - * we prioritize the "newer" packs because it's more likely they - * contain the objects we are looking for, and we prioritize local - * packs over remote ones. + * |-# pack_backend__refresh + * | The `multi-pack-index` is loaded if it exists and is valid. + * | Then we run a `dirent` callback through every file in the pack folder, + * | even those present in `multi-pack-index`. The unindexed packfiles are + * | then sorted according to a sorting callback. + * | + * |-# refresh_multi_pack_index + * | Detect the presence of the `multi-pack-index` file. If it needs to be + * | refreshed, frees the old copy and tries to load the new one, together + * | with all the packfiles it indexes. If the process fails, fall back to + * | the old behavior, as if the `multi-pack-index` file was not there. + * | + * |-# packfile_load__cb + * | | This callback is called from `dirent` with every single file + * | | inside the pack folder. We find the packs by actually locating + * | | their index (ends in ".idx"). From that index, we verify that + * | | the corresponding packfile exists and is valid, and if so, we + * | | add it to the pack list. + * | | + * | # git_mwindow_get_pack + * | Make sure that there's a packfile to back this index, and store + * | some very basic information regarding the packfile itself, + * | such as the full path, the size, and the modification time. + * | We don't actually open the packfile to check for internal consistency. + * | + * |-# packfile_sort__cb + * Sort all the preloaded packs according to some specific criteria: + * we prioritize the "newer" packs because it's more likely they + * contain the objects we are looking for, and we prioritize local + * packs over remote ones. * * * @@ -84,48 +94,66 @@ struct pack_writepack { * A standard packed `exist` query for an OID * -------------------------------------------------- * - * # pack_backend__exists - * | Check if the given SHA1 oid exists in any of the packs - * | that have been loaded for our ODB. + * # pack_backend__exists / pack_backend__exists_prefix + * | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the + * | packs that have been loaded for our ODB. * | - * |-# pack_entry_find - * | Iterate through all the packs that have been preloaded - * | (starting by the pack where the latest object was found) - * | to try to find the OID in one of them. - * | - * |-# pack_entry_find1 - * | Check the index of an individual pack to see if the SHA1 - * | OID can be found. If we can find the offset to that SHA1 - * | inside of the index, that means the object is contained - * | inside of the packfile and we can stop searching. - * | Before returning, we verify that the packfile behing the - * | index we are searching still exists on disk. - * | - * |-# pack_entry_find_offset - * | | Mmap the actual index file to disk if it hasn't been opened - * | | yet, and run a binary search through it to find the OID. - * | | See <http://book.git-scm.com/7_the_packfile.html> for specifics - * | | on the Packfile Index format and how do we find entries in it. - * | | - * | |-# pack_index_open - * | | Guess the name of the index based on the full path to the - * | | packfile, open it and verify its contents. Only if the index - * | | has not been opened already. - * | | - * | |-# pack_index_check - * | Mmap the index file and do a quick run through the header - * | to guess the index version (right now we support v1 and v2), - * | and to verify that the size of the index makes sense. - * | - * |-# packfile_open - * See `packfile_open` in Chapter 3 + * |-# pack_entry_find / pack_entry_find_prefix + * | If there is a multi-pack-index present, search the SHA1 oid in that + * | index first. If it is not found there, iterate through all the unindexed + * | packs that have been preloaded (starting by the pack where the latest + * | object was found) to try to find the OID in one of them. + * | + * |-# git_midx_entry_find + * | Search for the SHA1 oid in the multi-pack-index. See + * | <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt> + * | for specifics on the multi-pack-index format and how do we find + * | entries in it. + * | + * |-# git_pack_entry_find + * | Check the index of an individual unindexed pack to see if the SHA1 + * | OID can be found. If we can find the offset to that SHA1 inside of the + * | index, that means the object is contained inside of the packfile and + * | we can stop searching. Before returning, we verify that the + * | packfile behing the index we are searching still exists on disk. + * | + * |-# pack_entry_find_offset + * | Mmap the actual index file to disk if it hasn't been opened + * | yet, and run a binary search through it to find the OID. + * | See <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt> + * | for specifics on the Packfile Index format and how do we find + * | entries in it. + * | + * |-# pack_index_open + * | Guess the name of the index based on the full path to the + * | packfile, open it and verify its contents. Only if the index + * | has not been opened already. + * | + * |-# pack_index_check + * Mmap the index file and do a quick run through the header + * to guess the index version (right now we support v1 and v2), + * and to verify that the size of the index makes sense. * * * * Chapter 3: The neverending story... * A standard packed `lookup` query for an OID * -------------------------------------------------- - * TODO + * + * # pack_backend__read / pack_backend__read_prefix + * | Check if the given SHA1 oid (or a SHA1 oid prefix) exists in any of the + * | packs that have been loaded for our ODB. If it does, open the packfile and + * | read from it. + * | + * |-# git_packfile_unpack + * Armed with a packfile and the offset within it, we can finally unpack + * the object pointed at by the SHA1 oid. This involves mmapping part of + * the `.pack` file, and uncompressing the object within it (if it is + * stored in the undelfitied representation), or finding a base object and + * applying some deltas to its uncompressed representation (if it is stored + * in the deltified representation). See + * <https://github.com/git/git/blob/master/Documentation/technical/pack-format.txt> + * for specifics on the Packfile format and how do we read from it. * */ @@ -140,6 +168,8 @@ static int packfile_sort__cb(const void *a_, const void *b_); static int packfile_load__cb(void *_data, git_buf *path); +static int packfile_byname_search_cmp(const void *path, const void *pack_entry); + static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid); @@ -163,6 +193,14 @@ static int pack_entry_find_prefix( * ***********************************************************/ +static int packfile_byname_search_cmp(const void *path_, const void *p_) +{ + const git_buf *path = (const git_buf *)path_; + const struct git_pack_file *p = (const struct git_pack_file *)p_; + + return strncmp(p->pack_name, git_buf_cstr(path), git_buf_len(path)); +} + static int packfile_sort__cb(const void *a_, const void *b_) { const struct git_pack_file *a = a_; @@ -198,20 +236,20 @@ static int packfile_load__cb(void *data, git_buf *path) struct pack_backend *backend = data; struct git_pack_file *pack; const char *path_str = git_buf_cstr(path); - size_t i, cmp_len = git_buf_len(path); + git_buf index_prefix = GIT_BUF_INIT; + size_t cmp_len = git_buf_len(path); int error; if (cmp_len <= strlen(".idx") || git__suffixcmp(path_str, ".idx") != 0) return 0; /* not an index */ cmp_len -= strlen(".idx"); + git_buf_attach_notowned(&index_prefix, path_str, cmp_len); - for (i = 0; i < backend->packs.length; ++i) { - struct git_pack_file *p = git_vector_get(&backend->packs, i); - - if (strncmp(p->pack_name, path_str, cmp_len) == 0) - return 0; - } + if (git_vector_search2(NULL, &backend->midx_packs, packfile_byname_search_cmp, &index_prefix) == 0) + return 0; + if (git_vector_search2(NULL, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0) + return 0; error = git_mwindow_get_pack(&pack, path->ptr); @@ -228,22 +266,26 @@ static int packfile_load__cb(void *data, git_buf *path) } -static int pack_entry_find_inner( - struct git_pack_entry *e, - struct pack_backend *backend, - const git_oid *oid, - struct git_pack_file *last_found) +static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid) { + struct git_pack_file *last_found = backend->last_found, *p; + git_midx_entry midx_entry; size_t i; + if (backend->midx && + git_midx_entry_find(&midx_entry, backend->midx, oid, GIT_OID_HEXSZ) == 0 && + midx_entry.pack_index < git_vector_length(&backend->midx_packs)) { + e->offset = midx_entry.offset; + git_oid_cpy(&e->sha1, &midx_entry.sha1); + e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index); + return 0; + } + if (last_found && git_pack_entry_find(e, last_found, oid, GIT_OID_HEXSZ) == 0) return 0; - for (i = 0; i < backend->packs.length; ++i) { - struct git_pack_file *p; - - p = git_vector_get(&backend->packs, i); + git_vector_foreach(&backend->packs, i, p) { if (p == last_found) continue; @@ -253,20 +295,6 @@ static int pack_entry_find_inner( } } - return -1; -} - -static int pack_entry_find(struct git_pack_entry *e, struct pack_backend *backend, const git_oid *oid) -{ - struct git_pack_file *last_found = backend->last_found; - - if (backend->last_found && - git_pack_entry_find(e, backend->last_found, oid, GIT_OID_HEXSZ) == 0) - return 0; - - if (!pack_entry_find_inner(e, backend, oid, last_found)) - return 0; - return git_odb__error_notfound( "failed to find pack entry", oid, GIT_OID_HEXSZ); } @@ -281,22 +309,35 @@ static int pack_entry_find_prefix( size_t i; git_oid found_full_oid = {{0}}; bool found = false; - struct git_pack_file *last_found = backend->last_found; + struct git_pack_file *last_found = backend->last_found, *p; + git_midx_entry midx_entry; + + if (backend->midx) { + error = git_midx_entry_find(&midx_entry, backend->midx, short_oid, len); + if (error == GIT_EAMBIGUOUS) + return error; + if (!error && midx_entry.pack_index < git_vector_length(&backend->midx_packs)) { + e->offset = midx_entry.offset; + git_oid_cpy(&e->sha1, &midx_entry.sha1); + e->p = git_vector_get(&backend->midx_packs, midx_entry.pack_index); + git_oid_cpy(&found_full_oid, &e->sha1); + found = true; + } + } if (last_found) { error = git_pack_entry_find(e, last_found, short_oid, len); if (error == GIT_EAMBIGUOUS) return error; if (!error) { + if (found && git_oid_cmp(&e->sha1, &found_full_oid)) + return git_odb__error_ambiguous("found multiple pack entries"); git_oid_cpy(&found_full_oid, &e->sha1); found = true; } } - for (i = 0; i < backend->packs.length; ++i) { - struct git_pack_file *p; - - p = git_vector_get(&backend->packs, i); + git_vector_foreach(&backend->packs, i, p) { if (p == last_found) continue; @@ -319,6 +360,141 @@ static int pack_entry_find_prefix( return 0; } +/*********************************************************** + * + * MULTI-PACK-INDEX SUPPORT + * + * Functions needed to support the multi-pack-index. + * + ***********************************************************/ + +/* + * Remove the multi-pack-index, and move all midx_packs to packs. + */ +static int remove_multi_pack_index(struct pack_backend *backend) +{ + size_t i, j = git_vector_length(&backend->packs); + struct pack_backend *p; + int error = git_vector_size_hint( + &backend->packs, + j + git_vector_length(&backend->midx_packs)); + if (error < 0) + return error; + + git_vector_foreach(&backend->midx_packs, i, p) + git_vector_set(NULL, &backend->packs, j++, p); + git_vector_clear(&backend->midx_packs); + + git_midx_free(backend->midx); + backend->midx = NULL; + + return 0; +} + +/* + * Loads a single .pack file referred to by the multi-pack-index. These must + * match the order in which they are declared in the multi-pack-index file, + * since these files are referred to by their index. + */ +static int process_multi_pack_index_pack( + struct pack_backend *backend, + size_t i, + const char *packfile_name) +{ + int error; + size_t cmp_len = strlen(packfile_name); + struct git_pack_file *pack; + size_t found_position; + git_buf pack_path = GIT_BUF_INIT, index_prefix = GIT_BUF_INIT; + + error = git_buf_joinpath(&pack_path, backend->pack_folder, packfile_name); + if (error < 0) + return error; + + /* This is ensured by midx__parse_packfile_name() */ + if (cmp_len <= strlen(".idx") || git__suffixcmp(git_buf_cstr(&pack_path), ".idx") != 0) + return git_odb__error_notfound("midx file contained a non-index", NULL, 0); + + cmp_len -= strlen(".idx"); + git_buf_attach_notowned(&index_prefix, git_buf_cstr(&pack_path), cmp_len); + + if (git_vector_search2(&found_position, &backend->packs, packfile_byname_search_cmp, &index_prefix) == 0) { + /* Pack was found in the packs list. Moving it to the midx_packs list. */ + git_buf_dispose(&pack_path); + git_vector_set(NULL, &backend->midx_packs, i, git_vector_get(&backend->packs, found_position)); + git_vector_remove(&backend->packs, found_position); + return 0; + } + + /* Pack was not found. Allocate a new one. */ + error = git_mwindow_get_pack(&pack, git_buf_cstr(&pack_path)); + git_buf_dispose(&pack_path); + if (error < 0) + return error; + + git_vector_set(NULL, &backend->midx_packs, i, pack); + return 0; +} + +/* + * Reads the multi-pack-index. If this fails for whatever reason, the + * multi-pack-index object is freed, and all the packfiles that are related to + * it are moved to the unindexed packfiles vector. + */ +static int refresh_multi_pack_index(struct pack_backend *backend) +{ + int error; + git_buf midx_path = GIT_BUF_INIT; + const char *packfile_name; + size_t i; + + error = git_buf_joinpath(&midx_path, backend->pack_folder, "multi-pack-index"); + if (error < 0) + return error; + + /* + * Check whether the multi-pack-index has changed. If it has, close any + * old multi-pack-index and move all the packfiles to the unindexed + * packs. This is done to prevent losing any open packfiles in case + * refreshing the new multi-pack-index fails, or the file is deleted. + */ + if (backend->midx) { + if (!git_midx_needs_refresh(backend->midx, git_buf_cstr(&midx_path))) { + git_buf_dispose(&midx_path); + return 0; + } + error = remove_multi_pack_index(backend); + if (error < 0) { + git_buf_dispose(&midx_path); + return error; + } + } + + error = git_midx_open(&backend->midx, git_buf_cstr(&midx_path)); + git_buf_dispose(&midx_path); + if (error < 0) + return error; + + git_vector_resize_to(&backend->midx_packs, git_vector_length(&backend->midx->packfile_names)); + + git_vector_foreach(&backend->midx->packfile_names, i, packfile_name) { + error = process_multi_pack_index_pack(backend, i, packfile_name); + if (error < 0) { + /* + * Something failed during reading multi-pack-index. + * Restore the state of backend as if the + * multi-pack-index was never there, and move all + * packfiles that have been processed so far to the + * unindexed packs. + */ + git_vector_resize_to(&backend->midx_packs, i); + remove_multi_pack_index(backend); + return error; + } + } + + return 0; +} /*********************************************************** * @@ -340,9 +516,16 @@ static int pack_backend__refresh(git_odb_backend *backend_) if (p_stat(backend->pack_folder, &st) < 0 || !S_ISDIR(st.st_mode)) return git_odb__error_notfound("failed to refresh packfiles", NULL, 0); - git_buf_sets(&path, backend->pack_folder); + if (refresh_multi_pack_index(backend) < 0) { + /* + * It is okay if this fails. We will just not use the + * multi-pack-index in this case. + */ + git_error_clear(); + } /* reload all packs */ + git_buf_sets(&path, backend->pack_folder); error = git_path_direach(&path, 0, packfile_load__cb, backend); git_buf_dispose(&path); @@ -478,9 +661,11 @@ static int pack_backend__foreach(git_odb_backend *_backend, git_odb_foreach_cb c backend = (struct pack_backend *)_backend; /* Make sure we know about the packfiles */ - if ((error = pack_backend__refresh(_backend)) < 0) + if ((error = pack_backend__refresh(_backend)) != 0) return error; + if (backend->midx && (error = git_midx_foreach_entry(backend->midx, cb, data)) != 0) + return error; git_vector_foreach(&backend->packs, i, p) { if ((error = git_pack_foreach_entry(p, cb, data)) != 0) return error; @@ -562,6 +747,7 @@ static int pack_backend__writepack(struct git_odb_writepack **out, static void pack_backend__free(git_odb_backend *_backend) { struct pack_backend *backend; + struct git_pack_file *p; size_t i; if (!_backend) @@ -569,11 +755,13 @@ static void pack_backend__free(git_odb_backend *_backend) backend = (struct pack_backend *)_backend; - for (i = 0; i < backend->packs.length; ++i) { - struct git_pack_file *p = git_vector_get(&backend->packs, i); + git_vector_foreach(&backend->midx_packs, i, p) + git_mwindow_put_pack(p); + git_vector_foreach(&backend->packs, i, p) git_mwindow_put_pack(p); - } + git_midx_free(backend->midx); + git_vector_free(&backend->midx_packs); git_vector_free(&backend->packs); git__free(backend->pack_folder); git__free(backend); @@ -584,7 +772,12 @@ static int pack_backend__alloc(struct pack_backend **out, size_t initial_size) struct pack_backend *backend = git__calloc(1, sizeof(struct pack_backend)); GIT_ERROR_CHECK_ALLOC(backend); + if (git_vector_init(&backend->midx_packs, 0, NULL) < 0) { + git__free(backend); + return -1; + } if (git_vector_init(&backend->packs, initial_size, packfile_sort__cb) < 0) { + git_vector_free(&backend->midx_packs); git__free(backend); return -1; } diff --git a/src/pack.c b/src/pack.c index 982ad1770..f81bc91f2 100644 --- a/src/pack.c +++ b/src/pack.c @@ -481,6 +481,9 @@ int git_packfile_resolve_header( off64_t base_offset; int error; + if (p->mwf.fd == -1 && (error = packfile_open(p)) < 0) + return error; + error = git_packfile_unpack_header(&size, &type, &p->mwf, &w_curs, &curpos); if (error < 0) return error; @@ -631,6 +634,9 @@ int git_packfile_unpack( size_t stack_size = 0, elem_pos, alloclen; git_object_t base_type; + if (p->mwf.fd == -1 && (error = packfile_open(p)) < 0) + return error; + /* * TODO: optionally check the CRC on the packfile */ diff --git a/tests/pack/midx.c b/tests/pack/midx.c index 1f47d9502..92d9ae24c 100644 --- a/tests/pack/midx.c +++ b/tests/pack/midx.c @@ -27,3 +27,19 @@ void test_pack_midx__parse(void) git_repository_free(repo); git_buf_dispose(&midx_path); } + +void test_pack_midx__lookup(void) +{ + git_repository *repo; + git_commit *commit; + git_oid id; + + cl_git_pass(git_repository_open(&repo, cl_fixture("testrepo.git"))); + + cl_git_pass(git_oid_fromstr(&id, "5001298e0c09ad9c34e4249bc5801c75e9754fa5")); + cl_git_pass(git_commit_lookup_prefix(&commit, repo, &id, GIT_OID_HEXSZ)); + cl_assert_equal_s(git_commit_message(commit), "packed commit one\n"); + + git_commit_free(commit); + git_repository_free(repo); +} |