diff options
| -rw-r--r-- | Documentation/technical/commit-graph.txt | 17 | ||||
| -rw-r--r-- | builtin/am.c | 2 | ||||
| -rw-r--r-- | builtin/clone.c | 2 | ||||
| -rw-r--r-- | builtin/commit-graph.c | 22 | ||||
| -rw-r--r-- | builtin/commit.c | 5 | ||||
| -rw-r--r-- | builtin/fetch.c | 2 | ||||
| -rw-r--r-- | builtin/gc.c | 11 | ||||
| -rw-r--r-- | builtin/merge.c | 2 | ||||
| -rw-r--r-- | builtin/rebase.c | 2 | ||||
| -rw-r--r-- | builtin/receive-pack.c | 2 | ||||
| -rw-r--r-- | builtin/repack.c | 2 | ||||
| -rw-r--r-- | commit-graph.c | 613 | ||||
| -rw-r--r-- | commit-graph.h | 22 | ||||
| -rw-r--r-- | commit.c | 2 | ||||
| -rw-r--r-- | object.c | 2 | ||||
| -rw-r--r-- | packfile.c | 5 | ||||
| -rw-r--r-- | packfile.h | 2 | ||||
| -rwxr-xr-x | t/t5318-commit-graph.sh | 8 | ||||
| -rw-r--r-- | upload-pack.c | 2 |
19 files changed, 398 insertions, 327 deletions
diff --git a/Documentation/technical/commit-graph.txt b/Documentation/technical/commit-graph.txt index 7805b0968c..fb53341d5e 100644 --- a/Documentation/technical/commit-graph.txt +++ b/Documentation/technical/commit-graph.txt @@ -127,23 +127,6 @@ Design Details helpful for these clones, anyway. The commit-graph will not be read or written when shallow commits are present. -Future Work ------------ - -- After computing and storing generation numbers, we must make graph - walks aware of generation numbers to gain the performance benefits they - enable. This will mostly be accomplished by swapping a commit-date-ordered - priority queue with one ordered by generation number. The following - operations are important candidates: - - - 'log --topo-order' - - 'tag --merged' - -- A server could provide a commit-graph file as part of the network protocol - to avoid extra calculations by clients. This feature is only of benefit if - the user is willing to trust the file, because verifying the file is correct - is as hard as computing it from scratch. - Related Links ------------- [0] https://bugs.chromium.org/p/git/issues/detail?id=8 diff --git a/builtin/am.c b/builtin/am.c index 78389d08b6..252e37ddf0 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -1801,7 +1801,7 @@ next: */ if (!state->rebasing) { am_destroy(state); - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); run_command_v_opt(argv_gc_auto, RUN_GIT_CMD); } } diff --git a/builtin/clone.c b/builtin/clone.c index 356bae5ed7..3623f040d4 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1245,7 +1245,7 @@ int cmd_clone(int argc, const char **argv, const char *prefix) transport_disconnect(transport); if (option_dissociate) { - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); dissociate_from_references(); } diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 537fdfd0f0..d8efa5bab2 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -141,6 +141,8 @@ static int graph_write(int argc, const char **argv) struct string_list *pack_indexes = NULL; struct string_list *commit_hex = NULL; struct string_list lines; + int result = 0; + unsigned int flags = COMMIT_GRAPH_PROGRESS; static struct option builtin_commit_graph_write_options[] = { OPT_STRING(0, "object-dir", &opts.obj_dir, @@ -165,13 +167,13 @@ static int graph_write(int argc, const char **argv) die(_("use at most one of --reachable, --stdin-commits, or --stdin-packs")); if (!opts.obj_dir) opts.obj_dir = get_object_directory(); + if (opts.append) + flags |= COMMIT_GRAPH_APPEND; read_replace_refs = 0; - if (opts.reachable) { - write_commit_graph_reachable(opts.obj_dir, opts.append, 1); - return 0; - } + if (opts.reachable) + return write_commit_graph_reachable(opts.obj_dir, flags); string_list_init(&lines, 0); if (opts.stdin_packs || opts.stdin_commits) { @@ -188,14 +190,14 @@ static int graph_write(int argc, const char **argv) UNLEAK(buf); } - write_commit_graph(opts.obj_dir, - pack_indexes, - commit_hex, - opts.append, - 1); + if (write_commit_graph(opts.obj_dir, + pack_indexes, + commit_hex, + flags)) + result = 1; UNLEAK(lines); - return 0; + return result; } int cmd_commit_graph(int argc, const char **argv, const char *prefix) diff --git a/builtin/commit.c b/builtin/commit.c index 1c9e8e2228..1921401117 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1669,8 +1669,9 @@ int cmd_commit(int argc, const char **argv, const char *prefix) "new_index file. Check that disk is not full and quota is\n" "not exceeded, and then \"git reset HEAD\" to recover.")); - if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0)) - write_commit_graph_reachable(get_object_directory(), 0, 0); + if (git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) && + write_commit_graph_reachable(get_object_directory(), 0)) + return 1; repo_rerere(the_repository, 0); run_command_v_opt(argv_gc_auto, RUN_GIT_CMD); diff --git a/builtin/fetch.c b/builtin/fetch.c index 4ba63d5ac6..fc6c879bcf 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1672,7 +1672,7 @@ int cmd_fetch(int argc, const char **argv, const char *prefix) string_list_clear(&list, 0); - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); argv_array_pushl(&argv_gc_auto, "gc", "--auto", NULL); if (verbosity < 0) diff --git a/builtin/gc.c b/builtin/gc.c index 8943bcc300..be8e0bfcbe 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -653,7 +653,7 @@ int cmd_gc(int argc, const char **argv, const char *prefix) gc_before_repack(); if (!repository_format_precious_objects) { - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); if (run_command_v_opt(repack.argv, RUN_GIT_CMD)) die(FAILED_RUN, repack.argv[0]); @@ -681,13 +681,14 @@ int cmd_gc(int argc, const char **argv, const char *prefix) report_garbage = report_pack_garbage; reprepare_packed_git(the_repository); if (pack_garbage.nr > 0) { - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); clean_pack_garbage(); } - if (gc_write_commit_graph) - write_commit_graph_reachable(get_object_directory(), 0, - !quiet && !daemonized); + if (gc_write_commit_graph && + write_commit_graph_reachable(get_object_directory(), + !quiet && !daemonized ? COMMIT_GRAPH_PROGRESS : 0)) + return 1; if (auto_gc && too_many_loose_objects()) warning(_("There are too many unreachable loose objects; " diff --git a/builtin/merge.c b/builtin/merge.c index 57c2a24f6d..29988e54c5 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -457,7 +457,7 @@ static void finish(struct commit *head_commit, * We ignore errors in 'gc --auto', since the * user should see them. */ - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); run_command_v_opt(argv_gc_auto, RUN_GIT_CMD); } } diff --git a/builtin/rebase.c b/builtin/rebase.c index 3d6219d022..2748fa6f2e 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -742,7 +742,7 @@ static int finish_rebase(struct rebase_options *opts) delete_ref(NULL, "REBASE_HEAD", NULL, REF_NO_DEREF); apply_autostash(opts); - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); /* * We ignore errors in 'gc --auto', since the * user should see them. diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index 29f165d8bd..c5f5da940b 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -2043,7 +2043,7 @@ int cmd_receive_pack(int argc, const char **argv, const char *prefix) proc.git_cmd = 1; proc.argv = argv_gc_auto; - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); if (!start_command(&proc)) { if (use_sideband) copy_to_sideband(proc.err, -1, NULL); diff --git a/builtin/repack.c b/builtin/repack.c index caca113927..f834b5551b 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -422,7 +422,7 @@ int cmd_repack(int argc, const char **argv, const char *prefix) if (!names.nr && !po_args.quiet) printf_ln(_("Nothing new to pack.")); - close_all_packs(the_repository->objects); + close_object_store(the_repository->objects); /* * Ok we have prepared all new packfiles. diff --git a/commit-graph.c b/commit-graph.c index 7c5e54875f..1752341098 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -361,10 +361,10 @@ int generation_numbers_enabled(struct repository *r) return !!first_generation; } -void close_commit_graph(struct repository *r) +void close_commit_graph(struct raw_object_store *o) { - free_commit_graph(r->objects->commit_graph); - r->objects->commit_graph = NULL; + free_commit_graph(o->commit_graph); + o->commit_graph = NULL; } static int bsearch_graph(struct commit_graph *g, struct object_id *oid, uint32_t *pos) @@ -525,14 +525,38 @@ struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c); } +struct packed_commit_list { + struct commit **list; + int nr; + int alloc; +}; + +struct packed_oid_list { + struct object_id *list; + int nr; + int alloc; +}; + +struct write_commit_graph_context { + struct repository *r; + const char *obj_dir; + char *graph_name; + struct packed_oid_list oids; + struct packed_commit_list commits; + int num_extra_edges; + unsigned long approx_nr_objects; + struct progress *progress; + int progress_done; + uint64_t progress_cnt; + unsigned append:1, + report_progress:1; +}; + static void write_graph_chunk_fanout(struct hashfile *f, - struct commit **commits, - int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { int i, count = 0; - struct commit **list = commits; + struct commit **list = ctx->commits.list; /* * Write the first-level table (the list is sorted, @@ -540,10 +564,10 @@ static void write_graph_chunk_fanout(struct hashfile *f, * having to do eight extra binary search iterations). */ for (i = 0; i < 256; i++) { - while (count < nr_commits) { + while (count < ctx->commits.nr) { if ((*list)->object.oid.hash[0] != i) break; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); count++; list++; } @@ -553,14 +577,12 @@ static void write_graph_chunk_fanout(struct hashfile *f, } static void write_graph_chunk_oids(struct hashfile *f, int hash_len, - struct commit **commits, int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; + struct commit **list = ctx->commits.list; int count; - for (count = 0; count < nr_commits; count++, list++) { - display_progress(progress, ++*progress_cnt); + for (count = 0; count < ctx->commits.nr; count++, list++) { + display_progress(ctx->progress, ++ctx->progress_cnt); hashwrite(f, (*list)->object.oid.hash, (int)hash_len); } } @@ -572,19 +594,17 @@ static const unsigned char *commit_to_sha1(size_t index, void *table) } static void write_graph_chunk_data(struct hashfile *f, int hash_len, - struct commit **commits, int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; - struct commit **last = commits + nr_commits; + struct commit **list = ctx->commits.list; + struct commit **last = ctx->commits.list + ctx->commits.nr; uint32_t num_extra_edges = 0; while (list < last) { struct commit_list *parent; int edge_value; uint32_t packedDate[2]; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); parse_commit_no_graph(*list); hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len); @@ -595,8 +615,8 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len, edge_value = GRAPH_PARENT_NONE; else { edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) @@ -616,8 +636,8 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len, edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges; else { edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) BUG("missing parent %s for commit %s", @@ -649,19 +669,16 @@ static void write_graph_chunk_data(struct hashfile *f, int hash_len, } static void write_graph_chunk_extra_edges(struct hashfile *f, - struct commit **commits, - int nr_commits, - struct progress *progress, - uint64_t *progress_cnt) + struct write_commit_graph_context *ctx) { - struct commit **list = commits; - struct commit **last = commits + nr_commits; + struct commit **list = ctx->commits.list; + struct commit **last = ctx->commits.list + ctx->commits.nr; struct commit_list *parent; while (list < last) { int num_parents = 0; - display_progress(progress, ++*progress_cnt); + display_progress(ctx->progress, ++ctx->progress_cnt); for (parent = (*list)->parents; num_parents < 3 && parent; parent = parent->next) @@ -675,8 +692,8 @@ static void write_graph_chunk_extra_edges(struct hashfile *f, /* Since num_parents > 2, this initializer is safe. */ for (parent = (*list)->parents->next; parent; parent = parent->next) { int edge_value = sha1_pos(parent->item->object.oid.hash, - commits, - nr_commits, + ctx->commits.list, + ctx->commits.nr, commit_to_sha1); if (edge_value < 0) @@ -700,125 +717,111 @@ static int commit_compare(const void *_a, const void *_b) return oidcmp(a, b); } -struct packed_commit_list { - struct commit **list; - int nr; - int alloc; -}; - -struct packed_oid_list { - struct object_id *list; - int nr; - int alloc; - struct progress *progress; - int progress_done; -}; - static int add_packed_commits(const struct object_id *oid, struct packed_git *pack, uint32_t pos, void *data) { - struct packed_oid_list *list = (struct packed_oid_list*)data; + struct write_commit_graph_context *ctx = (struct write_commit_graph_context*)data; enum object_type type; off_t offset = nth_packed_object_offset(pack, pos); struct object_info oi = OBJECT_INFO_INIT; - if (list->progress) - display_progress(list->progress, ++list->progress_done); + if (ctx->progress) + display_progress(ctx->progress, ++ctx->progress_done); oi.typep = &type; - if (packed_object_info(the_repository, pack, offset, &oi) < 0) + if (packed_object_info(ctx->r, pack, offset, &oi) < 0) die(_("unable to get type of object %s"), oid_to_hex(oid)); if (type != OBJ_COMMIT) return 0; - ALLOC_GROW(list->list, list->nr + 1, list->alloc); - oidcpy(&(list->list[list->nr]), oid); - list->nr++; + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&(ctx->oids.list[ctx->oids.nr]), oid); + ctx->oids.nr++; return 0; } -static void add_missing_parents(struct packed_oid_list *oids, struct commit *commit) +static void add_missing_parents(struct write_commit_graph_context *ctx, struct commit *commit) { struct commit_list *parent; for (parent = commit->parents; parent; parent = parent->next) { if (!(parent->item->object.flags & UNINTERESTING)) { - ALLOC_GROW(oids->list, oids->nr + 1, oids->alloc); - oidcpy(&oids->list[oids->nr], &(parent->item->object.oid)); - oids->nr++; + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&ctx->oids.list[ctx->oids.nr], &(parent->item->object.oid)); + ctx->oids.nr++; parent->item->object.flags |= UNINTERESTING; } } } -static void close_reachable(struct packed_oid_list *oids, int report_progress) +static void close_reachable(struct write_commit_graph_context *ctx) { int i; struct commit *commit; - struct progress *progress = NULL; - if (report_progress) - progress = start_delayed_progress( - _("Loading known commits in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Loading known commits in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit) commit->object.flags |= UNINTERESTING; } - stop_progress(&progress); + stop_progress(&ctx->progress); /* - * As this loop runs, oids->nr may grow, but not more + * As this loop runs, ctx->oids.nr may grow, but not more * than the number of missing commits in the reachable * closure. */ - if (report_progress) - progress = start_delayed_progress( - _("Expanding reachable commits in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Expanding reachable commits in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit && !parse_commit_no_graph(commit)) - add_missing_parents(oids, commit); + add_missing_parents(ctx, commit); } - stop_progress(&progress); + stop_progress(&ctx->progress); - if (report_progress) - progress = start_delayed_progress( - _("Clearing commit marks in commit graph"), oids->nr); - for (i = 0; i < oids->nr; i++) { - display_progress(progress, i + 1); - commit = lookup_commit(the_repository, &oids->list[i]); + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Clearing commit marks in commit graph"), + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + commit = lookup_commit(ctx->r, &ctx->oids.list[i]); if (commit) commit->object.flags &= ~UNINTERESTING; } - stop_progress(&progress); + stop_progress(&ctx->progress); } -static void compute_generation_numbers(struct packed_commit_list* commits, - int report_progress) +static void compute_generation_numbers(struct write_commit_graph_context *ctx) { int i; struct commit_list *list = NULL; - struct progress *progress = NULL; - if (report_progress) - progress = start_progress( - _("Computing commit graph generation numbers"), - commits->nr); - for (i = 0; i < commits->nr; i++) { - display_progress(progress, i + 1); - if (commits->list[i]->generation != GENERATION_NUMBER_INFINITY && - commits->list[i]->generation != GENERATION_NUMBER_ZERO) + if (ctx->report_progress) + ctx->progress = start_progress( + _("Computing commit graph generation numbers"), + ctx->commits.nr); + for (i = 0; i < ctx->commits.nr; i++) { + display_progress(ctx->progress, i + 1); + if (ctx->commits.list[i]->generation != GENERATION_NUMBER_INFINITY && + ctx->commits.list[i]->generation != GENERATION_NUMBER_ZERO) continue; - commit_list_insert(commits->list[i], &list); + commit_list_insert(ctx->commits.list[i], &list); while (list) { struct commit *current = list->item; struct commit_list *parent; @@ -845,7 +848,7 @@ static void compute_generation_numbers(struct packed_commit_list* commits, } } } - stop_progress(&progress); + stop_progress(&ctx->progress); } static int add_ref_to_list(const char *refname, @@ -858,207 +861,187 @@ static int add_ref_to_list(const char *refname, return 0; } -void write_commit_graph_reachable(const char *obj_dir, int append, - int report_progress) +int write_commit_graph_reachable(const char *obj_dir, unsigned int flags) { struct string_list list = STRING_LIST_INIT_DUP; + int result; for_each_ref(add_ref_to_list, &list); - write_commit_graph(obj_dir, NULL, &list, append, report_progress); + result = write_commit_graph(obj_dir, NULL, &list, + flags); string_list_clear(&list, 0); + return result; } -void write_commit_graph(const char *obj_dir, - struct string_list *pack_indexes, - struct string_list *commit_hex, - int append, int report_progress) +static int fill_oids_from_packs(struct write_commit_graph_context *ctx, + struct string_list *pack_indexes) { - struct packed_oid_list oids; - struct packed_commit_list commits; - struct hashfile *f; - uint32_t i, count_distinct = 0; - char *graph_name; - struct lock_file lk = LOCK_INIT; - uint32_t chunk_ids[5]; - uint64_t chunk_offsets[5]; - int num_chunks; - int num_extra_edges; - struct commit_list *parent; - struct progress *progress = NULL; - const unsigned hashsz = the_hash_algo->rawsz; - uint64_t progress_cnt = 0; + uint32_t i; struct strbuf progress_title = STRBUF_INIT; - unsigned long approx_nr_objects; - - if (!commit_graph_compatible(the_repository)) - return; - - oids.nr = 0; - approx_nr_objects = approximate_object_count(); - oids.alloc = approx_nr_objects / 32; - oids.progress = NULL; - oids.progress_done = 0; + struct strbuf packname = STRBUF_INIT; + int dirlen; - if (append) { - prepare_commit_graph_one(the_repository, obj_dir); - if (the_repository->objects->commit_graph) - oids.alloc += the_repository->objects->commit_graph->num_commits; + strbuf_addf(&packname, "%s/pack/", ctx->obj_dir); + dirlen = packname.len; + if (ctx->report_progress) { + strbuf_addf(&progress_title, + Q_("Finding commits for commit graph in %d pack", + "Finding commits for commit graph in %d packs", + pack_indexes->nr), + pack_indexes->nr); + ctx->progress = start_delayed_progress(progress_title.buf, 0); + ctx->progress_done = 0; } - - if (oids.alloc < 1024) - oids.alloc = 1024; - ALLOC_ARRAY(oids.list, oids.alloc); - - if (append && the_repository->objects->commit_graph) { - struct commit_graph *commit_graph = - the_repository->objects->commit_graph; - for (i = 0; i < commit_graph->num_commits; i++) { - const unsigned char *hash = commit_graph->chunk_oid_lookup + - commit_graph->hash_len * i; - hashcpy(oids.list[oids.nr++].hash, hash); + for (i = 0; i < pack_indexes->nr; i++) { + struct packed_git *p; + strbuf_setlen(&packname, dirlen); + strbuf_addstr(&packname, pack_indexes->items[i].string); + p = add_packed_git(packname.buf, packname.len, 1); + if (!p) { + error(_("error adding pack %s"), packname.buf); + return -1; + } + if (open_pack_index(p)) { + error(_("error opening index for %s"), packname.buf); + return -1; } + for_each_object_in_pack(p, add_packed_commits, ctx, + FOR_EACH_OBJECT_PACK_ORDER); + close_pack(p); + free(p); } - if (pack_indexes) { - struct strbuf packname = STRBUF_INIT; - int dirlen; - strbuf_addf(&packname, "%s/pack/", obj_dir); - dirlen = packname.len; - if (report_progress) { - strbuf_addf(&progress_title, - Q_("Finding commits for commit graph in %d pack", - "Finding commits for commit graph in %d packs", - pack_indexes->nr), - pack_indexes->nr); - oids.progress = start_delayed_progress(progress_title.buf, 0); - oids.progress_done = 0; - } - for (i = 0; i < pack_indexes->nr; i++) { - struct packed_git *p; - strbuf_setlen(&packname, dirlen); - strbuf_addstr(&packname, pack_indexes->items[i].string); - p = add_packed_git(packname.buf, packname.len, 1); - if (!p) - die(_("error adding pack %s"), packname.buf); - if (open_pack_index(p)) - die(_("error opening index for %s"), packname.buf); - for_each_object_in_pack(p, add_packed_commits, &oids, - FOR_EACH_OBJECT_PACK_ORDER); - close_pack(p); - free(p); - } - stop_progress(&oids.progress); - strbuf_reset(&progress_title); - strbuf_release(&packname); + stop_progress(&ctx->progress); + strbuf_reset(&progress_title); + strbuf_release(&packname); + + return 0; +} + +static void fill_oids_from_commit_hex(struct write_commit_graph_context *ctx, + struct string_list *commit_hex) +{ + uint32_t i; + struct strbuf progress_title = STRBUF_INIT; + + if (ctx->report_progress) { + strbuf_addf(&progress_title, + Q_("Finding commits for commit graph from %d ref", + "Finding commits for commit graph from %d refs", + commit_hex->nr), + commit_hex->nr); + ctx->progress = start_delayed_progress( + progress_title.buf, + commit_hex->nr); } + for (i = 0; i < commit_hex->nr; i++) { + const char *end; + struct object_id oid; + struct commit *result; + + display_progress(ctx->progress, i + 1); + if (commit_hex->items[i].string && + parse_oid_hex(commit_hex->items[i].string, &oid, &end)) + continue; - if (commit_hex) { - if (report_progress) { - strbuf_addf(&progress_title, - Q_("Finding commits for commit graph from %d ref", - "Finding commits for commit graph from %d refs", - commit_hex->nr), - commit_hex->nr); - progress = start_delayed_progress(progress_title.buf, - commit_hex->nr); - } - for (i = 0; i < commit_hex->nr; i++) { - const char *end; - struct object_id oid; - struct commit *result; - - display_progress(progress, i + 1); - if (commit_hex->items[i].string && - parse_oid_hex(commit_hex->items[i].string, &oid, &end)) - continue; - - result = lookup_commit_reference_gently(the_repository, &oid, 1); - - if (result) { - ALLOC_GROW(oids.list, oids.nr + 1, oids.alloc); - oidcpy(&oids.list[oids.nr], &(result->object.oid)); - oids.nr++; - } + result = lookup_commit_reference_gently(ctx->r, &oid, 1); + + if (result) { + ALLOC_GROW(ctx->oids.list, ctx->oids.nr + 1, ctx->oids.alloc); + oidcpy(&ctx->oids.list[ctx->oids.nr], &(result->object.oid)); + ctx->oids.nr++; } - stop_progress(&progress); - strbuf_reset(&progress_title); } + stop_progress(&ctx->progress); + strbuf_release(&progress_title); +} - if (!pack_indexes && !commit_hex) { - if (report_progress) - oids.progress = start_delayed_progress( - _("Finding commits for commit graph among packed objects"), - approx_nr_objects); - for_each_packed_object(add_packed_commits, &oids, - FOR_EACH_OBJECT_PACK_ORDER); - if (oids.progress_done < approx_nr_objects) - display_progress(oids.progress, approx_nr_objects); - stop_progress(&oids.progress); - } +static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx) +{ + if (ctx->report_progress) + ctx->progress = start_delayed_progress( + _("Finding commits for commit graph among packed objects"), + ctx->approx_nr_objects); + for_each_packed_object(add_packed_commits, ctx, + FOR_EACH_OBJECT_PACK_ORDER); + if (ctx->progress_done < ctx->approx_nr_objects) + display_progress(ctx->progress, ctx->approx_nr_objects); + stop_progress(&ctx->progress); +} - close_reachable(&oids, report_progress); +static uint32_t count_distinct_commits(struct write_commit_graph_context *ctx) +{ + uint32_t i, count_distinct = 1; - if (report_progress) - progress = start_delayed_progress( + if (ctx->report_progress) + ctx->progress = start_delayed_progress( _("Counting distinct commits in commit graph"), - oids.nr); - display_progress(progress, 0); /* TODO: Measure QSORT() progress */ - QSORT(oids.list, oids.nr, commit_compare); - count_distinct = 1; - for (i = 1; i < oids.nr; i++) { - display_progress(progress, i + 1); - if (!oideq(&oids.list[i - 1], &oids.list[i])) + ctx->oids.nr); + display_progress(ctx->progress, 0); /* TODO: Measure QSORT() progress */ + QSORT(ctx->oids.list, ctx->oids.nr, commit_compare); + + for (i = 1; i < ctx->oids.nr; i++) { + display_progress(ctx->progress, i + 1); + if (!oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) count_distinct++; } - stop_progress(&progress); + stop_progress(&ctx->progress); - if (count_distinct >= GRAPH_EDGE_LAST_MASK) - die(_("the commit graph format cannot write %d commits"), count_distinct); + return count_distinct; +} - commits.nr = 0; - commits.alloc = count_distinct; - ALLOC_ARRAY(commits.list, commits.alloc); +static void copy_oids_to_commits(struct write_commit_graph_context *ctx) +{ + uint32_t i; + struct commit_list *parent; - num_extra_edges = 0; - if (report_progress) - progress = start_delayed_progress( + ctx->num_extra_edges = 0; + if (ctx->report_progress) + ctx->progress = start_delayed_progress( _("Finding extra edges in commit graph"), - oids.nr); - for (i = 0; i < oids.nr; i++) { + ctx->oids.nr); + for (i = 0; i < ctx->oids.nr; i++) { int num_parents = 0; - display_progress(progress, i + 1); - if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i])) + display_progress(ctx->progress, i + 1); + if (i > 0 && oideq(&ctx->oids.list[i - 1], &ctx->oids.list[i])) continue; - commits.list[commits.nr] = lookup_commit(the_repository, &oids.list[i]); - parse_commit_no_graph(commits.list[commits.nr]); + ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.list[i]); + parse_commit_no_graph(ctx->commits.list[ctx->commits.nr]); - for (parent = commits.list[commits.nr]->parents; + for (parent = ctx->commits.list[ctx->commits.nr]->parents; parent; parent = parent->next) num_parents++; if (num_parents > 2) - num_extra_edges += num_parents - 1; + ctx->num_extra_edges += num_parents - 1; - commits.nr++; + ctx->commits.nr++; } - num_chunks = num_extra_edges ? 4 : 3; - stop_progress(&progress); - - if (commits.nr >= GRAPH_EDGE_LAST_MASK) - die(_("too many commits to write graph")); - - compute_generation_numbers(&commits, report_progress); + stop_progress(&ctx->progress); +} - graph_name = get_commit_graph_filename(obj_dir); - if (safe_create_leading_directories(graph_name)) { - UNLEAK(graph_name); - die_errno(_("unable to create leading directories of %s"), - graph_name); +static int write_commit_graph_file(struct write_commit_graph_context *ctx) +{ + uint32_t i; + struct hashfile *f; + struct lock_file lk = LOCK_INIT; + uint32_t chunk_ids[5]; + uint64_t chunk_offsets[5]; + const unsigned hashsz = the_hash_algo->rawsz; + struct strbuf progress_title = STRBUF_INIT; + int num_chunks = ctx->num_extra_edges ? 4 : 3; + + ctx->graph_name = get_commit_graph_filename(ctx->obj_dir); + if (safe_create_leading_directories(ctx->graph_name)) { + UNLEAK(ctx->graph_name); + error(_("unable to create leading directories of %s"), + ctx->graph_name); + return -1; } - hold_lock_file_for_update(&lk, graph_name, LOCK_DIE_ON_ERROR); + hold_lock_file_for_update(&lk, ctx->graph_name, LOCK_DIE_ON_ERROR); f = hashfd(lk.tempfile->fd, lk.tempfile->filename.buf); hashwrite_be32(f, GRAPH_SIGNATURE); @@ -1071,7 +1054,7 @@ void write_commit_graph(const char *obj_dir, chunk_ids[0] = GRAPH_CHUNKID_OIDFANOUT; chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP; chunk_ids[2] = GRAPH_CHUNKID_DATA; - if (num_extra_edges) + if (ctx->num_extra_edges) chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES; else chunk_ids[3] = 0; @@ -1079,9 +1062,9 @@ void write_commit_graph(const char *obj_dir, chunk_offsets[0] = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH; chunk_offsets[1] = chunk_offsets[0] + GRAPH_FANOUT_SIZE; - chunk_offsets[2] = chunk_offsets[1] + hashsz * commits.nr; - chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * commits.nr; - chunk_offsets[4] = chunk_offsets[3] + 4 * num_extra_edges; + chunk_offsets[2] = chunk_offsets[1] + hashsz * ctx->commits.nr; + chunk_offsets[3] = chunk_offsets[2] + (hashsz + 16) * ctx->commits.nr; + chunk_offsets[4] = chunk_offsets[3] + 4 * ctx->num_extra_edges; for (i = 0; i <= num_chunks; i++) { uint32_t chunk_write[3]; @@ -1092,31 +1075,113 @@ void write_commit_graph(const char *obj_dir, hashwrite(f, chunk_write, 12); } - if (report_progress) { + if (ctx->report_progress) { strbuf_addf(&progress_title, Q_("Writing out commit graph in %d pass", "Writing out commit graph in %d passes", num_chunks), num_chunks); - progress = start_delayed_progress( + ctx->progress = start_delayed_progress( progress_title.buf, - num_chunks * commits.nr); + num_chunks * ctx->commits.nr); } - write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt); - write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt); - write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt); - if (num_extra_edges) - write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt); - stop_progress(&progress); + write_graph_chunk_fanout(f, ctx); + write_graph_chunk_oids(f, hashsz, ctx); + write_graph_chunk_data(f, hashsz, ctx); + if (ctx->num_extra_edges) + write_graph_chunk_extra_edges(f, ctx); + stop_progress(&ctx->progress); strbuf_release(&progress_title); - close_commit_graph(the_repository); + close_commit_graph(ctx->r->objects); finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC); commit_lock_file(&lk); - free(graph_name); - free(commits.list); - free(oids.list); + return 0; +} + +int write_commit_graph(const char *obj_dir, + struct string_list *pack_indexes, + struct string_list *commit_hex, + unsigned int flags) +{ + struct write_commit_graph_context *ctx; + uint32_t i, count_distinct = 0; + int res = 0; + + if (!commit_graph_compatible(the_repository)) + return 0; + + ctx = xcalloc(1, sizeof(struct write_commit_graph_context)); + ctx->r = the_repository; + ctx->obj_dir = obj_dir; + ctx->append = flags & COMMIT_GRAPH_APPEND ? 1 : 0; + ctx->report_progress = flags & COMMIT_GRAPH_PROGRESS ? 1 : 0; + + ctx->approx_nr_objects = approximate_object_count(); + ctx->oids.alloc = ctx->approx_nr_objects / 32; + + if (ctx->append) { + prepare_commit_graph_one(ctx->r, ctx->obj_dir); + if (ctx->r->objects->commit_graph) + ctx->oids.alloc += ctx->r->objects->commit_graph->num_commits; + } + + if (ctx->oids.alloc < 1024) + ctx->oids.alloc = 1024; + ALLOC_ARRAY(ctx->oids.list, ctx->oids.alloc); + + if (ctx->append && ctx->r->objects->commit_graph) { + struct commit_graph *g = ctx->r->objects->commit_graph; + for (i = 0; i < g->num_commits; i++) { + const unsigned char *hash = g->chunk_oid_lookup + g->hash_len * i; + hashcpy(ctx->oids.list[ctx->oids.nr++].hash, hash); + } + } + + if (pack_indexes) { + if ((res = fill_oids_from_packs(ctx, pack_indexes))) + goto cleanup; + } + + if (commit_hex) + fill_oids_from_commit_hex(ctx, commit_hex); + + if (!pack_indexes && !commit_hex) + fill_oids_from_all_packs(ctx); + + close_reachable(ctx); + + count_distinct = count_distinct_commits(ctx); + + if (count_distinct >= GRAPH_EDGE_LAST_MASK) { + error(_("the commit graph format cannot write %d commits"), count_distinct); + res = -1; + goto cleanup; + } + + ctx->commits.alloc = count_distinct; + ALLOC_ARRAY(ctx->commits.list, ctx->commits.alloc); + + copy_oids_to_commits(ctx); + + if (ctx->commits.nr >= GRAPH_EDGE_LAST_MASK) { + error(_("too many commits to write graph")); + res = -1; + goto cleanup; + } + + compute_generation_numbers(ctx); + + res = write_commit_graph_file(ctx); + +cleanup: + free(ctx->graph_name); + free(ctx->commits.list); + free(ctx->oids.list); + free(ctx); + + return res; } #define VERIFY_COMMIT_GRAPH_ERROR_HASH 2 diff --git a/commit-graph.h b/commit-graph.h index 7dfb8c896f..390c7f6961 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -65,16 +65,24 @@ struct commit_graph *parse_commit_graph(void *graph_map, int fd, */ int generation_numbers_enabled(struct repository *r); -void write_commit_graph_reachable(const char *obj_dir, int append, - int report_progress); -void write_commit_graph(const char *obj_dir, - struct string_list *pack_indexes, - struct string_list *commit_hex, - int append, int report_progress); +#define COMMIT_GRAPH_APPEND (1 << 0) +#define COMMIT_GRAPH_PROGRESS (1 << 1) + +/* + * The write_commit_graph* methods return zero on success + * and a negative value on failure. Note that if the repository + * is not compatible with the commit-graph feature, then the + * methods will return 0 without writing a commit-graph. + */ +int write_commit_graph_reachable(const char *obj_dir, unsigned int flags); +int write_commit_graph(const char *obj_dir, + struct string_list *pack_indexes, + struct string_list *commit_hex, + unsigned int flags); int verify_commit_graph(struct repository *r, struct commit_graph *g); -void close_commit_graph(struct repository *); +void close_commit_graph(struct raw_object_store *); void free_commit_graph(struct commit_graph *); #endif @@ -449,7 +449,7 @@ int parse_commit_buffer(struct repository *r, struct commit *item, const void *b item->date = parse_commit_date(bufptr, tail); if (check_graph) - load_commit_graph_info(the_repository, item); + load_commit_graph_info(r, item); return 0; } @@ -517,7 +517,7 @@ void raw_object_store_clear(struct raw_object_store *o) o->loaded_alternates = 0; INIT_LIST_HEAD(&o->packed_git_mru); - close_all_packs(o); + close_object_store(o); o->packed_git = NULL; } diff --git a/packfile.c b/packfile.c index d55cb7f013..c0d83fdfed 100644 --- a/packfile.c +++ b/packfile.c @@ -16,6 +16,7 @@ #include "tree.h" #include "object-store.h" #include "midx.h" +#include "commit-graph.h" char *odb_pack_name(struct strbuf *buf, const unsigned char *sha1, @@ -336,7 +337,7 @@ void close_pack(struct packed_git *p) close_pack_index(p); } -void close_all_packs(struct raw_object_store *o) +void close_object_store(struct raw_object_store *o) { struct packed_git *p; @@ -350,6 +351,8 @@ void close_all_packs(struct raw_object_store *o) close_midx(o->multi_pack_index); o->multi_pack_index = NULL; } + + close_commit_graph(o); } /* diff --git a/packfile.h b/packfile.h index b678d35c0b..81e868d55a 100644 --- a/packfile.h +++ b/packfile.h @@ -90,7 +90,7 @@ uint32_t get_pack_fanout(struct packed_git *p, uint32_t value); unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned long *); void close_pack_windows(struct packed_git *); void close_pack(struct packed_git *); -void close_all_packs(struct raw_object_store *o); +void close_object_store(struct raw_object_store *o); void unuse_pack(struct pack_window **); void clear_delta_base_cache(void); struct packed_git *add_packed_git(const char *path, size_t path_len, int local); diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh index 840ad4d8ac..5267c4be20 100755 --- a/t/t5318-commit-graph.sh +++ b/t/t5318-commit-graph.sh @@ -23,6 +23,14 @@ test_expect_success 'write graph with no packs' ' test_path_is_file info/commit-graph ' +test_expect_success 'close with correct error on bad input' ' + cd "$TRASH_DIRECTORY/full" && + echo doesnotexist >in && + { git commit-graph write --stdin-packs <in 2>stderr; ret=$?; } && + test "$ret" = 1 && + test_i18ngrep "error adding pack" stderr +' + test_expect_success 'create commits and repack' ' cd "$TRASH_DIRECTORY/full" && for i in $(test_seq 3) diff --git a/upload-pack.c b/upload-pack.c index 4d2129e7fc..b2a9f368ec 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -722,7 +722,7 @@ static void deepen_by_rev_list(struct packet_writer *writer, int ac, { struct commit_list *result; - close_commit_graph(the_repository); + close_commit_graph(the_repository->objects); result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW); send_shallow(writer, result); free_commit_list(result); |