freshlybakedca.ke
Git fork
1#define USE_THE_REPOSITORY_VARIABLE
2#define DISABLE_SIGN_COMPARE_WARNINGS
3
4#include "git-compat-util.h"
5#include "config.h"
6#include "csum-file.h"
7#include "environment.h"
8#include "gettext.h"
9#include "hex.h"
10#include "lockfile.h"
11#include "packfile.h"
12#include "commit.h"
13#include "object.h"
14#include "refs.h"
15#include "hash-lookup.h"
16#include "commit-graph.h"
17#include "odb.h"
18#include "oid-array.h"
19#include "path.h"
20#include "alloc.h"
21#include "hashmap.h"
22#include "replace-object.h"
23#include "progress.h"
24#include "bloom.h"
25#include "commit-slab.h"
26#include "shallow.h"
27#include "json-writer.h"
28#include "trace2.h"
29#include "tree.h"
30#include "chunk-format.h"
31
32void git_test_write_commit_graph_or_die(void)
33{
34 int flags = 0;
35 if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0))
36 return;
37
38 if (git_env_bool(GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS, 0))
39 flags = COMMIT_GRAPH_WRITE_BLOOM_FILTERS;
40
41 if (write_commit_graph_reachable(the_repository->objects->sources,
42 flags, NULL))
43 die("failed to write commit-graph under GIT_TEST_COMMIT_GRAPH");
44}
45
46#define GRAPH_SIGNATURE 0x43475048 /* "CGPH" */
47#define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
48#define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
49#define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
50#define GRAPH_CHUNKID_GENERATION_DATA 0x47444132 /* "GDA2" */
51#define GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW 0x47444f32 /* "GDO2" */
52#define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
53#define GRAPH_CHUNKID_BLOOMINDEXES 0x42494458 /* "BIDX" */
54#define GRAPH_CHUNKID_BLOOMDATA 0x42444154 /* "BDAT" */
55#define GRAPH_CHUNKID_BASE 0x42415345 /* "BASE" */
56
57#define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
58
59#define GRAPH_VERSION_1 0x1
60#define GRAPH_VERSION GRAPH_VERSION_1
61
62#define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
63#define GRAPH_EDGE_LAST_MASK 0x7fffffff
64#define GRAPH_PARENT_NONE 0x70000000
65
66#define GRAPH_LAST_EDGE 0x80000000
67
68#define GRAPH_HEADER_SIZE 8
69#define GRAPH_FANOUT_SIZE (4 * 256)
70#define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * CHUNK_TOC_ENTRY_SIZE \
71 + GRAPH_FANOUT_SIZE + the_hash_algo->rawsz)
72
73#define CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW (1ULL << 31)
74
75/* Remember to update object flag allocation in object.h */
76#define REACHABLE (1u<<15)
77
78define_commit_slab(topo_level_slab, uint32_t);
79
80/* Keep track of the order in which commits are added to our list. */
81define_commit_slab(commit_pos, int);
82static struct commit_pos commit_pos = COMMIT_SLAB_INIT(1, commit_pos);
83
84static void set_commit_pos(struct repository *r, const struct object_id *oid)
85{
86 static int32_t max_pos;
87 struct commit *commit = lookup_commit(r, oid);
88
89 if (!commit)
90 return; /* should never happen, but be lenient */
91
92 *commit_pos_at(&commit_pos, commit) = max_pos++;
93}
94
95static int commit_pos_cmp(const void *va, const void *vb)
96{
97 const struct commit *a = *(const struct commit **)va;
98 const struct commit *b = *(const struct commit **)vb;
99 return commit_pos_at(&commit_pos, a) -
100 commit_pos_at(&commit_pos, b);
101}
102
103define_commit_slab(commit_graph_data_slab, struct commit_graph_data);
104static struct commit_graph_data_slab commit_graph_data_slab =
105 COMMIT_SLAB_INIT(1, commit_graph_data_slab);
106
107static int get_configured_generation_version(struct repository *r)
108{
109 int version = 2;
110 repo_config_get_int(r, "commitgraph.generationversion", &version);
111 return version;
112}
113
114uint32_t commit_graph_position(const struct commit *c)
115{
116 struct commit_graph_data *data =
117 commit_graph_data_slab_peek(&commit_graph_data_slab, c);
118
119 return data ? data->graph_pos : COMMIT_NOT_FROM_GRAPH;
120}
121
122timestamp_t commit_graph_generation(const struct commit *c)
123{
124 struct commit_graph_data *data =
125 commit_graph_data_slab_peek(&commit_graph_data_slab, c);
126
127 if (data && data->generation)
128 return data->generation;
129
130 return GENERATION_NUMBER_INFINITY;
131}
132
133static timestamp_t commit_graph_generation_from_graph(const struct commit *c)
134{
135 struct commit_graph_data *data =
136 commit_graph_data_slab_peek(&commit_graph_data_slab, c);
137
138 if (!data || data->graph_pos == COMMIT_NOT_FROM_GRAPH)
139 return GENERATION_NUMBER_INFINITY;
140 return data->generation;
141}
142
143static struct commit_graph_data *commit_graph_data_at(const struct commit *c)
144{
145 unsigned int i, nth_slab;
146 struct commit_graph_data *data =
147 commit_graph_data_slab_peek(&commit_graph_data_slab, c);
148
149 if (data)
150 return data;
151
152 nth_slab = c->index / commit_graph_data_slab.slab_size;
153 data = commit_graph_data_slab_at(&commit_graph_data_slab, c);
154
155 /*
156 * commit-slab initializes elements with zero, overwrite this with
157 * COMMIT_NOT_FROM_GRAPH for graph_pos.
158 *
159 * We avoid initializing generation with checking if graph position
160 * is not COMMIT_NOT_FROM_GRAPH.
161 */
162 for (i = 0; i < commit_graph_data_slab.slab_size; i++) {
163 commit_graph_data_slab.slab[nth_slab][i].graph_pos =
164 COMMIT_NOT_FROM_GRAPH;
165 }
166
167 return data;
168}
169
170/*
171 * Should be used only while writing commit-graph as it compares
172 * generation value of commits by directly accessing commit-slab.
173 */
174static int commit_gen_cmp(const void *va, const void *vb)
175{
176 const struct commit *a = *(const struct commit **)va;
177 const struct commit *b = *(const struct commit **)vb;
178
179 const timestamp_t generation_a = commit_graph_data_at(a)->generation;
180 const timestamp_t generation_b = commit_graph_data_at(b)->generation;
181 /* lower generation commits first */
182 if (generation_a < generation_b)
183 return -1;
184 else if (generation_a > generation_b)
185 return 1;
186
187 /* use date as a heuristic when generations are equal */
188 if (a->date < b->date)
189 return -1;
190 else if (a->date > b->date)
191 return 1;
192 return 0;
193}
194
195char *get_commit_graph_filename(struct odb_source *source)
196{
197 return xstrfmt("%s/info/commit-graph", source->path);
198}
199
200static char *get_split_graph_filename(struct odb_source *source,
201 const char *oid_hex)
202{
203 return xstrfmt("%s/info/commit-graphs/graph-%s.graph", source->path,
204 oid_hex);
205}
206
207char *get_commit_graph_chain_filename(struct odb_source *source)
208{
209 return xstrfmt("%s/info/commit-graphs/commit-graph-chain", source->path);
210}
211
212static struct commit_graph *alloc_commit_graph(void)
213{
214 struct commit_graph *g = xcalloc(1, sizeof(*g));
215
216 return g;
217}
218
219static int commit_graph_compatible(struct repository *r)
220{
221 if (!r->gitdir)
222 return 0;
223
224 if (replace_refs_enabled(r)) {
225 prepare_replace_object(r);
226 if (oidmap_get_size(&r->objects->replace_map))
227 return 0;
228 }
229
230 prepare_commit_graft(r);
231 if (r->parsed_objects &&
232 (r->parsed_objects->grafts_nr || r->parsed_objects->substituted_parent))
233 return 0;
234 if (is_repository_shallow(r))
235 return 0;
236
237 return 1;
238}
239
240int open_commit_graph(const char *graph_file, int *fd, struct stat *st)
241{
242 *fd = git_open(graph_file);
243 if (*fd < 0)
244 return 0;
245 if (fstat(*fd, st)) {
246 close(*fd);
247 return 0;
248 }
249 return 1;
250}
251
252struct commit_graph *load_commit_graph_one_fd_st(struct repository *r,
253 int fd, struct stat *st,
254 struct odb_source *source)
255{
256 void *graph_map;
257 size_t graph_size;
258 struct commit_graph *ret;
259
260 graph_size = xsize_t(st->st_size);
261
262 if (graph_size < GRAPH_MIN_SIZE) {
263 close(fd);
264 error(_("commit-graph file is too small"));
265 return NULL;
266 }
267 graph_map = xmmap(NULL, graph_size, PROT_READ, MAP_PRIVATE, fd, 0);
268 close(fd);
269 prepare_repo_settings(r);
270 ret = parse_commit_graph(&r->settings, graph_map, graph_size);
271
272 if (ret)
273 ret->odb_source = source;
274 else
275 munmap(graph_map, graph_size);
276
277 return ret;
278}
279
280static int graph_read_oid_fanout(const unsigned char *chunk_start,
281 size_t chunk_size, void *data)
282{
283 struct commit_graph *g = data;
284 int i;
285
286 if (chunk_size != 256 * sizeof(uint32_t))
287 return error(_("commit-graph oid fanout chunk is wrong size"));
288 g->chunk_oid_fanout = (const uint32_t *)chunk_start;
289 g->num_commits = ntohl(g->chunk_oid_fanout[255]);
290
291 for (i = 0; i < 255; i++) {
292 uint32_t oid_fanout1 = ntohl(g->chunk_oid_fanout[i]);
293 uint32_t oid_fanout2 = ntohl(g->chunk_oid_fanout[i + 1]);
294
295 if (oid_fanout1 > oid_fanout2) {
296 error(_("commit-graph fanout values out of order"));
297 return 1;
298 }
299 }
300
301 return 0;
302}
303
304static int graph_read_oid_lookup(const unsigned char *chunk_start,
305 size_t chunk_size, void *data)
306{
307 struct commit_graph *g = data;
308 g->chunk_oid_lookup = chunk_start;
309 if (chunk_size / g->hash_len != g->num_commits)
310 return error(_("commit-graph OID lookup chunk is the wrong size"));
311 return 0;
312}
313
314static int graph_read_commit_data(const unsigned char *chunk_start,
315 size_t chunk_size, void *data)
316{
317 struct commit_graph *g = data;
318 if (chunk_size / GRAPH_DATA_WIDTH != g->num_commits)
319 return error(_("commit-graph commit data chunk is wrong size"));
320 g->chunk_commit_data = chunk_start;
321 return 0;
322}
323
324static int graph_read_generation_data(const unsigned char *chunk_start,
325 size_t chunk_size, void *data)
326{
327 struct commit_graph *g = data;
328 if (chunk_size / sizeof(uint32_t) != g->num_commits)
329 return error(_("commit-graph generations chunk is wrong size"));
330 g->chunk_generation_data = chunk_start;
331 return 0;
332}
333
334static int graph_read_bloom_index(const unsigned char *chunk_start,
335 size_t chunk_size, void *data)
336{
337 struct commit_graph *g = data;
338 if (chunk_size / 4 != g->num_commits) {
339 warning(_("commit-graph changed-path index chunk is too small"));
340 return -1;
341 }
342 g->chunk_bloom_indexes = chunk_start;
343 return 0;
344}
345
346static int graph_read_bloom_data(const unsigned char *chunk_start,
347 size_t chunk_size, void *data)
348{
349 struct commit_graph *g = data;
350
351 if (chunk_size < BLOOMDATA_CHUNK_HEADER_SIZE) {
352 warning(_("ignoring too-small changed-path chunk"
353 " (%"PRIuMAX" < %"PRIuMAX") in commit-graph file"),
354 (uintmax_t)chunk_size,
355 (uintmax_t)BLOOMDATA_CHUNK_HEADER_SIZE);
356 return -1;
357 }
358
359 g->chunk_bloom_data = chunk_start;
360 g->chunk_bloom_data_size = chunk_size;
361
362 g->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings));
363 g->bloom_filter_settings->hash_version = get_be32(chunk_start);
364 g->bloom_filter_settings->num_hashes = get_be32(chunk_start + 4);
365 g->bloom_filter_settings->bits_per_entry = get_be32(chunk_start + 8);
366 g->bloom_filter_settings->max_changed_paths = DEFAULT_BLOOM_MAX_CHANGES;
367
368 return 0;
369}
370
371struct commit_graph *parse_commit_graph(struct repo_settings *s,
372 void *graph_map, size_t graph_size)
373{
374 const unsigned char *data;
375 struct commit_graph *graph;
376 uint32_t graph_signature;
377 unsigned char graph_version, hash_version;
378 struct chunkfile *cf = NULL;
379
380 if (!graph_map)
381 return NULL;
382
383 if (graph_size < GRAPH_MIN_SIZE)
384 return NULL;
385
386 data = (const unsigned char *)graph_map;
387
388 graph_signature = get_be32(data);
389 if (graph_signature != GRAPH_SIGNATURE) {
390 error(_("commit-graph signature %X does not match signature %X"),
391 graph_signature, GRAPH_SIGNATURE);
392 return NULL;
393 }
394
395 graph_version = *(unsigned char*)(data + 4);
396 if (graph_version != GRAPH_VERSION) {
397 error(_("commit-graph version %X does not match version %X"),
398 graph_version, GRAPH_VERSION);
399 return NULL;
400 }
401
402 hash_version = *(unsigned char*)(data + 5);
403 if (hash_version != oid_version(the_hash_algo)) {
404 error(_("commit-graph hash version %X does not match version %X"),
405 hash_version, oid_version(the_hash_algo));
406 return NULL;
407 }
408
409 graph = alloc_commit_graph();
410
411 graph->hash_len = the_hash_algo->rawsz;
412 graph->num_chunks = *(unsigned char*)(data + 6);
413 graph->data = graph_map;
414 graph->data_len = graph_size;
415
416 if (graph_size < GRAPH_HEADER_SIZE +
417 (graph->num_chunks + 1) * CHUNK_TOC_ENTRY_SIZE +
418 GRAPH_FANOUT_SIZE + the_hash_algo->rawsz) {
419 error(_("commit-graph file is too small to hold %u chunks"),
420 graph->num_chunks);
421 free(graph);
422 return NULL;
423 }
424
425 cf = init_chunkfile(NULL);
426
427 if (read_table_of_contents(cf, graph->data, graph_size,
428 GRAPH_HEADER_SIZE, graph->num_chunks, 1))
429 goto free_and_return;
430
431 if (read_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, graph_read_oid_fanout, graph)) {
432 error(_("commit-graph required OID fanout chunk missing or corrupted"));
433 goto free_and_return;
434 }
435 if (read_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, graph_read_oid_lookup, graph)) {
436 error(_("commit-graph required OID lookup chunk missing or corrupted"));
437 goto free_and_return;
438 }
439 if (read_chunk(cf, GRAPH_CHUNKID_DATA, graph_read_commit_data, graph)) {
440 error(_("commit-graph required commit data chunk missing or corrupted"));
441 goto free_and_return;
442 }
443
444 pair_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, &graph->chunk_extra_edges,
445 &graph->chunk_extra_edges_size);
446 pair_chunk(cf, GRAPH_CHUNKID_BASE, &graph->chunk_base_graphs,
447 &graph->chunk_base_graphs_size);
448
449 if (s->commit_graph_generation_version >= 2) {
450 read_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA,
451 graph_read_generation_data, graph);
452 pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW,
453 &graph->chunk_generation_data_overflow,
454 &graph->chunk_generation_data_overflow_size);
455
456 if (graph->chunk_generation_data)
457 graph->read_generation_data = 1;
458 }
459
460 if (s->commit_graph_changed_paths_version) {
461 read_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES,
462 graph_read_bloom_index, graph);
463 read_chunk(cf, GRAPH_CHUNKID_BLOOMDATA,
464 graph_read_bloom_data, graph);
465 }
466
467 if (graph->chunk_bloom_indexes && graph->chunk_bloom_data) {
468 init_bloom_filters();
469 } else {
470 /* We need both the bloom chunks to exist together. Else ignore the data */
471 graph->chunk_bloom_indexes = NULL;
472 graph->chunk_bloom_data = NULL;
473 FREE_AND_NULL(graph->bloom_filter_settings);
474 }
475
476 oidread(&graph->oid, graph->data + graph->data_len - graph->hash_len,
477 the_repository->hash_algo);
478
479 free_chunkfile(cf);
480 return graph;
481
482free_and_return:
483 free_chunkfile(cf);
484 free(graph->bloom_filter_settings);
485 free(graph);
486 return NULL;
487}
488
489static struct commit_graph *load_commit_graph_one(struct repository *r,
490 const char *graph_file,
491 struct odb_source *source)
492{
493
494 struct stat st;
495 int fd;
496 struct commit_graph *g;
497 int open_ok = open_commit_graph(graph_file, &fd, &st);
498
499 if (!open_ok)
500 return NULL;
501
502 g = load_commit_graph_one_fd_st(r, fd, &st, source);
503
504 if (g)
505 g->filename = xstrdup(graph_file);
506
507 return g;
508}
509
510static struct commit_graph *load_commit_graph_v1(struct repository *r,
511 struct odb_source *source)
512{
513 char *graph_name = get_commit_graph_filename(source);
514 struct commit_graph *g = load_commit_graph_one(r, graph_name, source);
515 free(graph_name);
516
517 return g;
518}
519
520/*
521 * returns 1 if and only if all graphs in the chain have
522 * corrected commit dates stored in the generation_data chunk.
523 */
524static int validate_mixed_generation_chain(struct commit_graph *g)
525{
526 int read_generation_data = 1;
527 struct commit_graph *p = g;
528
529 while (read_generation_data && p) {
530 read_generation_data = p->read_generation_data;
531 p = p->base_graph;
532 }
533
534 if (read_generation_data)
535 return 1;
536
537 while (g) {
538 g->read_generation_data = 0;
539 g = g->base_graph;
540 }
541
542 return 0;
543}
544
545static void validate_mixed_bloom_settings(struct commit_graph *g)
546{
547 struct bloom_filter_settings *settings = NULL;
548 for (; g; g = g->base_graph) {
549 if (!g->bloom_filter_settings)
550 continue;
551 if (!settings) {
552 settings = g->bloom_filter_settings;
553 continue;
554 }
555
556 if (g->bloom_filter_settings->bits_per_entry != settings->bits_per_entry ||
557 g->bloom_filter_settings->num_hashes != settings->num_hashes ||
558 g->bloom_filter_settings->hash_version != settings->hash_version) {
559 g->chunk_bloom_indexes = NULL;
560 g->chunk_bloom_data = NULL;
561 FREE_AND_NULL(g->bloom_filter_settings);
562
563 warning(_("disabling Bloom filters for commit-graph "
564 "layer '%s' due to incompatible settings"),
565 oid_to_hex(&g->oid));
566 }
567 }
568}
569
570static int add_graph_to_chain(struct commit_graph *g,
571 struct commit_graph *chain,
572 struct object_id *oids,
573 int n)
574{
575 struct commit_graph *cur_g = chain;
576
577 if (n && !g->chunk_base_graphs) {
578 warning(_("commit-graph has no base graphs chunk"));
579 return 0;
580 }
581
582 if (g->chunk_base_graphs_size / g->hash_len < n) {
583 warning(_("commit-graph base graphs chunk is too small"));
584 return 0;
585 }
586
587 while (n) {
588 n--;
589
590 if (!cur_g ||
591 !oideq(&oids[n], &cur_g->oid) ||
592 !hasheq(oids[n].hash, g->chunk_base_graphs + st_mult(g->hash_len, n),
593 the_repository->hash_algo)) {
594 warning(_("commit-graph chain does not match"));
595 return 0;
596 }
597
598 cur_g = cur_g->base_graph;
599 }
600
601 if (chain) {
602 if (unsigned_add_overflows(chain->num_commits,
603 chain->num_commits_in_base)) {
604 warning(_("commit count in base graph too high: %"PRIuMAX),
605 (uintmax_t)chain->num_commits_in_base);
606 return 0;
607 }
608 g->num_commits_in_base = chain->num_commits + chain->num_commits_in_base;
609 }
610
611 g->base_graph = chain;
612
613 return 1;
614}
615
616int open_commit_graph_chain(const char *chain_file,
617 int *fd, struct stat *st)
618{
619 *fd = git_open(chain_file);
620 if (*fd < 0)
621 return 0;
622 if (fstat(*fd, st)) {
623 close(*fd);
624 return 0;
625 }
626 if (st->st_size < the_hash_algo->hexsz) {
627 close(*fd);
628 if (!st->st_size) {
629 /* treat empty files the same as missing */
630 errno = ENOENT;
631 } else {
632 warning(_("commit-graph chain file too small"));
633 errno = EINVAL;
634 }
635 return 0;
636 }
637 return 1;
638}
639
640struct commit_graph *load_commit_graph_chain_fd_st(struct repository *r,
641 int fd, struct stat *st,
642 int *incomplete_chain)
643{
644 struct commit_graph *graph_chain = NULL;
645 struct strbuf line = STRBUF_INIT;
646 struct object_id *oids;
647 int i = 0, valid = 1, count;
648 FILE *fp = xfdopen(fd, "r");
649
650 count = st->st_size / (the_hash_algo->hexsz + 1);
651 CALLOC_ARRAY(oids, count);
652
653 odb_prepare_alternates(r->objects);
654
655 for (i = 0; i < count; i++) {
656 struct odb_source *source;
657
658 if (strbuf_getline_lf(&line, fp) == EOF)
659 break;
660
661 if (get_oid_hex(line.buf, &oids[i])) {
662 warning(_("invalid commit-graph chain: line '%s' not a hash"),
663 line.buf);
664 valid = 0;
665 break;
666 }
667
668 valid = 0;
669 for (source = r->objects->sources; source; source = source->next) {
670 char *graph_name = get_split_graph_filename(source, line.buf);
671 struct commit_graph *g = load_commit_graph_one(r, graph_name, source);
672
673 free(graph_name);
674
675 if (g) {
676 if (add_graph_to_chain(g, graph_chain, oids, i)) {
677 graph_chain = g;
678 valid = 1;
679 } else {
680 free_commit_graph(g);
681 }
682
683 break;
684 }
685 }
686
687 if (!valid) {
688 warning(_("unable to find all commit-graph files"));
689 break;
690 }
691 }
692
693 validate_mixed_generation_chain(graph_chain);
694 validate_mixed_bloom_settings(graph_chain);
695
696 free(oids);
697 fclose(fp);
698 strbuf_release(&line);
699
700 *incomplete_chain = !valid;
701 return graph_chain;
702}
703
704static struct commit_graph *load_commit_graph_chain(struct repository *r,
705 struct odb_source *source)
706{
707 char *chain_file = get_commit_graph_chain_filename(source);
708 struct stat st;
709 int fd;
710 struct commit_graph *g = NULL;
711
712 if (open_commit_graph_chain(chain_file, &fd, &st)) {
713 int incomplete;
714 /* ownership of fd is taken over by load function */
715 g = load_commit_graph_chain_fd_st(r, fd, &st, &incomplete);
716 }
717
718 free(chain_file);
719 return g;
720}
721
722struct commit_graph *read_commit_graph_one(struct repository *r,
723 struct odb_source *source)
724{
725 struct commit_graph *g = load_commit_graph_v1(r, source);
726
727 if (!g)
728 g = load_commit_graph_chain(r, source);
729
730 return g;
731}
732
733static void prepare_commit_graph_one(struct repository *r,
734 struct odb_source *source)
735{
736
737 if (r->objects->commit_graph)
738 return;
739
740 r->objects->commit_graph = read_commit_graph_one(r, source);
741}
742
743/*
744 * Return 1 if commit_graph is non-NULL, and 0 otherwise.
745 *
746 * On the first invocation, this function attempts to load the commit
747 * graph if the_repository is configured to have one.
748 */
749static int prepare_commit_graph(struct repository *r)
750{
751 struct odb_source *source;
752
753 /*
754 * Early return if there is no git dir or if the commit graph is
755 * disabled.
756 *
757 * This must come before the "already attempted?" check below, because
758 * we want to disable even an already-loaded graph file.
759 */
760 if (!r->gitdir || r->commit_graph_disabled)
761 return 0;
762
763 if (r->objects->commit_graph_attempted)
764 return !!r->objects->commit_graph;
765 r->objects->commit_graph_attempted = 1;
766
767 prepare_repo_settings(r);
768
769 if (!git_env_bool(GIT_TEST_COMMIT_GRAPH, 0) &&
770 r->settings.core_commit_graph != 1)
771 /*
772 * This repository is not configured to use commit graphs, so
773 * do not load one. (But report commit_graph_attempted anyway
774 * so that commit graph loading is not attempted again for this
775 * repository.)
776 */
777 return 0;
778
779 if (!commit_graph_compatible(r))
780 return 0;
781
782 odb_prepare_alternates(r->objects);
783 for (source = r->objects->sources;
784 !r->objects->commit_graph && source;
785 source = source->next)
786 prepare_commit_graph_one(r, source);
787 return !!r->objects->commit_graph;
788}
789
790int generation_numbers_enabled(struct repository *r)
791{
792 uint32_t first_generation;
793 struct commit_graph *g;
794 if (!prepare_commit_graph(r))
795 return 0;
796
797 g = r->objects->commit_graph;
798
799 if (!g->num_commits)
800 return 0;
801
802 first_generation = get_be32(g->chunk_commit_data +
803 g->hash_len + 8) >> 2;
804
805 return !!first_generation;
806}
807
808int corrected_commit_dates_enabled(struct repository *r)
809{
810 struct commit_graph *g;
811 if (!prepare_commit_graph(r))
812 return 0;
813
814 g = r->objects->commit_graph;
815
816 if (!g->num_commits)
817 return 0;
818
819 return g->read_generation_data;
820}
821
822struct bloom_filter_settings *get_bloom_filter_settings(struct repository *r)
823{
824 struct commit_graph *g = r->objects->commit_graph;
825 while (g) {
826 if (g->bloom_filter_settings)
827 return g->bloom_filter_settings;
828 g = g->base_graph;
829 }
830 return NULL;
831}
832
833void close_commit_graph(struct object_database *o)
834{
835 if (!o->commit_graph)
836 return;
837
838 clear_commit_graph_data_slab(&commit_graph_data_slab);
839 deinit_bloom_filters();
840 free_commit_graph(o->commit_graph);
841 o->commit_graph = NULL;
842}
843
844static int bsearch_graph(struct commit_graph *g, const struct object_id *oid, uint32_t *pos)
845{
846 return bsearch_hash(oid->hash, g->chunk_oid_fanout,
847 g->chunk_oid_lookup, g->hash_len, pos);
848}
849
850static void load_oid_from_graph(struct commit_graph *g,
851 uint32_t pos,
852 struct object_id *oid)
853{
854 uint32_t lex_index;
855
856 while (g && pos < g->num_commits_in_base)
857 g = g->base_graph;
858
859 if (!g)
860 BUG("NULL commit-graph");
861
862 if (pos >= g->num_commits + g->num_commits_in_base)
863 die(_("invalid commit position. commit-graph is likely corrupt"));
864
865 lex_index = pos - g->num_commits_in_base;
866
867 oidread(oid, g->chunk_oid_lookup + st_mult(g->hash_len, lex_index),
868 the_repository->hash_algo);
869}
870
871static struct commit_list **insert_parent_or_die(struct repository *r,
872 struct commit_graph *g,
873 uint32_t pos,
874 struct commit_list **pptr)
875{
876 struct commit *c;
877 struct object_id oid;
878
879 if (pos >= g->num_commits + g->num_commits_in_base)
880 die("invalid parent position %"PRIu32, pos);
881
882 load_oid_from_graph(g, pos, &oid);
883 c = lookup_commit(r, &oid);
884 if (!c)
885 die(_("could not find commit %s"), oid_to_hex(&oid));
886 commit_graph_data_at(c)->graph_pos = pos;
887 return &commit_list_insert(c, pptr)->next;
888}
889
890static void fill_commit_graph_info(struct commit *item, struct commit_graph *g, uint32_t pos)
891{
892 const unsigned char *commit_data;
893 struct commit_graph_data *graph_data;
894 uint32_t lex_index, offset_pos;
895 uint64_t date_high, date_low, offset;
896
897 while (pos < g->num_commits_in_base)
898 g = g->base_graph;
899
900 if (pos >= g->num_commits + g->num_commits_in_base)
901 die(_("invalid commit position. commit-graph is likely corrupt"));
902
903 lex_index = pos - g->num_commits_in_base;
904 commit_data = g->chunk_commit_data + st_mult(GRAPH_DATA_WIDTH, lex_index);
905
906 graph_data = commit_graph_data_at(item);
907 graph_data->graph_pos = pos;
908
909 date_high = get_be32(commit_data + g->hash_len + 8) & 0x3;
910 date_low = get_be32(commit_data + g->hash_len + 12);
911 item->date = (timestamp_t)((date_high << 32) | date_low);
912
913 if (g->read_generation_data) {
914 offset = (timestamp_t)get_be32(g->chunk_generation_data + st_mult(sizeof(uint32_t), lex_index));
915
916 if (offset & CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW) {
917 if (!g->chunk_generation_data_overflow)
918 die(_("commit-graph requires overflow generation data but has none"));
919
920 offset_pos = offset ^ CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW;
921 if (g->chunk_generation_data_overflow_size / sizeof(uint64_t) <= offset_pos)
922 die(_("commit-graph overflow generation data is too small"));
923 graph_data->generation = item->date +
924 get_be64(g->chunk_generation_data_overflow + sizeof(uint64_t) * offset_pos);
925 } else
926 graph_data->generation = item->date + offset;
927 } else
928 graph_data->generation = get_be32(commit_data + g->hash_len + 8) >> 2;
929
930 if (g->topo_levels)
931 *topo_level_slab_at(g->topo_levels, item) = get_be32(commit_data + g->hash_len + 8) >> 2;
932}
933
934static inline void set_commit_tree(struct commit *c, struct tree *t)
935{
936 c->maybe_tree = t;
937}
938
939static int fill_commit_in_graph(struct repository *r,
940 struct commit *item,
941 struct commit_graph *g, uint32_t pos)
942{
943 uint32_t edge_value;
944 uint32_t parent_data_pos;
945 struct commit_list **pptr;
946 const unsigned char *commit_data;
947 uint32_t lex_index;
948
949 while (pos < g->num_commits_in_base)
950 g = g->base_graph;
951
952 fill_commit_graph_info(item, g, pos);
953
954 lex_index = pos - g->num_commits_in_base;
955 commit_data = g->chunk_commit_data + st_mult(g->hash_len + 16, lex_index);
956
957 item->object.parsed = 1;
958
959 set_commit_tree(item, NULL);
960
961 pptr = &item->parents;
962
963 edge_value = get_be32(commit_data + g->hash_len);
964 if (edge_value == GRAPH_PARENT_NONE)
965 return 1;
966 pptr = insert_parent_or_die(r, g, edge_value, pptr);
967
968 edge_value = get_be32(commit_data + g->hash_len + 4);
969 if (edge_value == GRAPH_PARENT_NONE)
970 return 1;
971 if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) {
972 pptr = insert_parent_or_die(r, g, edge_value, pptr);
973 return 1;
974 }
975
976 parent_data_pos = edge_value & GRAPH_EDGE_LAST_MASK;
977 do {
978 if (g->chunk_extra_edges_size / sizeof(uint32_t) <= parent_data_pos) {
979 error(_("commit-graph extra-edges pointer out of bounds"));
980 free_commit_list(item->parents);
981 item->parents = NULL;
982 item->object.parsed = 0;
983 return 0;
984 }
985 edge_value = get_be32(g->chunk_extra_edges +
986 sizeof(uint32_t) * parent_data_pos);
987 pptr = insert_parent_or_die(r, g,
988 edge_value & GRAPH_EDGE_LAST_MASK,
989 pptr);
990 parent_data_pos++;
991 } while (!(edge_value & GRAPH_LAST_EDGE));
992
993 return 1;
994}
995
996static int search_commit_pos_in_graph(const struct object_id *id, struct commit_graph *g, uint32_t *pos)
997{
998 struct commit_graph *cur_g = g;
999 uint32_t lex_index;
1000
1001 while (cur_g && !bsearch_graph(cur_g, id, &lex_index))
1002 cur_g = cur_g->base_graph;
1003
1004 if (cur_g) {
1005 *pos = lex_index + cur_g->num_commits_in_base;
1006 return 1;
1007 }
1008
1009 return 0;
1010}
1011
1012static int find_commit_pos_in_graph(struct commit *item, struct commit_graph *g, uint32_t *pos)
1013{
1014 uint32_t graph_pos = commit_graph_position(item);
1015 if (graph_pos != COMMIT_NOT_FROM_GRAPH) {
1016 *pos = graph_pos;
1017 return 1;
1018 } else {
1019 return search_commit_pos_in_graph(&item->object.oid, g, pos);
1020 }
1021}
1022
1023int repo_find_commit_pos_in_graph(struct repository *r, struct commit *c,
1024 uint32_t *pos)
1025{
1026 if (!prepare_commit_graph(r))
1027 return 0;
1028 return find_commit_pos_in_graph(c, r->objects->commit_graph, pos);
1029}
1030
1031struct commit *lookup_commit_in_graph(struct repository *repo, const struct object_id *id)
1032{
1033 static int commit_graph_paranoia = -1;
1034 struct commit *commit;
1035 uint32_t pos;
1036
1037 if (commit_graph_paranoia == -1)
1038 commit_graph_paranoia = git_env_bool(GIT_COMMIT_GRAPH_PARANOIA, 0);
1039
1040 if (!prepare_commit_graph(repo))
1041 return NULL;
1042 if (!search_commit_pos_in_graph(id, repo->objects->commit_graph, &pos))
1043 return NULL;
1044 if (commit_graph_paranoia && !odb_has_object(repo->objects, id, 0))
1045 return NULL;
1046
1047 commit = lookup_commit(repo, id);
1048 if (!commit)
1049 return NULL;
1050 if (commit->object.parsed)
1051 return commit;
1052
1053 if (!fill_commit_in_graph(repo, commit, repo->objects->commit_graph, pos))
1054 return NULL;
1055
1056 return commit;
1057}
1058
1059static int parse_commit_in_graph_one(struct repository *r,
1060 struct commit_graph *g,
1061 struct commit *item)
1062{
1063 uint32_t pos;
1064
1065 if (item->object.parsed)
1066 return 1;
1067
1068 if (find_commit_pos_in_graph(item, g, &pos))
1069 return fill_commit_in_graph(r, item, g, pos);
1070
1071 return 0;
1072}
1073
1074int parse_commit_in_graph(struct repository *r, struct commit *item)
1075{
1076 static int checked_env = 0;
1077
1078 if (!checked_env &&
1079 git_env_bool(GIT_TEST_COMMIT_GRAPH_DIE_ON_PARSE, 0))
1080 die("dying as requested by the '%s' variable on commit-graph parse!",
1081 GIT_TEST_COMMIT_GRAPH_DIE_ON_PARSE);
1082 checked_env = 1;
1083
1084 if (!prepare_commit_graph(r))
1085 return 0;
1086 return parse_commit_in_graph_one(r, r->objects->commit_graph, item);
1087}
1088
1089void load_commit_graph_info(struct repository *r, struct commit *item)
1090{
1091 uint32_t pos;
1092 if (repo_find_commit_pos_in_graph(r, item, &pos))
1093 fill_commit_graph_info(item, r->objects->commit_graph, pos);
1094}
1095
1096static struct tree *load_tree_for_commit(struct repository *r,
1097 struct commit_graph *g,
1098 struct commit *c)
1099{
1100 struct object_id oid;
1101 const unsigned char *commit_data;
1102 uint32_t graph_pos = commit_graph_position(c);
1103
1104 while (graph_pos < g->num_commits_in_base)
1105 g = g->base_graph;
1106
1107 commit_data = g->chunk_commit_data +
1108 st_mult(GRAPH_DATA_WIDTH, graph_pos - g->num_commits_in_base);
1109
1110 oidread(&oid, commit_data, the_repository->hash_algo);
1111 set_commit_tree(c, lookup_tree(r, &oid));
1112
1113 return c->maybe_tree;
1114}
1115
1116static struct tree *get_commit_tree_in_graph_one(struct repository *r,
1117 struct commit_graph *g,
1118 const struct commit *c)
1119{
1120 if (c->maybe_tree)
1121 return c->maybe_tree;
1122 if (commit_graph_position(c) == COMMIT_NOT_FROM_GRAPH)
1123 BUG("get_commit_tree_in_graph_one called from non-commit-graph commit");
1124
1125 return load_tree_for_commit(r, g, (struct commit *)c);
1126}
1127
1128struct tree *get_commit_tree_in_graph(struct repository *r, const struct commit *c)
1129{
1130 return get_commit_tree_in_graph_one(r, r->objects->commit_graph, c);
1131}
1132
1133struct packed_commit_list {
1134 struct commit **list;
1135 size_t nr;
1136 size_t alloc;
1137};
1138
1139struct write_commit_graph_context {
1140 struct repository *r;
1141 struct odb_source *odb_source;
1142 char *graph_name;
1143 struct oid_array oids;
1144 struct packed_commit_list commits;
1145 int num_extra_edges;
1146 int num_generation_data_overflows;
1147 unsigned long approx_nr_objects;
1148 struct progress *progress;
1149 int progress_done;
1150 uint64_t progress_cnt;
1151
1152 char *base_graph_name;
1153 int num_commit_graphs_before;
1154 int num_commit_graphs_after;
1155 char **commit_graph_filenames_before;
1156 char **commit_graph_filenames_after;
1157 char **commit_graph_hash_after;
1158 uint32_t new_num_commits_in_base;
1159 struct commit_graph *new_base_graph;
1160
1161 unsigned append:1,
1162 report_progress:1,
1163 split:1,
1164 changed_paths:1,
1165 order_by_pack:1,
1166 write_generation_data:1,
1167 trust_generation_numbers:1;
1168
1169 struct topo_level_slab *topo_levels;
1170 const struct commit_graph_opts *opts;
1171 size_t total_bloom_filter_data_size;
1172 const struct bloom_filter_settings *bloom_settings;
1173
1174 int count_bloom_filter_computed;
1175 int count_bloom_filter_not_computed;
1176 int count_bloom_filter_trunc_empty;
1177 int count_bloom_filter_trunc_large;
1178 int count_bloom_filter_upgraded;
1179};
1180
1181static int write_graph_chunk_fanout(struct hashfile *f,
1182 void *data)
1183{
1184 struct write_commit_graph_context *ctx = data;
1185 int i, count = 0;
1186 struct commit **list = ctx->commits.list;
1187
1188 /*
1189 * Write the first-level table (the list is sorted,
1190 * but we use a 256-entry lookup to be able to avoid
1191 * having to do eight extra binary search iterations).
1192 */
1193 for (i = 0; i < 256; i++) {
1194 while (count < ctx->commits.nr) {
1195 if ((*list)->object.oid.hash[0] != i)
1196 break;
1197 display_progress(ctx->progress, ++ctx->progress_cnt);
1198 count++;
1199 list++;
1200 }
1201
1202 hashwrite_be32(f, count);
1203 }
1204
1205 return 0;
1206}
1207
1208static int write_graph_chunk_oids(struct hashfile *f,
1209 void *data)
1210{
1211 struct write_commit_graph_context *ctx = data;
1212 struct commit **list = ctx->commits.list;
1213 int count;
1214 for (count = 0; count < ctx->commits.nr; count++, list++) {
1215 display_progress(ctx->progress, ++ctx->progress_cnt);
1216 hashwrite(f, (*list)->object.oid.hash, the_hash_algo->rawsz);
1217 }
1218
1219 return 0;
1220}
1221
1222static const struct object_id *commit_to_oid(size_t index, const void *table)
1223{
1224 const struct commit * const *commits = table;
1225 return &commits[index]->object.oid;
1226}
1227
1228static int write_graph_chunk_data(struct hashfile *f,
1229 void *data)
1230{
1231 struct write_commit_graph_context *ctx = data;
1232 struct commit **list = ctx->commits.list;
1233 struct commit **last = ctx->commits.list + ctx->commits.nr;
1234 uint32_t num_extra_edges = 0;
1235
1236 while (list < last) {
1237 struct commit_list *parent;
1238 struct object_id *tree;
1239 int edge_value;
1240 uint32_t packedDate[2];
1241 display_progress(ctx->progress, ++ctx->progress_cnt);
1242
1243 if (repo_parse_commit_no_graph(ctx->r, *list))
1244 die(_("unable to parse commit %s"),
1245 oid_to_hex(&(*list)->object.oid));
1246 tree = get_commit_tree_oid(*list);
1247 hashwrite(f, tree->hash, the_hash_algo->rawsz);
1248
1249 parent = (*list)->parents;
1250
1251 if (!parent)
1252 edge_value = GRAPH_PARENT_NONE;
1253 else {
1254 edge_value = oid_pos(&parent->item->object.oid,
1255 ctx->commits.list,
1256 ctx->commits.nr,
1257 commit_to_oid);
1258
1259 if (edge_value >= 0)
1260 edge_value += ctx->new_num_commits_in_base;
1261 else if (ctx->new_base_graph) {
1262 uint32_t pos;
1263 if (find_commit_pos_in_graph(parent->item,
1264 ctx->new_base_graph,
1265 &pos))
1266 edge_value = pos;
1267 }
1268
1269 if (edge_value < 0)
1270 BUG("missing parent %s for commit %s",
1271 oid_to_hex(&parent->item->object.oid),
1272 oid_to_hex(&(*list)->object.oid));
1273 }
1274
1275 hashwrite_be32(f, edge_value);
1276
1277 if (parent)
1278 parent = parent->next;
1279
1280 if (!parent)
1281 edge_value = GRAPH_PARENT_NONE;
1282 else if (parent->next)
1283 edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges;
1284 else {
1285 edge_value = oid_pos(&parent->item->object.oid,
1286 ctx->commits.list,
1287 ctx->commits.nr,
1288 commit_to_oid);
1289
1290 if (edge_value >= 0)
1291 edge_value += ctx->new_num_commits_in_base;
1292 else if (ctx->new_base_graph) {
1293 uint32_t pos;
1294 if (find_commit_pos_in_graph(parent->item,
1295 ctx->new_base_graph,
1296 &pos))
1297 edge_value = pos;
1298 }
1299
1300 if (edge_value < 0)
1301 BUG("missing parent %s for commit %s",
1302 oid_to_hex(&parent->item->object.oid),
1303 oid_to_hex(&(*list)->object.oid));
1304 }
1305
1306 hashwrite_be32(f, edge_value);
1307
1308 if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) {
1309 do {
1310 num_extra_edges++;
1311 parent = parent->next;
1312 } while (parent);
1313 }
1314
1315 if (sizeof((*list)->date) > 4)
1316 packedDate[0] = htonl(((*list)->date >> 32) & 0x3);
1317 else
1318 packedDate[0] = 0;
1319
1320 packedDate[0] |= htonl(*topo_level_slab_at(ctx->topo_levels, *list) << 2);
1321
1322 packedDate[1] = htonl((*list)->date);
1323 hashwrite(f, packedDate, 8);
1324
1325 list++;
1326 }
1327
1328 return 0;
1329}
1330
1331static int write_graph_chunk_generation_data(struct hashfile *f,
1332 void *data)
1333{
1334 struct write_commit_graph_context *ctx = data;
1335 int i, num_generation_data_overflows = 0;
1336
1337 for (i = 0; i < ctx->commits.nr; i++) {
1338 struct commit *c = ctx->commits.list[i];
1339 timestamp_t offset;
1340 repo_parse_commit(ctx->r, c);
1341 offset = commit_graph_data_at(c)->generation - c->date;
1342 display_progress(ctx->progress, ++ctx->progress_cnt);
1343
1344 if (offset > GENERATION_NUMBER_V2_OFFSET_MAX) {
1345 offset = CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW | num_generation_data_overflows;
1346 num_generation_data_overflows++;
1347 }
1348
1349 hashwrite_be32(f, offset);
1350 }
1351
1352 return 0;
1353}
1354
1355static int write_graph_chunk_generation_data_overflow(struct hashfile *f,
1356 void *data)
1357{
1358 struct write_commit_graph_context *ctx = data;
1359 int i;
1360 for (i = 0; i < ctx->commits.nr; i++) {
1361 struct commit *c = ctx->commits.list[i];
1362 timestamp_t offset = commit_graph_data_at(c)->generation - c->date;
1363 display_progress(ctx->progress, ++ctx->progress_cnt);
1364
1365 if (offset > GENERATION_NUMBER_V2_OFFSET_MAX) {
1366 hashwrite_be32(f, offset >> 32);
1367 hashwrite_be32(f, (uint32_t) offset);
1368 }
1369 }
1370
1371 return 0;
1372}
1373
1374static int write_graph_chunk_extra_edges(struct hashfile *f,
1375 void *data)
1376{
1377 struct write_commit_graph_context *ctx = data;
1378 struct commit **list = ctx->commits.list;
1379 struct commit **last = ctx->commits.list + ctx->commits.nr;
1380 struct commit_list *parent;
1381
1382 while (list < last) {
1383 int num_parents = 0;
1384
1385 display_progress(ctx->progress, ++ctx->progress_cnt);
1386
1387 for (parent = (*list)->parents; num_parents < 3 && parent;
1388 parent = parent->next)
1389 num_parents++;
1390
1391 if (num_parents <= 2) {
1392 list++;
1393 continue;
1394 }
1395
1396 /* Since num_parents > 2, this initializer is safe. */
1397 for (parent = (*list)->parents->next; parent; parent = parent->next) {
1398 int edge_value = oid_pos(&parent->item->object.oid,
1399 ctx->commits.list,
1400 ctx->commits.nr,
1401 commit_to_oid);
1402
1403 if (edge_value >= 0)
1404 edge_value += ctx->new_num_commits_in_base;
1405 else if (ctx->new_base_graph) {
1406 uint32_t pos;
1407 if (find_commit_pos_in_graph(parent->item,
1408 ctx->new_base_graph,
1409 &pos))
1410 edge_value = pos;
1411 }
1412
1413 if (edge_value < 0)
1414 BUG("missing parent %s for commit %s",
1415 oid_to_hex(&parent->item->object.oid),
1416 oid_to_hex(&(*list)->object.oid));
1417 else if (!parent->next)
1418 edge_value |= GRAPH_LAST_EDGE;
1419
1420 hashwrite_be32(f, edge_value);
1421 }
1422
1423 list++;
1424 }
1425
1426 return 0;
1427}
1428
1429static int write_graph_chunk_bloom_indexes(struct hashfile *f,
1430 void *data)
1431{
1432 struct write_commit_graph_context *ctx = data;
1433 struct commit **list = ctx->commits.list;
1434 struct commit **last = ctx->commits.list + ctx->commits.nr;
1435 uint32_t cur_pos = 0;
1436
1437 while (list < last) {
1438 struct bloom_filter *filter = get_bloom_filter(ctx->r, *list);
1439 size_t len = filter ? filter->len : 0;
1440 cur_pos += len;
1441 display_progress(ctx->progress, ++ctx->progress_cnt);
1442 hashwrite_be32(f, cur_pos);
1443 list++;
1444 }
1445
1446 return 0;
1447}
1448
1449static void trace2_bloom_filter_settings(struct write_commit_graph_context *ctx)
1450{
1451 struct json_writer jw = JSON_WRITER_INIT;
1452
1453 jw_object_begin(&jw, 0);
1454 jw_object_intmax(&jw, "hash_version", ctx->bloom_settings->hash_version);
1455 jw_object_intmax(&jw, "num_hashes", ctx->bloom_settings->num_hashes);
1456 jw_object_intmax(&jw, "bits_per_entry", ctx->bloom_settings->bits_per_entry);
1457 jw_object_intmax(&jw, "max_changed_paths", ctx->bloom_settings->max_changed_paths);
1458 jw_end(&jw);
1459
1460 trace2_data_json("bloom", ctx->r, "settings", &jw);
1461
1462 jw_release(&jw);
1463}
1464
1465static int write_graph_chunk_bloom_data(struct hashfile *f,
1466 void *data)
1467{
1468 struct write_commit_graph_context *ctx = data;
1469 struct commit **list = ctx->commits.list;
1470 struct commit **last = ctx->commits.list + ctx->commits.nr;
1471
1472 trace2_bloom_filter_settings(ctx);
1473
1474 hashwrite_be32(f, ctx->bloom_settings->hash_version);
1475 hashwrite_be32(f, ctx->bloom_settings->num_hashes);
1476 hashwrite_be32(f, ctx->bloom_settings->bits_per_entry);
1477
1478 while (list < last) {
1479 struct bloom_filter *filter = get_bloom_filter(ctx->r, *list);
1480 size_t len = filter ? filter->len : 0;
1481
1482 display_progress(ctx->progress, ++ctx->progress_cnt);
1483 if (len)
1484 hashwrite(f, filter->data, len * sizeof(unsigned char));
1485 list++;
1486 }
1487
1488 return 0;
1489}
1490
1491static int add_packed_commits(const struct object_id *oid,
1492 struct packed_git *pack,
1493 uint32_t pos,
1494 void *data)
1495{
1496 struct write_commit_graph_context *ctx = (struct write_commit_graph_context*)data;
1497 enum object_type type;
1498 off_t offset = nth_packed_object_offset(pack, pos);
1499 struct object_info oi = OBJECT_INFO_INIT;
1500
1501 if (ctx->progress)
1502 display_progress(ctx->progress, ++ctx->progress_done);
1503
1504 oi.typep = &type;
1505 if (packed_object_info(ctx->r, pack, offset, &oi) < 0)
1506 die(_("unable to get type of object %s"), oid_to_hex(oid));
1507
1508 if (type != OBJ_COMMIT)
1509 return 0;
1510
1511 oid_array_append(&ctx->oids, oid);
1512 set_commit_pos(ctx->r, oid);
1513
1514 return 0;
1515}
1516
1517static void add_missing_parents(struct write_commit_graph_context *ctx, struct commit *commit)
1518{
1519 struct commit_list *parent;
1520 for (parent = commit->parents; parent; parent = parent->next) {
1521 if (!(parent->item->object.flags & REACHABLE)) {
1522 oid_array_append(&ctx->oids, &parent->item->object.oid);
1523 parent->item->object.flags |= REACHABLE;
1524 }
1525 }
1526}
1527
1528static void close_reachable(struct write_commit_graph_context *ctx)
1529{
1530 int i;
1531 struct commit *commit;
1532 enum commit_graph_split_flags flags = ctx->opts ?
1533 ctx->opts->split_flags : COMMIT_GRAPH_SPLIT_UNSPECIFIED;
1534
1535 if (ctx->report_progress)
1536 ctx->progress = start_delayed_progress(
1537 the_repository,
1538 _("Loading known commits in commit graph"),
1539 ctx->oids.nr);
1540 for (i = 0; i < ctx->oids.nr; i++) {
1541 display_progress(ctx->progress, i + 1);
1542 commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
1543 if (commit)
1544 commit->object.flags |= REACHABLE;
1545 }
1546 stop_progress(&ctx->progress);
1547
1548 /*
1549 * As this loop runs, ctx->oids.nr may grow, but not more
1550 * than the number of missing commits in the reachable
1551 * closure.
1552 */
1553 if (ctx->report_progress)
1554 ctx->progress = start_delayed_progress(
1555 the_repository,
1556 _("Expanding reachable commits in commit graph"),
1557 0);
1558 for (i = 0; i < ctx->oids.nr; i++) {
1559 display_progress(ctx->progress, i + 1);
1560 commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
1561
1562 if (!commit)
1563 continue;
1564 if (ctx->split) {
1565 if ((!repo_parse_commit(ctx->r, commit) &&
1566 commit_graph_position(commit) == COMMIT_NOT_FROM_GRAPH) ||
1567 flags == COMMIT_GRAPH_SPLIT_REPLACE)
1568 add_missing_parents(ctx, commit);
1569 } else if (!repo_parse_commit_no_graph(ctx->r, commit))
1570 add_missing_parents(ctx, commit);
1571 }
1572 stop_progress(&ctx->progress);
1573
1574 if (ctx->report_progress)
1575 ctx->progress = start_delayed_progress(
1576 the_repository,
1577 _("Clearing commit marks in commit graph"),
1578 ctx->oids.nr);
1579 for (i = 0; i < ctx->oids.nr; i++) {
1580 display_progress(ctx->progress, i + 1);
1581 commit = lookup_commit(ctx->r, &ctx->oids.oid[i]);
1582
1583 if (commit)
1584 commit->object.flags &= ~REACHABLE;
1585 }
1586 stop_progress(&ctx->progress);
1587}
1588
1589struct compute_generation_info {
1590 struct repository *r;
1591 struct packed_commit_list *commits;
1592 struct progress *progress;
1593 int progress_cnt;
1594
1595 timestamp_t (*get_generation)(struct commit *c, void *data);
1596 void (*set_generation)(struct commit *c, timestamp_t gen, void *data);
1597 void *data;
1598};
1599
1600static timestamp_t compute_generation_from_max(struct commit *c,
1601 timestamp_t max_gen,
1602 int generation_version)
1603{
1604 switch (generation_version) {
1605 case 1: /* topological levels */
1606 if (max_gen > GENERATION_NUMBER_V1_MAX - 1)
1607 max_gen = GENERATION_NUMBER_V1_MAX - 1;
1608 return max_gen + 1;
1609
1610 case 2: /* corrected commit date */
1611 if (c->date && c->date > max_gen)
1612 max_gen = c->date - 1;
1613 return max_gen + 1;
1614
1615 default:
1616 BUG("attempting unimplemented version");
1617 }
1618}
1619
1620static void compute_reachable_generation_numbers(
1621 struct compute_generation_info *info,
1622 int generation_version)
1623{
1624 int i;
1625 struct commit_list *list = NULL;
1626
1627 for (i = 0; i < info->commits->nr; i++) {
1628 struct commit *c = info->commits->list[i];
1629 timestamp_t gen;
1630 repo_parse_commit(info->r, c);
1631 gen = info->get_generation(c, info->data);
1632 display_progress(info->progress, ++info->progress_cnt);
1633
1634 if (gen != GENERATION_NUMBER_ZERO && gen != GENERATION_NUMBER_INFINITY)
1635 continue;
1636
1637 commit_list_insert(c, &list);
1638 while (list) {
1639 struct commit *current = list->item;
1640 struct commit_list *parent;
1641 int all_parents_computed = 1;
1642 uint32_t max_gen = 0;
1643
1644 for (parent = current->parents; parent; parent = parent->next) {
1645 repo_parse_commit(info->r, parent->item);
1646 gen = info->get_generation(parent->item, info->data);
1647
1648 if (gen == GENERATION_NUMBER_ZERO) {
1649 all_parents_computed = 0;
1650 commit_list_insert(parent->item, &list);
1651 break;
1652 }
1653
1654 if (gen > max_gen)
1655 max_gen = gen;
1656 }
1657
1658 if (all_parents_computed) {
1659 pop_commit(&list);
1660 gen = compute_generation_from_max(
1661 current, max_gen,
1662 generation_version);
1663 info->set_generation(current, gen, info->data);
1664 }
1665 }
1666 }
1667}
1668
1669static timestamp_t get_topo_level(struct commit *c, void *data)
1670{
1671 struct write_commit_graph_context *ctx = data;
1672 return *topo_level_slab_at(ctx->topo_levels, c);
1673}
1674
1675static void set_topo_level(struct commit *c, timestamp_t t, void *data)
1676{
1677 struct write_commit_graph_context *ctx = data;
1678 *topo_level_slab_at(ctx->topo_levels, c) = (uint32_t)t;
1679}
1680
1681static void compute_topological_levels(struct write_commit_graph_context *ctx)
1682{
1683 struct compute_generation_info info = {
1684 .r = ctx->r,
1685 .commits = &ctx->commits,
1686 .get_generation = get_topo_level,
1687 .set_generation = set_topo_level,
1688 .data = ctx,
1689 };
1690
1691 if (ctx->report_progress)
1692 info.progress = ctx->progress
1693 = start_delayed_progress(
1694 the_repository,
1695 _("Computing commit graph topological levels"),
1696 ctx->commits.nr);
1697
1698 compute_reachable_generation_numbers(&info, 1);
1699
1700 stop_progress(&ctx->progress);
1701}
1702
1703static timestamp_t get_generation_from_graph_data(struct commit *c,
1704 void *data UNUSED)
1705{
1706 return commit_graph_data_at(c)->generation;
1707}
1708
1709static void set_generation_v2(struct commit *c, timestamp_t t,
1710 void *data UNUSED)
1711{
1712 struct commit_graph_data *g = commit_graph_data_at(c);
1713 g->generation = t;
1714}
1715
1716static void compute_generation_numbers(struct write_commit_graph_context *ctx)
1717{
1718 int i;
1719 struct compute_generation_info info = {
1720 .r = ctx->r,
1721 .commits = &ctx->commits,
1722 .get_generation = get_generation_from_graph_data,
1723 .set_generation = set_generation_v2,
1724 };
1725
1726 if (ctx->report_progress)
1727 info.progress = ctx->progress
1728 = start_delayed_progress(
1729 the_repository,
1730 _("Computing commit graph generation numbers"),
1731 ctx->commits.nr);
1732
1733 if (!ctx->trust_generation_numbers) {
1734 for (i = 0; i < ctx->commits.nr; i++) {
1735 struct commit *c = ctx->commits.list[i];
1736 repo_parse_commit(ctx->r, c);
1737 commit_graph_data_at(c)->generation = GENERATION_NUMBER_ZERO;
1738 }
1739 }
1740
1741 compute_reachable_generation_numbers(&info, 2);
1742
1743 for (i = 0; i < ctx->commits.nr; i++) {
1744 struct commit *c = ctx->commits.list[i];
1745 timestamp_t offset = commit_graph_data_at(c)->generation - c->date;
1746 if (offset > GENERATION_NUMBER_V2_OFFSET_MAX)
1747 ctx->num_generation_data_overflows++;
1748 }
1749 stop_progress(&ctx->progress);
1750}
1751
1752static void set_generation_in_graph_data(struct commit *c, timestamp_t t,
1753 void *data UNUSED)
1754{
1755 commit_graph_data_at(c)->generation = t;
1756}
1757
1758/*
1759 * After this method, all commits reachable from those in the given
1760 * list will have non-zero, non-infinite generation numbers.
1761 */
1762void ensure_generations_valid(struct repository *r,
1763 struct commit **commits, size_t nr)
1764{
1765 int generation_version = get_configured_generation_version(r);
1766 struct packed_commit_list list = {
1767 .list = commits,
1768 .alloc = nr,
1769 .nr = nr,
1770 };
1771 struct compute_generation_info info = {
1772 .r = r,
1773 .commits = &list,
1774 .get_generation = get_generation_from_graph_data,
1775 .set_generation = set_generation_in_graph_data,
1776 };
1777
1778 compute_reachable_generation_numbers(&info, generation_version);
1779}
1780
1781static void trace2_bloom_filter_write_statistics(struct write_commit_graph_context *ctx)
1782{
1783 trace2_data_intmax("commit-graph", ctx->r, "filter-computed",
1784 ctx->count_bloom_filter_computed);
1785 trace2_data_intmax("commit-graph", ctx->r, "filter-not-computed",
1786 ctx->count_bloom_filter_not_computed);
1787 trace2_data_intmax("commit-graph", ctx->r, "filter-trunc-empty",
1788 ctx->count_bloom_filter_trunc_empty);
1789 trace2_data_intmax("commit-graph", ctx->r, "filter-trunc-large",
1790 ctx->count_bloom_filter_trunc_large);
1791 trace2_data_intmax("commit-graph", ctx->r, "filter-upgraded",
1792 ctx->count_bloom_filter_upgraded);
1793}
1794
1795static void compute_bloom_filters(struct write_commit_graph_context *ctx)
1796{
1797 int i;
1798 struct progress *progress = NULL;
1799 struct commit **sorted_commits;
1800 int max_new_filters;
1801
1802 init_bloom_filters();
1803
1804 if (ctx->report_progress)
1805 progress = start_delayed_progress(
1806 the_repository,
1807 _("Computing commit changed paths Bloom filters"),
1808 ctx->commits.nr);
1809
1810 DUP_ARRAY(sorted_commits, ctx->commits.list, ctx->commits.nr);
1811
1812 if (ctx->order_by_pack)
1813 QSORT(sorted_commits, ctx->commits.nr, commit_pos_cmp);
1814 else
1815 QSORT(sorted_commits, ctx->commits.nr, commit_gen_cmp);
1816
1817 max_new_filters = ctx->opts && ctx->opts->max_new_filters >= 0 ?
1818 ctx->opts->max_new_filters : ctx->commits.nr;
1819
1820 for (i = 0; i < ctx->commits.nr; i++) {
1821 enum bloom_filter_computed computed = 0;
1822 struct commit *c = sorted_commits[i];
1823 struct bloom_filter *filter = get_or_compute_bloom_filter(
1824 ctx->r,
1825 c,
1826 ctx->count_bloom_filter_computed < max_new_filters,
1827 ctx->bloom_settings,
1828 &computed);
1829 if (computed & BLOOM_COMPUTED) {
1830 ctx->count_bloom_filter_computed++;
1831 if (computed & BLOOM_TRUNC_EMPTY)
1832 ctx->count_bloom_filter_trunc_empty++;
1833 if (computed & BLOOM_TRUNC_LARGE)
1834 ctx->count_bloom_filter_trunc_large++;
1835 } else if (computed & BLOOM_UPGRADED) {
1836 ctx->count_bloom_filter_upgraded++;
1837 } else if (computed & BLOOM_NOT_COMPUTED)
1838 ctx->count_bloom_filter_not_computed++;
1839 ctx->total_bloom_filter_data_size += filter
1840 ? sizeof(unsigned char) * filter->len : 0;
1841 display_progress(progress, i + 1);
1842 }
1843
1844 if (trace2_is_enabled())
1845 trace2_bloom_filter_write_statistics(ctx);
1846
1847 free(sorted_commits);
1848 stop_progress(&progress);
1849}
1850
1851struct refs_cb_data {
1852 struct oidset *commits;
1853 struct progress *progress;
1854};
1855
1856static int add_ref_to_set(const char *refname UNUSED,
1857 const char *referent UNUSED,
1858 const struct object_id *oid,
1859 int flags UNUSED, void *cb_data)
1860{
1861 struct object_id peeled;
1862 struct refs_cb_data *data = (struct refs_cb_data *)cb_data;
1863
1864 if (!peel_iterated_oid(the_repository, oid, &peeled))
1865 oid = &peeled;
1866 if (odb_read_object_info(the_repository->objects, oid, NULL) == OBJ_COMMIT)
1867 oidset_insert(data->commits, oid);
1868
1869 display_progress(data->progress, oidset_size(data->commits));
1870
1871 return 0;
1872}
1873
1874int write_commit_graph_reachable(struct odb_source *source,
1875 enum commit_graph_write_flags flags,
1876 const struct commit_graph_opts *opts)
1877{
1878 struct oidset commits = OIDSET_INIT;
1879 struct refs_cb_data data;
1880 int result;
1881
1882 memset(&data, 0, sizeof(data));
1883 data.commits = &commits;
1884 if (flags & COMMIT_GRAPH_WRITE_PROGRESS)
1885 data.progress = start_delayed_progress(
1886 the_repository,
1887 _("Collecting referenced commits"), 0);
1888
1889 refs_for_each_ref(get_main_ref_store(the_repository), add_ref_to_set,
1890 &data);
1891
1892 stop_progress(&data.progress);
1893
1894 result = write_commit_graph(source, NULL, &commits,
1895 flags, opts);
1896
1897 oidset_clear(&commits);
1898 return result;
1899}
1900
1901static int fill_oids_from_packs(struct write_commit_graph_context *ctx,
1902 const struct string_list *pack_indexes)
1903{
1904 uint32_t i;
1905 struct strbuf progress_title = STRBUF_INIT;
1906 struct strbuf packname = STRBUF_INIT;
1907 int dirlen;
1908 int ret = 0;
1909
1910 strbuf_addf(&packname, "%s/pack/", ctx->odb_source->path);
1911 dirlen = packname.len;
1912 if (ctx->report_progress) {
1913 strbuf_addf(&progress_title,
1914 Q_("Finding commits for commit graph in %"PRIuMAX" pack",
1915 "Finding commits for commit graph in %"PRIuMAX" packs",
1916 pack_indexes->nr),
1917 (uintmax_t)pack_indexes->nr);
1918 ctx->progress = start_delayed_progress(the_repository,
1919 progress_title.buf, 0);
1920 ctx->progress_done = 0;
1921 }
1922 for (i = 0; i < pack_indexes->nr; i++) {
1923 struct packed_git *p;
1924 strbuf_setlen(&packname, dirlen);
1925 strbuf_addstr(&packname, pack_indexes->items[i].string);
1926 p = add_packed_git(ctx->r, packname.buf, packname.len, 1);
1927 if (!p) {
1928 ret = error(_("error adding pack %s"), packname.buf);
1929 goto cleanup;
1930 }
1931 if (open_pack_index(p)) {
1932 ret = error(_("error opening index for %s"), packname.buf);
1933 close_pack(p);
1934 free(p);
1935 goto cleanup;
1936 }
1937 for_each_object_in_pack(p, add_packed_commits, ctx,
1938 FOR_EACH_OBJECT_PACK_ORDER);
1939 close_pack(p);
1940 free(p);
1941 }
1942
1943cleanup:
1944 stop_progress(&ctx->progress);
1945 strbuf_release(&progress_title);
1946 strbuf_release(&packname);
1947
1948 return ret;
1949}
1950
1951static int fill_oids_from_commits(struct write_commit_graph_context *ctx,
1952 struct oidset *commits)
1953{
1954 struct oidset_iter iter;
1955 struct object_id *oid;
1956
1957 if (!oidset_size(commits))
1958 return 0;
1959
1960 oidset_iter_init(commits, &iter);
1961 while ((oid = oidset_iter_next(&iter))) {
1962 oid_array_append(&ctx->oids, oid);
1963 }
1964
1965 return 0;
1966}
1967
1968static void fill_oids_from_all_packs(struct write_commit_graph_context *ctx)
1969{
1970 if (ctx->report_progress)
1971 ctx->progress = start_delayed_progress(
1972 the_repository,
1973 _("Finding commits for commit graph among packed objects"),
1974 ctx->approx_nr_objects);
1975 for_each_packed_object(ctx->r, add_packed_commits, ctx,
1976 FOR_EACH_OBJECT_PACK_ORDER);
1977 if (ctx->progress_done < ctx->approx_nr_objects)
1978 display_progress(ctx->progress, ctx->approx_nr_objects);
1979 stop_progress(&ctx->progress);
1980}
1981
1982static void copy_oids_to_commits(struct write_commit_graph_context *ctx)
1983{
1984 uint32_t i;
1985 enum commit_graph_split_flags flags = ctx->opts ?
1986 ctx->opts->split_flags : COMMIT_GRAPH_SPLIT_UNSPECIFIED;
1987
1988 ctx->num_extra_edges = 0;
1989 if (ctx->report_progress)
1990 ctx->progress = start_delayed_progress(
1991 the_repository,
1992 _("Finding extra edges in commit graph"),
1993 ctx->oids.nr);
1994 oid_array_sort(&ctx->oids);
1995 for (i = 0; i < ctx->oids.nr; i = oid_array_next_unique(&ctx->oids, i)) {
1996 unsigned int num_parents;
1997
1998 display_progress(ctx->progress, i + 1);
1999
2000 ALLOC_GROW(ctx->commits.list, ctx->commits.nr + 1, ctx->commits.alloc);
2001 ctx->commits.list[ctx->commits.nr] = lookup_commit(ctx->r, &ctx->oids.oid[i]);
2002
2003 if (ctx->split && flags != COMMIT_GRAPH_SPLIT_REPLACE &&
2004 commit_graph_position(ctx->commits.list[ctx->commits.nr]) != COMMIT_NOT_FROM_GRAPH)
2005 continue;
2006
2007 if (ctx->split && flags == COMMIT_GRAPH_SPLIT_REPLACE)
2008 repo_parse_commit(ctx->r, ctx->commits.list[ctx->commits.nr]);
2009 else
2010 repo_parse_commit_no_graph(ctx->r, ctx->commits.list[ctx->commits.nr]);
2011
2012 num_parents = commit_list_count(ctx->commits.list[ctx->commits.nr]->parents);
2013 if (num_parents > 2)
2014 ctx->num_extra_edges += num_parents - 1;
2015
2016 ctx->commits.nr++;
2017 }
2018 stop_progress(&ctx->progress);
2019}
2020
2021static int write_graph_chunk_base_1(struct hashfile *f,
2022 struct commit_graph *g)
2023{
2024 int num = 0;
2025
2026 if (!g)
2027 return 0;
2028
2029 num = write_graph_chunk_base_1(f, g->base_graph);
2030 hashwrite(f, g->oid.hash, the_hash_algo->rawsz);
2031 return num + 1;
2032}
2033
2034static int write_graph_chunk_base(struct hashfile *f,
2035 void *data)
2036{
2037 struct write_commit_graph_context *ctx = data;
2038 int num = write_graph_chunk_base_1(f, ctx->new_base_graph);
2039
2040 if (num != ctx->num_commit_graphs_after - 1) {
2041 error(_("failed to write correct number of base graph ids"));
2042 return -1;
2043 }
2044
2045 return 0;
2046}
2047
2048static int write_commit_graph_file(struct write_commit_graph_context *ctx)
2049{
2050 uint32_t i;
2051 struct hashfile *f;
2052 struct tempfile *graph_layer; /* when ctx->split is non-zero */
2053 struct lock_file lk = LOCK_INIT;
2054 const unsigned hashsz = the_hash_algo->rawsz;
2055 struct strbuf progress_title = STRBUF_INIT;
2056 struct chunkfile *cf;
2057 unsigned char file_hash[GIT_MAX_RAWSZ];
2058
2059 if (ctx->split) {
2060 struct strbuf tmp_file = STRBUF_INIT;
2061
2062 strbuf_addf(&tmp_file,
2063 "%s/info/commit-graphs/tmp_graph_XXXXXX",
2064 ctx->odb_source->path);
2065 ctx->graph_name = strbuf_detach(&tmp_file, NULL);
2066 } else {
2067 ctx->graph_name = get_commit_graph_filename(ctx->odb_source);
2068 }
2069
2070 if (safe_create_leading_directories(the_repository, ctx->graph_name)) {
2071 error(_("unable to create leading directories of %s"),
2072 ctx->graph_name);
2073 return -1;
2074 }
2075
2076 if (ctx->split) {
2077 char *lock_name = get_commit_graph_chain_filename(ctx->odb_source);
2078
2079 hold_lock_file_for_update_mode(&lk, lock_name,
2080 LOCK_DIE_ON_ERROR, 0444);
2081 free(lock_name);
2082
2083 graph_layer = mks_tempfile_m(ctx->graph_name, 0444);
2084 if (!graph_layer) {
2085 error(_("unable to create temporary graph layer"));
2086 return -1;
2087 }
2088
2089 if (adjust_shared_perm(the_repository, get_tempfile_path(graph_layer))) {
2090 error(_("unable to adjust shared permissions for '%s'"),
2091 get_tempfile_path(graph_layer));
2092 return -1;
2093 }
2094
2095 f = hashfd(the_repository->hash_algo,
2096 get_tempfile_fd(graph_layer), get_tempfile_path(graph_layer));
2097 } else {
2098 hold_lock_file_for_update_mode(&lk, ctx->graph_name,
2099 LOCK_DIE_ON_ERROR, 0444);
2100 f = hashfd(the_repository->hash_algo,
2101 get_lock_file_fd(&lk), get_lock_file_path(&lk));
2102 }
2103
2104 cf = init_chunkfile(f);
2105
2106 add_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, GRAPH_FANOUT_SIZE,
2107 write_graph_chunk_fanout);
2108 add_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, st_mult(hashsz, ctx->commits.nr),
2109 write_graph_chunk_oids);
2110 add_chunk(cf, GRAPH_CHUNKID_DATA, st_mult(hashsz + 16, ctx->commits.nr),
2111 write_graph_chunk_data);
2112
2113 if (ctx->write_generation_data)
2114 add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA,
2115 st_mult(sizeof(uint32_t), ctx->commits.nr),
2116 write_graph_chunk_generation_data);
2117 if (ctx->num_generation_data_overflows)
2118 add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW,
2119 st_mult(sizeof(timestamp_t), ctx->num_generation_data_overflows),
2120 write_graph_chunk_generation_data_overflow);
2121 if (ctx->num_extra_edges)
2122 add_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES,
2123 st_mult(4, ctx->num_extra_edges),
2124 write_graph_chunk_extra_edges);
2125 if (ctx->changed_paths) {
2126 add_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES,
2127 st_mult(sizeof(uint32_t), ctx->commits.nr),
2128 write_graph_chunk_bloom_indexes);
2129 add_chunk(cf, GRAPH_CHUNKID_BLOOMDATA,
2130 st_add(sizeof(uint32_t) * 3,
2131 ctx->total_bloom_filter_data_size),
2132 write_graph_chunk_bloom_data);
2133 }
2134 if (ctx->num_commit_graphs_after > 1)
2135 add_chunk(cf, GRAPH_CHUNKID_BASE,
2136 st_mult(hashsz, ctx->num_commit_graphs_after - 1),
2137 write_graph_chunk_base);
2138
2139 hashwrite_be32(f, GRAPH_SIGNATURE);
2140
2141 hashwrite_u8(f, GRAPH_VERSION);
2142 hashwrite_u8(f, oid_version(the_hash_algo));
2143 hashwrite_u8(f, get_num_chunks(cf));
2144 hashwrite_u8(f, ctx->num_commit_graphs_after - 1);
2145
2146 if (ctx->report_progress) {
2147 strbuf_addf(&progress_title,
2148 Q_("Writing out commit graph in %d pass",
2149 "Writing out commit graph in %d passes",
2150 get_num_chunks(cf)),
2151 get_num_chunks(cf));
2152 ctx->progress = start_delayed_progress(
2153 the_repository,
2154 progress_title.buf,
2155 st_mult(get_num_chunks(cf), ctx->commits.nr));
2156 }
2157
2158 write_chunkfile(cf, ctx);
2159
2160 stop_progress(&ctx->progress);
2161 strbuf_release(&progress_title);
2162
2163 if (ctx->split && ctx->base_graph_name && ctx->num_commit_graphs_after > 1) {
2164 char *new_base_hash = xstrdup(oid_to_hex(&ctx->new_base_graph->oid));
2165 char *new_base_name = get_split_graph_filename(ctx->new_base_graph->odb_source, new_base_hash);
2166
2167 free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2]);
2168 free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2]);
2169 ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 2] = new_base_name;
2170 ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 2] = new_base_hash;
2171 }
2172
2173 close_commit_graph(ctx->r->objects);
2174 finalize_hashfile(f, file_hash, FSYNC_COMPONENT_COMMIT_GRAPH,
2175 CSUM_HASH_IN_STREAM | CSUM_FSYNC);
2176 free_chunkfile(cf);
2177
2178 if (ctx->split) {
2179 FILE *chainf = fdopen_lock_file(&lk, "w");
2180 char *final_graph_name;
2181 int result;
2182
2183 if (!chainf) {
2184 error(_("unable to open commit-graph chain file"));
2185 return -1;
2186 }
2187
2188 if (ctx->base_graph_name) {
2189 const char *dest;
2190 int idx = ctx->num_commit_graphs_after - 1;
2191 if (ctx->num_commit_graphs_after > 1)
2192 idx--;
2193
2194 dest = ctx->commit_graph_filenames_after[idx];
2195
2196 if (strcmp(ctx->base_graph_name, dest)) {
2197 result = rename(ctx->base_graph_name, dest);
2198
2199 if (result) {
2200 error(_("failed to rename base commit-graph file"));
2201 return -1;
2202 }
2203 }
2204 } else {
2205 char *graph_name = get_commit_graph_filename(ctx->odb_source);
2206 unlink(graph_name);
2207 free(graph_name);
2208 }
2209
2210 free(ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]);
2211 ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1] = xstrdup(hash_to_hex(file_hash));
2212 final_graph_name = get_split_graph_filename(ctx->odb_source,
2213 ctx->commit_graph_hash_after[ctx->num_commit_graphs_after - 1]);
2214 free(ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1]);
2215 ctx->commit_graph_filenames_after[ctx->num_commit_graphs_after - 1] = final_graph_name;
2216
2217 result = rename_tempfile(&graph_layer, final_graph_name);
2218
2219 for (i = 0; i < ctx->num_commit_graphs_after; i++)
2220 fprintf(get_lock_file_fp(&lk), "%s\n", ctx->commit_graph_hash_after[i]);
2221
2222 if (result) {
2223 error(_("failed to rename temporary commit-graph file"));
2224 return -1;
2225 }
2226 }
2227
2228 commit_lock_file(&lk);
2229
2230 return 0;
2231}
2232
2233static void split_graph_merge_strategy(struct write_commit_graph_context *ctx)
2234{
2235 struct commit_graph *g;
2236 uint32_t num_commits;
2237 enum commit_graph_split_flags flags = COMMIT_GRAPH_SPLIT_UNSPECIFIED;
2238 uint32_t i;
2239
2240 int max_commits = 0;
2241 int size_mult = 2;
2242
2243 if (ctx->opts) {
2244 max_commits = ctx->opts->max_commits;
2245
2246 if (ctx->opts->size_multiple)
2247 size_mult = ctx->opts->size_multiple;
2248
2249 flags = ctx->opts->split_flags;
2250 }
2251
2252 g = ctx->r->objects->commit_graph;
2253 num_commits = ctx->commits.nr;
2254 if (flags == COMMIT_GRAPH_SPLIT_REPLACE)
2255 ctx->num_commit_graphs_after = 1;
2256 else
2257 ctx->num_commit_graphs_after = ctx->num_commit_graphs_before + 1;
2258
2259 if (flags != COMMIT_GRAPH_SPLIT_MERGE_PROHIBITED &&
2260 flags != COMMIT_GRAPH_SPLIT_REPLACE) {
2261 while (g && (g->num_commits <= st_mult(size_mult, num_commits) ||
2262 (max_commits && num_commits > max_commits))) {
2263 if (g->odb_source != ctx->odb_source)
2264 break;
2265
2266 if (unsigned_add_overflows(num_commits, g->num_commits))
2267 die(_("cannot merge graphs with %"PRIuMAX", "
2268 "%"PRIuMAX" commits"),
2269 (uintmax_t)num_commits,
2270 (uintmax_t)g->num_commits);
2271 num_commits += g->num_commits;
2272 g = g->base_graph;
2273
2274 ctx->num_commit_graphs_after--;
2275 }
2276 }
2277
2278 if (flags != COMMIT_GRAPH_SPLIT_REPLACE)
2279 ctx->new_base_graph = g;
2280 else if (ctx->num_commit_graphs_after != 1)
2281 BUG("split_graph_merge_strategy: num_commit_graphs_after "
2282 "should be 1 with --split=replace");
2283
2284 if (ctx->num_commit_graphs_after == 2) {
2285 char *old_graph_name = get_commit_graph_filename(g->odb_source);
2286
2287 if (!strcmp(g->filename, old_graph_name) &&
2288 g->odb_source != ctx->odb_source) {
2289 ctx->num_commit_graphs_after = 1;
2290 ctx->new_base_graph = NULL;
2291 }
2292
2293 free(old_graph_name);
2294 }
2295
2296 CALLOC_ARRAY(ctx->commit_graph_filenames_after, ctx->num_commit_graphs_after);
2297 CALLOC_ARRAY(ctx->commit_graph_hash_after, ctx->num_commit_graphs_after);
2298
2299 for (i = 0; i < ctx->num_commit_graphs_after &&
2300 i < ctx->num_commit_graphs_before; i++)
2301 ctx->commit_graph_filenames_after[i] = xstrdup(ctx->commit_graph_filenames_before[i]);
2302
2303 i = ctx->num_commit_graphs_before - 1;
2304 g = ctx->r->objects->commit_graph;
2305
2306 while (g) {
2307 if (i < ctx->num_commit_graphs_after)
2308 ctx->commit_graph_hash_after[i] = xstrdup(oid_to_hex(&g->oid));
2309
2310 /*
2311 * If the topmost remaining layer has generation data chunk, the
2312 * resultant layer also has generation data chunk.
2313 */
2314 if (i == ctx->num_commit_graphs_after - 2)
2315 ctx->write_generation_data = !!g->chunk_generation_data;
2316
2317 i--;
2318 g = g->base_graph;
2319 }
2320}
2321
2322static void merge_commit_graph(struct write_commit_graph_context *ctx,
2323 struct commit_graph *g)
2324{
2325 uint32_t i;
2326 uint32_t offset = g->num_commits_in_base;
2327
2328 if (unsigned_add_overflows(ctx->commits.nr, g->num_commits))
2329 die(_("cannot merge graph %s, too many commits: %"PRIuMAX),
2330 oid_to_hex(&g->oid),
2331 (uintmax_t)st_add(ctx->commits.nr, g->num_commits));
2332
2333 ALLOC_GROW(ctx->commits.list, ctx->commits.nr + g->num_commits, ctx->commits.alloc);
2334
2335 for (i = 0; i < g->num_commits; i++) {
2336 struct object_id oid;
2337 struct commit *result;
2338
2339 display_progress(ctx->progress, i + 1);
2340
2341 load_oid_from_graph(g, i + offset, &oid);
2342
2343 /* only add commits if they still exist in the repo */
2344 result = lookup_commit_reference_gently(ctx->r, &oid, 1);
2345
2346 if (result) {
2347 ctx->commits.list[ctx->commits.nr] = result;
2348 ctx->commits.nr++;
2349 }
2350 }
2351}
2352
2353static int commit_compare(const void *_a, const void *_b)
2354{
2355 const struct commit *a = *(const struct commit **)_a;
2356 const struct commit *b = *(const struct commit **)_b;
2357 return oidcmp(&a->object.oid, &b->object.oid);
2358}
2359
2360static void sort_and_scan_merged_commits(struct write_commit_graph_context *ctx)
2361{
2362 uint32_t i, dedup_i = 0;
2363
2364 if (ctx->report_progress)
2365 ctx->progress = start_delayed_progress(
2366 the_repository,
2367 _("Scanning merged commits"),
2368 ctx->commits.nr);
2369
2370 QSORT(ctx->commits.list, ctx->commits.nr, commit_compare);
2371
2372 ctx->num_extra_edges = 0;
2373 for (i = 0; i < ctx->commits.nr; i++) {
2374 display_progress(ctx->progress, i + 1);
2375
2376 if (i && oideq(&ctx->commits.list[i - 1]->object.oid,
2377 &ctx->commits.list[i]->object.oid)) {
2378 /*
2379 * Silently ignore duplicates. These were likely
2380 * created due to a commit appearing in multiple
2381 * layers of the chain, which is unexpected but
2382 * not invalid. We should make sure there is a
2383 * unique copy in the new layer.
2384 */
2385 } else {
2386 unsigned int num_parents;
2387
2388 ctx->commits.list[dedup_i] = ctx->commits.list[i];
2389 dedup_i++;
2390
2391 num_parents = commit_list_count(ctx->commits.list[i]->parents);
2392 if (num_parents > 2)
2393 ctx->num_extra_edges += num_parents - 1;
2394 }
2395 }
2396
2397 ctx->commits.nr = dedup_i;
2398
2399 stop_progress(&ctx->progress);
2400}
2401
2402static void merge_commit_graphs(struct write_commit_graph_context *ctx)
2403{
2404 struct commit_graph *g = ctx->r->objects->commit_graph;
2405 uint32_t current_graph_number = ctx->num_commit_graphs_before;
2406
2407 while (g && current_graph_number >= ctx->num_commit_graphs_after) {
2408 current_graph_number--;
2409
2410 if (ctx->report_progress)
2411 ctx->progress = start_delayed_progress(the_repository,
2412 _("Merging commit-graph"), 0);
2413
2414 merge_commit_graph(ctx, g);
2415 stop_progress(&ctx->progress);
2416
2417 g = g->base_graph;
2418 }
2419
2420 if (g) {
2421 ctx->new_base_graph = g;
2422 ctx->new_num_commits_in_base = g->num_commits + g->num_commits_in_base;
2423 }
2424
2425 if (ctx->new_base_graph)
2426 ctx->base_graph_name = xstrdup(ctx->new_base_graph->filename);
2427
2428 sort_and_scan_merged_commits(ctx);
2429}
2430
2431static void mark_commit_graphs(struct write_commit_graph_context *ctx)
2432{
2433 uint32_t i;
2434 time_t now = time(NULL);
2435
2436 for (i = ctx->num_commit_graphs_after - 1; i < ctx->num_commit_graphs_before; i++) {
2437 struct stat st;
2438 struct utimbuf updated_time;
2439
2440 if (stat(ctx->commit_graph_filenames_before[i], &st) < 0)
2441 continue;
2442
2443 updated_time.actime = st.st_atime;
2444 updated_time.modtime = now;
2445 utime(ctx->commit_graph_filenames_before[i], &updated_time);
2446 }
2447}
2448
2449static void expire_commit_graphs(struct write_commit_graph_context *ctx)
2450{
2451 struct strbuf path = STRBUF_INIT;
2452 DIR *dir;
2453 struct dirent *de;
2454 size_t dirnamelen;
2455 timestamp_t expire_time = time(NULL);
2456
2457 if (ctx->opts && ctx->opts->expire_time)
2458 expire_time = ctx->opts->expire_time;
2459 if (!ctx->split) {
2460 char *chain_file_name = get_commit_graph_chain_filename(ctx->odb_source);
2461 unlink(chain_file_name);
2462 free(chain_file_name);
2463 ctx->num_commit_graphs_after = 0;
2464 }
2465
2466 strbuf_addstr(&path, ctx->odb_source->path);
2467 strbuf_addstr(&path, "/info/commit-graphs");
2468 dir = opendir(path.buf);
2469
2470 if (!dir)
2471 goto out;
2472
2473 strbuf_addch(&path, '/');
2474 dirnamelen = path.len;
2475 while ((de = readdir(dir)) != NULL) {
2476 struct stat st;
2477 uint32_t i, found = 0;
2478
2479 strbuf_setlen(&path, dirnamelen);
2480 strbuf_addstr(&path, de->d_name);
2481
2482 if (stat(path.buf, &st) < 0)
2483 continue;
2484
2485 if (st.st_mtime > expire_time)
2486 continue;
2487 if (path.len < 6 || strcmp(path.buf + path.len - 6, ".graph"))
2488 continue;
2489
2490 for (i = 0; i < ctx->num_commit_graphs_after; i++) {
2491 if (!strcmp(ctx->commit_graph_filenames_after[i],
2492 path.buf)) {
2493 found = 1;
2494 break;
2495 }
2496 }
2497
2498 if (!found)
2499 unlink(path.buf);
2500 }
2501
2502out:
2503 if(dir)
2504 closedir(dir);
2505 strbuf_release(&path);
2506}
2507
2508int write_commit_graph(struct odb_source *source,
2509 const struct string_list *const pack_indexes,
2510 struct oidset *commits,
2511 enum commit_graph_write_flags flags,
2512 const struct commit_graph_opts *opts)
2513{
2514 struct repository *r = the_repository;
2515 struct write_commit_graph_context ctx = {
2516 .r = r,
2517 .odb_source = source,
2518 .append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0,
2519 .report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0,
2520 .split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0,
2521 .opts = opts,
2522 .total_bloom_filter_data_size = 0,
2523 .write_generation_data = (get_configured_generation_version(r) == 2),
2524 .num_generation_data_overflows = 0,
2525 };
2526 uint32_t i;
2527 int res = 0;
2528 int replace = 0;
2529 struct bloom_filter_settings bloom_settings = DEFAULT_BLOOM_FILTER_SETTINGS;
2530 struct topo_level_slab topo_levels;
2531
2532 prepare_repo_settings(r);
2533 if (!r->settings.core_commit_graph) {
2534 warning(_("attempting to write a commit-graph, but 'core.commitGraph' is disabled"));
2535 return 0;
2536 }
2537 if (!commit_graph_compatible(r))
2538 return 0;
2539 if (r->settings.commit_graph_changed_paths_version < -1
2540 || r->settings.commit_graph_changed_paths_version > 2) {
2541 warning(_("attempting to write a commit-graph, but "
2542 "'commitGraph.changedPathsVersion' (%d) is not supported"),
2543 r->settings.commit_graph_changed_paths_version);
2544 return 0;
2545 }
2546
2547 bloom_settings.hash_version = r->settings.commit_graph_changed_paths_version;
2548 bloom_settings.bits_per_entry = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY",
2549 bloom_settings.bits_per_entry);
2550 bloom_settings.num_hashes = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_NUM_HASHES",
2551 bloom_settings.num_hashes);
2552 bloom_settings.max_changed_paths = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS",
2553 bloom_settings.max_changed_paths);
2554 ctx.bloom_settings = &bloom_settings;
2555
2556 init_topo_level_slab(&topo_levels);
2557 ctx.topo_levels = &topo_levels;
2558
2559 prepare_commit_graph(ctx.r);
2560 if (ctx.r->objects->commit_graph) {
2561 struct commit_graph *g = ctx.r->objects->commit_graph;
2562
2563 while (g) {
2564 g->topo_levels = &topo_levels;
2565 g = g->base_graph;
2566 }
2567 }
2568
2569 if (flags & COMMIT_GRAPH_WRITE_BLOOM_FILTERS)
2570 ctx.changed_paths = 1;
2571 if (!(flags & COMMIT_GRAPH_NO_WRITE_BLOOM_FILTERS)) {
2572 struct commit_graph *g;
2573
2574 g = ctx.r->objects->commit_graph;
2575
2576 /* We have changed-paths already. Keep them in the next graph */
2577 if (g && g->bloom_filter_settings) {
2578 ctx.changed_paths = 1;
2579
2580 /* don't propagate the hash_version unless unspecified */
2581 if (bloom_settings.hash_version == -1)
2582 bloom_settings.hash_version = g->bloom_filter_settings->hash_version;
2583 bloom_settings.bits_per_entry = g->bloom_filter_settings->bits_per_entry;
2584 bloom_settings.num_hashes = g->bloom_filter_settings->num_hashes;
2585 bloom_settings.max_changed_paths = g->bloom_filter_settings->max_changed_paths;
2586 }
2587 }
2588
2589 bloom_settings.hash_version = bloom_settings.hash_version == 2 ? 2 : 1;
2590
2591 if (ctx.split) {
2592 struct commit_graph *g = ctx.r->objects->commit_graph;
2593
2594 while (g) {
2595 ctx.num_commit_graphs_before++;
2596 g = g->base_graph;
2597 }
2598
2599 if (ctx.num_commit_graphs_before) {
2600 ALLOC_ARRAY(ctx.commit_graph_filenames_before, ctx.num_commit_graphs_before);
2601 i = ctx.num_commit_graphs_before;
2602 g = ctx.r->objects->commit_graph;
2603
2604 while (g) {
2605 ctx.commit_graph_filenames_before[--i] = xstrdup(g->filename);
2606 g = g->base_graph;
2607 }
2608 }
2609
2610 if (ctx.opts)
2611 replace = ctx.opts->split_flags & COMMIT_GRAPH_SPLIT_REPLACE;
2612 }
2613
2614 ctx.approx_nr_objects = repo_approximate_object_count(the_repository);
2615
2616 if (ctx.append && ctx.r->objects->commit_graph) {
2617 struct commit_graph *g = ctx.r->objects->commit_graph;
2618 for (i = 0; i < g->num_commits; i++) {
2619 struct object_id oid;
2620 oidread(&oid, g->chunk_oid_lookup + st_mult(g->hash_len, i),
2621 the_repository->hash_algo);
2622 oid_array_append(&ctx.oids, &oid);
2623 }
2624 }
2625
2626 if (pack_indexes) {
2627 ctx.order_by_pack = 1;
2628 if ((res = fill_oids_from_packs(&ctx, pack_indexes)))
2629 goto cleanup;
2630 }
2631
2632 if (commits) {
2633 if ((res = fill_oids_from_commits(&ctx, commits)))
2634 goto cleanup;
2635 }
2636
2637 if (!pack_indexes && !commits) {
2638 ctx.order_by_pack = 1;
2639 fill_oids_from_all_packs(&ctx);
2640 }
2641
2642 close_reachable(&ctx);
2643
2644 copy_oids_to_commits(&ctx);
2645
2646 if (ctx.commits.nr >= GRAPH_EDGE_LAST_MASK) {
2647 error(_("too many commits to write graph"));
2648 res = -1;
2649 goto cleanup;
2650 }
2651
2652 if (!ctx.commits.nr && !replace)
2653 goto cleanup;
2654
2655 if (ctx.split) {
2656 split_graph_merge_strategy(&ctx);
2657
2658 if (!replace)
2659 merge_commit_graphs(&ctx);
2660 } else
2661 ctx.num_commit_graphs_after = 1;
2662
2663 ctx.trust_generation_numbers = validate_mixed_generation_chain(ctx.r->objects->commit_graph);
2664
2665 compute_topological_levels(&ctx);
2666 if (ctx.write_generation_data)
2667 compute_generation_numbers(&ctx);
2668
2669 if (ctx.changed_paths)
2670 compute_bloom_filters(&ctx);
2671
2672 res = write_commit_graph_file(&ctx);
2673
2674 if (ctx.changed_paths)
2675 deinit_bloom_filters();
2676
2677 if (ctx.split)
2678 mark_commit_graphs(&ctx);
2679
2680 expire_commit_graphs(&ctx);
2681
2682cleanup:
2683 free(ctx.graph_name);
2684 free(ctx.base_graph_name);
2685 free(ctx.commits.list);
2686 oid_array_clear(&ctx.oids);
2687 clear_topo_level_slab(&topo_levels);
2688
2689 if (ctx.r->objects->commit_graph) {
2690 struct commit_graph *g = ctx.r->objects->commit_graph;
2691
2692 while (g) {
2693 g->topo_levels = NULL;
2694 g = g->base_graph;
2695 }
2696 }
2697
2698 for (i = 0; i < ctx.num_commit_graphs_before; i++)
2699 free(ctx.commit_graph_filenames_before[i]);
2700 free(ctx.commit_graph_filenames_before);
2701
2702 for (i = 0; i < ctx.num_commit_graphs_after; i++) {
2703 free(ctx.commit_graph_filenames_after[i]);
2704 free(ctx.commit_graph_hash_after[i]);
2705 }
2706 free(ctx.commit_graph_filenames_after);
2707 free(ctx.commit_graph_hash_after);
2708
2709 return res;
2710}
2711
2712#define VERIFY_COMMIT_GRAPH_ERROR_HASH 2
2713static int verify_commit_graph_error;
2714
2715__attribute__((format (printf, 1, 2)))
2716static void graph_report(const char *fmt, ...)
2717{
2718 va_list ap;
2719
2720 verify_commit_graph_error = 1;
2721 va_start(ap, fmt);
2722 vfprintf(stderr, fmt, ap);
2723 fprintf(stderr, "\n");
2724 va_end(ap);
2725}
2726
2727static int commit_graph_checksum_valid(struct commit_graph *g)
2728{
2729 return hashfile_checksum_valid(the_repository->hash_algo,
2730 g->data, g->data_len);
2731}
2732
2733static int verify_one_commit_graph(struct repository *r,
2734 struct commit_graph *g,
2735 struct progress *progress,
2736 uint64_t *seen)
2737{
2738 uint32_t i, cur_fanout_pos = 0;
2739 struct object_id prev_oid, cur_oid;
2740 struct commit *seen_gen_zero = NULL;
2741 struct commit *seen_gen_non_zero = NULL;
2742
2743 if (!commit_graph_checksum_valid(g)) {
2744 graph_report(_("the commit-graph file has incorrect checksum and is likely corrupt"));
2745 verify_commit_graph_error = VERIFY_COMMIT_GRAPH_ERROR_HASH;
2746 }
2747
2748 for (i = 0; i < g->num_commits; i++) {
2749 struct commit *graph_commit;
2750
2751 oidread(&cur_oid, g->chunk_oid_lookup + st_mult(g->hash_len, i),
2752 the_repository->hash_algo);
2753
2754 if (i && oidcmp(&prev_oid, &cur_oid) >= 0)
2755 graph_report(_("commit-graph has incorrect OID order: %s then %s"),
2756 oid_to_hex(&prev_oid),
2757 oid_to_hex(&cur_oid));
2758
2759 oidcpy(&prev_oid, &cur_oid);
2760
2761 while (cur_oid.hash[0] > cur_fanout_pos) {
2762 uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
2763
2764 if (i != fanout_value)
2765 graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
2766 cur_fanout_pos, fanout_value, i);
2767 cur_fanout_pos++;
2768 }
2769
2770 graph_commit = lookup_commit(r, &cur_oid);
2771 if (!parse_commit_in_graph_one(r, g, graph_commit))
2772 graph_report(_("failed to parse commit %s from commit-graph"),
2773 oid_to_hex(&cur_oid));
2774 }
2775
2776 while (cur_fanout_pos < 256) {
2777 uint32_t fanout_value = get_be32(g->chunk_oid_fanout + cur_fanout_pos);
2778
2779 if (g->num_commits != fanout_value)
2780 graph_report(_("commit-graph has incorrect fanout value: fanout[%d] = %u != %u"),
2781 cur_fanout_pos, fanout_value, i);
2782
2783 cur_fanout_pos++;
2784 }
2785
2786 if (verify_commit_graph_error & ~VERIFY_COMMIT_GRAPH_ERROR_HASH)
2787 return verify_commit_graph_error;
2788
2789 for (i = 0; i < g->num_commits; i++) {
2790 struct commit *graph_commit, *odb_commit;
2791 struct commit_list *graph_parents, *odb_parents;
2792 timestamp_t max_generation = 0;
2793 timestamp_t generation;
2794
2795 display_progress(progress, ++(*seen));
2796 oidread(&cur_oid, g->chunk_oid_lookup + st_mult(g->hash_len, i),
2797 the_repository->hash_algo);
2798
2799 graph_commit = lookup_commit(r, &cur_oid);
2800 odb_commit = (struct commit *)create_object(r, &cur_oid, alloc_commit_node(r));
2801 if (repo_parse_commit_internal(r, odb_commit, 0, 0)) {
2802 graph_report(_("failed to parse commit %s from object database for commit-graph"),
2803 oid_to_hex(&cur_oid));
2804 continue;
2805 }
2806
2807 if (!oideq(&get_commit_tree_in_graph_one(r, g, graph_commit)->object.oid,
2808 get_commit_tree_oid(odb_commit)))
2809 graph_report(_("root tree OID for commit %s in commit-graph is %s != %s"),
2810 oid_to_hex(&cur_oid),
2811 oid_to_hex(get_commit_tree_oid(graph_commit)),
2812 oid_to_hex(get_commit_tree_oid(odb_commit)));
2813
2814 graph_parents = graph_commit->parents;
2815 odb_parents = odb_commit->parents;
2816
2817 while (graph_parents) {
2818 if (!odb_parents) {
2819 graph_report(_("commit-graph parent list for commit %s is too long"),
2820 oid_to_hex(&cur_oid));
2821 break;
2822 }
2823
2824 /* parse parent in case it is in a base graph */
2825 parse_commit_in_graph_one(r, g, graph_parents->item);
2826
2827 if (!oideq(&graph_parents->item->object.oid, &odb_parents->item->object.oid))
2828 graph_report(_("commit-graph parent for %s is %s != %s"),
2829 oid_to_hex(&cur_oid),
2830 oid_to_hex(&graph_parents->item->object.oid),
2831 oid_to_hex(&odb_parents->item->object.oid));
2832
2833 generation = commit_graph_generation_from_graph(graph_parents->item);
2834 if (generation > max_generation)
2835 max_generation = generation;
2836
2837 graph_parents = graph_parents->next;
2838 odb_parents = odb_parents->next;
2839 }
2840
2841 if (odb_parents)
2842 graph_report(_("commit-graph parent list for commit %s terminates early"),
2843 oid_to_hex(&cur_oid));
2844
2845 if (commit_graph_generation_from_graph(graph_commit))
2846 seen_gen_non_zero = graph_commit;
2847 else
2848 seen_gen_zero = graph_commit;
2849
2850 if (seen_gen_zero)
2851 continue;
2852
2853 /*
2854 * If we are using topological level and one of our parents has
2855 * generation GENERATION_NUMBER_V1_MAX, then our generation is
2856 * also GENERATION_NUMBER_V1_MAX. Decrement to avoid extra logic
2857 * in the following condition.
2858 */
2859 if (!g->read_generation_data && max_generation == GENERATION_NUMBER_V1_MAX)
2860 max_generation--;
2861
2862 generation = commit_graph_generation(graph_commit);
2863 if (generation < max_generation + 1)
2864 graph_report(_("commit-graph generation for commit %s is %"PRItime" < %"PRItime),
2865 oid_to_hex(&cur_oid),
2866 generation,
2867 max_generation + 1);
2868
2869 if (graph_commit->date != odb_commit->date)
2870 graph_report(_("commit date for commit %s in commit-graph is %"PRItime" != %"PRItime),
2871 oid_to_hex(&cur_oid),
2872 graph_commit->date,
2873 odb_commit->date);
2874 }
2875
2876 if (seen_gen_zero && seen_gen_non_zero)
2877 graph_report(_("commit-graph has both zero and non-zero "
2878 "generations (e.g., commits '%s' and '%s')"),
2879 oid_to_hex(&seen_gen_zero->object.oid),
2880 oid_to_hex(&seen_gen_non_zero->object.oid));
2881
2882 return verify_commit_graph_error;
2883}
2884
2885int verify_commit_graph(struct repository *r, struct commit_graph *g, int flags)
2886{
2887 struct progress *progress = NULL;
2888 int local_error = 0;
2889 uint64_t seen = 0;
2890
2891 if (!g) {
2892 graph_report("no commit-graph file loaded");
2893 return 1;
2894 }
2895
2896 if (flags & COMMIT_GRAPH_WRITE_PROGRESS) {
2897 uint64_t total = g->num_commits;
2898 if (!(flags & COMMIT_GRAPH_VERIFY_SHALLOW))
2899 total += g->num_commits_in_base;
2900
2901 progress = start_progress(the_repository,
2902 _("Verifying commits in commit graph"),
2903 total);
2904 }
2905
2906 for (; g; g = g->base_graph) {
2907 local_error |= verify_one_commit_graph(r, g, progress, &seen);
2908 if (flags & COMMIT_GRAPH_VERIFY_SHALLOW)
2909 break;
2910 }
2911
2912 stop_progress(&progress);
2913
2914 return local_error;
2915}
2916
2917void free_commit_graph(struct commit_graph *g)
2918{
2919 while (g) {
2920 struct commit_graph *next = g->base_graph;
2921
2922 if (g->data)
2923 munmap((void *)g->data, g->data_len);
2924 free(g->filename);
2925 free(g->bloom_filter_settings);
2926 free(g);
2927
2928 g = next;
2929 }
2930}
2931
2932void disable_commit_graph(struct repository *r)
2933{
2934 r->commit_graph_disabled = 1;
2935}