freshlybakedca.ke
Git fork
1#define DISABLE_SIGN_COMPARE_WARNINGS
2
3#include "git-compat-util.h"
4#include "config.h"
5#include "dir.h"
6#include "hex.h"
7#include "packfile.h"
8#include "hash-lookup.h"
9#include "midx.h"
10#include "progress.h"
11#include "trace2.h"
12#include "chunk-format.h"
13#include "pack-bitmap.h"
14#include "pack-revindex.h"
15
16#define MIDX_PACK_ERROR ((void *)(intptr_t)-1)
17
18int midx_checksum_valid(struct multi_pack_index *m);
19void clear_midx_files_ext(struct odb_source *source, const char *ext,
20 const char *keep_hash);
21void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext,
22 char **keep_hashes,
23 uint32_t hashes_nr);
24int cmp_idx_or_pack_name(const char *idx_or_pack_name,
25 const char *idx_name);
26
27const unsigned char *get_midx_checksum(struct multi_pack_index *m)
28{
29 return m->data + m->data_len - m->source->odb->repo->hash_algo->rawsz;
30}
31
32void get_midx_filename(struct odb_source *source, struct strbuf *out)
33{
34 get_midx_filename_ext(source, out, NULL, NULL);
35}
36
37void get_midx_filename_ext(struct odb_source *source, struct strbuf *out,
38 const unsigned char *hash, const char *ext)
39{
40 strbuf_addf(out, "%s/pack/multi-pack-index", source->path);
41 if (ext)
42 strbuf_addf(out, "-%s.%s", hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext);
43}
44
45static int midx_read_oid_fanout(const unsigned char *chunk_start,
46 size_t chunk_size, void *data)
47{
48 int i;
49 struct multi_pack_index *m = data;
50 m->chunk_oid_fanout = (uint32_t *)chunk_start;
51
52 if (chunk_size != 4 * 256) {
53 error(_("multi-pack-index OID fanout is of the wrong size"));
54 return 1;
55 }
56 for (i = 0; i < 255; i++) {
57 uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]);
58 uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i+1]);
59
60 if (oid_fanout1 > oid_fanout2) {
61 error(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"),
62 i, oid_fanout1, oid_fanout2, i + 1);
63 return 1;
64 }
65 }
66 m->num_objects = ntohl(m->chunk_oid_fanout[255]);
67 return 0;
68}
69
70static int midx_read_oid_lookup(const unsigned char *chunk_start,
71 size_t chunk_size, void *data)
72{
73 struct multi_pack_index *m = data;
74 m->chunk_oid_lookup = chunk_start;
75
76 if (chunk_size != st_mult(m->hash_len, m->num_objects)) {
77 error(_("multi-pack-index OID lookup chunk is the wrong size"));
78 return 1;
79 }
80 return 0;
81}
82
83static int midx_read_object_offsets(const unsigned char *chunk_start,
84 size_t chunk_size, void *data)
85{
86 struct multi_pack_index *m = data;
87 m->chunk_object_offsets = chunk_start;
88
89 if (chunk_size != st_mult(m->num_objects, MIDX_CHUNK_OFFSET_WIDTH)) {
90 error(_("multi-pack-index object offset chunk is the wrong size"));
91 return 1;
92 }
93 return 0;
94}
95
96struct multi_pack_index *get_multi_pack_index(struct odb_source *source)
97{
98 packfile_store_prepare(source->odb->packfiles);
99 return source->midx;
100}
101
102static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source,
103 const char *midx_name)
104{
105 struct repository *r = source->odb->repo;
106 struct multi_pack_index *m = NULL;
107 int fd;
108 struct stat st;
109 size_t midx_size;
110 void *midx_map = NULL;
111 uint32_t hash_version;
112 uint32_t i;
113 const char *cur_pack_name;
114 struct chunkfile *cf = NULL;
115
116 fd = git_open(midx_name);
117
118 if (fd < 0)
119 goto cleanup_fail;
120 if (fstat(fd, &st)) {
121 error_errno(_("failed to read %s"), midx_name);
122 goto cleanup_fail;
123 }
124
125 midx_size = xsize_t(st.st_size);
126
127 if (midx_size < (MIDX_HEADER_SIZE + r->hash_algo->rawsz)) {
128 error(_("multi-pack-index file %s is too small"), midx_name);
129 goto cleanup_fail;
130 }
131
132 midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
133 close(fd);
134
135 CALLOC_ARRAY(m, 1);
136 m->data = midx_map;
137 m->data_len = midx_size;
138 m->source = source;
139
140 m->signature = get_be32(m->data);
141 if (m->signature != MIDX_SIGNATURE)
142 die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
143 m->signature, MIDX_SIGNATURE);
144
145 m->version = m->data[MIDX_BYTE_FILE_VERSION];
146 if (m->version != MIDX_VERSION)
147 die(_("multi-pack-index version %d not recognized"),
148 m->version);
149
150 hash_version = m->data[MIDX_BYTE_HASH_VERSION];
151 if (hash_version != oid_version(r->hash_algo)) {
152 error(_("multi-pack-index hash version %u does not match version %u"),
153 hash_version, oid_version(r->hash_algo));
154 goto cleanup_fail;
155 }
156 m->hash_len = r->hash_algo->rawsz;
157
158 m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
159
160 m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
161
162 m->preferred_pack_idx = -1;
163
164 cf = init_chunkfile(NULL);
165
166 if (read_table_of_contents(cf, m->data, midx_size,
167 MIDX_HEADER_SIZE, m->num_chunks,
168 MIDX_CHUNK_ALIGNMENT))
169 goto cleanup_fail;
170
171 if (pair_chunk(cf, MIDX_CHUNKID_PACKNAMES, &m->chunk_pack_names, &m->chunk_pack_names_len))
172 die(_("multi-pack-index required pack-name chunk missing or corrupted"));
173 if (read_chunk(cf, MIDX_CHUNKID_OIDFANOUT, midx_read_oid_fanout, m))
174 die(_("multi-pack-index required OID fanout chunk missing or corrupted"));
175 if (read_chunk(cf, MIDX_CHUNKID_OIDLOOKUP, midx_read_oid_lookup, m))
176 die(_("multi-pack-index required OID lookup chunk missing or corrupted"));
177 if (read_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS, midx_read_object_offsets, m))
178 die(_("multi-pack-index required object offsets chunk missing or corrupted"));
179
180 pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets,
181 &m->chunk_large_offsets_len);
182 if (git_env_bool("GIT_TEST_MIDX_READ_BTMP", 1))
183 pair_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS,
184 (const unsigned char **)&m->chunk_bitmapped_packs,
185 &m->chunk_bitmapped_packs_len);
186
187 if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1))
188 pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex,
189 &m->chunk_revindex_len);
190
191 CALLOC_ARRAY(m->pack_names, m->num_packs);
192 CALLOC_ARRAY(m->packs, m->num_packs);
193
194 cur_pack_name = (const char *)m->chunk_pack_names;
195 for (i = 0; i < m->num_packs; i++) {
196 const char *end;
197 size_t avail = m->chunk_pack_names_len -
198 (cur_pack_name - (const char *)m->chunk_pack_names);
199
200 m->pack_names[i] = cur_pack_name;
201
202 end = memchr(cur_pack_name, '\0', avail);
203 if (!end)
204 die(_("multi-pack-index pack-name chunk is too short"));
205 cur_pack_name = end + 1;
206
207 if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0)
208 die(_("multi-pack-index pack names out of order: '%s' before '%s'"),
209 m->pack_names[i - 1],
210 m->pack_names[i]);
211 }
212
213 trace2_data_intmax("midx", r, "load/num_packs", m->num_packs);
214 trace2_data_intmax("midx", r, "load/num_objects", m->num_objects);
215
216 free_chunkfile(cf);
217 return m;
218
219cleanup_fail:
220 free(m);
221 free_chunkfile(cf);
222 if (midx_map)
223 munmap(midx_map, midx_size);
224 if (0 <= fd)
225 close(fd);
226 return NULL;
227}
228
229void get_midx_chain_dirname(struct odb_source *source, struct strbuf *buf)
230{
231 strbuf_addf(buf, "%s/pack/multi-pack-index.d", source->path);
232}
233
234void get_midx_chain_filename(struct odb_source *source, struct strbuf *buf)
235{
236 get_midx_chain_dirname(source, buf);
237 strbuf_addstr(buf, "/multi-pack-index-chain");
238}
239
240void get_split_midx_filename_ext(struct odb_source *source, struct strbuf *buf,
241 const unsigned char *hash, const char *ext)
242{
243 get_midx_chain_dirname(source, buf);
244 strbuf_addf(buf, "/multi-pack-index-%s.%s",
245 hash_to_hex_algop(hash, source->odb->repo->hash_algo), ext);
246}
247
248static int open_multi_pack_index_chain(const struct git_hash_algo *hash_algo,
249 const char *chain_file, int *fd,
250 struct stat *st)
251{
252 *fd = git_open(chain_file);
253 if (*fd < 0)
254 return 0;
255 if (fstat(*fd, st)) {
256 close(*fd);
257 return 0;
258 }
259 if (st->st_size < hash_algo->hexsz) {
260 close(*fd);
261 if (!st->st_size) {
262 /* treat empty files the same as missing */
263 errno = ENOENT;
264 } else {
265 warning(_("multi-pack-index chain file too small"));
266 errno = EINVAL;
267 }
268 return 0;
269 }
270 return 1;
271}
272
273static int add_midx_to_chain(struct multi_pack_index *midx,
274 struct multi_pack_index *midx_chain)
275{
276 if (midx_chain) {
277 if (unsigned_add_overflows(midx_chain->num_packs,
278 midx_chain->num_packs_in_base)) {
279 warning(_("pack count in base MIDX too high: %"PRIuMAX),
280 (uintmax_t)midx_chain->num_packs_in_base);
281 return 0;
282 }
283 if (unsigned_add_overflows(midx_chain->num_objects,
284 midx_chain->num_objects_in_base)) {
285 warning(_("object count in base MIDX too high: %"PRIuMAX),
286 (uintmax_t)midx_chain->num_objects_in_base);
287 return 0;
288 }
289 midx->num_packs_in_base = midx_chain->num_packs +
290 midx_chain->num_packs_in_base;
291 midx->num_objects_in_base = midx_chain->num_objects +
292 midx_chain->num_objects_in_base;
293 }
294
295 midx->base_midx = midx_chain;
296 midx->has_chain = 1;
297
298 return 1;
299}
300
301static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source,
302 int fd, struct stat *st,
303 int *incomplete_chain)
304{
305 const struct git_hash_algo *hash_algo = source->odb->repo->hash_algo;
306 struct multi_pack_index *midx_chain = NULL;
307 struct strbuf buf = STRBUF_INIT;
308 int valid = 1;
309 uint32_t i, count;
310 FILE *fp = xfdopen(fd, "r");
311
312 count = st->st_size / (hash_algo->hexsz + 1);
313
314 for (i = 0; i < count; i++) {
315 struct multi_pack_index *m;
316 struct object_id layer;
317
318 if (strbuf_getline_lf(&buf, fp) == EOF)
319 break;
320
321 if (get_oid_hex_algop(buf.buf, &layer, hash_algo)) {
322 warning(_("invalid multi-pack-index chain: line '%s' "
323 "not a hash"),
324 buf.buf);
325 valid = 0;
326 break;
327 }
328
329 valid = 0;
330
331 strbuf_reset(&buf);
332 get_split_midx_filename_ext(source, &buf,
333 layer.hash, MIDX_EXT_MIDX);
334 m = load_multi_pack_index_one(source, buf.buf);
335
336 if (m) {
337 if (add_midx_to_chain(m, midx_chain)) {
338 midx_chain = m;
339 valid = 1;
340 } else {
341 close_midx(m);
342 }
343 }
344 if (!valid) {
345 warning(_("unable to find all multi-pack index files"));
346 break;
347 }
348 }
349
350 fclose(fp);
351 strbuf_release(&buf);
352
353 *incomplete_chain = !valid;
354 return midx_chain;
355}
356
357static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *source)
358{
359 struct strbuf chain_file = STRBUF_INIT;
360 struct stat st;
361 int fd;
362 struct multi_pack_index *m = NULL;
363
364 get_midx_chain_filename(source, &chain_file);
365 if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) {
366 int incomplete;
367 /* ownership of fd is taken over by load function */
368 m = load_midx_chain_fd_st(source, fd, &st, &incomplete);
369 }
370
371 strbuf_release(&chain_file);
372 return m;
373}
374
375struct multi_pack_index *load_multi_pack_index(struct odb_source *source)
376{
377 struct strbuf midx_name = STRBUF_INIT;
378 struct multi_pack_index *m;
379
380 get_midx_filename(source, &midx_name);
381
382 m = load_multi_pack_index_one(source, midx_name.buf);
383 if (!m)
384 m = load_multi_pack_index_chain(source);
385
386 strbuf_release(&midx_name);
387
388 return m;
389}
390
391void close_midx(struct multi_pack_index *m)
392{
393 uint32_t i;
394
395 if (!m)
396 return;
397
398 close_midx(m->base_midx);
399
400 munmap((unsigned char *)m->data, m->data_len);
401
402 for (i = 0; i < m->num_packs; i++) {
403 if (m->packs[i] && m->packs[i] != MIDX_PACK_ERROR)
404 m->packs[i]->multi_pack_index = 0;
405 }
406 FREE_AND_NULL(m->packs);
407 FREE_AND_NULL(m->pack_names);
408 free(m);
409}
410
411static uint32_t midx_for_object(struct multi_pack_index **_m, uint32_t pos)
412{
413 struct multi_pack_index *m = *_m;
414 while (m && pos < m->num_objects_in_base)
415 m = m->base_midx;
416
417 if (!m)
418 BUG("NULL multi-pack-index for object position: %"PRIu32, pos);
419
420 if (pos >= m->num_objects + m->num_objects_in_base)
421 die(_("invalid MIDX object position, MIDX is likely corrupt"));
422
423 *_m = m;
424
425 return pos - m->num_objects_in_base;
426}
427
428static uint32_t midx_for_pack(struct multi_pack_index **_m,
429 uint32_t pack_int_id)
430{
431 struct multi_pack_index *m = *_m;
432 while (m && pack_int_id < m->num_packs_in_base)
433 m = m->base_midx;
434
435 if (!m)
436 BUG("NULL multi-pack-index for pack ID: %"PRIu32, pack_int_id);
437
438 if (pack_int_id >= m->num_packs + m->num_packs_in_base)
439 die(_("bad pack-int-id: %u (%u total packs)"),
440 pack_int_id, m->num_packs + m->num_packs_in_base);
441
442 *_m = m;
443
444 return pack_int_id - m->num_packs_in_base;
445}
446
447int prepare_midx_pack(struct multi_pack_index *m,
448 uint32_t pack_int_id)
449{
450 struct repository *r = m->source->odb->repo;
451 struct strbuf pack_name = STRBUF_INIT;
452 struct packed_git *p;
453
454 pack_int_id = midx_for_pack(&m, pack_int_id);
455
456 if (m->packs[pack_int_id] == MIDX_PACK_ERROR)
457 return 1;
458 if (m->packs[pack_int_id])
459 return 0;
460
461 strbuf_addf(&pack_name, "%s/pack/%s", m->source->path,
462 m->pack_names[pack_int_id]);
463 p = packfile_store_load_pack(r->objects->packfiles,
464 pack_name.buf, m->source->local);
465 if (p)
466 list_add_tail(&p->mru, &r->objects->packfiles->mru);
467 strbuf_release(&pack_name);
468
469 if (!p) {
470 m->packs[pack_int_id] = MIDX_PACK_ERROR;
471 return 1;
472 }
473
474 p->multi_pack_index = 1;
475 m->packs[pack_int_id] = p;
476
477 return 0;
478}
479
480struct packed_git *nth_midxed_pack(struct multi_pack_index *m,
481 uint32_t pack_int_id)
482{
483 uint32_t local_pack_int_id = midx_for_pack(&m, pack_int_id);
484 if (m->packs[local_pack_int_id] == MIDX_PACK_ERROR)
485 return NULL;
486 return m->packs[local_pack_int_id];
487}
488
489#define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t))
490
491int nth_bitmapped_pack(struct multi_pack_index *m,
492 struct bitmapped_pack *bp, uint32_t pack_int_id)
493{
494 uint32_t local_pack_int_id = midx_for_pack(&m, pack_int_id);
495
496 if (!m->chunk_bitmapped_packs)
497 return error(_("MIDX does not contain the BTMP chunk"));
498
499 if (prepare_midx_pack(m, pack_int_id))
500 return error(_("could not load bitmapped pack %"PRIu32), pack_int_id);
501
502 bp->p = m->packs[local_pack_int_id];
503 bp->bitmap_pos = get_be32((char *)m->chunk_bitmapped_packs +
504 MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * local_pack_int_id);
505 bp->bitmap_nr = get_be32((char *)m->chunk_bitmapped_packs +
506 MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * local_pack_int_id +
507 sizeof(uint32_t));
508 bp->pack_int_id = pack_int_id;
509 bp->from_midx = m;
510
511 return 0;
512}
513
514int bsearch_one_midx(const struct object_id *oid, struct multi_pack_index *m,
515 uint32_t *result)
516{
517 int ret = bsearch_hash(oid->hash, m->chunk_oid_fanout,
518 m->chunk_oid_lookup,
519 m->source->odb->repo->hash_algo->rawsz,
520 result);
521 if (result)
522 *result += m->num_objects_in_base;
523 return ret;
524}
525
526int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m,
527 uint32_t *result)
528{
529 for (; m; m = m->base_midx)
530 if (bsearch_one_midx(oid, m, result))
531 return 1;
532 return 0;
533}
534
535int midx_has_oid(struct multi_pack_index *m, const struct object_id *oid)
536{
537 return bsearch_midx(oid, m, NULL);
538}
539
540struct object_id *nth_midxed_object_oid(struct object_id *oid,
541 struct multi_pack_index *m,
542 uint32_t n)
543{
544 if (n >= m->num_objects + m->num_objects_in_base)
545 return NULL;
546
547 n = midx_for_object(&m, n);
548
549 oidread(oid, m->chunk_oid_lookup + st_mult(m->hash_len, n),
550 m->source->odb->repo->hash_algo);
551 return oid;
552}
553
554off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos)
555{
556 const unsigned char *offset_data;
557 uint32_t offset32;
558
559 pos = midx_for_object(&m, pos);
560
561 offset_data = m->chunk_object_offsets + (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH;
562 offset32 = get_be32(offset_data + sizeof(uint32_t));
563
564 if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) {
565 if (sizeof(off_t) < sizeof(uint64_t))
566 die(_("multi-pack-index stores a 64-bit offset, but off_t is too small"));
567
568 offset32 ^= MIDX_LARGE_OFFSET_NEEDED;
569 if (offset32 >= m->chunk_large_offsets_len / sizeof(uint64_t))
570 die(_("multi-pack-index large offset out of bounds"));
571 return get_be64(m->chunk_large_offsets + sizeof(uint64_t) * offset32);
572 }
573
574 return offset32;
575}
576
577uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos)
578{
579 pos = midx_for_object(&m, pos);
580
581 return m->num_packs_in_base + get_be32(m->chunk_object_offsets +
582 (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH);
583}
584
585int fill_midx_entry(struct multi_pack_index *m,
586 const struct object_id *oid,
587 struct pack_entry *e)
588{
589 uint32_t pos;
590 uint32_t pack_int_id;
591 struct packed_git *p;
592
593 if (!bsearch_midx(oid, m, &pos))
594 return 0;
595
596 midx_for_object(&m, pos);
597 pack_int_id = nth_midxed_pack_int_id(m, pos);
598
599 if (prepare_midx_pack(m, pack_int_id))
600 return 0;
601 p = m->packs[pack_int_id - m->num_packs_in_base];
602
603 /*
604 * We are about to tell the caller where they can locate the
605 * requested object. We better make sure the packfile is
606 * still here and can be accessed before supplying that
607 * answer, as it may have been deleted since the MIDX was
608 * loaded!
609 */
610 if (!is_pack_valid(p))
611 return 0;
612
613 if (oidset_size(&p->bad_objects) &&
614 oidset_contains(&p->bad_objects, oid))
615 return 0;
616
617 e->offset = nth_midxed_offset(m, pos);
618 e->p = p;
619
620 return 1;
621}
622
623/* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */
624int cmp_idx_or_pack_name(const char *idx_or_pack_name,
625 const char *idx_name)
626{
627 /* Skip past any initial matching prefix. */
628 while (*idx_name && *idx_name == *idx_or_pack_name) {
629 idx_name++;
630 idx_or_pack_name++;
631 }
632
633 /*
634 * If we didn't match completely, we may have matched "pack-1234." and
635 * be left with "idx" and "pack" respectively, which is also OK. We do
636 * not have to check for "idx" and "idx", because that would have been
637 * a complete match (and in that case these strcmps will be false, but
638 * we'll correctly return 0 from the final strcmp() below.
639 *
640 * Technically this matches "fooidx" and "foopack", but we'd never have
641 * such names in the first place.
642 */
643 if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack"))
644 return 0;
645
646 /*
647 * This not only checks for a complete match, but also orders based on
648 * the first non-identical character, which means our ordering will
649 * match a raw strcmp(). That makes it OK to use this to binary search
650 * a naively-sorted list.
651 */
652 return strcmp(idx_or_pack_name, idx_name);
653}
654
655static int midx_contains_pack_1(struct multi_pack_index *m,
656 const char *idx_or_pack_name)
657{
658 uint32_t first = 0, last = m->num_packs;
659
660 while (first < last) {
661 uint32_t mid = first + (last - first) / 2;
662 const char *current;
663 int cmp;
664
665 current = m->pack_names[mid];
666 cmp = cmp_idx_or_pack_name(idx_or_pack_name, current);
667 if (!cmp)
668 return 1;
669 if (cmp > 0) {
670 first = mid + 1;
671 continue;
672 }
673 last = mid;
674 }
675
676 return 0;
677}
678
679int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
680{
681 for (; m; m = m->base_midx)
682 if (midx_contains_pack_1(m, idx_or_pack_name))
683 return 1;
684 return 0;
685}
686
687int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id)
688{
689 if (m->preferred_pack_idx == -1) {
690 uint32_t midx_pos;
691 if (load_midx_revindex(m) < 0) {
692 m->preferred_pack_idx = -2;
693 return -1;
694 }
695
696 midx_pos = pack_pos_to_midx(m, m->num_objects_in_base);
697
698 m->preferred_pack_idx = nth_midxed_pack_int_id(m, midx_pos);
699
700 } else if (m->preferred_pack_idx == -2)
701 return -1; /* no revindex */
702
703 *pack_int_id = m->preferred_pack_idx;
704 return 0;
705}
706
707int prepare_multi_pack_index_one(struct odb_source *source)
708{
709 struct repository *r = source->odb->repo;
710
711 prepare_repo_settings(r);
712 if (!r->settings.core_multi_pack_index)
713 return 0;
714
715 if (source->midx)
716 return 1;
717
718 source->midx = load_multi_pack_index(source);
719
720 return !!source->midx;
721}
722
723int midx_checksum_valid(struct multi_pack_index *m)
724{
725 return hashfile_checksum_valid(m->source->odb->repo->hash_algo,
726 m->data, m->data_len);
727}
728
729struct clear_midx_data {
730 char **keep;
731 uint32_t keep_nr;
732 const char *ext;
733};
734
735static void clear_midx_file_ext(const char *full_path, size_t full_path_len UNUSED,
736 const char *file_name, void *_data)
737{
738 struct clear_midx_data *data = _data;
739 uint32_t i;
740
741 if (!(starts_with(file_name, "multi-pack-index-") &&
742 ends_with(file_name, data->ext)))
743 return;
744 for (i = 0; i < data->keep_nr; i++) {
745 if (!strcmp(data->keep[i], file_name))
746 return;
747 }
748 if (unlink(full_path))
749 die_errno(_("failed to remove %s"), full_path);
750}
751
752void clear_midx_files_ext(struct odb_source *source, const char *ext,
753 const char *keep_hash)
754{
755 struct clear_midx_data data;
756 memset(&data, 0, sizeof(struct clear_midx_data));
757
758 if (keep_hash) {
759 ALLOC_ARRAY(data.keep, 1);
760
761 data.keep[0] = xstrfmt("multi-pack-index-%s.%s", keep_hash, ext);
762 data.keep_nr = 1;
763 }
764 data.ext = ext;
765
766 for_each_file_in_pack_dir(source->path,
767 clear_midx_file_ext,
768 &data);
769
770 if (keep_hash)
771 free(data.keep[0]);
772 free(data.keep);
773}
774
775void clear_incremental_midx_files_ext(struct odb_source *source, const char *ext,
776 char **keep_hashes,
777 uint32_t hashes_nr)
778{
779 struct clear_midx_data data;
780 uint32_t i;
781
782 memset(&data, 0, sizeof(struct clear_midx_data));
783
784 ALLOC_ARRAY(data.keep, hashes_nr);
785 for (i = 0; i < hashes_nr; i++)
786 data.keep[i] = xstrfmt("multi-pack-index-%s.%s", keep_hashes[i],
787 ext);
788 data.keep_nr = hashes_nr;
789 data.ext = ext;
790
791 for_each_file_in_pack_subdir(source->path, "multi-pack-index.d",
792 clear_midx_file_ext, &data);
793
794 for (i = 0; i < hashes_nr; i++)
795 free(data.keep[i]);
796 free(data.keep);
797}
798
799void clear_midx_file(struct repository *r)
800{
801 struct strbuf midx = STRBUF_INIT;
802
803 get_midx_filename(r->objects->sources, &midx);
804
805 if (r->objects) {
806 struct odb_source *source;
807
808 for (source = r->objects->sources; source; source = source->next) {
809 if (source->midx)
810 close_midx(source->midx);
811 source->midx = NULL;
812 }
813 }
814
815 if (remove_path(midx.buf))
816 die(_("failed to clear multi-pack-index at %s"), midx.buf);
817
818 clear_midx_files_ext(r->objects->sources, MIDX_EXT_BITMAP, NULL);
819 clear_midx_files_ext(r->objects->sources, MIDX_EXT_REV, NULL);
820
821 strbuf_release(&midx);
822}
823
824static int verify_midx_error;
825
826__attribute__((format (printf, 1, 2)))
827static void midx_report(const char *fmt, ...)
828{
829 va_list ap;
830 verify_midx_error = 1;
831 va_start(ap, fmt);
832 vfprintf(stderr, fmt, ap);
833 fprintf(stderr, "\n");
834 va_end(ap);
835}
836
837struct pair_pos_vs_id
838{
839 uint32_t pos;
840 uint32_t pack_int_id;
841};
842
843static int compare_pair_pos_vs_id(const void *_a, const void *_b)
844{
845 struct pair_pos_vs_id *a = (struct pair_pos_vs_id *)_a;
846 struct pair_pos_vs_id *b = (struct pair_pos_vs_id *)_b;
847
848 return b->pack_int_id - a->pack_int_id;
849}
850
851/*
852 * Limit calls to display_progress() for performance reasons.
853 * The interval here was arbitrarily chosen.
854 */
855#define SPARSE_PROGRESS_INTERVAL (1 << 12)
856#define midx_display_sparse_progress(progress, n) \
857 do { \
858 uint64_t _n = (n); \
859 if ((_n & (SPARSE_PROGRESS_INTERVAL - 1)) == 0) \
860 display_progress(progress, _n); \
861 } while (0)
862
863int verify_midx_file(struct odb_source *source, unsigned flags)
864{
865 struct repository *r = source->odb->repo;
866 struct pair_pos_vs_id *pairs = NULL;
867 uint32_t i;
868 struct progress *progress = NULL;
869 struct multi_pack_index *m = load_multi_pack_index(source);
870 struct multi_pack_index *curr;
871 verify_midx_error = 0;
872
873 if (!m) {
874 int result = 0;
875 struct stat sb;
876 struct strbuf filename = STRBUF_INIT;
877
878 get_midx_filename(source, &filename);
879
880 if (!stat(filename.buf, &sb)) {
881 error(_("multi-pack-index file exists, but failed to parse"));
882 result = 1;
883 }
884 strbuf_release(&filename);
885 return result;
886 }
887
888 if (!midx_checksum_valid(m))
889 midx_report(_("incorrect checksum"));
890
891 if (flags & MIDX_PROGRESS)
892 progress = start_delayed_progress(r,
893 _("Looking for referenced packfiles"),
894 m->num_packs + m->num_packs_in_base);
895 for (i = 0; i < m->num_packs + m->num_packs_in_base; i++) {
896 if (prepare_midx_pack(m, i))
897 midx_report("failed to load pack in position %d", i);
898
899 display_progress(progress, i + 1);
900 }
901 stop_progress(&progress);
902
903 if (m->num_objects == 0) {
904 midx_report(_("the midx contains no oid"));
905 /*
906 * Remaining tests assume that we have objects, so we can
907 * return here.
908 */
909 goto cleanup;
910 }
911
912 if (flags & MIDX_PROGRESS)
913 progress = start_sparse_progress(r,
914 _("Verifying OID order in multi-pack-index"),
915 m->num_objects - 1);
916
917 for (curr = m; curr; curr = curr->base_midx) {
918 for (i = 0; i < m->num_objects - 1; i++) {
919 struct object_id oid1, oid2;
920
921 nth_midxed_object_oid(&oid1, m, m->num_objects_in_base + i);
922 nth_midxed_object_oid(&oid2, m, m->num_objects_in_base + i + 1);
923
924 if (oidcmp(&oid1, &oid2) >= 0)
925 midx_report(_("oid lookup out of order: oid[%d] = %s >= %s = oid[%d]"),
926 i, oid_to_hex(&oid1), oid_to_hex(&oid2), i + 1);
927
928 midx_display_sparse_progress(progress, i + 1);
929 }
930 }
931 stop_progress(&progress);
932
933 /*
934 * Create an array mapping each object to its packfile id. Sort it
935 * to group the objects by packfile. Use this permutation to visit
936 * each of the objects and only require 1 packfile to be open at a
937 * time.
938 */
939 ALLOC_ARRAY(pairs, m->num_objects + m->num_objects_in_base);
940 for (i = 0; i < m->num_objects + m->num_objects_in_base; i++) {
941 pairs[i].pos = i;
942 pairs[i].pack_int_id = nth_midxed_pack_int_id(m, i);
943 }
944
945 if (flags & MIDX_PROGRESS)
946 progress = start_sparse_progress(r,
947 _("Sorting objects by packfile"),
948 m->num_objects);
949 display_progress(progress, 0); /* TODO: Measure QSORT() progress */
950 QSORT(pairs, m->num_objects, compare_pair_pos_vs_id);
951 stop_progress(&progress);
952
953 if (flags & MIDX_PROGRESS)
954 progress = start_sparse_progress(r,
955 _("Verifying object offsets"),
956 m->num_objects);
957 for (i = 0; i < m->num_objects + m->num_objects_in_base; i++) {
958 struct object_id oid;
959 struct pack_entry e;
960 off_t m_offset, p_offset;
961
962 if (i > 0 && pairs[i-1].pack_int_id != pairs[i].pack_int_id &&
963 nth_midxed_pack(m, pairs[i-1].pack_int_id)) {
964 uint32_t pack_int_id = pairs[i-1].pack_int_id;
965 struct packed_git *p = nth_midxed_pack(m, pack_int_id);
966
967 close_pack_fd(p);
968 close_pack_index(p);
969 }
970
971 nth_midxed_object_oid(&oid, m, pairs[i].pos);
972
973 if (!fill_midx_entry(m, &oid, &e)) {
974 midx_report(_("failed to load pack entry for oid[%d] = %s"),
975 pairs[i].pos, oid_to_hex(&oid));
976 continue;
977 }
978
979 if (open_pack_index(e.p)) {
980 midx_report(_("failed to load pack-index for packfile %s"),
981 e.p->pack_name);
982 break;
983 }
984
985 m_offset = e.offset;
986 p_offset = find_pack_entry_one(&oid, e.p);
987
988 if (m_offset != p_offset)
989 midx_report(_("incorrect object offset for oid[%d] = %s: %"PRIx64" != %"PRIx64),
990 pairs[i].pos, oid_to_hex(&oid), m_offset, p_offset);
991
992 midx_display_sparse_progress(progress, i + 1);
993 }
994 stop_progress(&progress);
995
996cleanup:
997 free(pairs);
998 close_midx(m);
999
1000 return verify_midx_error;
1001}