freshlybakedca.ke
Git fork
1#!/bin/sh
2
3test_description='exercise basic multi-pack bitmap functionality'
4
5. ./test-lib.sh
6. "${TEST_DIRECTORY}/lib-bitmap.sh"
7
8# We'll be writing our own MIDX, so avoid getting confused by the
9# automatic ones.
10GIT_TEST_MULTI_PACK_INDEX=0
11GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
12
13# This test exercise multi-pack bitmap functionality where the object order is
14# stored and read from a special chunk within the MIDX, so use the default
15# behavior here.
16sane_unset GIT_TEST_MIDX_WRITE_REV
17sane_unset GIT_TEST_MIDX_READ_RIDX
18
19bitmap_reuse_tests() {
20 from=$1
21 to=$2
22 writeLookupTable=false
23
24 for i in $3-${$#}
25 do
26 case $i in
27 "pack.writeBitmapLookupTable") writeLookupTable=true;;
28 esac
29 done
30
31 test_expect_success "setup pack reuse tests ($from -> $to)" '
32 rm -fr repo &&
33 git init repo &&
34 (
35 cd repo &&
36 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
37 test_commit_bulk 16 &&
38 git tag old-tip &&
39
40 git config core.multiPackIndex true &&
41 if test "MIDX" = "$from"
42 then
43 git repack -Ad &&
44 git multi-pack-index write --bitmap
45 else
46 git repack -Adb
47 fi
48 )
49 '
50
51 test_expect_success "build bitmap from existing ($from -> $to)" '
52 (
53 cd repo &&
54 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
55 test_commit_bulk --id=further 16 &&
56 git tag new-tip &&
57
58 if test "MIDX" = "$to"
59 then
60 git repack -d &&
61 git multi-pack-index write --bitmap
62 else
63 git repack -Adb
64 fi
65 )
66 '
67
68 test_expect_success "verify resulting bitmaps ($from -> $to)" '
69 (
70 cd repo &&
71 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
72 git for-each-ref &&
73 git rev-list --test-bitmap refs/tags/old-tip &&
74 git rev-list --test-bitmap refs/tags/new-tip
75 )
76 '
77}
78
79test_midx_bitmap_cases () {
80 writeLookupTable=false
81 writeBitmapLookupTable=
82
83 for i in "$@"
84 do
85 case $i in
86 "pack.writeBitmapLookupTable")
87 writeLookupTable=true
88 writeBitmapLookupTable="$i"
89 ;;
90 esac
91 done
92
93 test_expect_success 'setup test_repository' '
94 rm -rf * .git &&
95 git init &&
96 git config pack.writeBitmapLookupTable '"$writeLookupTable"'
97 '
98
99 midx_bitmap_core
100
101 bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
102 bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
103 bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"
104
105 test_expect_success 'missing object closure fails gracefully' '
106 rm -fr repo &&
107 git init repo &&
108 test_when_finished "rm -fr repo" &&
109 (
110 cd repo &&
111 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
112
113 test_commit loose &&
114 test_commit packed &&
115
116 # Do not pass "--revs"; we want a pack without the "loose"
117 # commit.
118 git pack-objects $objdir/pack/pack <<-EOF &&
119 $(git rev-parse packed)
120 EOF
121
122 test_must_fail git multi-pack-index write --bitmap 2>err &&
123 grep "doesn.t have full closure" err &&
124 test_path_is_missing $midx
125 )
126 '
127
128 midx_bitmap_partial_tests
129
130 test_expect_success 'removing a MIDX clears stale bitmaps' '
131 rm -fr repo &&
132 git init repo &&
133 test_when_finished "rm -fr repo" &&
134 (
135 cd repo &&
136 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
137 test_commit base &&
138 git repack &&
139 git multi-pack-index write --bitmap &&
140
141 # Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
142 stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
143 rm $midx &&
144
145 # Then write a new MIDX.
146 test_commit new &&
147 git repack &&
148 git multi-pack-index write --bitmap &&
149
150 test_path_is_file $midx &&
151 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
152 test_path_is_missing $stale_bitmap
153 )
154 '
155
156 test_expect_success 'pack.preferBitmapTips' '
157 git init repo &&
158 test_when_finished "rm -fr repo" &&
159 (
160 cd repo &&
161 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
162
163 test_commit_bulk --message="%s" 103 &&
164
165 git log --format="%H" >commits.raw &&
166 sort <commits.raw >commits &&
167
168 git log --format="create refs/tags/%s %H" HEAD >refs &&
169 git update-ref --stdin <refs &&
170
171 git multi-pack-index write --bitmap &&
172 test_path_is_file $midx &&
173 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
174
175 test-tool bitmap list-commits | sort >bitmaps &&
176 comm -13 bitmaps commits >before &&
177 test_line_count = 1 before &&
178
179 sed "s|\(.*\)|create refs/tags/include/\1 \1|" before |
180 git update-ref --stdin &&
181
182 rm -fr $midx-$(midx_checksum $objdir).bitmap &&
183 rm -fr $midx &&
184
185 git -c pack.preferBitmapTips=refs/tags/include \
186 multi-pack-index write --bitmap &&
187 test-tool bitmap list-commits | sort >bitmaps &&
188 comm -13 bitmaps commits >after &&
189
190 ! test_cmp before after
191 )
192 '
193
194 test_expect_success 'writing a bitmap with --refs-snapshot' '
195 git init repo &&
196 test_when_finished "rm -fr repo" &&
197 (
198 cd repo &&
199 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
200
201 test_commit one &&
202 test_commit two &&
203
204 git rev-parse one >snapshot &&
205
206 git repack -ad &&
207
208 # First, write a MIDX which see both refs/tags/one and
209 # refs/tags/two (causing both of those commits to receive
210 # bitmaps).
211 git multi-pack-index write --bitmap &&
212
213 test_path_is_file $midx &&
214 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
215
216 test-tool bitmap list-commits | sort >bitmaps &&
217 grep "$(git rev-parse one)" bitmaps &&
218 grep "$(git rev-parse two)" bitmaps &&
219
220 rm -fr $midx-$(midx_checksum $objdir).bitmap &&
221 rm -fr $midx &&
222
223 # Then again, but with a refs snapshot which only sees
224 # refs/tags/one.
225 git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
226
227 test_path_is_file $midx &&
228 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
229
230 test-tool bitmap list-commits | sort >bitmaps &&
231 grep "$(git rev-parse one)" bitmaps &&
232 ! grep "$(git rev-parse two)" bitmaps
233 )
234 '
235
236 test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
237 git init repo &&
238 test_when_finished "rm -fr repo" &&
239 (
240 cd repo &&
241 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
242
243 test_commit_bulk --message="%s" 103 &&
244
245 git log --format="%H" >commits.raw &&
246 sort <commits.raw >commits &&
247
248 git log --format="create refs/tags/%s %H" HEAD >refs &&
249 git update-ref --stdin <refs &&
250
251 git multi-pack-index write --bitmap &&
252 test_path_is_file $midx &&
253 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
254
255 test-tool bitmap list-commits | sort >bitmaps &&
256 comm -13 bitmaps commits >before &&
257 test_line_count = 1 before &&
258
259 (
260 grep -vf before commits.raw &&
261 # mark missing commits as preferred
262 sed "s/^/+/" before
263 ) >snapshot &&
264
265 rm -fr $midx-$(midx_checksum $objdir).bitmap &&
266 rm -fr $midx &&
267
268 git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
269 test-tool bitmap list-commits | sort >bitmaps &&
270 comm -13 bitmaps commits >after &&
271
272 ! test_cmp before after
273 )
274 '
275
276 test_expect_success 'hash-cache values are propagated from pack bitmaps' '
277 rm -fr repo &&
278 git init repo &&
279 test_when_finished "rm -fr repo" &&
280 (
281 cd repo &&
282 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
283
284 test_commit base &&
285 test_commit base2 &&
286 git repack -adb &&
287
288 test-tool bitmap dump-hashes >pack.raw &&
289 test_file_not_empty pack.raw &&
290 sort pack.raw >pack.hashes &&
291
292 test_commit new &&
293 git repack &&
294 git multi-pack-index write --bitmap &&
295
296 test-tool bitmap dump-hashes >midx.raw &&
297 sort midx.raw >midx.hashes &&
298
299 # ensure that every namehash in the pack bitmap can be found in
300 # the midx bitmap (i.e., that there are no oid-namehash pairs
301 # unique to the pack bitmap).
302 comm -23 pack.hashes midx.hashes >dropped.hashes &&
303 test_must_be_empty dropped.hashes
304 )
305 '
306
307 test_expect_success 'no .bitmap is written without any objects' '
308 rm -fr repo &&
309 git init repo &&
310 test_when_finished "rm -fr repo" &&
311 (
312 cd repo &&
313 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
314
315 empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
316 cat >packs <<-EOF &&
317 pack-$empty.idx
318 EOF
319
320 git multi-pack-index write --bitmap --stdin-packs \
321 <packs 2>err &&
322
323 grep "bitmap without any objects" err &&
324
325 test_path_is_file $midx &&
326 test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
327 )
328 '
329
330 test_expect_success 'graceful fallback when missing reverse index' '
331 rm -fr repo &&
332 git init repo &&
333 test_when_finished "rm -fr repo" &&
334 (
335 cd repo &&
336 git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
337
338 test_commit base &&
339
340 # write a pack and MIDX bitmap containing base
341 git repack -adb &&
342 git multi-pack-index write --bitmap &&
343
344 GIT_TEST_MIDX_READ_RIDX=0 \
345 git rev-list --use-bitmap-index HEAD 2>err &&
346 ! grep "ignoring extra bitmap file" err
347 )
348 '
349}
350
351test_midx_bitmap_cases
352
353test_midx_bitmap_cases "pack.writeBitmapLookupTable"
354
355test_expect_success 'multi-pack-index write writes lookup table if enabled' '
356 rm -fr repo &&
357 git init repo &&
358 test_when_finished "rm -fr repo" &&
359 (
360 cd repo &&
361 test_commit base &&
362 git config pack.writeBitmapLookupTable true &&
363 git repack -ad &&
364 GIT_TRACE2_EVENT="$(pwd)/trace" \
365 git multi-pack-index write --bitmap &&
366 grep "\"label\":\"writing_lookup_table\"" trace
367 )
368'
369
370test_expect_success 'preferred pack change with existing MIDX bitmap' '
371 git init preferred-pack-with-existing &&
372 (
373 cd preferred-pack-with-existing &&
374
375 test_commit base &&
376 test_commit other &&
377
378 git rev-list --objects --no-object-names base >p1.objects &&
379 git rev-list --objects --no-object-names other >p2.objects &&
380
381 p1="$(git pack-objects "$objdir/pack/pack" \
382 --delta-base-offset <p1.objects)" &&
383 p2="$(git pack-objects "$objdir/pack/pack" \
384 --delta-base-offset <p2.objects)" &&
385
386 # Generate a MIDX containing the first two packs,
387 # marking p1 as preferred, and ensure that it can be
388 # successfully cloned.
389 git multi-pack-index write --bitmap \
390 --preferred-pack="pack-$p1.pack" &&
391 test_path_is_file $midx &&
392 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
393 git clone --no-local . clone1 &&
394
395 # Then generate a new pack which sorts ahead of any
396 # existing pack (by tweaking the pack prefix).
397 test_commit foo &&
398 git pack-objects --all --unpacked $objdir/pack/pack0 &&
399
400 # Generate a new MIDX which changes the preferred pack
401 # to a pack contained in the existing MIDX.
402 git multi-pack-index write --bitmap \
403 --preferred-pack="pack-$p2.pack" &&
404 test_path_is_file $midx &&
405 test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
406
407 # When the above circumstances are met, the preferred
408 # pack should change appropriately and clones should
409 # (still) succeed.
410 git clone --no-local . clone2
411 )
412'
413
414test_expect_success 'tagged commits are selected for bitmapping' '
415 rm -fr repo &&
416 git init repo &&
417 test_when_finished "rm -fr repo" &&
418 (
419 cd repo &&
420
421 test_commit --annotate base &&
422 git repack -d &&
423
424 # Remove refs/heads/main which points at the commit directly,
425 # leaving only a reference to the annotated tag.
426 git branch -M main &&
427 git checkout base &&
428 git branch -d main &&
429
430 git multi-pack-index write --bitmap &&
431
432 git rev-parse HEAD >want &&
433 test-tool bitmap list-commits >actual &&
434 grep $(cat want) actual
435 )
436'
437
438test_expect_success 'do not follow replace objects for MIDX bitmap' '
439 rm -fr repo &&
440 git init repo &&
441 test_when_finished "rm -fr repo" &&
442 (
443 cd repo &&
444
445 test_commit A &&
446 test_commit B &&
447 git checkout --orphan=orphan A &&
448 test_commit orphan &&
449
450 git replace A HEAD &&
451 git repack -ad --write-midx --write-bitmap-index &&
452
453 # generating reachability bitmaps with replace refs
454 # enabled will result in broken clones
455 git clone --no-local --bare . clone.git
456 )
457'
458
459corrupt_file () {
460 chmod a+w "$1" &&
461 printf "bogus" | dd of="$1" bs=1 seek="12" conv=notrunc
462}
463
464test_expect_success 'git fsck correctly identifies good and bad bitmaps' '
465 git init valid &&
466 test_when_finished rm -rf valid &&
467
468 test_commit_bulk 20 &&
469 git repack -adbf &&
470
471 # Move pack-bitmap aside so it is not deleted
472 # in next repack.
473 packbitmap=$(ls .git/objects/pack/pack-*.bitmap) &&
474 mv "$packbitmap" "$packbitmap.bak" &&
475
476 test_commit_bulk 10 &&
477 git repack -b --write-midx &&
478 midxbitmap=$(ls .git/objects/pack/multi-pack-index-*.bitmap) &&
479
480 # Copy MIDX bitmap to backup. Copy pack bitmap from backup.
481 cp "$midxbitmap" "$midxbitmap.bak" &&
482 cp "$packbitmap.bak" "$packbitmap" &&
483
484 # fsck works at first
485 git fsck 2>err &&
486 test_must_be_empty err &&
487
488 corrupt_file "$packbitmap" &&
489 test_must_fail git fsck 2>err &&
490 grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err &&
491
492 cp "$packbitmap.bak" "$packbitmap" &&
493 corrupt_file "$midxbitmap" &&
494 test_must_fail git fsck 2>err &&
495 grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
496
497 corrupt_file "$packbitmap" &&
498 test_must_fail git fsck 2>err &&
499 grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
500 grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err
501'
502
503test_expect_success 'corrupt MIDX with bitmap causes fallback' '
504 git init corrupt-midx-bitmap &&
505 (
506 cd corrupt-midx-bitmap &&
507
508 test_commit first &&
509 git repack -d &&
510 test_commit second &&
511 git repack -d &&
512
513 git multi-pack-index write --bitmap &&
514 checksum=$(midx_checksum $objdir) &&
515 for f in $midx $midx-$checksum.bitmap
516 do
517 mv $f $f.bak || return 1
518 done &&
519
520 # pack everything together, invalidating the MIDX
521 git repack -ad &&
522 # then restore the now-stale MIDX
523 for f in $midx $midx-$checksum.bitmap
524 do
525 mv $f.bak $f || return 1
526 done &&
527
528 git rev-list --count --objects --use-bitmap-index HEAD >out 2>err &&
529 # should attempt opening the broken pack twice (once
530 # from the attempt to load it via the stale bitmap, and
531 # again when attempting to load it from the stale MIDX)
532 # before falling back to the non-MIDX case
533 test 2 -eq $(grep -c "could not open pack" err) &&
534 test 6 -eq $(cat out)
535 )
536'
537
538for allow_pack_reuse in single multi
539do
540 test_expect_success "reading MIDX without BTMP chunk does not complain with $allow_pack_reuse pack reuse" '
541 test_when_finished "rm -rf midx-without-btmp" &&
542 git init midx-without-btmp &&
543 (
544 cd midx-without-btmp &&
545 test_commit initial &&
546
547 git repack -Adbl --write-bitmap-index --write-midx &&
548 GIT_TEST_MIDX_READ_BTMP=false git -c pack.allowPackReuse=$allow_pack_reuse \
549 pack-objects --all --use-bitmap-index --stdout </dev/null >/dev/null 2>err &&
550 test_must_be_empty err
551 )
552 '
553done
554
555test_expect_success 'remove one packfile between MIDX bitmap writes' '
556 git init remove-pack-between-writes &&
557 (
558 cd remove-pack-between-writes &&
559
560 test_commit A &&
561 test_commit B &&
562 test_commit C &&
563
564 # Create packs with the prefix "pack-A", "pack-B",
565 # "pack-C" to impose a lexicographic order on these
566 # packs so the pack being removed is always from the
567 # middle.
568 packdir=.git/objects/pack &&
569 A="$(echo A | git pack-objects $packdir/pack-A --revs)" &&
570 B="$(echo B | git pack-objects $packdir/pack-B --revs)" &&
571 C="$(echo C | git pack-objects $packdir/pack-C --revs)" &&
572
573 git multi-pack-index write --bitmap &&
574
575 cat >in <<-EOF &&
576 pack-A-$A.idx
577 pack-C-$C.idx
578 EOF
579 git multi-pack-index write --bitmap --stdin-packs <in &&
580
581 git rev-list --test-bitmap HEAD
582 )
583'
584
585test_done