freshlybakedca.ke
Git fork
midx: load multi-pack indices via their source
To load a multi-pack index the caller is expected to pass both the
repository and the object directory where the multi-pack index is
located. While this works, this layout has a couple of downsides:
- We need to pass in information reduntant with the owning source,
namely its object directory and whether the source is local or not.
- We don't have access to the source when loading the multi-pack
index. If we had that access, we could store a pointer to the owning
source in the MIDX and thus deduplicate some information.
- Multi-pack indices are inherently specific to the object source and
its format. With the goal of pluggable object backends in mind we
will eventually want the backends to own the logic of reading and
writing multi-pack indices. Making the logic work on top of object
sources is a step into that direction.
Refactor loading of multi-pack indices accordingly.
This surfaces one small problem though: git-multi-pack-index(1) and our
MIDX test helper both know to read and write multi-pack-indices located
in a different object directory. This issue is addressed by adding the
user-provided object directory as an in-memory alternate.
Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
authored by
Patrick Steinhardt
and committed by
Junio C Hamano
017db7bb
9ff21296
+16
-2
builtin/multi-pack-index.c
+16
-2
builtin/multi-pack-index.c
···
64
64
char **value = opt->value;
65
65
free(*value);
66
66
if (unset)
67
-
*value = xstrdup(repo_get_object_directory(the_repository));
67
+
*value = xstrdup(the_repository->objects->sources->path);
68
68
else
69
69
*value = real_pathdup(arg, 1);
70
70
return 0;
71
71
}
72
72
73
+
static struct odb_source *handle_object_dir_option(struct repository *repo)
74
+
{
75
+
struct odb_source *source = odb_find_source(repo->objects, opts.object_dir);
76
+
if (!source)
77
+
source = odb_add_to_alternates_memory(repo->objects, opts.object_dir);
78
+
return source;
79
+
}
80
+
73
81
static struct option common_opts[] = {
74
82
OPT_CALLBACK(0, "object-dir", &opts.object_dir,
75
83
N_("directory"),
···
157
165
if (argc)
158
166
usage_with_options(builtin_multi_pack_index_write_usage,
159
167
options);
168
+
handle_object_dir_option(repo);
160
169
161
170
FREE_AND_NULL(options);
162
171
···
193
202
N_("force progress reporting"), MIDX_PROGRESS),
194
203
OPT_END(),
195
204
};
205
+
struct odb_source *source;
206
+
196
207
options = add_common_options(builtin_multi_pack_index_verify_options);
197
208
198
209
trace2_cmd_mode(argv[0]);
···
205
216
if (argc)
206
217
usage_with_options(builtin_multi_pack_index_verify_usage,
207
218
options);
219
+
source = handle_object_dir_option(the_repository);
208
220
209
221
FREE_AND_NULL(options);
210
222
211
-
return verify_midx_file(the_repository, opts.object_dir, opts.flags);
223
+
return verify_midx_file(source, opts.flags);
212
224
}
213
225
214
226
static int cmd_multi_pack_index_expire(int argc, const char **argv,
···
233
245
if (argc)
234
246
usage_with_options(builtin_multi_pack_index_expire_usage,
235
247
options);
248
+
handle_object_dir_option(the_repository);
236
249
237
250
FREE_AND_NULL(options);
238
251
···
265
278
if (argc)
266
279
usage_with_options(builtin_multi_pack_index_repack_usage,
267
280
options);
281
+
handle_object_dir_option(the_repository);
268
282
269
283
FREE_AND_NULL(options);
270
284
+25
-32
midx.c
+25
-32
midx.c
···
95
95
return 0;
96
96
}
97
97
98
-
static struct multi_pack_index *load_multi_pack_index_one(struct repository *r,
99
-
const char *object_dir,
100
-
const char *midx_name,
101
-
int local)
98
+
static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source,
99
+
const char *midx_name)
102
100
{
101
+
struct repository *r = source->odb->repo;
103
102
struct multi_pack_index *m = NULL;
104
103
int fd;
105
104
struct stat st;
···
129
128
midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
130
129
close(fd);
131
130
132
-
FLEX_ALLOC_STR(m, object_dir, object_dir);
131
+
FLEX_ALLOC_STR(m, object_dir, source->path);
133
132
m->data = midx_map;
134
133
m->data_len = midx_size;
135
-
m->local = local;
134
+
m->local = source->local;
136
135
m->repo = r;
137
136
138
137
m->signature = get_be32(m->data);
···
297
296
return 1;
298
297
}
299
298
300
-
static struct multi_pack_index *load_midx_chain_fd_st(struct repository *r,
301
-
const char *object_dir,
302
-
int local,
299
+
static struct multi_pack_index *load_midx_chain_fd_st(struct odb_source *source,
303
300
int fd, struct stat *st,
304
301
int *incomplete_chain)
305
302
{
303
+
const struct git_hash_algo *hash_algo = source->odb->repo->hash_algo;
306
304
struct multi_pack_index *midx_chain = NULL;
307
305
struct strbuf buf = STRBUF_INIT;
308
306
int valid = 1;
309
307
uint32_t i, count;
310
308
FILE *fp = xfdopen(fd, "r");
311
309
312
-
count = st->st_size / (r->hash_algo->hexsz + 1);
310
+
count = st->st_size / (hash_algo->hexsz + 1);
313
311
314
312
for (i = 0; i < count; i++) {
315
313
struct multi_pack_index *m;
···
318
316
if (strbuf_getline_lf(&buf, fp) == EOF)
319
317
break;
320
318
321
-
if (get_oid_hex_algop(buf.buf, &layer, r->hash_algo)) {
319
+
if (get_oid_hex_algop(buf.buf, &layer, hash_algo)) {
322
320
warning(_("invalid multi-pack-index chain: line '%s' "
323
321
"not a hash"),
324
322
buf.buf);
···
329
327
valid = 0;
330
328
331
329
strbuf_reset(&buf);
332
-
get_split_midx_filename_ext(r->hash_algo, &buf, object_dir,
330
+
get_split_midx_filename_ext(hash_algo, &buf, source->path,
333
331
layer.hash, MIDX_EXT_MIDX);
334
-
m = load_multi_pack_index_one(r, object_dir, buf.buf, local);
332
+
m = load_multi_pack_index_one(source, buf.buf);
335
333
336
334
if (m) {
337
335
if (add_midx_to_chain(m, midx_chain)) {
···
354
352
return midx_chain;
355
353
}
356
354
357
-
static struct multi_pack_index *load_multi_pack_index_chain(struct repository *r,
358
-
const char *object_dir,
359
-
int local)
355
+
static struct multi_pack_index *load_multi_pack_index_chain(struct odb_source *source)
360
356
{
361
357
struct strbuf chain_file = STRBUF_INIT;
362
358
struct stat st;
363
359
int fd;
364
360
struct multi_pack_index *m = NULL;
365
361
366
-
get_midx_chain_filename(&chain_file, object_dir);
367
-
if (open_multi_pack_index_chain(r->hash_algo, chain_file.buf, &fd, &st)) {
362
+
get_midx_chain_filename(&chain_file, source->path);
363
+
if (open_multi_pack_index_chain(source->odb->repo->hash_algo, chain_file.buf, &fd, &st)) {
368
364
int incomplete;
369
365
/* ownership of fd is taken over by load function */
370
-
m = load_midx_chain_fd_st(r, object_dir, local, fd, &st,
371
-
&incomplete);
366
+
m = load_midx_chain_fd_st(source, fd, &st, &incomplete);
372
367
}
373
368
374
369
strbuf_release(&chain_file);
375
370
return m;
376
371
}
377
372
378
-
struct multi_pack_index *load_multi_pack_index(struct repository *r,
379
-
const char *object_dir,
380
-
int local)
373
+
struct multi_pack_index *load_multi_pack_index(struct odb_source *source)
381
374
{
382
375
struct strbuf midx_name = STRBUF_INIT;
383
376
struct multi_pack_index *m;
384
377
385
-
get_midx_filename(r->hash_algo, &midx_name, object_dir);
378
+
get_midx_filename(source->odb->repo->hash_algo, &midx_name,
379
+
source->path);
386
380
387
-
m = load_multi_pack_index_one(r, object_dir,
388
-
midx_name.buf, local);
381
+
m = load_multi_pack_index_one(source, midx_name.buf);
389
382
if (!m)
390
-
m = load_multi_pack_index_chain(r, object_dir, local);
383
+
m = load_multi_pack_index_chain(source);
391
384
392
385
strbuf_release(&midx_name);
393
386
···
734
727
if (source->midx)
735
728
return 1;
736
729
737
-
source->midx = load_multi_pack_index(r, source->path,
738
-
source->local);
730
+
source->midx = load_multi_pack_index(source);
739
731
740
732
return !!source->midx;
741
733
}
···
880
872
display_progress(progress, _n); \
881
873
} while (0)
882
874
883
-
int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags)
875
+
int verify_midx_file(struct odb_source *source, unsigned flags)
884
876
{
877
+
struct repository *r = source->odb->repo;
885
878
struct pair_pos_vs_id *pairs = NULL;
886
879
uint32_t i;
887
880
struct progress *progress = NULL;
888
-
struct multi_pack_index *m = load_multi_pack_index(r, object_dir, 1);
881
+
struct multi_pack_index *m = load_multi_pack_index(source);
889
882
struct multi_pack_index *curr;
890
883
verify_midx_error = 0;
891
884
···
894
887
struct stat sb;
895
888
struct strbuf filename = STRBUF_INIT;
896
889
897
-
get_midx_filename(r->hash_algo, &filename, object_dir);
890
+
get_midx_filename(r->hash_algo, &filename, source->path);
898
891
899
892
if (!stat(filename.buf, &sb)) {
900
893
error(_("multi-pack-index file exists, but failed to parse"));
+2
-4
midx.h
+2
-4
midx.h
···
100
100
struct strbuf *buf, const char *object_dir,
101
101
const unsigned char *hash, const char *ext);
102
102
103
-
struct multi_pack_index *load_multi_pack_index(struct repository *r,
104
-
const char *object_dir,
105
-
int local);
103
+
struct multi_pack_index *load_multi_pack_index(struct odb_source *source);
106
104
int prepare_midx_pack(struct multi_pack_index *m, uint32_t pack_int_id);
107
105
struct packed_git *nth_midxed_pack(struct multi_pack_index *m,
108
106
uint32_t pack_int_id);
···
136
134
const char *preferred_pack_name,
137
135
const char *refs_snapshot, unsigned flags);
138
136
void clear_midx_file(struct repository *r);
139
-
int verify_midx_file(struct repository *r, const char *object_dir, unsigned flags);
137
+
int verify_midx_file(struct odb_source *source, unsigned flags);
140
138
int expire_midx_packs(struct repository *r, const char *object_dir, unsigned flags);
141
139
int midx_repack(struct repository *r, const char *object_dir, size_t batch_size, unsigned flags);
142
140
+15
-10
t/helper/test-read-midx.c
+15
-10
t/helper/test-read-midx.c
···
11
11
#include "gettext.h"
12
12
#include "pack-revindex.h"
13
13
14
+
static struct multi_pack_index *setup_midx(const char *object_dir)
15
+
{
16
+
struct odb_source *source;
17
+
setup_git_directory();
18
+
source = odb_find_source(the_repository->objects, object_dir);
19
+
if (!source)
20
+
source = odb_add_to_alternates_memory(the_repository->objects,
21
+
object_dir);
22
+
return load_multi_pack_index(source);
23
+
}
24
+
14
25
static int read_midx_file(const char *object_dir, const char *checksum,
15
26
int show_objects)
16
27
{
17
28
uint32_t i;
18
29
struct multi_pack_index *m;
19
30
20
-
setup_git_directory();
21
-
m = load_multi_pack_index(the_repository, object_dir, 1);
31
+
m = setup_midx(object_dir);
22
32
23
33
if (!m)
24
34
return 1;
···
81
91
{
82
92
struct multi_pack_index *m;
83
93
84
-
setup_git_directory();
85
-
m = load_multi_pack_index(the_repository, object_dir, 1);
94
+
m = setup_midx(object_dir);
86
95
if (!m)
87
96
return 1;
88
97
printf("%s\n", hash_to_hex(get_midx_checksum(m)));
···
96
105
struct multi_pack_index *midx = NULL;
97
106
uint32_t preferred_pack;
98
107
99
-
setup_git_directory();
100
-
101
-
midx = load_multi_pack_index(the_repository, object_dir, 1);
108
+
midx = setup_midx(object_dir);
102
109
if (!midx)
103
110
return 1;
104
111
···
119
126
struct bitmapped_pack pack;
120
127
uint32_t i;
121
128
122
-
setup_git_directory();
123
-
124
-
midx = load_multi_pack_index(the_repository, object_dir, 1);
129
+
midx = setup_midx(object_dir);
125
130
if (!midx)
126
131
return 1;
127
132
+4
-4
t/t5319-multi-pack-index.sh
+4
-4
t/t5319-multi-pack-index.sh
···
28
28
EOF
29
29
if test $NUM_PACKS -ge 1
30
30
then
31
-
ls $OBJECT_DIR/pack/ | grep idx | sort
31
+
ls "$OBJECT_DIR"/pack/ | grep idx | sort
32
32
fi &&
33
33
printf "object-dir: $OBJECT_DIR\n"
34
34
} >expect &&
35
-
test-tool read-midx $OBJECT_DIR >actual &&
35
+
test-tool read-midx "$OBJECT_DIR" >actual &&
36
36
test_cmp expect actual
37
37
}
38
38
···
305
305
306
306
ofs=$(git show-index <objects/pack/test-BC-$bc.idx | grep $b |
307
307
cut -d" " -f1) &&
308
-
printf "%s %s\tobjects/pack/test-BC-%s.pack\n" \
308
+
printf "%s %s\t./objects/pack/test-BC-%s.pack\n" \
309
309
"$b" "$ofs" "$bc" >expect &&
310
310
grep ^$b out >actual &&
311
311
···
639
639
( cd ../objects64 && pwd ) >.git/objects/info/alternates &&
640
640
midx64=$(git multi-pack-index --object-dir=../objects64 write)
641
641
) &&
642
-
midx_read_expect 1 63 5 objects64 " large-offsets"
642
+
midx_read_expect 1 63 5 "$(pwd)/objects64" " large-offsets"
643
643
'
644
644
645
645
test_expect_success 'verify multi-pack-index with 64-bit offsets' '