freshlybakedca.ke
Git fork
1#define USE_THE_REPOSITORY_VARIABLE
2#define DISABLE_SIGN_COMPARE_WARNINGS
3
4#include "git-compat-util.h"
5#include "environment.h"
6#include "ewah/ewok.h"
7#include "gettext.h"
8#include "name-hash.h"
9#include "read-cache-ll.h"
10#include "repository.h"
11#include "sparse-index.h"
12#include "tree.h"
13#include "pathspec.h"
14#include "trace2.h"
15#include "cache-tree.h"
16#include "config.h"
17#include "dir.h"
18#include "fsmonitor-ll.h"
19#include "advice.h"
20
21/**
22 * This global is used by expand_index() to determine if we should give the
23 * advice for advice.sparseIndexExpanded when expanding a sparse index to a full
24 * one. However, this is sometimes done on purpose, such as in the sparse-checkout
25 * builtin, even when index.sparse=false. This may be disabled in
26 * convert_to_sparse() or by commands that know they will lead to a full
27 * expansion, but this message is not actionable.
28 */
29int give_advice_on_expansion = 1;
30#define ADVICE_MSG \
31 "The sparse index is expanding to a full index, a slow operation.\n" \
32 "Your working directory likely has contents that are outside of\n" \
33 "your sparse-checkout patterns. Use 'git sparse-checkout list' to\n" \
34 "see your sparse-checkout definition and compare it to your working\n" \
35 "directory contents. Cleaning up any merge conflicts or staged\n" \
36 "changes before running 'git sparse-checkout clean' or 'git\n" \
37 "sparse-checkout reapply' may assist in this cleanup."
38
39struct modify_index_context {
40 struct index_state *write;
41 struct pattern_list *pl;
42};
43
44static struct cache_entry *construct_sparse_dir_entry(
45 struct index_state *istate,
46 const char *sparse_dir,
47 struct cache_tree *tree)
48{
49 struct cache_entry *de;
50
51 de = make_cache_entry(istate, S_IFDIR, &tree->oid, sparse_dir, 0, 0);
52
53 de->ce_flags |= CE_SKIP_WORKTREE;
54 return de;
55}
56
57/*
58 * Returns the number of entries "inserted" into the index.
59 */
60static int convert_to_sparse_rec(struct index_state *istate,
61 int num_converted,
62 int start, int end,
63 const char *ct_path, size_t ct_pathlen,
64 struct cache_tree *ct)
65{
66 int i, can_convert = 1;
67 int start_converted = num_converted;
68 struct strbuf child_path = STRBUF_INIT;
69
70 /*
71 * Is the current path outside of the sparse cone?
72 * Then check if the region can be replaced by a sparse
73 * directory entry (everything is sparse and merged).
74 */
75 if (path_in_sparse_checkout(ct_path, istate))
76 can_convert = 0;
77
78 for (i = start; can_convert && i < end; i++) {
79 struct cache_entry *ce = istate->cache[i];
80
81 if (ce_stage(ce) ||
82 S_ISGITLINK(ce->ce_mode) ||
83 !(ce->ce_flags & CE_SKIP_WORKTREE))
84 can_convert = 0;
85 }
86
87 if (can_convert) {
88 struct cache_entry *se;
89 se = construct_sparse_dir_entry(istate, ct_path, ct);
90
91 istate->cache[num_converted++] = se;
92 return 1;
93 }
94
95 for (i = start; i < end; ) {
96 int count, span, pos = -1;
97 const char *base, *slash;
98 struct cache_entry *ce = istate->cache[i];
99
100 /*
101 * Detect if this is a normal entry outside of any subtree
102 * entry.
103 */
104 base = ce->name + ct_pathlen;
105 slash = strchr(base, '/');
106
107 if (slash)
108 pos = cache_tree_subtree_pos(ct, base, slash - base);
109
110 if (pos < 0) {
111 istate->cache[num_converted++] = ce;
112 i++;
113 continue;
114 }
115
116 strbuf_setlen(&child_path, 0);
117 strbuf_add(&child_path, ce->name, slash - ce->name + 1);
118
119 span = ct->down[pos]->cache_tree->entry_count;
120 count = convert_to_sparse_rec(istate,
121 num_converted, i, i + span,
122 child_path.buf, child_path.len,
123 ct->down[pos]->cache_tree);
124 num_converted += count;
125 i += span;
126 }
127
128 strbuf_release(&child_path);
129 return num_converted - start_converted;
130}
131
132int set_sparse_index_config(struct repository *repo, int enable)
133{
134 int res = repo_config_set_worktree_gently(repo,
135 "index.sparse",
136 enable ? "true" : "false");
137 prepare_repo_settings(repo);
138 repo->settings.sparse_index = enable;
139 return res;
140}
141
142static int index_has_unmerged_entries(struct index_state *istate)
143{
144 int i;
145 for (i = 0; i < istate->cache_nr; i++) {
146 if (ce_stage(istate->cache[i]))
147 return 1;
148 }
149
150 return 0;
151}
152
153int is_sparse_index_allowed(struct index_state *istate, int flags)
154{
155 if (!core_apply_sparse_checkout || !core_sparse_checkout_cone)
156 return 0;
157
158 if (!(flags & SPARSE_INDEX_MEMORY_ONLY)) {
159 int test_env;
160
161 /*
162 * The sparse index is not (yet) integrated with a split index.
163 */
164 if (istate->split_index || git_env_bool("GIT_TEST_SPLIT_INDEX", 0))
165 return 0;
166 /*
167 * The GIT_TEST_SPARSE_INDEX environment variable triggers the
168 * index.sparse config variable to be on.
169 */
170 test_env = git_env_bool("GIT_TEST_SPARSE_INDEX", -1);
171 if (test_env >= 0)
172 set_sparse_index_config(istate->repo, test_env);
173
174 /*
175 * Only convert to sparse if index.sparse is set.
176 */
177 prepare_repo_settings(istate->repo);
178 if (!istate->repo->settings.sparse_index)
179 return 0;
180 }
181
182 if (init_sparse_checkout_patterns(istate))
183 return 0;
184
185 /*
186 * We need cone-mode patterns to use sparse-index. If a user edits
187 * their sparse-checkout file manually, then we can detect during
188 * parsing that they are not actually using cone-mode patterns and
189 * hence we need to abort this conversion _without error_. Warnings
190 * already exist in the pattern parsing to inform the user of their
191 * bad patterns.
192 */
193 if (!istate->sparse_checkout_patterns->use_cone_patterns)
194 return 0;
195
196 return 1;
197}
198
199int convert_to_sparse(struct index_state *istate, int flags)
200{
201 /*
202 * If the index is already sparse, empty, or otherwise
203 * cannot be converted to sparse, do not convert.
204 */
205 if (istate->sparse_index == INDEX_COLLAPSED || !istate->cache_nr ||
206 !is_sparse_index_allowed(istate, flags))
207 return 0;
208
209 /*
210 * If we are purposefully collapsing a full index, then don't give
211 * advice when it is expanded later.
212 */
213 give_advice_on_expansion = 0;
214
215 /*
216 * NEEDSWORK: If we have unmerged entries, then stay full.
217 * Unmerged entries prevent the cache-tree extension from working.
218 */
219 if (index_has_unmerged_entries(istate))
220 return 0;
221
222 if (!cache_tree_fully_valid(istate->cache_tree)) {
223 /* Clear and recompute the cache-tree */
224 cache_tree_free(&istate->cache_tree);
225
226 /*
227 * Silently return if there is a problem with the cache tree update,
228 * which might just be due to a conflict state in some entry.
229 *
230 * This might create new tree objects, so be sure to use
231 * WRITE_TREE_MISSING_OK.
232 */
233 if (cache_tree_update(istate, WRITE_TREE_MISSING_OK))
234 return 0;
235 }
236
237 remove_fsmonitor(istate);
238
239 trace2_region_enter("index", "convert_to_sparse", istate->repo);
240 istate->cache_nr = convert_to_sparse_rec(istate,
241 0, 0, istate->cache_nr,
242 "", 0, istate->cache_tree);
243
244 /* Clear and recompute the cache-tree */
245 cache_tree_free(&istate->cache_tree);
246 cache_tree_update(istate, 0);
247
248 istate->fsmonitor_has_run_once = 0;
249 ewah_free(istate->fsmonitor_dirty);
250 istate->fsmonitor_dirty = NULL;
251 FREE_AND_NULL(istate->fsmonitor_last_update);
252
253 istate->sparse_index = INDEX_COLLAPSED;
254 trace2_region_leave("index", "convert_to_sparse", istate->repo);
255 return 0;
256}
257
258static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
259{
260 ALLOC_GROW(istate->cache, nr + 1, istate->cache_alloc);
261
262 istate->cache[nr] = ce;
263 add_name_hash(istate, ce);
264}
265
266static int add_path_to_index(const struct object_id *oid,
267 struct strbuf *base, const char *path,
268 unsigned int mode, void *context)
269{
270 struct modify_index_context *ctx = (struct modify_index_context *)context;
271 struct cache_entry *ce;
272 size_t len = base->len;
273
274 if (S_ISDIR(mode)) {
275 int dtype;
276 size_t baselen = base->len;
277 if (!ctx->pl)
278 return READ_TREE_RECURSIVE;
279
280 /*
281 * Have we expanded to a point outside of the sparse-checkout?
282 *
283 * Artificially pad the path name with a slash "/" to
284 * indicate it as a directory, and add an arbitrary file
285 * name ("-") so we can consider base->buf as a file name
286 * to match against the cone-mode patterns.
287 *
288 * If we compared just "path", then we would expand more
289 * than we should. Since every file at root is always
290 * included, we would expand every directory at root at
291 * least one level deep instead of using sparse directory
292 * entries.
293 */
294 strbuf_addstr(base, path);
295 strbuf_add(base, "/-", 2);
296
297 if (path_matches_pattern_list(base->buf, base->len,
298 NULL, &dtype,
299 ctx->pl, ctx->write)) {
300 strbuf_setlen(base, baselen);
301 return READ_TREE_RECURSIVE;
302 }
303
304 /*
305 * The path "{base}{path}/" is a sparse directory. Create the correct
306 * name for inserting the entry into the index.
307 */
308 strbuf_setlen(base, base->len - 1);
309 } else {
310 strbuf_addstr(base, path);
311 }
312
313 ce = make_cache_entry(ctx->write, mode, oid, base->buf, 0, 0);
314 ce->ce_flags |= CE_SKIP_WORKTREE | CE_EXTENDED;
315 set_index_entry(ctx->write, ctx->write->cache_nr++, ce);
316
317 strbuf_setlen(base, len);
318 return 0;
319}
320
321void expand_index(struct index_state *istate, struct pattern_list *pl)
322{
323 int i;
324 struct index_state *full;
325 struct strbuf base = STRBUF_INIT;
326 const char *tr_region;
327 struct modify_index_context ctx;
328
329 /*
330 * If the index is already full, then keep it full. We will convert
331 * it to a sparse index on write, if possible.
332 */
333 if (istate->sparse_index == INDEX_EXPANDED)
334 return;
335
336 /*
337 * If our index is sparse, but our new pattern set does not use
338 * cone mode patterns, then we need to expand the index before we
339 * continue. A NULL pattern set indicates a full expansion to a
340 * full index.
341 */
342 if (pl && !pl->use_cone_patterns) {
343 pl = NULL;
344 } else {
345 /*
346 * We might contract file entries into sparse-directory
347 * entries, and for that we will need the cache tree to
348 * be recomputed.
349 */
350 cache_tree_free(&istate->cache_tree);
351
352 /*
353 * If there is a problem creating the cache tree, then we
354 * need to expand to a full index since we cannot satisfy
355 * the current request as a sparse index.
356 */
357 if (cache_tree_update(istate, 0))
358 pl = NULL;
359 }
360
361 if (!pl && give_advice_on_expansion) {
362 give_advice_on_expansion = 0;
363 advise_if_enabled(ADVICE_SPARSE_INDEX_EXPANDED,
364 _(ADVICE_MSG));
365 }
366
367 /*
368 * A NULL pattern set indicates we are expanding a full index, so
369 * we use a special region name that indicates the full expansion.
370 * This is used by test cases, but also helps to differentiate the
371 * two cases.
372 */
373 tr_region = pl ? "expand_index" : "ensure_full_index";
374 trace2_region_enter("index", tr_region, istate->repo);
375
376 /* initialize basics of new index */
377 full = xcalloc(1, sizeof(struct index_state));
378 memcpy(full, istate, sizeof(struct index_state));
379
380 /*
381 * This slightly-misnamed 'full' index might still be sparse if we
382 * are only modifying the list of sparse directories. This hinges
383 * on whether we have a non-NULL pattern list.
384 */
385 full->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED;
386
387 /* then change the necessary things */
388 full->cache_alloc = (3 * istate->cache_alloc) / 2;
389 full->cache_nr = 0;
390 ALLOC_ARRAY(full->cache, full->cache_alloc);
391
392 ctx.write = full;
393 ctx.pl = pl;
394
395 for (i = 0; i < istate->cache_nr; i++) {
396 struct cache_entry *ce = istate->cache[i];
397 struct tree *tree;
398 struct pathspec ps;
399 int dtype;
400
401 if (!S_ISSPARSEDIR(ce->ce_mode)) {
402 set_index_entry(full, full->cache_nr++, ce);
403 continue;
404 }
405
406 /* We now have a sparse directory entry. Should we expand? */
407 if (pl &&
408 path_matches_pattern_list(ce->name, ce->ce_namelen,
409 NULL, &dtype,
410 pl, istate) == NOT_MATCHED) {
411 set_index_entry(full, full->cache_nr++, ce);
412 continue;
413 }
414
415 if (!(ce->ce_flags & CE_SKIP_WORKTREE))
416 warning(_("index entry is a directory, but not sparse (%08x)"),
417 ce->ce_flags);
418
419 /* recursively walk into cd->name */
420 tree = lookup_tree(istate->repo, &ce->oid);
421
422 memset(&ps, 0, sizeof(ps));
423 ps.recursive = 1;
424 ps.has_wildcard = 1;
425 ps.max_depth = -1;
426
427 strbuf_setlen(&base, 0);
428 strbuf_add(&base, ce->name, strlen(ce->name));
429
430 read_tree_at(istate->repo, tree, &base, 0, &ps,
431 add_path_to_index, &ctx);
432
433 /* free directory entries. full entries are re-used */
434 discard_cache_entry(ce);
435 }
436
437 /* Copy back into original index. */
438 memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash));
439 memcpy(&istate->dir_hash, &full->dir_hash, sizeof(full->dir_hash));
440 istate->sparse_index = pl ? INDEX_PARTIALLY_SPARSE : INDEX_EXPANDED;
441 free(istate->cache);
442 istate->cache = full->cache;
443 istate->cache_nr = full->cache_nr;
444 istate->cache_alloc = full->cache_alloc;
445 istate->fsmonitor_has_run_once = 0;
446 ewah_free(istate->fsmonitor_dirty);
447 istate->fsmonitor_dirty = NULL;
448 FREE_AND_NULL(istate->fsmonitor_last_update);
449
450 strbuf_release(&base);
451 free(full);
452
453 /* Clear and recompute the cache-tree */
454 cache_tree_free(&istate->cache_tree);
455 cache_tree_update(istate, 0);
456
457 trace2_region_leave("index", tr_region, istate->repo);
458}
459
460void ensure_full_index(struct index_state *istate)
461{
462 if (!istate)
463 BUG("ensure_full_index() must get an index!");
464 expand_index(istate, NULL);
465}
466
467void ensure_correct_sparsity(struct index_state *istate)
468{
469 /*
470 * If the index can be sparse, make it sparse. Otherwise,
471 * ensure the index is full.
472 */
473 if (is_sparse_index_allowed(istate, 0))
474 convert_to_sparse(istate, 0);
475 else
476 ensure_full_index(istate);
477}
478
479struct path_found_data {
480 /**
481 * The path stored in 'dir', if non-empty, corresponds to the most-
482 * recent path that we checked where:
483 *
484 * 1. The path should be a directory, according to the index.
485 * 2. The path does not exist.
486 * 3. The parent path _does_ exist. (This may be the root of the
487 * working directory.)
488 */
489 struct strbuf dir;
490 size_t lstat_count;
491};
492
493#define PATH_FOUND_DATA_INIT { \
494 .dir = STRBUF_INIT \
495}
496
497static void clear_path_found_data(struct path_found_data *data)
498{
499 strbuf_release(&data->dir);
500}
501
502/**
503 * Return the length of the longest common substring that ends in a
504 * slash ('/') to indicate the longest common parent directory. Returns
505 * zero if no common directory exists.
506 */
507static size_t max_common_dir_prefix(const char *path1, const char *path2)
508{
509 size_t common_prefix = 0;
510 for (size_t i = 0; path1[i] && path2[i]; i++) {
511 if (path1[i] != path2[i])
512 break;
513
514 /*
515 * If they agree at a directory separator, then add one
516 * to make sure it is included in the common prefix string.
517 */
518 if (path1[i] == '/')
519 common_prefix = i + 1;
520 }
521
522 return common_prefix;
523}
524
525static int path_found(const char *path, struct path_found_data *data)
526{
527 struct stat st;
528 size_t common_prefix;
529
530 /*
531 * If data->dir is non-empty, then it contains a path that doesn't
532 * exist, including an ending slash ('/'). If it is a prefix of 'path',
533 * then we can return 0.
534 */
535 if (data->dir.len && !memcmp(path, data->dir.buf, data->dir.len))
536 return 0;
537
538 /*
539 * Otherwise, we must check if the current path exists. If it does, then
540 * return 1. The cached directory will be skipped until we come across
541 * a missing path again.
542 */
543 data->lstat_count++;
544 if (!lstat(path, &st))
545 return 1;
546
547 /*
548 * At this point, we know that 'path' doesn't exist, and we know that
549 * the parent directory of 'data->dir' does exist. Let's set 'data->dir'
550 * to be the top-most non-existing directory of 'path'. If the first
551 * parent of 'path' exists, then we will act as though 'path'
552 * corresponds to a directory (by adding a slash).
553 */
554 common_prefix = max_common_dir_prefix(path, data->dir.buf);
555
556 /*
557 * At this point, 'path' and 'data->dir' have a common existing parent
558 * directory given by path[0..common_prefix] (which could have length 0).
559 * We "grow" the data->dir buffer by checking for existing directories
560 * along 'path'.
561 */
562
563 strbuf_setlen(&data->dir, common_prefix);
564 while (1) {
565 /* Find the next directory in 'path'. */
566 const char *rest = path + data->dir.len;
567 const char *next_slash = strchr(rest, '/');
568
569 /*
570 * If there are no more slashes, then 'path' doesn't contain a
571 * non-existent _parent_ directory. Set 'data->dir' to be equal
572 * to 'path' plus an additional slash, so it can be used for
573 * caching in the future. The filename of 'path' is considered
574 * a non-existent directory.
575 *
576 * Note: if "{path}/" exists as a directory, then it will never
577 * appear as a prefix of other callers to this method, assuming
578 * the context from the clear_skip_worktree... methods. If this
579 * method is reused, then this must be reconsidered.
580 */
581 if (!next_slash) {
582 strbuf_addstr(&data->dir, rest);
583 strbuf_addch(&data->dir, '/');
584 break;
585 }
586
587 /*
588 * Now that we have a slash, let's grow 'data->dir' to include
589 * this slash, then test if we should stop.
590 */
591 strbuf_add(&data->dir, rest, next_slash - rest + 1);
592
593 /* If the parent dir doesn't exist, then stop here. */
594 data->lstat_count++;
595 if (lstat(data->dir.buf, &st))
596 return 0;
597 }
598
599 /*
600 * At this point, 'data->dir' is equal to 'path' plus a slash character,
601 * and the parent directory of 'path' definitely exists. Moreover, we
602 * know that 'path' doesn't exist, or we would have returned 1 earlier.
603 */
604 return 0;
605}
606
607static int clear_skip_worktree_from_present_files_sparse(struct index_state *istate)
608{
609 struct path_found_data data = PATH_FOUND_DATA_INIT;
610
611 int path_count = 0;
612 int to_restart = 0;
613
614 trace2_region_enter("index", "clear_skip_worktree_from_present_files_sparse",
615 istate->repo);
616 for (int i = 0; i < istate->cache_nr; i++) {
617 struct cache_entry *ce = istate->cache[i];
618
619 if (ce_skip_worktree(ce)) {
620 path_count++;
621 if (path_found(ce->name, &data)) {
622 if (S_ISSPARSEDIR(ce->ce_mode)) {
623 to_restart = 1;
624 break;
625 }
626 ce->ce_flags &= ~CE_SKIP_WORKTREE;
627 }
628 }
629 }
630
631 trace2_data_intmax("index", istate->repo,
632 "sparse_path_count", path_count);
633 trace2_data_intmax("index", istate->repo,
634 "sparse_lstat_count", data.lstat_count);
635 trace2_region_leave("index", "clear_skip_worktree_from_present_files_sparse",
636 istate->repo);
637 clear_path_found_data(&data);
638 return to_restart;
639}
640
641static void clear_skip_worktree_from_present_files_full(struct index_state *istate)
642{
643 struct path_found_data data = PATH_FOUND_DATA_INIT;
644
645 int path_count = 0;
646
647 trace2_region_enter("index", "clear_skip_worktree_from_present_files_full",
648 istate->repo);
649 for (int i = 0; i < istate->cache_nr; i++) {
650 struct cache_entry *ce = istate->cache[i];
651
652 if (S_ISSPARSEDIR(ce->ce_mode))
653 BUG("ensure-full-index did not fully flatten?");
654
655 if (ce_skip_worktree(ce)) {
656 path_count++;
657 if (path_found(ce->name, &data))
658 ce->ce_flags &= ~CE_SKIP_WORKTREE;
659 }
660 }
661
662 trace2_data_intmax("index", istate->repo,
663 "full_path_count", path_count);
664 trace2_data_intmax("index", istate->repo,
665 "full_lstat_count", data.lstat_count);
666 trace2_region_leave("index", "clear_skip_worktree_from_present_files_full",
667 istate->repo);
668 clear_path_found_data(&data);
669}
670
671void clear_skip_worktree_from_present_files(struct index_state *istate)
672{
673 if (!core_apply_sparse_checkout ||
674 sparse_expect_files_outside_of_patterns)
675 return;
676
677 if (clear_skip_worktree_from_present_files_sparse(istate)) {
678 ensure_full_index(istate);
679 clear_skip_worktree_from_present_files_full(istate);
680 }
681}
682
683/*
684 * This static global helps avoid infinite recursion between
685 * expand_to_path() and index_file_exists().
686 */
687static int in_expand_to_path = 0;
688
689void expand_to_path(struct index_state *istate,
690 const char *path, size_t pathlen, int icase)
691{
692 struct strbuf path_mutable = STRBUF_INIT;
693 size_t substr_len;
694
695 /* prevent extra recursion */
696 if (in_expand_to_path)
697 return;
698
699 if (!istate->sparse_index)
700 return;
701
702 in_expand_to_path = 1;
703
704 /*
705 * We only need to actually expand a region if the
706 * following are both true:
707 *
708 * 1. 'path' is not already in the index.
709 * 2. Some parent directory of 'path' is a sparse directory.
710 */
711
712 if (index_file_exists(istate, path, pathlen, icase))
713 goto cleanup;
714
715 strbuf_add(&path_mutable, path, pathlen);
716 strbuf_addch(&path_mutable, '/');
717
718 /* Check the name hash for all parent directories */
719 substr_len = 0;
720 while (substr_len < pathlen) {
721 char temp;
722 char *replace = strchr(path_mutable.buf + substr_len, '/');
723
724 if (!replace)
725 break;
726
727 /* replace the character _after_ the slash */
728 replace++;
729 temp = *replace;
730 *replace = '\0';
731 substr_len = replace - path_mutable.buf;
732 if (index_file_exists(istate, path_mutable.buf,
733 substr_len, icase)) {
734 /*
735 * We found a parent directory in the name-hash
736 * hashtable, because only sparse directory entries
737 * have a trailing '/' character. Since "path" wasn't
738 * in the index, perhaps it exists within this
739 * sparse-directory. Expand accordingly.
740 */
741 ensure_full_index(istate);
742 break;
743 }
744
745 *replace = temp;
746 }
747
748cleanup:
749 strbuf_release(&path_mutable);
750 in_expand_to_path = 0;
751}