freshlybakedca.ke
Git fork
1#define USE_THE_REPOSITORY_VARIABLE
2
3#include "git-compat-util.h"
4#include "odb.h"
5#include "dir.h"
6#include "environment.h"
7#include "gettext.h"
8#include "hex.h"
9#include "name-hash.h"
10#include "sparse-index.h"
11#include "streaming.h"
12#include "submodule.h"
13#include "symlinks.h"
14#include "progress.h"
15#include "fsmonitor.h"
16#include "entry.h"
17#include "parallel-checkout.h"
18
19static void create_directories(const char *path, int path_len,
20 const struct checkout *state)
21{
22 char *buf = xmallocz(path_len);
23 int len = 0;
24
25 while (len < path_len) {
26 do {
27 buf[len] = path[len];
28 len++;
29 } while (len < path_len && path[len] != '/');
30 if (len >= path_len)
31 break;
32 buf[len] = 0;
33
34 /*
35 * For 'checkout-index --prefix=<dir>', <dir> is
36 * allowed to be a symlink to an existing directory,
37 * and we set 'state->base_dir_len' below, such that
38 * we test the path components of the prefix with the
39 * stat() function instead of the lstat() function.
40 */
41 if (has_dirs_only_path(buf, len, state->base_dir_len))
42 continue; /* ok, it is already a directory. */
43
44 /*
45 * If this mkdir() would fail, it could be that there
46 * is already a symlink or something else exists
47 * there, therefore we then try to unlink it and try
48 * one more time to create the directory.
49 */
50 if (mkdir(buf, 0777)) {
51 if (errno == EEXIST && state->force &&
52 !unlink_or_warn(buf) && !mkdir(buf, 0777))
53 continue;
54 die_errno("cannot create directory at '%s'", buf);
55 }
56 }
57 free(buf);
58}
59
60static void remove_subtree(struct strbuf *path)
61{
62 DIR *dir = opendir(path->buf);
63 struct dirent *de;
64 int origlen = path->len;
65
66 if (!dir)
67 die_errno("cannot opendir '%s'", path->buf);
68 while ((de = readdir_skip_dot_and_dotdot(dir)) != NULL) {
69 struct stat st;
70
71 strbuf_addch(path, '/');
72 strbuf_addstr(path, de->d_name);
73 if (lstat(path->buf, &st))
74 die_errno("cannot lstat '%s'", path->buf);
75 if (S_ISDIR(st.st_mode))
76 remove_subtree(path);
77 else if (unlink(path->buf))
78 die_errno("cannot unlink '%s'", path->buf);
79 strbuf_setlen(path, origlen);
80 }
81 closedir(dir);
82 if (rmdir(path->buf))
83 die_errno("cannot rmdir '%s'", path->buf);
84}
85
86static int create_file(const char *path, unsigned int mode)
87{
88 mode = (mode & 0100) ? 0777 : 0666;
89 return open(path, O_WRONLY | O_CREAT | O_EXCL, mode);
90}
91
92void *read_blob_entry(const struct cache_entry *ce, size_t *size)
93{
94 enum object_type type;
95 unsigned long ul;
96 void *blob_data = odb_read_object(the_repository->objects, &ce->oid,
97 &type, &ul);
98
99 *size = ul;
100 if (blob_data) {
101 if (type == OBJ_BLOB)
102 return blob_data;
103 free(blob_data);
104 }
105 return NULL;
106}
107
108static int open_output_fd(char *path, const struct cache_entry *ce, int to_tempfile)
109{
110 int symlink = (ce->ce_mode & S_IFMT) != S_IFREG;
111 if (to_tempfile) {
112 xsnprintf(path, TEMPORARY_FILENAME_LENGTH, "%s",
113 symlink ? ".merge_link_XXXXXX" : ".merge_file_XXXXXX");
114 return mkstemp(path);
115 } else {
116 return create_file(path, !symlink ? ce->ce_mode : 0666);
117 }
118}
119
120int fstat_checkout_output(int fd, const struct checkout *state, struct stat *st)
121{
122 /* use fstat() only when path == ce->name */
123 if (fstat_is_reliable() &&
124 state->refresh_cache && !state->base_dir_len) {
125 return !fstat(fd, st);
126 }
127 return 0;
128}
129
130static int streaming_write_entry(const struct cache_entry *ce, char *path,
131 struct stream_filter *filter,
132 const struct checkout *state, int to_tempfile,
133 int *fstat_done, struct stat *statbuf)
134{
135 int result = 0;
136 int fd;
137
138 fd = open_output_fd(path, ce, to_tempfile);
139 if (fd < 0)
140 return -1;
141
142 result |= stream_blob_to_fd(fd, &ce->oid, filter, 1);
143 *fstat_done = fstat_checkout_output(fd, state, statbuf);
144 result |= close(fd);
145
146 if (result)
147 unlink(path);
148 return result;
149}
150
151void enable_delayed_checkout(struct checkout *state)
152{
153 if (!state->delayed_checkout) {
154 state->delayed_checkout = xmalloc(sizeof(*state->delayed_checkout));
155 state->delayed_checkout->state = CE_CAN_DELAY;
156 string_list_init_nodup(&state->delayed_checkout->filters);
157 string_list_init_nodup(&state->delayed_checkout->paths);
158 }
159}
160
161static int remove_available_paths(struct string_list_item *item, void *cb_data)
162{
163 struct string_list *available_paths = cb_data;
164 struct string_list_item *available;
165
166 available = string_list_lookup(available_paths, item->string);
167 if (available)
168 available->util = item->util;
169 return !available;
170}
171
172static int string_is_not_null(struct string_list_item *item, void *data UNUSED)
173{
174 return !!item->string;
175}
176
177int finish_delayed_checkout(struct checkout *state, int show_progress)
178{
179 int errs = 0;
180 unsigned processed_paths = 0;
181 off_t filtered_bytes = 0;
182 struct string_list_item *filter, *path;
183 struct progress *progress = NULL;
184 struct delayed_checkout *dco = state->delayed_checkout;
185
186 if (!state->delayed_checkout)
187 return errs;
188
189 dco->state = CE_RETRY;
190 if (show_progress)
191 progress = start_delayed_progress(the_repository,
192 _("Filtering content"),
193 dco->paths.nr);
194 while (dco->filters.nr > 0) {
195 for_each_string_list_item(filter, &dco->filters) {
196 struct string_list available_paths = STRING_LIST_INIT_DUP;
197
198 if (!async_query_available_blobs(filter->string, &available_paths)) {
199 /* Filter reported an error */
200 errs = 1;
201 filter->string = NULL;
202 continue;
203 }
204 if (available_paths.nr <= 0) {
205 /*
206 * Filter responded with no entries. That means
207 * the filter is done and we can remove the
208 * filter from the list (see
209 * "string_list_remove_empty_items" call below).
210 */
211 filter->string = NULL;
212 continue;
213 }
214
215 /*
216 * In dco->paths we store a list of all delayed paths.
217 * The filter just send us a list of available paths.
218 * Remove them from the list.
219 */
220 filter_string_list(&dco->paths, 0,
221 &remove_available_paths, &available_paths);
222
223 for_each_string_list_item(path, &available_paths) {
224 struct cache_entry* ce;
225
226 if (!path->util) {
227 error("external filter '%s' signaled that '%s' "
228 "is now available although it has not been "
229 "delayed earlier",
230 filter->string, path->string);
231 errs |= 1;
232
233 /*
234 * Do not ask the filter for available blobs,
235 * again, as the filter is likely buggy.
236 */
237 filter->string = NULL;
238 continue;
239 }
240 ce = index_file_exists(state->istate, path->string,
241 strlen(path->string), 0);
242 if (ce) {
243 display_progress(progress, ++processed_paths);
244 errs |= checkout_entry(ce, state, NULL, path->util);
245 filtered_bytes += ce->ce_stat_data.sd_size;
246 display_throughput(progress, filtered_bytes);
247 } else
248 errs = 1;
249 }
250
251 string_list_clear(&available_paths, 0);
252 }
253
254 filter_string_list(&dco->filters, 0, string_is_not_null, NULL);
255 }
256 stop_progress(&progress);
257 string_list_clear(&dco->filters, 0);
258
259 /* At this point we should not have any delayed paths anymore. */
260 errs |= dco->paths.nr;
261 for_each_string_list_item(path, &dco->paths) {
262 error("'%s' was not filtered properly", path->string);
263 }
264 string_list_clear(&dco->paths, 0);
265
266 free(dco);
267 state->delayed_checkout = NULL;
268
269 return errs;
270}
271
272void update_ce_after_write(const struct checkout *state, struct cache_entry *ce,
273 struct stat *st)
274{
275 if (state->refresh_cache) {
276 assert(state->istate);
277 fill_stat_cache_info(state->istate, ce, st);
278 ce->ce_flags |= CE_UPDATE_IN_BASE;
279 mark_fsmonitor_invalid(state->istate, ce);
280 state->istate->cache_changed |= CE_ENTRY_CHANGED;
281 }
282}
283
284/* Note: ca is used (and required) iff the entry refers to a regular file. */
285static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca,
286 const struct checkout *state, int to_tempfile,
287 int *nr_checkouts)
288{
289 unsigned int ce_mode_s_ifmt = ce->ce_mode & S_IFMT;
290 struct delayed_checkout *dco = state->delayed_checkout;
291 int fd, ret, fstat_done = 0;
292 char *new_blob;
293 struct strbuf buf = STRBUF_INIT;
294 size_t size;
295 ssize_t wrote;
296 size_t newsize = 0;
297 struct stat st;
298 const struct submodule *sub;
299 struct checkout_metadata meta;
300 static int scratch_nr_checkouts;
301
302 clone_checkout_metadata(&meta, &state->meta, &ce->oid);
303
304 if (ce_mode_s_ifmt == S_IFREG) {
305 struct stream_filter *filter = get_stream_filter_ca(ca, &ce->oid);
306 if (filter &&
307 !streaming_write_entry(ce, path, filter,
308 state, to_tempfile,
309 &fstat_done, &st))
310 goto finish;
311 }
312
313 switch (ce_mode_s_ifmt) {
314 case S_IFLNK:
315 new_blob = read_blob_entry(ce, &size);
316 if (!new_blob)
317 return error("unable to read sha1 file of %s (%s)",
318 ce->name, oid_to_hex(&ce->oid));
319
320 /*
321 * We can't make a real symlink; write out a regular file entry
322 * with the symlink destination as its contents.
323 */
324 if (!has_symlinks || to_tempfile)
325 goto write_file_entry;
326
327 ret = symlink(new_blob, path);
328 free(new_blob);
329 if (ret)
330 return error_errno("unable to create symlink %s", path);
331 break;
332
333 case S_IFREG:
334 /*
335 * We do not send the blob in case of a retry, so do not
336 * bother reading it at all.
337 */
338 if (dco && dco->state == CE_RETRY) {
339 new_blob = NULL;
340 size = 0;
341 } else {
342 new_blob = read_blob_entry(ce, &size);
343 if (!new_blob)
344 return error("unable to read sha1 file of %s (%s)",
345 ce->name, oid_to_hex(&ce->oid));
346 }
347
348 /*
349 * Convert from git internal format to working tree format
350 */
351 if (dco && dco->state != CE_NO_DELAY) {
352 ret = async_convert_to_working_tree_ca(ca, ce->name,
353 new_blob, size,
354 &buf, &meta, dco);
355 if (ret) {
356 struct string_list_item *item =
357 string_list_lookup(&dco->paths, ce->name);
358 if (item) {
359 item->util = nr_checkouts ? nr_checkouts
360 : &scratch_nr_checkouts;
361 free(new_blob);
362 goto delayed;
363 }
364 }
365 } else {
366 ret = convert_to_working_tree_ca(ca, ce->name, new_blob,
367 size, &buf, &meta);
368 }
369
370 if (ret) {
371 free(new_blob);
372 new_blob = strbuf_detach(&buf, &newsize);
373 size = newsize;
374 }
375 /*
376 * No "else" here as errors from convert are OK at this
377 * point. If the error would have been fatal (e.g.
378 * filter is required), then we would have died already.
379 */
380
381 write_file_entry:
382 fd = open_output_fd(path, ce, to_tempfile);
383 if (fd < 0) {
384 free(new_blob);
385 return error_errno("unable to create file %s", path);
386 }
387
388 wrote = write_in_full(fd, new_blob, size);
389 if (!to_tempfile)
390 fstat_done = fstat_checkout_output(fd, state, &st);
391 close(fd);
392 free(new_blob);
393 if (wrote < 0)
394 return error("unable to write file %s", path);
395 break;
396
397 case S_IFGITLINK:
398 if (to_tempfile)
399 return error("cannot create temporary submodule %s", ce->name);
400 if (mkdir(path, 0777) < 0)
401 return error("cannot create submodule directory %s", path);
402 sub = submodule_from_ce(ce);
403 if (sub)
404 return submodule_move_head(ce->name, state->super_prefix,
405 NULL, oid_to_hex(&ce->oid),
406 state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
407 break;
408
409 default:
410 return error("unknown file mode for %s in index", ce->name);
411 }
412
413finish:
414 if (state->refresh_cache) {
415 if (!fstat_done && lstat(ce->name, &st) < 0)
416 return error_errno("unable to stat just-written file %s",
417 ce->name);
418 update_ce_after_write(state, ce , &st);
419 }
420 if (nr_checkouts)
421 (*nr_checkouts)++;
422delayed:
423 return 0;
424}
425
426/*
427 * This is like 'lstat()', except it refuses to follow symlinks
428 * in the path, after skipping "skiplen".
429 */
430static int check_path(const char *path, int len, struct stat *st, int skiplen)
431{
432 const char *slash = path + len;
433
434 while (path < slash && *slash != '/')
435 slash--;
436 if (!has_dirs_only_path(path, slash - path, skiplen)) {
437 errno = ENOENT;
438 return -1;
439 }
440 return lstat(path, st);
441}
442
443static void mark_colliding_entries(const struct checkout *state,
444 struct cache_entry *ce, struct stat *st)
445{
446 int trust_ino = check_stat;
447
448#if defined(GIT_WINDOWS_NATIVE) || defined(__CYGWIN__)
449 trust_ino = 0;
450#endif
451
452 ce->ce_flags |= CE_MATCHED;
453
454 /* TODO: audit for interaction with sparse-index. */
455 ensure_full_index(state->istate);
456 for (size_t i = 0; i < state->istate->cache_nr; i++) {
457 struct cache_entry *dup = state->istate->cache[i];
458
459 if (dup == ce) {
460 /*
461 * Parallel checkout doesn't create the files in index
462 * order. So the other side of the collision may appear
463 * after the given cache_entry in the array.
464 */
465 if (parallel_checkout_status() == PC_RUNNING)
466 continue;
467 else
468 break;
469 }
470
471 if (dup->ce_flags & (CE_MATCHED | CE_VALID | CE_SKIP_WORKTREE))
472 continue;
473
474 if ((trust_ino && !match_stat_data(&dup->ce_stat_data, st)) ||
475 paths_collide(ce->name, dup->name)) {
476 dup->ce_flags |= CE_MATCHED;
477 break;
478 }
479 }
480}
481
482int checkout_entry_ca(struct cache_entry *ce, struct conv_attrs *ca,
483 const struct checkout *state, char *topath,
484 int *nr_checkouts)
485{
486 static struct strbuf path = STRBUF_INIT;
487 struct stat st;
488 struct conv_attrs ca_buf;
489
490 if (ce->ce_flags & CE_WT_REMOVE) {
491 if (topath)
492 /*
493 * No content and thus no path to create, so we have
494 * no pathname to return.
495 */
496 BUG("Can't remove entry to a path");
497 unlink_entry(ce, state->super_prefix);
498 return 0;
499 }
500
501 if (topath) {
502 if (S_ISREG(ce->ce_mode) && !ca) {
503 convert_attrs(state->istate, &ca_buf, ce->name);
504 ca = &ca_buf;
505 }
506 return write_entry(ce, topath, ca, state, 1, nr_checkouts);
507 }
508
509 strbuf_reset(&path);
510 strbuf_add(&path, state->base_dir, state->base_dir_len);
511 strbuf_add(&path, ce->name, ce_namelen(ce));
512
513 if (!check_path(path.buf, path.len, &st, state->base_dir_len)) {
514 const struct submodule *sub;
515 unsigned changed = ie_match_stat(state->istate, ce, &st,
516 CE_MATCH_IGNORE_VALID | CE_MATCH_IGNORE_SKIP_WORKTREE);
517 /*
518 * Needs to be checked before !changed returns early,
519 * as the possibly empty directory was not changed
520 */
521 sub = submodule_from_ce(ce);
522 if (sub) {
523 int err;
524 if (!is_submodule_populated_gently(ce->name, &err)) {
525 struct stat sb;
526 if (lstat(ce->name, &sb))
527 die(_("could not stat file '%s'"), ce->name);
528 if (!(st.st_mode & S_IFDIR))
529 unlink_or_warn(ce->name);
530
531 return submodule_move_head(ce->name, state->super_prefix,
532 NULL, oid_to_hex(&ce->oid), 0);
533 } else
534 return submodule_move_head(ce->name, state->super_prefix,
535 "HEAD", oid_to_hex(&ce->oid),
536 state->force ? SUBMODULE_MOVE_HEAD_FORCE : 0);
537 }
538
539 if (!changed)
540 return 0;
541 if (!state->force) {
542 if (!state->quiet)
543 fprintf(stderr,
544 "%s already exists, no checkout\n",
545 path.buf);
546 return -1;
547 }
548
549 if (state->clone)
550 mark_colliding_entries(state, ce, &st);
551
552 /*
553 * We unlink the old file, to get the new one with the
554 * right permissions (including umask, which is nasty
555 * to emulate by hand - much easier to let the system
556 * just do the right thing)
557 */
558 if (S_ISDIR(st.st_mode)) {
559 /* If it is a gitlink, leave it alone! */
560 if (S_ISGITLINK(ce->ce_mode))
561 return 0;
562 /*
563 * We must avoid replacing submodules' leading
564 * directories with symbolic links, lest recursive
565 * clones can write into arbitrary locations.
566 *
567 * Technically, this logic is not limited
568 * to recursive clones, or for that matter to
569 * submodules' paths colliding with symbolic links'
570 * paths. Yet it strikes a balance in favor of
571 * simplicity, and if paths are colliding, we might
572 * just as well keep the directories during a clone.
573 */
574 if (state->clone && S_ISLNK(ce->ce_mode))
575 return 0;
576 remove_subtree(&path);
577 } else if (unlink(path.buf))
578 return error_errno("unable to unlink old '%s'", path.buf);
579 } else if (state->not_new)
580 return 0;
581
582 create_directories(path.buf, path.len, state);
583
584 if (S_ISREG(ce->ce_mode) && !ca) {
585 convert_attrs(state->istate, &ca_buf, ce->name);
586 ca = &ca_buf;
587 }
588
589 if (!enqueue_checkout(ce, ca, nr_checkouts))
590 return 0;
591
592 return write_entry(ce, path.buf, ca, state, 0, nr_checkouts);
593}
594
595void unlink_entry(const struct cache_entry *ce, const char *super_prefix)
596{
597 const struct submodule *sub = submodule_from_ce(ce);
598 if (sub) {
599 /* state.force is set at the caller. */
600 submodule_move_head(ce->name, super_prefix, "HEAD", NULL,
601 SUBMODULE_MOVE_HEAD_FORCE);
602 }
603 if (check_leading_path(ce->name, ce_namelen(ce), 1) >= 0)
604 return;
605 if (remove_or_warn(ce->ce_mode, ce->name))
606 return;
607 schedule_dir_for_removal(ce->name, ce_namelen(ce));
608}
609
610int remove_or_warn(unsigned int mode, const char *file)
611{
612 return S_ISGITLINK(mode) ? rmdir_or_warn(file) : unlink_or_warn(file);
613}