freshlybakedca.ke
Git fork
1#define USE_THE_REPOSITORY_VARIABLE
2
3#include "git-compat-util.h"
4#include "gettext.h"
5#include "hex.h"
6#include "odb.h"
7#include "promisor-remote.h"
8#include "config.h"
9#include "trace2.h"
10#include "transport.h"
11#include "strvec.h"
12#include "packfile.h"
13#include "environment.h"
14#include "url.h"
15#include "version.h"
16
17struct promisor_remote_config {
18 struct promisor_remote *promisors;
19 struct promisor_remote **promisors_tail;
20};
21
22static int fetch_objects(struct repository *repo,
23 const char *remote_name,
24 const struct object_id *oids,
25 int oid_nr)
26{
27 struct child_process child = CHILD_PROCESS_INIT;
28 int i;
29 FILE *child_in;
30 int quiet;
31
32 if (git_env_bool(NO_LAZY_FETCH_ENVIRONMENT, 0)) {
33 static int warning_shown;
34 if (!warning_shown) {
35 warning_shown = 1;
36 warning(_("lazy fetching disabled; some objects may not be available"));
37 }
38 return -1;
39 }
40
41 child.git_cmd = 1;
42 child.in = -1;
43 if (repo != the_repository)
44 prepare_other_repo_env(&child.env, repo->gitdir);
45 strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
46 "fetch", remote_name, "--no-tags",
47 "--no-write-fetch-head", "--recurse-submodules=no",
48 "--filter=blob:none", "--stdin", NULL);
49 if (!repo_config_get_bool(the_repository, "promisor.quiet", &quiet) && quiet)
50 strvec_push(&child.args, "--quiet");
51 if (start_command(&child))
52 die(_("promisor-remote: unable to fork off fetch subprocess"));
53 child_in = xfdopen(child.in, "w");
54
55 trace2_data_intmax("promisor", repo, "fetch_count", oid_nr);
56
57 for (i = 0; i < oid_nr; i++) {
58 if (fputs(oid_to_hex(&oids[i]), child_in) < 0)
59 die_errno(_("promisor-remote: could not write to fetch subprocess"));
60 if (fputc('\n', child_in) < 0)
61 die_errno(_("promisor-remote: could not write to fetch subprocess"));
62 }
63
64 if (fclose(child_in) < 0)
65 die_errno(_("promisor-remote: could not close stdin to fetch subprocess"));
66 return finish_command(&child) ? -1 : 0;
67}
68
69static struct promisor_remote *promisor_remote_new(struct promisor_remote_config *config,
70 const char *remote_name)
71{
72 struct promisor_remote *r;
73
74 if (*remote_name == '/') {
75 warning(_("promisor remote name cannot begin with '/': %s"),
76 remote_name);
77 return NULL;
78 }
79
80 FLEX_ALLOC_STR(r, name, remote_name);
81
82 *config->promisors_tail = r;
83 config->promisors_tail = &r->next;
84
85 return r;
86}
87
88static struct promisor_remote *promisor_remote_lookup(struct promisor_remote_config *config,
89 const char *remote_name,
90 struct promisor_remote **previous)
91{
92 struct promisor_remote *r, *p;
93
94 for (p = NULL, r = config->promisors; r; p = r, r = r->next)
95 if (!strcmp(r->name, remote_name)) {
96 if (previous)
97 *previous = p;
98 return r;
99 }
100
101 return NULL;
102}
103
104static void promisor_remote_move_to_tail(struct promisor_remote_config *config,
105 struct promisor_remote *r,
106 struct promisor_remote *previous)
107{
108 if (!r->next)
109 return;
110
111 if (previous)
112 previous->next = r->next;
113 else
114 config->promisors = r->next ? r->next : r;
115 r->next = NULL;
116 *config->promisors_tail = r;
117 config->promisors_tail = &r->next;
118}
119
120static int promisor_remote_config(const char *var, const char *value,
121 const struct config_context *ctx UNUSED,
122 void *data)
123{
124 struct promisor_remote_config *config = data;
125 const char *name;
126 size_t namelen;
127 const char *subkey;
128
129 if (parse_config_key(var, "remote", &name, &namelen, &subkey) < 0)
130 return 0;
131
132 if (!strcmp(subkey, "promisor")) {
133 char *remote_name;
134
135 if (!git_config_bool(var, value))
136 return 0;
137
138 remote_name = xmemdupz(name, namelen);
139
140 if (!promisor_remote_lookup(config, remote_name, NULL))
141 promisor_remote_new(config, remote_name);
142
143 free(remote_name);
144 return 0;
145 }
146 if (!strcmp(subkey, "partialclonefilter")) {
147 struct promisor_remote *r;
148 char *remote_name = xmemdupz(name, namelen);
149
150 r = promisor_remote_lookup(config, remote_name, NULL);
151 if (!r)
152 r = promisor_remote_new(config, remote_name);
153
154 free(remote_name);
155
156 if (!r)
157 return 0;
158
159 FREE_AND_NULL(r->partial_clone_filter);
160 return git_config_string(&r->partial_clone_filter, var, value);
161 }
162
163 return 0;
164}
165
166static void promisor_remote_init(struct repository *r)
167{
168 struct promisor_remote_config *config;
169
170 if (r->promisor_remote_config)
171 return;
172 config = r->promisor_remote_config =
173 xcalloc(1, sizeof(*r->promisor_remote_config));
174 config->promisors_tail = &config->promisors;
175
176 repo_config(r, promisor_remote_config, config);
177
178 if (r->repository_format_partial_clone) {
179 struct promisor_remote *o, *previous;
180
181 o = promisor_remote_lookup(config,
182 r->repository_format_partial_clone,
183 &previous);
184 if (o)
185 promisor_remote_move_to_tail(config, o, previous);
186 else
187 promisor_remote_new(config, r->repository_format_partial_clone);
188 }
189}
190
191void promisor_remote_clear(struct promisor_remote_config *config)
192{
193 while (config->promisors) {
194 struct promisor_remote *r = config->promisors;
195 free(r->partial_clone_filter);
196 config->promisors = config->promisors->next;
197 free(r);
198 }
199
200 config->promisors_tail = &config->promisors;
201}
202
203void repo_promisor_remote_reinit(struct repository *r)
204{
205 promisor_remote_clear(r->promisor_remote_config);
206 FREE_AND_NULL(r->promisor_remote_config);
207 promisor_remote_init(r);
208}
209
210struct promisor_remote *repo_promisor_remote_find(struct repository *r,
211 const char *remote_name)
212{
213 promisor_remote_init(r);
214
215 if (!remote_name)
216 return r->promisor_remote_config->promisors;
217
218 return promisor_remote_lookup(r->promisor_remote_config, remote_name, NULL);
219}
220
221int repo_has_promisor_remote(struct repository *r)
222{
223 return !!repo_promisor_remote_find(r, NULL);
224}
225
226int repo_has_accepted_promisor_remote(struct repository *r)
227{
228 struct promisor_remote *p;
229
230 promisor_remote_init(r);
231
232 for (p = r->promisor_remote_config->promisors; p; p = p->next)
233 if (p->accepted)
234 return 1;
235 return 0;
236}
237
238static int remove_fetched_oids(struct repository *repo,
239 struct object_id **oids,
240 int oid_nr, int to_free)
241{
242 int i, remaining_nr = 0;
243 int *remaining = xcalloc(oid_nr, sizeof(*remaining));
244 struct object_id *old_oids = *oids;
245 struct object_id *new_oids;
246
247 for (i = 0; i < oid_nr; i++)
248 if (odb_read_object_info_extended(repo->objects, &old_oids[i], NULL,
249 OBJECT_INFO_SKIP_FETCH_OBJECT)) {
250 remaining[i] = 1;
251 remaining_nr++;
252 }
253
254 if (remaining_nr) {
255 int j = 0;
256 CALLOC_ARRAY(new_oids, remaining_nr);
257 for (i = 0; i < oid_nr; i++)
258 if (remaining[i])
259 oidcpy(&new_oids[j++], &old_oids[i]);
260 *oids = new_oids;
261 if (to_free)
262 free(old_oids);
263 }
264
265 free(remaining);
266
267 return remaining_nr;
268}
269
270void promisor_remote_get_direct(struct repository *repo,
271 const struct object_id *oids,
272 int oid_nr)
273{
274 struct promisor_remote *r;
275 struct object_id *remaining_oids = (struct object_id *)oids;
276 int remaining_nr = oid_nr;
277 int to_free = 0;
278 int i;
279
280 if (oid_nr == 0)
281 return;
282
283 promisor_remote_init(repo);
284
285 for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
286 if (fetch_objects(repo, r->name, remaining_oids, remaining_nr) < 0) {
287 if (remaining_nr == 1)
288 continue;
289 remaining_nr = remove_fetched_oids(repo, &remaining_oids,
290 remaining_nr, to_free);
291 if (remaining_nr) {
292 to_free = 1;
293 continue;
294 }
295 }
296 goto all_fetched;
297 }
298
299 for (i = 0; i < remaining_nr; i++) {
300 if (is_promisor_object(repo, &remaining_oids[i]))
301 die(_("could not fetch %s from promisor remote"),
302 oid_to_hex(&remaining_oids[i]));
303 }
304
305all_fetched:
306 if (to_free)
307 free(remaining_oids);
308}
309
310static int allow_unsanitized(char ch)
311{
312 if (ch == ',' || ch == ';' || ch == '%')
313 return 0;
314 return ch > 32 && ch < 127;
315}
316
317/*
318 * All the fields used in "promisor-remote" protocol capability,
319 * including the mandatory "name" and "url" ones.
320 */
321static const char promisor_field_name[] = "name";
322static const char promisor_field_url[] = "url";
323static const char promisor_field_filter[] = "partialCloneFilter";
324static const char promisor_field_token[] = "token";
325
326/*
327 * List of optional field names that can be used in the
328 * "promisor-remote" protocol capability (others must be
329 * ignored). Each field should correspond to a configurable property
330 * of a remote that can be relevant for the client.
331 */
332static const char *known_fields[] = {
333 promisor_field_filter, /* Filter used for partial clone */
334 promisor_field_token, /* Authentication token for the remote */
335 NULL
336};
337
338/*
339 * Check if 'field' is in the list of the known field names for the
340 * "promisor-remote" protocol capability.
341 */
342static int is_known_field(const char *field)
343{
344 const char **p;
345
346 for (p = known_fields; *p; p++)
347 if (!strcasecmp(*p, field))
348 return 1;
349 return 0;
350}
351
352static int is_valid_field(struct string_list_item *item, void *cb_data)
353{
354 const char *field = item->string;
355 const char *config_key = (const char *)cb_data;
356
357 if (!is_known_field(field)) {
358 warning(_("unsupported field '%s' in '%s' config"), field, config_key);
359 return 0;
360 }
361 return 1;
362}
363
364static char *fields_from_config(struct string_list *fields_list, const char *config_key)
365{
366 char *fields = NULL;
367
368 if (!repo_config_get_string(the_repository, config_key, &fields) && *fields) {
369 string_list_split_in_place_f(fields_list, fields, ",", -1,
370 STRING_LIST_SPLIT_TRIM |
371 STRING_LIST_SPLIT_NONEMPTY);
372 filter_string_list(fields_list, 0, is_valid_field, (void *)config_key);
373 }
374
375 return fields;
376}
377
378static struct string_list *fields_sent(void)
379{
380 static struct string_list fields_list = STRING_LIST_INIT_NODUP;
381 static int initialized;
382
383 if (!initialized) {
384 fields_list.cmp = strcasecmp;
385 fields_from_config(&fields_list, "promisor.sendFields");
386 initialized = 1;
387 }
388
389 return &fields_list;
390}
391
392static struct string_list *fields_checked(void)
393{
394 static struct string_list fields_list = STRING_LIST_INIT_NODUP;
395 static int initialized;
396
397 if (!initialized) {
398 fields_list.cmp = strcasecmp;
399 fields_from_config(&fields_list, "promisor.checkFields");
400 initialized = 1;
401 }
402
403 return &fields_list;
404}
405
406/*
407 * Struct for promisor remotes involved in the "promisor-remote"
408 * protocol capability.
409 *
410 * Except for "name", each <member> in this struct and its <value>
411 * should correspond (either on the client side or on the server side)
412 * to a "remote.<name>.<member>" config variable set to <value> where
413 * "<name>" is a promisor remote name.
414 */
415struct promisor_info {
416 const char *name;
417 const char *url;
418 const char *filter;
419 const char *token;
420};
421
422static void promisor_info_free(struct promisor_info *p)
423{
424 free((char *)p->name);
425 free((char *)p->url);
426 free((char *)p->filter);
427 free((char *)p->token);
428 free(p);
429}
430
431static void promisor_info_list_clear(struct string_list *list)
432{
433 for (size_t i = 0; i < list->nr; i++)
434 promisor_info_free(list->items[i].util);
435 string_list_clear(list, 0);
436}
437
438static void set_one_field(struct promisor_info *p,
439 const char *field, const char *value)
440{
441 if (!strcasecmp(field, promisor_field_filter))
442 p->filter = xstrdup(value);
443 else if (!strcasecmp(field, promisor_field_token))
444 p->token = xstrdup(value);
445 else
446 BUG("invalid field '%s'", field);
447}
448
449static void set_fields(struct promisor_info *p,
450 struct string_list *field_names)
451{
452 struct string_list_item *item;
453
454 for_each_string_list_item(item, field_names) {
455 char *key = xstrfmt("remote.%s.%s", p->name, item->string);
456 const char *val;
457 if (!repo_config_get_string_tmp(the_repository, key, &val) && *val)
458 set_one_field(p, item->string, val);
459 free(key);
460 }
461}
462
463/*
464 * Populate 'list' with promisor remote information from the config.
465 * The 'util' pointer of each list item will hold a 'struct
466 * promisor_info'. Except "name" and "url", only members of that
467 * struct specified by the 'field_names' list are set (using values
468 * from the configuration).
469 */
470static void promisor_config_info_list(struct repository *repo,
471 struct string_list *list,
472 struct string_list *field_names)
473{
474 struct promisor_remote *r;
475
476 promisor_remote_init(repo);
477
478 for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
479 const char *url;
480 char *url_key = xstrfmt("remote.%s.url", r->name);
481
482 /* Only add remotes with a non empty URL */
483 if (!repo_config_get_string_tmp(the_repository, url_key, &url) && *url) {
484 struct promisor_info *new_info = xcalloc(1, sizeof(*new_info));
485 struct string_list_item *item;
486
487 new_info->name = xstrdup(r->name);
488 new_info->url = xstrdup(url);
489
490 if (field_names)
491 set_fields(new_info, field_names);
492
493 item = string_list_append(list, new_info->name);
494 item->util = new_info;
495 }
496
497 free(url_key);
498 }
499}
500
501char *promisor_remote_info(struct repository *repo)
502{
503 struct strbuf sb = STRBUF_INIT;
504 int advertise_promisors = 0;
505 struct string_list config_info = STRING_LIST_INIT_NODUP;
506 struct string_list_item *item;
507
508 repo_config_get_bool(the_repository, "promisor.advertise", &advertise_promisors);
509
510 if (!advertise_promisors)
511 return NULL;
512
513 promisor_config_info_list(repo, &config_info, fields_sent());
514
515 if (!config_info.nr)
516 return NULL;
517
518 for_each_string_list_item(item, &config_info) {
519 struct promisor_info *p = item->util;
520
521 if (item != config_info.items)
522 strbuf_addch(&sb, ';');
523
524 strbuf_addf(&sb, "%s=", promisor_field_name);
525 strbuf_addstr_urlencode(&sb, p->name, allow_unsanitized);
526 strbuf_addf(&sb, ",%s=", promisor_field_url);
527 strbuf_addstr_urlencode(&sb, p->url, allow_unsanitized);
528
529 if (p->filter) {
530 strbuf_addf(&sb, ",%s=", promisor_field_filter);
531 strbuf_addstr_urlencode(&sb, p->filter, allow_unsanitized);
532 }
533 if (p->token) {
534 strbuf_addf(&sb, ",%s=", promisor_field_token);
535 strbuf_addstr_urlencode(&sb, p->token, allow_unsanitized);
536 }
537 }
538
539 promisor_info_list_clear(&config_info);
540
541 return strbuf_detach(&sb, NULL);
542}
543
544enum accept_promisor {
545 ACCEPT_NONE = 0,
546 ACCEPT_KNOWN_URL,
547 ACCEPT_KNOWN_NAME,
548 ACCEPT_ALL
549};
550
551static int match_field_against_config(const char *field, const char *value,
552 struct promisor_info *config_info)
553{
554 if (config_info->filter && !strcasecmp(field, promisor_field_filter))
555 return !strcmp(config_info->filter, value);
556 else if (config_info->token && !strcasecmp(field, promisor_field_token))
557 return !strcmp(config_info->token, value);
558
559 return 0;
560}
561
562static int all_fields_match(struct promisor_info *advertised,
563 struct string_list *config_info,
564 int in_list)
565{
566 struct string_list *fields = fields_checked();
567 struct string_list_item *item_checked;
568
569 for_each_string_list_item(item_checked, fields) {
570 int match = 0;
571 const char *field = item_checked->string;
572 const char *value = NULL;
573 struct string_list_item *item;
574
575 if (!strcasecmp(field, promisor_field_filter))
576 value = advertised->filter;
577 else if (!strcasecmp(field, promisor_field_token))
578 value = advertised->token;
579
580 if (!value)
581 return 0;
582
583 if (in_list) {
584 for_each_string_list_item(item, config_info) {
585 struct promisor_info *p = item->util;
586 if (match_field_against_config(field, value, p)) {
587 match = 1;
588 break;
589 }
590 }
591 } else {
592 item = string_list_lookup(config_info, advertised->name);
593 if (item) {
594 struct promisor_info *p = item->util;
595 match = match_field_against_config(field, value, p);
596 }
597 }
598
599 if (!match)
600 return 0;
601 }
602
603 return 1;
604}
605
606static int should_accept_remote(enum accept_promisor accept,
607 struct promisor_info *advertised,
608 struct string_list *config_info)
609{
610 struct promisor_info *p;
611 struct string_list_item *item;
612 const char *remote_name = advertised->name;
613 const char *remote_url = advertised->url;
614
615 if (accept == ACCEPT_ALL)
616 return all_fields_match(advertised, config_info, 1);
617
618 /* Get config info for that promisor remote */
619 item = string_list_lookup(config_info, remote_name);
620
621 if (!item)
622 /* We don't know about that remote */
623 return 0;
624
625 p = item->util;
626
627 if (accept == ACCEPT_KNOWN_NAME)
628 return all_fields_match(advertised, config_info, 0);
629
630 if (accept != ACCEPT_KNOWN_URL)
631 BUG("Unhandled 'enum accept_promisor' value '%d'", accept);
632
633 if (!remote_url || !*remote_url) {
634 warning(_("no or empty URL advertised for remote '%s'"), remote_name);
635 return 0;
636 }
637
638 if (!strcmp(p->url, remote_url))
639 return all_fields_match(advertised, config_info, 0);
640
641 warning(_("known remote named '%s' but with URL '%s' instead of '%s'"),
642 remote_name, p->url, remote_url);
643
644 return 0;
645}
646
647static int skip_field_name_prefix(const char *elem, const char *field_name, const char **value)
648{
649 const char *p;
650 if (!skip_prefix(elem, field_name, &p) || *p != '=')
651 return 0;
652 *value = p + 1;
653 return 1;
654}
655
656static struct promisor_info *parse_one_advertised_remote(const char *remote_info)
657{
658 struct promisor_info *info = xcalloc(1, sizeof(*info));
659 struct string_list elem_list = STRING_LIST_INIT_DUP;
660 struct string_list_item *item;
661
662 string_list_split(&elem_list, remote_info, ",", -1);
663
664 for_each_string_list_item(item, &elem_list) {
665 const char *elem = item->string;
666 const char *p = strchr(elem, '=');
667
668 if (!p) {
669 warning(_("invalid element '%s' from remote info"), elem);
670 continue;
671 }
672
673 if (skip_field_name_prefix(elem, promisor_field_name, &p))
674 info->name = url_percent_decode(p);
675 else if (skip_field_name_prefix(elem, promisor_field_url, &p))
676 info->url = url_percent_decode(p);
677 else if (skip_field_name_prefix(elem, promisor_field_filter, &p))
678 info->filter = url_percent_decode(p);
679 else if (skip_field_name_prefix(elem, promisor_field_token, &p))
680 info->token = url_percent_decode(p);
681 }
682
683 string_list_clear(&elem_list, 0);
684
685 if (!info->name || !info->url) {
686 warning(_("server advertised a promisor remote without a name or URL: %s"),
687 remote_info);
688 promisor_info_free(info);
689 return NULL;
690 }
691
692 return info;
693}
694
695static void filter_promisor_remote(struct repository *repo,
696 struct strvec *accepted,
697 const char *info)
698{
699 const char *accept_str;
700 enum accept_promisor accept = ACCEPT_NONE;
701 struct string_list config_info = STRING_LIST_INIT_NODUP;
702 struct string_list remote_info = STRING_LIST_INIT_DUP;
703 struct string_list_item *item;
704
705 if (!repo_config_get_string_tmp(the_repository, "promisor.acceptfromserver", &accept_str)) {
706 if (!*accept_str || !strcasecmp("None", accept_str))
707 accept = ACCEPT_NONE;
708 else if (!strcasecmp("KnownUrl", accept_str))
709 accept = ACCEPT_KNOWN_URL;
710 else if (!strcasecmp("KnownName", accept_str))
711 accept = ACCEPT_KNOWN_NAME;
712 else if (!strcasecmp("All", accept_str))
713 accept = ACCEPT_ALL;
714 else
715 warning(_("unknown '%s' value for '%s' config option"),
716 accept_str, "promisor.acceptfromserver");
717 }
718
719 if (accept == ACCEPT_NONE)
720 return;
721
722 /* Parse remote info received */
723
724 string_list_split(&remote_info, info, ";", -1);
725
726 for_each_string_list_item(item, &remote_info) {
727 struct promisor_info *advertised;
728
729 advertised = parse_one_advertised_remote(item->string);
730
731 if (!advertised)
732 continue;
733
734 if (!config_info.nr) {
735 promisor_config_info_list(repo, &config_info, fields_checked());
736 string_list_sort(&config_info);
737 }
738
739 if (should_accept_remote(accept, advertised, &config_info))
740 strvec_push(accepted, advertised->name);
741
742 promisor_info_free(advertised);
743 }
744
745 promisor_info_list_clear(&config_info);
746 string_list_clear(&remote_info, 0);
747}
748
749char *promisor_remote_reply(const char *info)
750{
751 struct strvec accepted = STRVEC_INIT;
752 struct strbuf reply = STRBUF_INIT;
753
754 filter_promisor_remote(the_repository, &accepted, info);
755
756 if (!accepted.nr)
757 return NULL;
758
759 for (size_t i = 0; i < accepted.nr; i++) {
760 if (i)
761 strbuf_addch(&reply, ';');
762 strbuf_addstr_urlencode(&reply, accepted.v[i], allow_unsanitized);
763 }
764
765 strvec_clear(&accepted);
766
767 return strbuf_detach(&reply, NULL);
768}
769
770void mark_promisor_remotes_as_accepted(struct repository *r, const char *remotes)
771{
772 struct string_list accepted_remotes = STRING_LIST_INIT_DUP;
773 struct string_list_item *item;
774
775 string_list_split(&accepted_remotes, remotes, ";", -1);
776
777 for_each_string_list_item(item, &accepted_remotes) {
778 char *decoded_remote = url_percent_decode(item->string);
779 struct promisor_remote *p = repo_promisor_remote_find(r, decoded_remote);
780
781 if (p)
782 p->accepted = 1;
783 else
784 warning(_("accepted promisor remote '%s' not found"),
785 decoded_remote);
786
787 free(decoded_remote);
788 }
789
790 string_list_clear(&accepted_remotes, 0);
791}