freshlybakedca.ke
Git fork
1#define USE_THE_REPOSITORY_VARIABLE
2#define DISABLE_SIGN_COMPARE_WARNINGS
3
4#include "git-compat-util.h"
5#include "repository.h"
6#include "hex.h"
7#include "walker.h"
8#include "http.h"
9#include "list.h"
10#include "transport.h"
11#include "packfile.h"
12#include "object-file.h"
13#include "odb.h"
14
15struct alt_base {
16 char *base;
17 int got_indices;
18 struct packed_git *packs;
19 struct alt_base *next;
20};
21
22enum object_request_state {
23 WAITING,
24 ABORTED,
25 ACTIVE,
26 COMPLETE
27};
28
29struct object_request {
30 struct walker *walker;
31 struct object_id oid;
32 struct alt_base *repo;
33 enum object_request_state state;
34 struct http_object_request *req;
35 struct list_head node;
36};
37
38struct alternates_request {
39 struct walker *walker;
40 const char *base;
41 struct strbuf *url;
42 struct strbuf *buffer;
43 struct active_request_slot *slot;
44 int http_specific;
45};
46
47struct walker_data {
48 const char *url;
49 int got_alternates;
50 struct alt_base *alt;
51};
52
53static LIST_HEAD(object_queue_head);
54
55static void fetch_alternates(struct walker *walker, const char *base);
56
57static void process_object_response(void *callback_data);
58
59static void start_object_request(struct object_request *obj_req)
60{
61 struct active_request_slot *slot;
62 struct http_object_request *req;
63
64 req = new_http_object_request(obj_req->repo->base, &obj_req->oid);
65 if (!req) {
66 obj_req->state = ABORTED;
67 return;
68 }
69 obj_req->req = req;
70
71 slot = req->slot;
72 slot->callback_func = process_object_response;
73 slot->callback_data = obj_req;
74
75 /* Try to get the request started, abort the request on error */
76 obj_req->state = ACTIVE;
77 if (!start_active_slot(slot)) {
78 obj_req->state = ABORTED;
79 release_http_object_request(&req);
80 return;
81 }
82}
83
84static void finish_object_request(struct object_request *obj_req)
85{
86 if (finish_http_object_request(obj_req->req))
87 return;
88
89 if (obj_req->req->rename == 0)
90 walker_say(obj_req->walker, "got %s\n", oid_to_hex(&obj_req->oid));
91}
92
93static void process_object_response(void *callback_data)
94{
95 struct object_request *obj_req =
96 (struct object_request *)callback_data;
97 struct walker *walker = obj_req->walker;
98 struct walker_data *data = walker->data;
99 struct alt_base *alt = data->alt;
100
101 process_http_object_request(obj_req->req);
102 obj_req->state = COMPLETE;
103
104 normalize_curl_result(&obj_req->req->curl_result,
105 obj_req->req->http_code,
106 obj_req->req->errorstr,
107 sizeof(obj_req->req->errorstr));
108
109 /* Use alternates if necessary */
110 if (missing_target(obj_req->req)) {
111 fetch_alternates(walker, alt->base);
112 if (obj_req->repo->next) {
113 obj_req->repo =
114 obj_req->repo->next;
115 release_http_object_request(&obj_req->req);
116 start_object_request(obj_req);
117 return;
118 }
119 }
120
121 finish_object_request(obj_req);
122}
123
124static void release_object_request(struct object_request *obj_req)
125{
126 if (obj_req->req !=NULL && obj_req->req->localfile != -1)
127 error("fd leakage in release: %d", obj_req->req->localfile);
128
129 list_del(&obj_req->node);
130 free(obj_req);
131}
132
133static int fill_active_slot(void *data UNUSED)
134{
135 struct object_request *obj_req;
136 struct list_head *pos, *tmp, *head = &object_queue_head;
137
138 list_for_each_safe(pos, tmp, head) {
139 obj_req = list_entry(pos, struct object_request, node);
140 if (obj_req->state == WAITING) {
141 if (odb_has_object(the_repository->objects, &obj_req->oid,
142 HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR))
143 obj_req->state = COMPLETE;
144 else {
145 start_object_request(obj_req);
146 return 1;
147 }
148 }
149 }
150 return 0;
151}
152
153static void prefetch(struct walker *walker, const struct object_id *oid)
154{
155 struct object_request *newreq;
156 struct walker_data *data = walker->data;
157
158 newreq = xmalloc(sizeof(*newreq));
159 newreq->walker = walker;
160 oidcpy(&newreq->oid, oid);
161 newreq->repo = data->alt;
162 newreq->state = WAITING;
163 newreq->req = NULL;
164
165 http_is_verbose = walker->get_verbosely;
166 list_add_tail(&newreq->node, &object_queue_head);
167
168 fill_active_slots();
169 step_active_slots();
170}
171
172static int is_alternate_allowed(const char *url)
173{
174 const char *protocols[] = {
175 "http", "https", "ftp", "ftps"
176 };
177 int i;
178
179 if (http_follow_config != HTTP_FOLLOW_ALWAYS) {
180 warning("alternate disabled by http.followRedirects: %s", url);
181 return 0;
182 }
183
184 for (i = 0; i < ARRAY_SIZE(protocols); i++) {
185 const char *end;
186 if (skip_prefix(url, protocols[i], &end) &&
187 starts_with(end, "://"))
188 break;
189 }
190
191 if (i >= ARRAY_SIZE(protocols)) {
192 warning("ignoring alternate with unknown protocol: %s", url);
193 return 0;
194 }
195 if (!is_transport_allowed(protocols[i], 0)) {
196 warning("ignoring alternate with restricted protocol: %s", url);
197 return 0;
198 }
199
200 return 1;
201}
202
203static void process_alternates_response(void *callback_data)
204{
205 struct alternates_request *alt_req =
206 (struct alternates_request *)callback_data;
207 struct walker *walker = alt_req->walker;
208 struct walker_data *cdata = walker->data;
209 struct active_request_slot *slot = alt_req->slot;
210 struct alt_base *tail = cdata->alt;
211 const char *base = alt_req->base;
212 const char null_byte = '\0';
213 char *data;
214 int i = 0;
215
216 normalize_curl_result(&slot->curl_result, slot->http_code,
217 curl_errorstr, sizeof(curl_errorstr));
218
219 if (alt_req->http_specific) {
220 if (slot->curl_result != CURLE_OK ||
221 !alt_req->buffer->len) {
222
223 /* Try reusing the slot to get non-http alternates */
224 alt_req->http_specific = 0;
225 strbuf_reset(alt_req->url);
226 strbuf_addf(alt_req->url, "%s/objects/info/alternates",
227 base);
228 curl_easy_setopt(slot->curl, CURLOPT_URL,
229 alt_req->url->buf);
230 active_requests++;
231 slot->in_use = 1;
232 if (slot->finished)
233 (*slot->finished) = 0;
234 if (!start_active_slot(slot)) {
235 cdata->got_alternates = -1;
236 slot->in_use = 0;
237 if (slot->finished)
238 (*slot->finished) = 1;
239 }
240 return;
241 }
242 } else if (slot->curl_result != CURLE_OK) {
243 if (!missing_target(slot)) {
244 cdata->got_alternates = -1;
245 return;
246 }
247 }
248
249 fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
250 alt_req->buffer->len--;
251 data = alt_req->buffer->buf;
252
253 while (i < alt_req->buffer->len) {
254 int posn = i;
255 while (posn < alt_req->buffer->len && data[posn] != '\n')
256 posn++;
257 if (data[posn] == '\n') {
258 int okay = 0;
259 int serverlen = 0;
260 struct alt_base *newalt;
261 if (data[i] == '/') {
262 /*
263 * This counts
264 * http://git.host/pub/scm/linux.git/
265 * -----------here^
266 * so memcpy(dst, base, serverlen) will
267 * copy up to "...git.host".
268 */
269 const char *colon_ss = strstr(base,"://");
270 if (colon_ss) {
271 serverlen = (strchr(colon_ss + 3, '/')
272 - base);
273 okay = 1;
274 }
275 } else if (!memcmp(data + i, "../", 3)) {
276 /*
277 * Relative URL; chop the corresponding
278 * number of subpath from base (and ../
279 * from data), and concatenate the result.
280 *
281 * The code first drops ../ from data, and
282 * then drops one ../ from data and one path
283 * from base. IOW, one extra ../ is dropped
284 * from data than path is dropped from base.
285 *
286 * This is not wrong. The alternate in
287 * http://git.host/pub/scm/linux.git/
288 * to borrow from
289 * http://git.host/pub/scm/linus.git/
290 * is ../../linus.git/objects/. You need
291 * two ../../ to borrow from your direct
292 * neighbour.
293 */
294 i += 3;
295 serverlen = strlen(base);
296 while (i + 2 < posn &&
297 !memcmp(data + i, "../", 3)) {
298 do {
299 serverlen--;
300 } while (serverlen &&
301 base[serverlen - 1] != '/');
302 i += 3;
303 }
304 /* If the server got removed, give up. */
305 okay = strchr(base, ':') - base + 3 <
306 serverlen;
307 } else if (alt_req->http_specific) {
308 char *colon = strchr(data + i, ':');
309 char *slash = strchr(data + i, '/');
310 if (colon && slash && colon < data + posn &&
311 slash < data + posn && colon < slash) {
312 okay = 1;
313 }
314 }
315 if (okay) {
316 struct strbuf target = STRBUF_INIT;
317 strbuf_add(&target, base, serverlen);
318 strbuf_add(&target, data + i, posn - i);
319 if (!strbuf_strip_suffix(&target, "objects")) {
320 warning("ignoring alternate that does"
321 " not end in 'objects': %s",
322 target.buf);
323 strbuf_release(&target);
324 } else if (is_alternate_allowed(target.buf)) {
325 warning("adding alternate object store: %s",
326 target.buf);
327 newalt = xmalloc(sizeof(*newalt));
328 newalt->next = NULL;
329 newalt->base = strbuf_detach(&target, NULL);
330 newalt->got_indices = 0;
331 newalt->packs = NULL;
332
333 while (tail->next != NULL)
334 tail = tail->next;
335 tail->next = newalt;
336 } else {
337 strbuf_release(&target);
338 }
339 }
340 }
341 i = posn + 1;
342 }
343
344 cdata->got_alternates = 1;
345}
346
347static void fetch_alternates(struct walker *walker, const char *base)
348{
349 struct strbuf buffer = STRBUF_INIT;
350 struct strbuf url = STRBUF_INIT;
351 struct active_request_slot *slot;
352 struct alternates_request alt_req;
353 struct walker_data *cdata = walker->data;
354
355 /*
356 * If another request has already started fetching alternates,
357 * wait for them to arrive and return to processing this request's
358 * curl message
359 */
360 while (cdata->got_alternates == 0) {
361 step_active_slots();
362 }
363
364 /* Nothing to do if they've already been fetched */
365 if (cdata->got_alternates == 1)
366 return;
367
368 /* Start the fetch */
369 cdata->got_alternates = 0;
370
371 if (walker->get_verbosely)
372 fprintf(stderr, "Getting alternates list for %s\n", base);
373
374 strbuf_addf(&url, "%s/objects/info/http-alternates", base);
375
376 /*
377 * Use a callback to process the result, since another request
378 * may fail and need to have alternates loaded before continuing
379 */
380 slot = get_active_slot();
381 slot->callback_func = process_alternates_response;
382 alt_req.walker = walker;
383 slot->callback_data = &alt_req;
384
385 curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, &buffer);
386 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
387 curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);
388
389 alt_req.base = base;
390 alt_req.url = &url;
391 alt_req.buffer = &buffer;
392 alt_req.http_specific = 1;
393 alt_req.slot = slot;
394
395 if (start_active_slot(slot))
396 run_active_slot(slot);
397 else
398 cdata->got_alternates = -1;
399
400 strbuf_release(&buffer);
401 strbuf_release(&url);
402}
403
404static int fetch_indices(struct walker *walker, struct alt_base *repo)
405{
406 int ret;
407
408 if (repo->got_indices)
409 return 0;
410
411 if (walker->get_verbosely)
412 fprintf(stderr, "Getting pack list for %s\n", repo->base);
413
414 switch (http_get_info_packs(repo->base, &repo->packs)) {
415 case HTTP_OK:
416 case HTTP_MISSING_TARGET:
417 repo->got_indices = 1;
418 ret = 0;
419 break;
420 default:
421 repo->got_indices = 0;
422 ret = -1;
423 }
424
425 return ret;
426}
427
428static int http_fetch_pack(struct walker *walker, struct alt_base *repo,
429 const struct object_id *oid)
430{
431 struct packed_git *target;
432 int ret;
433 struct slot_results results;
434 struct http_pack_request *preq;
435
436 if (fetch_indices(walker, repo))
437 return -1;
438 target = find_oid_pack(oid, repo->packs);
439 if (!target)
440 return -1;
441 close_pack_index(target);
442
443 if (walker->get_verbosely) {
444 fprintf(stderr, "Getting pack %s\n",
445 hash_to_hex(target->hash));
446 fprintf(stderr, " which contains %s\n",
447 oid_to_hex(oid));
448 }
449
450 preq = new_http_pack_request(target->hash, repo->base);
451 if (!preq)
452 goto abort;
453 preq->slot->results = &results;
454
455 if (start_active_slot(preq->slot)) {
456 run_active_slot(preq->slot);
457 if (results.curl_result != CURLE_OK) {
458 error("Unable to get pack file %s\n%s", preq->url,
459 curl_errorstr);
460 goto abort;
461 }
462 } else {
463 error("Unable to start request");
464 goto abort;
465 }
466
467 ret = finish_http_pack_request(preq);
468 release_http_pack_request(preq);
469 if (ret)
470 return ret;
471 http_install_packfile(target, &repo->packs);
472
473 return 0;
474
475abort:
476 return -1;
477}
478
479static void abort_object_request(struct object_request *obj_req)
480{
481 release_object_request(obj_req);
482}
483
484static int fetch_object(struct walker *walker, const struct object_id *oid)
485{
486 char *hex = oid_to_hex(oid);
487 int ret = 0;
488 struct object_request *obj_req = NULL;
489 struct http_object_request *req;
490 struct list_head *pos, *head = &object_queue_head;
491
492 list_for_each(pos, head) {
493 obj_req = list_entry(pos, struct object_request, node);
494 if (oideq(&obj_req->oid, oid))
495 break;
496 }
497 if (!obj_req)
498 return error("Couldn't find request for %s in the queue", hex);
499
500 if (odb_has_object(the_repository->objects, &obj_req->oid,
501 HAS_OBJECT_RECHECK_PACKED | HAS_OBJECT_FETCH_PROMISOR)) {
502 if (obj_req->req)
503 abort_http_object_request(&obj_req->req);
504 abort_object_request(obj_req);
505 return 0;
506 }
507
508 while (obj_req->state == WAITING)
509 step_active_slots();
510
511 /*
512 * obj_req->req might change when fetching alternates in the callback
513 * process_object_response; therefore, the "shortcut" variable, req,
514 * is used only after we're done with slots.
515 */
516 while (obj_req->state == ACTIVE)
517 run_active_slot(obj_req->req->slot);
518
519 req = obj_req->req;
520
521 if (req->localfile != -1) {
522 close(req->localfile);
523 req->localfile = -1;
524 }
525
526 normalize_curl_result(&req->curl_result, req->http_code,
527 req->errorstr, sizeof(req->errorstr));
528
529 if (obj_req->state == ABORTED) {
530 ret = error("Request for %s aborted", hex);
531 } else if (req->curl_result != CURLE_OK &&
532 req->http_code != 416) {
533 if (missing_target(req))
534 ret = -1; /* Be silent, it is probably in a pack. */
535 else
536 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
537 req->errorstr, req->curl_result,
538 req->http_code, hex);
539 } else if (req->zret != Z_STREAM_END) {
540 walker->corrupt_object_found++;
541 ret = error("File %s (%s) corrupt", hex, req->url);
542 } else if (!oideq(&obj_req->oid, &req->real_oid)) {
543 ret = error("File %s has bad hash", hex);
544 } else if (req->rename < 0) {
545 struct strbuf buf = STRBUF_INIT;
546 odb_loose_path(the_repository->objects->sources, &buf, &req->oid);
547 ret = error("unable to write sha1 filename %s", buf.buf);
548 strbuf_release(&buf);
549 }
550
551 release_http_object_request(&obj_req->req);
552 release_object_request(obj_req);
553 return ret;
554}
555
556static int fetch(struct walker *walker, const struct object_id *oid)
557{
558 struct walker_data *data = walker->data;
559 struct alt_base *altbase = data->alt;
560
561 if (!fetch_object(walker, oid))
562 return 0;
563 while (altbase) {
564 if (!http_fetch_pack(walker, altbase, oid))
565 return 0;
566 fetch_alternates(walker, data->alt->base);
567 altbase = altbase->next;
568 }
569 return error("Unable to find %s under %s", oid_to_hex(oid),
570 data->alt->base);
571}
572
573static int fetch_ref(struct walker *walker, struct ref *ref)
574{
575 struct walker_data *data = walker->data;
576 return http_fetch_ref(data->alt->base, ref);
577}
578
579static void cleanup(struct walker *walker)
580{
581 struct walker_data *data = walker->data;
582 struct alt_base *alt, *alt_next;
583
584 if (data) {
585 alt = data->alt;
586 while (alt) {
587 struct packed_git *pack;
588
589 alt_next = alt->next;
590
591 pack = alt->packs;
592 while (pack) {
593 struct packed_git *pack_next = pack->next;
594 close_pack(pack);
595 free(pack);
596 pack = pack_next;
597 }
598
599 free(alt->base);
600 free(alt);
601
602 alt = alt_next;
603 }
604 free(data);
605 walker->data = NULL;
606 }
607}
608
609struct walker *get_http_walker(const char *url)
610{
611 char *s;
612 struct walker_data *data = xmalloc(sizeof(struct walker_data));
613 struct walker *walker = xmalloc(sizeof(struct walker));
614
615 data->alt = xmalloc(sizeof(*data->alt));
616 data->alt->base = xstrdup(url);
617 for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
618 *s = 0;
619
620 data->alt->got_indices = 0;
621 data->alt->packs = NULL;
622 data->alt->next = NULL;
623 data->got_alternates = -1;
624
625 walker->corrupt_object_found = 0;
626 walker->fetch = fetch;
627 walker->fetch_ref = fetch_ref;
628 walker->prefetch = prefetch;
629 walker->cleanup = cleanup;
630 walker->data = data;
631
632 add_fill_function(NULL, fill_active_slot);
633
634 return walker;
635}