Merge branch 'ps/reftable-iterator-reuse' · freshlybakedca.ke/git@7041902

+261 -148

refs/reftable-backend.c

··· 34 34 */ 35 35 #define REF_UPDATE_VIA_HEAD (1 << 8) 36 36 37 + struct reftable_backend { 38 + struct reftable_stack *stack; 39 + struct reftable_iterator it; 40 + }; 41 + 42 + static void reftable_backend_on_reload(void *payload) 43 + { 44 + struct reftable_backend *be = payload; 45 + reftable_iterator_destroy(&be->it); 46 + } 47 + 48 + static int reftable_backend_init(struct reftable_backend *be, 49 + const char *path, 50 + const struct reftable_write_options *_opts) 51 + { 52 + struct reftable_write_options opts = *_opts; 53 + opts.on_reload = reftable_backend_on_reload; 54 + opts.on_reload_payload = be; 55 + return reftable_new_stack(&be->stack, path, &opts); 56 + } 57 + 58 + static void reftable_backend_release(struct reftable_backend *be) 59 + { 60 + reftable_stack_destroy(be->stack); 61 + be->stack = NULL; 62 + reftable_iterator_destroy(&be->it); 63 + } 64 + 65 + static int reftable_backend_read_ref(struct reftable_backend *be, 66 + const char *refname, 67 + struct object_id *oid, 68 + struct strbuf *referent, 69 + unsigned int *type) 70 + { 71 + struct reftable_ref_record ref = {0}; 72 + int ret; 73 + 74 + if (!be->it.ops) { 75 + ret = reftable_stack_init_ref_iterator(be->stack, &be->it); 76 + if (ret) 77 + goto done; 78 + } 79 + 80 + ret = reftable_iterator_seek_ref(&be->it, refname); 81 + if (ret) 82 + goto done; 83 + 84 + ret = reftable_iterator_next_ref(&be->it, &ref); 85 + if (ret) 86 + goto done; 87 + 88 + if (strcmp(ref.refname, refname)) { 89 + ret = 1; 90 + goto done; 91 + } 92 + 93 + if (ref.value_type == REFTABLE_REF_SYMREF) { 94 + strbuf_reset(referent); 95 + strbuf_addstr(referent, ref.value.symref); 96 + *type |= REF_ISSYMREF; 97 + } else if (reftable_ref_record_val1(&ref)) { 98 + unsigned int hash_id; 99 + 100 + switch (reftable_stack_hash_id(be->stack)) { 101 + case REFTABLE_HASH_SHA1: 102 + hash_id = GIT_HASH_SHA1; 103 + break; 104 + case REFTABLE_HASH_SHA256: 105 + hash_id = GIT_HASH_SHA256; 106 + break; 107 + default: 108 + BUG("unhandled hash ID %d", reftable_stack_hash_id(be->stack)); 109 + } 110 + 111 + oidread(oid, reftable_ref_record_val1(&ref), 112 + &hash_algos[hash_id]); 113 + } else { 114 + /* We got a tombstone, which should not happen. */ 115 + BUG("unhandled reference value type %d", ref.value_type); 116 + } 117 + 118 + done: 119 + assert(ret != REFTABLE_API_ERROR); 120 + reftable_ref_record_release(&ref); 121 + return ret; 122 + } 123 + 37 124 struct reftable_ref_store { 38 125 struct ref_store base; 39 126 40 127 /* 41 - * The main stack refers to the common dir and thus contains common 128 + * The main backend refers to the common dir and thus contains common 42 129 * refs as well as refs of the main repository. 43 130 */ 44 - struct reftable_stack *main_stack; 131 + struct reftable_backend main_backend; 45 132 /* 46 - * The worktree stack refers to the gitdir in case the refdb is opened 133 + * The worktree backend refers to the gitdir in case the refdb is opened 47 134 * via a worktree. It thus contains the per-worktree refs. 48 135 */ 49 - struct reftable_stack *worktree_stack; 136 + struct reftable_backend worktree_backend; 50 137 /* 51 - * Map of worktree stacks by their respective worktree names. The map 138 + * Map of worktree backends by their respective worktree names. The map 52 139 * is populated lazily when we try to resolve `worktrees/$worktree` refs. 53 140 */ 54 - struct strmap worktree_stacks; 141 + struct strmap worktree_backends; 55 142 struct reftable_write_options write_options; 56 143 57 144 unsigned int store_flags; ··· 97 184 * like `worktrees/$worktree/refs/heads/foo` as worktree stacks will store 98 185 * those references in their normalized form. 99 186 */ 100 - static struct reftable_stack *stack_for(struct reftable_ref_store *store, 101 - const char *refname, 102 - const char **rewritten_ref) 187 + static int backend_for(struct reftable_backend **out, 188 + struct reftable_ref_store *store, 189 + const char *refname, 190 + const char **rewritten_ref, 191 + int reload) 103 192 { 193 + struct reftable_backend *be; 104 194 const char *wtname; 105 195 int wtname_len; 106 196 107 - if (!refname) 108 - return store->main_stack; 197 + if (!refname) { 198 + be = &store->main_backend; 199 + goto out; 200 + } 109 201 110 202 switch (parse_worktree_ref(refname, &wtname, &wtname_len, rewritten_ref)) { 111 203 case REF_WORKTREE_OTHER: { 112 204 static struct strbuf wtname_buf = STRBUF_INIT; 113 205 struct strbuf wt_dir = STRBUF_INIT; 114 - struct reftable_stack *stack; 115 206 116 207 /* 117 208 * We're using a static buffer here so that we don't need to ··· 125 216 /* 126 217 * There is an edge case here: when the worktree references the 127 218 * current worktree, then we set up the stack once via 128 - * `worktree_stacks` and once via `worktree_stack`. This is 219 + * `worktree_backends` and once via `worktree_backend`. This is 129 220 * wasteful, but in the reading case it shouldn't matter. And 130 221 * in the writing case we would notice that the stack is locked 131 222 * already and error out when trying to write a reference via 132 223 * both stacks. 133 224 */ 134 - stack = strmap_get(&store->worktree_stacks, wtname_buf.buf); 135 - if (!stack) { 225 + be = strmap_get(&store->worktree_backends, wtname_buf.buf); 226 + if (!be) { 136 227 strbuf_addf(&wt_dir, "%s/worktrees/%s/reftable", 137 228 store->base.repo->commondir, wtname_buf.buf); 138 229 139 - store->err = reftable_new_stack(&stack, wt_dir.buf, 140 - &store->write_options); 230 + CALLOC_ARRAY(be, 1); 231 + store->err = reftable_backend_init(be, wt_dir.buf, 232 + &store->write_options); 141 233 assert(store->err != REFTABLE_API_ERROR); 142 - strmap_put(&store->worktree_stacks, wtname_buf.buf, stack); 234 + 235 + strmap_put(&store->worktree_backends, wtname_buf.buf, be); 143 236 } 144 237 145 238 strbuf_release(&wt_dir); 146 - return stack; 239 + goto out; 147 240 } 148 241 case REF_WORKTREE_CURRENT: 149 242 /* 150 243 * If there is no worktree stack then we're currently in the 151 244 * main worktree. We thus return the main stack in that case. 152 245 */ 153 - if (!store->worktree_stack) 154 - return store->main_stack; 155 - return store->worktree_stack; 246 + if (!store->worktree_backend.stack) 247 + be = &store->main_backend; 248 + else 249 + be = &store->worktree_backend; 250 + goto out; 156 251 case REF_WORKTREE_MAIN: 157 252 case REF_WORKTREE_SHARED: 158 - return store->main_stack; 253 + be = &store->main_backend; 254 + goto out; 159 255 default: 160 256 BUG("unhandled worktree reference type"); 161 257 } 258 + 259 + out: 260 + if (reload) { 261 + int ret = reftable_stack_reload(be->stack); 262 + if (ret) 263 + return ret; 264 + } 265 + *out = be; 266 + 267 + return 0; 162 268 } 163 269 164 270 static int should_write_log(struct reftable_ref_store *refs, const char *refname) ··· 207 313 log->value.update.tz_offset = sign * atoi(tz_begin); 208 314 } 209 315 210 - static int read_ref_without_reload(struct reftable_ref_store *refs, 211 - struct reftable_stack *stack, 212 - const char *refname, 213 - struct object_id *oid, 214 - struct strbuf *referent, 215 - unsigned int *type) 216 - { 217 - struct reftable_ref_record ref = {0}; 218 - int ret; 219 - 220 - ret = reftable_stack_read_ref(stack, refname, &ref); 221 - if (ret) 222 - goto done; 223 - 224 - if (ref.value_type == REFTABLE_REF_SYMREF) { 225 - strbuf_reset(referent); 226 - strbuf_addstr(referent, ref.value.symref); 227 - *type |= REF_ISSYMREF; 228 - } else if (reftable_ref_record_val1(&ref)) { 229 - oidread(oid, reftable_ref_record_val1(&ref), 230 - refs->base.repo->hash_algo); 231 - } else { 232 - /* We got a tombstone, which should not happen. */ 233 - BUG("unhandled reference value type %d", ref.value_type); 234 - } 235 - 236 - done: 237 - assert(ret != REFTABLE_API_ERROR); 238 - reftable_ref_record_release(&ref); 239 - return ret; 240 - } 241 - 242 316 static int reftable_be_config(const char *var, const char *value, 243 317 const struct config_context *ctx, 244 318 void *_opts) ··· 292 366 umask(mask); 293 367 294 368 base_ref_store_init(&refs->base, repo, gitdir, &refs_be_reftable); 295 - strmap_init(&refs->worktree_stacks); 369 + strmap_init(&refs->worktree_backends); 296 370 refs->store_flags = store_flags; 297 371 refs->log_all_ref_updates = repo_settings_get_log_all_ref_updates(repo); 298 372 ··· 337 411 strbuf_realpath(&path, gitdir, 0); 338 412 } 339 413 strbuf_addstr(&path, "/reftable"); 340 - refs->err = reftable_new_stack(&refs->main_stack, path.buf, 341 - &refs->write_options); 414 + refs->err = reftable_backend_init(&refs->main_backend, path.buf, 415 + &refs->write_options); 342 416 if (refs->err) 343 417 goto done; 344 418 ··· 354 428 strbuf_reset(&path); 355 429 strbuf_addf(&path, "%s/reftable", gitdir); 356 430 357 - refs->err = reftable_new_stack(&refs->worktree_stack, path.buf, 358 - &refs->write_options); 431 + refs->err = reftable_backend_init(&refs->worktree_backend, path.buf, 432 + &refs->write_options); 359 433 if (refs->err) 360 434 goto done; 361 435 } ··· 374 448 struct strmap_entry *entry; 375 449 struct hashmap_iter iter; 376 450 377 - if (refs->main_stack) { 378 - reftable_stack_destroy(refs->main_stack); 379 - refs->main_stack = NULL; 380 - } 451 + if (refs->main_backend.stack) 452 + reftable_backend_release(&refs->main_backend); 453 + if (refs->worktree_backend.stack) 454 + reftable_backend_release(&refs->worktree_backend); 381 455 382 - if (refs->worktree_stack) { 383 - reftable_stack_destroy(refs->worktree_stack); 384 - refs->worktree_stack = NULL; 456 + strmap_for_each_entry(&refs->worktree_backends, &iter, entry) { 457 + struct reftable_backend *be = entry->value; 458 + reftable_backend_release(be); 459 + free(be); 385 460 } 386 - 387 - strmap_for_each_entry(&refs->worktree_stacks, &iter, entry) 388 - reftable_stack_destroy(entry->value); 389 - strmap_clear(&refs->worktree_stacks, 0); 461 + strmap_clear(&refs->worktree_backends, 0); 390 462 } 391 463 392 464 static int reftable_be_create_on_disk(struct ref_store *ref_store, ··· 781 853 required_flags |= REF_STORE_ODB; 782 854 refs = reftable_be_downcast(ref_store, required_flags, "ref_iterator_begin"); 783 855 784 - main_iter = ref_iterator_for_stack(refs, refs->main_stack, prefix, 856 + main_iter = ref_iterator_for_stack(refs, refs->main_backend.stack, prefix, 785 857 exclude_patterns, flags); 786 858 787 859 /* ··· 789 861 * right now. If we aren't, then we return the common reftable 790 862 * iterator, only. 791 863 */ 792 - if (!refs->worktree_stack) 864 + if (!refs->worktree_backend.stack) 793 865 return &main_iter->base; 794 866 795 867 /* 796 868 * Otherwise we merge both the common and the per-worktree refs into a 797 869 * single iterator. 798 870 */ 799 - worktree_iter = ref_iterator_for_stack(refs, refs->worktree_stack, prefix, 871 + worktree_iter = ref_iterator_for_stack(refs, refs->worktree_backend.stack, prefix, 800 872 exclude_patterns, flags); 801 873 return merge_ref_iterator_begin(&worktree_iter->base, &main_iter->base, 802 874 ref_iterator_select, NULL); ··· 811 883 { 812 884 struct reftable_ref_store *refs = 813 885 reftable_be_downcast(ref_store, REF_STORE_READ, "read_raw_ref"); 814 - struct reftable_stack *stack = stack_for(refs, refname, &refname); 886 + struct reftable_backend *be; 815 887 int ret; 816 888 817 889 if (refs->err < 0) 818 890 return refs->err; 819 891 820 - ret = reftable_stack_reload(stack); 892 + ret = backend_for(&be, refs, refname, &refname, 1); 821 893 if (ret) 822 894 return ret; 823 895 824 - ret = read_ref_without_reload(refs, stack, refname, oid, referent, type); 896 + ret = reftable_backend_read_ref(be, refname, oid, referent, type); 825 897 if (ret < 0) 826 898 return ret; 827 899 if (ret > 0) { ··· 838 910 { 839 911 struct reftable_ref_store *refs = 840 912 reftable_be_downcast(ref_store, REF_STORE_READ, "read_symbolic_ref"); 841 - struct reftable_stack *stack = stack_for(refs, refname, &refname); 842 - struct reftable_ref_record ref = {0}; 913 + struct reftable_backend *be; 914 + struct object_id oid; 915 + unsigned int type = 0; 843 916 int ret; 844 917 845 - ret = reftable_stack_reload(stack); 918 + ret = backend_for(&be, refs, refname, &refname, 1); 846 919 if (ret) 847 920 return ret; 848 921 849 - ret = reftable_stack_read_ref(stack, refname, &ref); 850 - if (ret == 0 && ref.value_type == REFTABLE_REF_SYMREF) 851 - strbuf_addstr(referent, ref.value.symref); 852 - else 922 + ret = reftable_backend_read_ref(be, refname, &oid, referent, &type); 923 + if (type != REF_ISSYMREF) 853 924 ret = -1; 854 - 855 - reftable_ref_record_release(&ref); 856 925 return ret; 857 926 } 858 927 ··· 863 932 864 933 struct write_transaction_table_arg { 865 934 struct reftable_ref_store *refs; 866 - struct reftable_stack *stack; 935 + struct reftable_backend *be; 867 936 struct reftable_addition *addition; 868 937 struct reftable_transaction_update *updates; 869 938 size_t updates_nr; ··· 898 967 struct ref_update *update, 899 968 struct strbuf *err) 900 969 { 901 - struct reftable_stack *stack = stack_for(refs, update->refname, NULL); 902 970 struct write_transaction_table_arg *arg = NULL; 971 + struct reftable_backend *be; 903 972 size_t i; 904 973 int ret; 905 974 906 975 /* 976 + * This function gets called in a loop, and we don't want to repeatedly 977 + * reload the stack for every single ref update. Instead, we manually 978 + * reload further down in the case where we haven't yet prepared the 979 + * specific `reftable_backend`. 980 + */ 981 + ret = backend_for(&be, refs, update->refname, NULL, 0); 982 + if (ret) 983 + return ret; 984 + 985 + /* 907 986 * Search for a preexisting stack update. If there is one then we add 908 987 * the update to it, otherwise we set up a new stack update. 909 988 */ 910 989 for (i = 0; !arg && i < tx_data->args_nr; i++) 911 - if (tx_data->args[i].stack == stack) 990 + if (tx_data->args[i].be == be) 912 991 arg = &tx_data->args[i]; 913 992 914 993 if (!arg) { 915 994 struct reftable_addition *addition; 916 995 917 - ret = reftable_stack_reload(stack); 996 + ret = reftable_stack_reload(be->stack); 918 997 if (ret) 919 998 return ret; 920 999 921 - ret = reftable_stack_new_addition(&addition, stack, 1000 + ret = reftable_stack_new_addition(&addition, be->stack, 922 1001 REFTABLE_STACK_NEW_ADDITION_RELOAD); 923 1002 if (ret) { 924 1003 if (ret == REFTABLE_LOCK_ERROR) ··· 930 1009 tx_data->args_alloc); 931 1010 arg = &tx_data->args[tx_data->args_nr++]; 932 1011 arg->refs = refs; 933 - arg->stack = stack; 1012 + arg->be = be; 934 1013 arg->addition = addition; 935 1014 arg->updates = NULL; 936 1015 arg->updates_nr = 0; ··· 985 1064 struct strbuf referent = STRBUF_INIT, head_referent = STRBUF_INIT; 986 1065 struct string_list affected_refnames = STRING_LIST_INIT_NODUP; 987 1066 struct reftable_transaction_data *tx_data = NULL; 1067 + struct reftable_backend *be; 988 1068 struct object_id head_oid; 989 1069 unsigned int head_type = 0; 990 1070 size_t i; ··· 1031 1111 goto done; 1032 1112 } 1033 1113 1034 - ret = read_ref_without_reload(refs, stack_for(refs, "HEAD", NULL), "HEAD", 1035 - &head_oid, &head_referent, &head_type); 1114 + /* 1115 + * TODO: it's dubious whether we should reload the stack that "HEAD" 1116 + * belongs to or not. In theory, it may happen that we only modify 1117 + * stacks which are _not_ part of the "HEAD" stack. In that case we 1118 + * wouldn't have prepared any transaction for its stack and would not 1119 + * have reloaded it, which may mean that it is stale. 1120 + * 1121 + * On the other hand, reloading that stack without locking it feels 1122 + * wrong, too, as the value of "HEAD" could be modified concurrently at 1123 + * any point in time. 1124 + */ 1125 + ret = backend_for(&be, refs, "HEAD", NULL, 0); 1126 + if (ret) 1127 + goto done; 1128 + 1129 + ret = reftable_backend_read_ref(be, "HEAD", &head_oid, 1130 + &head_referent, &head_type); 1036 1131 if (ret < 0) 1037 1132 goto done; 1038 1133 ret = 0; ··· 1040 1135 for (i = 0; i < transaction->nr; i++) { 1041 1136 struct ref_update *u = transaction->updates[i]; 1042 1137 struct object_id current_oid = {0}; 1043 - struct reftable_stack *stack; 1044 1138 const char *rewritten_ref; 1045 1139 1046 - stack = stack_for(refs, u->refname, &rewritten_ref); 1140 + /* 1141 + * There is no need to reload the respective backends here as 1142 + * we have already reloaded them when preparing the transaction 1143 + * update. And given that the stacks have been locked there 1144 + * shouldn't have been any concurrent modifications of the 1145 + * stack. 1146 + */ 1147 + ret = backend_for(&be, refs, u->refname, &rewritten_ref, 0); 1148 + if (ret) 1149 + goto done; 1047 1150 1048 1151 /* Verify that the new object ID is valid. */ 1049 1152 if ((u->flags & REF_HAVE_NEW) && !is_null_oid(&u->new_oid) && ··· 1099 1202 string_list_insert(&affected_refnames, new_update->refname); 1100 1203 } 1101 1204 1102 - ret = read_ref_without_reload(refs, stack, rewritten_ref, 1103 - &current_oid, &referent, &u->type); 1205 + ret = reftable_backend_read_ref(be, rewritten_ref, 1206 + &current_oid, &referent, &u->type); 1104 1207 if (ret < 0) 1105 1208 goto done; 1106 1209 if (ret > 0 && !ref_update_expects_existing_old_ref(u)) { ··· 1303 1406 static int write_transaction_table(struct reftable_writer *writer, void *cb_data) 1304 1407 { 1305 1408 struct write_transaction_table_arg *arg = cb_data; 1306 - uint64_t ts = reftable_stack_next_update_index(arg->stack); 1409 + uint64_t ts = reftable_stack_next_update_index(arg->be->stack); 1307 1410 struct reftable_log_record *logs = NULL; 1308 1411 struct ident_split committer_ident = {0}; 1309 1412 size_t logs_nr = 0, logs_alloc = 0, i; ··· 1339 1442 struct reftable_log_record log = {0}; 1340 1443 struct reftable_iterator it = {0}; 1341 1444 1342 - ret = reftable_stack_init_log_iterator(arg->stack, &it); 1445 + ret = reftable_stack_init_log_iterator(arg->be->stack, &it); 1343 1446 if (ret < 0) 1344 1447 goto done; 1345 1448 ··· 1520 1623 if (refs->err) 1521 1624 return refs->err; 1522 1625 1523 - stack = refs->worktree_stack; 1626 + stack = refs->worktree_backend.stack; 1524 1627 if (!stack) 1525 - stack = refs->main_stack; 1628 + stack = refs->main_backend.stack; 1526 1629 1527 1630 if (opts->flags & PACK_REFS_AUTO) 1528 1631 ret = reftable_stack_auto_compact(stack); ··· 1553 1656 1554 1657 struct write_copy_arg { 1555 1658 struct reftable_ref_store *refs; 1556 - struct reftable_stack *stack; 1659 + struct reftable_backend *be; 1557 1660 const char *oldname; 1558 1661 const char *newname; 1559 1662 const char *logmsg; ··· 1578 1681 if (split_ident_line(&committer_ident, committer_info, strlen(committer_info))) 1579 1682 BUG("failed splitting committer info"); 1580 1683 1581 - if (reftable_stack_read_ref(arg->stack, arg->oldname, &old_ref)) { 1684 + if (reftable_stack_read_ref(arg->be->stack, arg->oldname, &old_ref)) { 1582 1685 ret = error(_("refname %s not found"), arg->oldname); 1583 1686 goto done; 1584 1687 } ··· 1617 1720 * the old branch and the creation of the new branch, and we cannot do 1618 1721 * two changes to a reflog in a single update. 1619 1722 */ 1620 - deletion_ts = creation_ts = reftable_stack_next_update_index(arg->stack); 1723 + deletion_ts = creation_ts = reftable_stack_next_update_index(arg->be->stack); 1621 1724 if (arg->delete_old) 1622 1725 creation_ts++; 1623 1726 reftable_writer_set_limits(writer, deletion_ts, creation_ts); ··· 1660 1763 memcpy(logs[logs_nr].value.update.old_hash, old_ref.value.val1, GIT_MAX_RAWSZ); 1661 1764 logs_nr++; 1662 1765 1663 - ret = read_ref_without_reload(arg->refs, arg->stack, "HEAD", &head_oid, 1664 - &head_referent, &head_type); 1766 + ret = reftable_backend_read_ref(arg->be, "HEAD", &head_oid, 1767 + &head_referent, &head_type); 1665 1768 if (ret < 0) 1666 1769 goto done; 1667 1770 append_head_reflog = (head_type & REF_ISSYMREF) && !strcmp(head_referent.buf, arg->oldname); ··· 1704 1807 * copy over all log entries from the old reflog. Last but not least, 1705 1808 * when renaming we also have to delete all the old reflog entries. 1706 1809 */ 1707 - ret = reftable_stack_init_log_iterator(arg->stack, &it); 1810 + ret = reftable_stack_init_log_iterator(arg->be->stack, &it); 1708 1811 if (ret < 0) 1709 1812 goto done; 1710 1813 ··· 1777 1880 { 1778 1881 struct reftable_ref_store *refs = 1779 1882 reftable_be_downcast(ref_store, REF_STORE_WRITE, "rename_ref"); 1780 - struct reftable_stack *stack = stack_for(refs, newrefname, &newrefname); 1781 1883 struct write_copy_arg arg = { 1782 1884 .refs = refs, 1783 - .stack = stack, 1784 1885 .oldname = oldrefname, 1785 1886 .newname = newrefname, 1786 1887 .logmsg = logmsg, ··· 1792 1893 if (ret < 0) 1793 1894 goto done; 1794 1895 1795 - ret = reftable_stack_reload(stack); 1896 + ret = backend_for(&arg.be, refs, newrefname, &newrefname, 1); 1796 1897 if (ret) 1797 1898 goto done; 1798 - ret = reftable_stack_add(stack, &write_copy_table, &arg); 1899 + ret = reftable_stack_add(arg.be->stack, &write_copy_table, &arg); 1799 1900 1800 1901 done: 1801 1902 assert(ret != REFTABLE_API_ERROR); ··· 1809 1910 { 1810 1911 struct reftable_ref_store *refs = 1811 1912 reftable_be_downcast(ref_store, REF_STORE_WRITE, "copy_ref"); 1812 - struct reftable_stack *stack = stack_for(refs, newrefname, &newrefname); 1813 1913 struct write_copy_arg arg = { 1814 1914 .refs = refs, 1815 - .stack = stack, 1816 1915 .oldname = oldrefname, 1817 1916 .newname = newrefname, 1818 1917 .logmsg = logmsg, ··· 1823 1922 if (ret < 0) 1824 1923 goto done; 1825 1924 1826 - ret = reftable_stack_reload(stack); 1925 + ret = backend_for(&arg.be, refs, newrefname, &newrefname, 1); 1827 1926 if (ret) 1828 1927 goto done; 1829 - ret = reftable_stack_add(stack, &write_copy_table, &arg); 1928 + ret = reftable_stack_add(arg.be->stack, &write_copy_table, &arg); 1830 1929 1831 1930 done: 1832 1931 assert(ret != REFTABLE_API_ERROR); ··· 1947 2046 reftable_be_downcast(ref_store, REF_STORE_READ, "reflog_iterator_begin"); 1948 2047 struct reftable_reflog_iterator *main_iter, *worktree_iter; 1949 2048 1950 - main_iter = reflog_iterator_for_stack(refs, refs->main_stack); 1951 - if (!refs->worktree_stack) 2049 + main_iter = reflog_iterator_for_stack(refs, refs->main_backend.stack); 2050 + if (!refs->worktree_backend.stack) 1952 2051 return &main_iter->base; 1953 2052 1954 - worktree_iter = reflog_iterator_for_stack(refs, refs->worktree_stack); 2053 + worktree_iter = reflog_iterator_for_stack(refs, refs->worktree_backend.stack); 1955 2054 1956 2055 return merge_ref_iterator_begin(&worktree_iter->base, &main_iter->base, 1957 2056 ref_iterator_select, NULL); ··· 1990 2089 { 1991 2090 struct reftable_ref_store *refs = 1992 2091 reftable_be_downcast(ref_store, REF_STORE_READ, "for_each_reflog_ent_reverse"); 1993 - struct reftable_stack *stack = stack_for(refs, refname, &refname); 1994 2092 struct reftable_log_record log = {0}; 1995 2093 struct reftable_iterator it = {0}; 2094 + struct reftable_backend *be; 1996 2095 int ret; 1997 2096 1998 2097 if (refs->err < 0) 1999 2098 return refs->err; 2000 2099 2001 - ret = reftable_stack_init_log_iterator(stack, &it); 2100 + /* 2101 + * TODO: we should adapt this callsite to reload the stack. There is no 2102 + * obvious reason why we shouldn't. 2103 + */ 2104 + ret = backend_for(&be, refs, refname, &refname, 0); 2105 + if (ret) 2106 + goto done; 2107 + 2108 + ret = reftable_stack_init_log_iterator(be->stack, &it); 2002 2109 if (ret < 0) 2003 2110 goto done; 2004 2111 ··· 2030 2137 { 2031 2138 struct reftable_ref_store *refs = 2032 2139 reftable_be_downcast(ref_store, REF_STORE_READ, "for_each_reflog_ent"); 2033 - struct reftable_stack *stack = stack_for(refs, refname, &refname); 2034 2140 struct reftable_log_record *logs = NULL; 2035 2141 struct reftable_iterator it = {0}; 2142 + struct reftable_backend *be; 2036 2143 size_t logs_alloc = 0, logs_nr = 0, i; 2037 2144 int ret; 2038 2145 2039 2146 if (refs->err < 0) 2040 2147 return refs->err; 2041 2148 2042 - ret = reftable_stack_init_log_iterator(stack, &it); 2149 + /* 2150 + * TODO: we should adapt this callsite to reload the stack. There is no 2151 + * obvious reason why we shouldn't. 2152 + */ 2153 + ret = backend_for(&be, refs, refname, &refname, 0); 2154 + if (ret) 2155 + goto done; 2156 + 2157 + ret = reftable_stack_init_log_iterator(be->stack, &it); 2043 2158 if (ret < 0) 2044 2159 goto done; 2045 2160 ··· 2079 2194 { 2080 2195 struct reftable_ref_store *refs = 2081 2196 reftable_be_downcast(ref_store, REF_STORE_READ, "reflog_exists"); 2082 - struct reftable_stack *stack = stack_for(refs, refname, &refname); 2083 2197 struct reftable_log_record log = {0}; 2084 2198 struct reftable_iterator it = {0}; 2199 + struct reftable_backend *be; 2085 2200 int ret; 2086 2201 2087 2202 ret = refs->err; 2088 2203 if (ret < 0) 2089 2204 goto done; 2090 2205 2091 - ret = reftable_stack_reload(stack); 2206 + ret = backend_for(&be, refs, refname, &refname, 1); 2092 2207 if (ret < 0) 2093 2208 goto done; 2094 2209 2095 - ret = reftable_stack_init_log_iterator(stack, &it); 2210 + ret = reftable_stack_init_log_iterator(be->stack, &it); 2096 2211 if (ret < 0) 2097 2212 goto done; 2098 2213 ··· 2164 2279 { 2165 2280 struct reftable_ref_store *refs = 2166 2281 reftable_be_downcast(ref_store, REF_STORE_WRITE, "create_reflog"); 2167 - struct reftable_stack *stack = stack_for(refs, refname, &refname); 2282 + struct reftable_backend *be; 2168 2283 struct write_reflog_existence_arg arg = { 2169 2284 .refs = refs, 2170 - .stack = stack, 2171 2285 .refname = refname, 2172 2286 }; 2173 2287 int ret; ··· 2176 2290 if (ret < 0) 2177 2291 goto done; 2178 2292 2179 - ret = reftable_stack_reload(stack); 2293 + ret = backend_for(&be, refs, refname, &refname, 1); 2180 2294 if (ret) 2181 2295 goto done; 2296 + arg.stack = be->stack; 2182 2297 2183 - ret = reftable_stack_add(stack, &write_reflog_existence_table, &arg); 2298 + ret = reftable_stack_add(be->stack, &write_reflog_existence_table, &arg); 2184 2299 2185 2300 done: 2186 2301 return ret; ··· 2238 2353 { 2239 2354 struct reftable_ref_store *refs = 2240 2355 reftable_be_downcast(ref_store, REF_STORE_WRITE, "delete_reflog"); 2241 - struct reftable_stack *stack = stack_for(refs, refname, &refname); 2356 + struct reftable_backend *be; 2242 2357 struct write_reflog_delete_arg arg = { 2243 - .stack = stack, 2244 2358 .refname = refname, 2245 2359 }; 2246 2360 int ret; 2247 2361 2248 - ret = reftable_stack_reload(stack); 2362 + ret = backend_for(&be, refs, refname, &refname, 1); 2249 2363 if (ret) 2250 2364 return ret; 2251 - ret = reftable_stack_add(stack, &write_reflog_delete_table, &arg); 2365 + arg.stack = be->stack; 2366 + 2367 + ret = reftable_stack_add(be->stack, &write_reflog_delete_table, &arg); 2252 2368 2253 2369 assert(ret != REFTABLE_API_ERROR); 2254 2370 return ret; ··· 2347 2463 */ 2348 2464 struct reftable_ref_store *refs = 2349 2465 reftable_be_downcast(ref_store, REF_STORE_WRITE, "reflog_expire"); 2350 - struct reftable_stack *stack = stack_for(refs, refname, &refname); 2351 2466 struct reftable_log_record *logs = NULL; 2352 2467 struct reftable_log_record *rewritten = NULL; 2353 - struct reftable_ref_record ref_record = {0}; 2354 2468 struct reftable_iterator it = {0}; 2355 2469 struct reftable_addition *add = NULL; 2356 2470 struct reflog_expiry_arg arg = {0}; 2471 + struct reftable_backend *be; 2357 2472 struct object_id oid = {0}; 2473 + struct strbuf referent = STRBUF_INIT; 2358 2474 uint8_t *last_hash = NULL; 2359 2475 size_t logs_nr = 0, logs_alloc = 0, i; 2476 + unsigned int type = 0; 2360 2477 int ret; 2361 2478 2362 2479 if (refs->err < 0) 2363 2480 return refs->err; 2364 2481 2365 - ret = reftable_stack_reload(stack); 2482 + ret = backend_for(&be, refs, refname, &refname, 1); 2366 2483 if (ret < 0) 2367 2484 goto done; 2368 2485 2369 - ret = reftable_stack_init_log_iterator(stack, &it); 2486 + ret = reftable_stack_init_log_iterator(be->stack, &it); 2370 2487 if (ret < 0) 2371 2488 goto done; 2372 2489 ··· 2374 2491 if (ret < 0) 2375 2492 goto done; 2376 2493 2377 - ret = reftable_stack_new_addition(&add, stack, 0); 2494 + ret = reftable_stack_new_addition(&add, be->stack, 0); 2378 2495 if (ret < 0) 2379 2496 goto done; 2380 2497 2381 - ret = reftable_stack_read_ref(stack, refname, &ref_record); 2498 + ret = reftable_backend_read_ref(be, refname, &oid, &referent, &type); 2382 2499 if (ret < 0) 2383 2500 goto done; 2384 - if (reftable_ref_record_val1(&ref_record)) 2385 - oidread(&oid, reftable_ref_record_val1(&ref_record), 2386 - ref_store->repo->hash_algo); 2387 2501 prepare_fn(refname, &oid, policy_cb_data); 2388 2502 2389 2503 while (1) { ··· 2450 2564 } 2451 2565 } 2452 2566 2453 - if (flags & EXPIRE_REFLOGS_UPDATE_REF && last_hash && 2454 - reftable_ref_record_val1(&ref_record)) 2567 + if (flags & EXPIRE_REFLOGS_UPDATE_REF && last_hash && !is_null_oid(&oid)) 2455 2568 oidread(&arg.update_oid, last_hash, ref_store->repo->hash_algo); 2456 2569 2457 2570 arg.refs = refs; 2458 2571 arg.records = rewritten; 2459 2572 arg.len = logs_nr; 2460 - arg.stack = stack, 2461 - arg.refname = refname, 2573 + arg.stack = be->stack; 2574 + arg.refname = refname; 2462 2575 2463 2576 ret = reftable_addition_add(add, &write_reflog_expiry_table, &arg); 2464 2577 if (ret < 0) ··· 2476 2589 cleanup_fn(policy_cb_data); 2477 2590 assert(ret != REFTABLE_API_ERROR); 2478 2591 2479 - reftable_ref_record_release(&ref_record); 2480 2592 reftable_iterator_destroy(&it); 2481 2593 reftable_addition_destroy(add); 2482 2594 for (i = 0; i < logs_nr; i++) 2483 2595 reftable_log_record_release(&logs[i]); 2596 + strbuf_release(&referent); 2484 2597 free(logs); 2485 2598 free(rewritten); 2486 2599 return ret;

+2

reftable/merged.c

··· 66 66 int err; 67 67 68 68 mi->advance_index = -1; 69 + while (!merged_iter_pqueue_is_empty(mi->pq)) 70 + merged_iter_pqueue_remove(&mi->pq); 69 71 70 72 for (size_t i = 0; i < mi->subiters_len; i++) { 71 73 err = iterator_seek(&mi->subiters[i].iter, want);

+3

reftable/reftable-stack.h

··· 149 149 struct reftable_compaction_stats * 150 150 reftable_stack_compaction_stats(struct reftable_stack *st); 151 151 152 + /* Return the hash of the stack. */ 153 + enum reftable_hash reftable_stack_hash_id(struct reftable_stack *st); 154 + 152 155 #endif

+9

reftable/reftable-writer.h

··· 68 68 * fsync(3P) when unset. 69 69 */ 70 70 int (*fsync)(int fd); 71 + 72 + /* 73 + * Callback function to execute whenever the stack is being reloaded. 74 + * This can be used e.g. to discard cached information that relies on 75 + * the old stack's data. The payload data will be passed as argument to 76 + * the callback. 77 + */ 78 + void (*on_reload)(void *payload); 79 + void *on_reload_payload; 71 80 }; 72 81 73 82 /* reftable_block_stats holds statistics for a single block type */

+9

reftable/stack.c

··· 548 548 close(fd); 549 549 free_names(names); 550 550 free_names(names_after); 551 + 552 + if (st->opts.on_reload) 553 + st->opts.on_reload(st->opts.on_reload_payload); 554 + 551 555 return err; 552 556 } 553 557 ··· 1791 1795 reftable_addition_destroy(add); 1792 1796 return err; 1793 1797 } 1798 + 1799 + enum reftable_hash reftable_stack_hash_id(struct reftable_stack *st) 1800 + { 1801 + return reftable_merged_table_hash_id(st->merged); 1802 + }

+73

t/unit-tests/t-reftable-merged.c

··· 273 273 reftable_free(sources); 274 274 } 275 275 276 + static void t_merged_seek_multiple_times_without_draining(void) 277 + { 278 + struct reftable_ref_record r1[] = { 279 + { 280 + .refname = (char *) "a", 281 + .update_index = 1, 282 + .value_type = REFTABLE_REF_VAL1, 283 + .value.val1 = { 1 }, 284 + }, 285 + { 286 + .refname = (char *) "c", 287 + .update_index = 1, 288 + .value_type = REFTABLE_REF_VAL1, 289 + .value.val1 = { 2 }, 290 + } 291 + }; 292 + struct reftable_ref_record r2[] = { 293 + { 294 + .refname = (char *) "b", 295 + .update_index = 2, 296 + .value_type = REFTABLE_REF_VAL1, 297 + .value.val1 = { 3 }, 298 + }, 299 + { 300 + .refname = (char *) "d", 301 + .update_index = 2, 302 + .value_type = REFTABLE_REF_VAL1, 303 + .value.val1 = { 4 }, 304 + }, 305 + }; 306 + struct reftable_ref_record *refs[] = { 307 + r1, r2, 308 + }; 309 + size_t sizes[] = { 310 + ARRAY_SIZE(r1), ARRAY_SIZE(r2), 311 + }; 312 + struct reftable_buf bufs[] = { 313 + REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, 314 + }; 315 + struct reftable_block_source *sources = NULL; 316 + struct reftable_reader **readers = NULL; 317 + struct reftable_ref_record rec = { 0 }; 318 + struct reftable_iterator it = { 0 }; 319 + struct reftable_merged_table *mt; 320 + int err; 321 + 322 + mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2); 323 + merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); 324 + 325 + err = reftable_iterator_seek_ref(&it, "b"); 326 + check(!err); 327 + err = reftable_iterator_next_ref(&it, &rec); 328 + check(!err); 329 + err = reftable_ref_record_equal(&rec, &r2[0], REFTABLE_HASH_SIZE_SHA1); 330 + check(err == 1); 331 + 332 + err = reftable_iterator_seek_ref(&it, "a"); 333 + check(!err); 334 + err = reftable_iterator_next_ref(&it, &rec); 335 + check(!err); 336 + err = reftable_ref_record_equal(&rec, &r1[0], REFTABLE_HASH_SIZE_SHA1); 337 + check(err == 1); 338 + 339 + for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) 340 + reftable_buf_release(&bufs[i]); 341 + readers_destroy(readers, ARRAY_SIZE(refs)); 342 + reftable_ref_record_release(&rec); 343 + reftable_iterator_destroy(&it); 344 + reftable_merged_table_free(mt); 345 + reftable_free(sources); 346 + } 347 + 276 348 static struct reftable_merged_table * 277 349 merged_table_from_log_records(struct reftable_log_record **logs, 278 350 struct reftable_block_source **source, ··· 467 539 TEST(t_merged_logs(), "merged table with multiple log updates for same ref"); 468 540 TEST(t_merged_refs(), "merged table with multiple updates to same ref"); 469 541 TEST(t_merged_seek_multiple_times(), "merged table can seek multiple times"); 542 + TEST(t_merged_seek_multiple_times_without_draining(), "merged table can seek multiple times without draining"); 470 543 TEST(t_merged_single_record(), "ref occurring in only one record can be fetched"); 471 544 472 545 return test_done();