+2 -2

Documentation/howto/recover-corrupted-object-harder.adoc

··· 125 { 126 /* make this absurdly large so we don't have to loop */ 127 static unsigned char out[1024*1024]; 128 - z_stream z; 129 int ret; 130 131 memset(&z, 0, sizeof(z)); ··· 278 static unsigned char buf[25 * 1024 * 1024]; 279 static unsigned char out[25 * 1024 * 1024]; 280 int len; 281 - z_stream z; 282 int ret; 283 284 len = read(0, buf, sizeof(buf));

··· 125 { 126 /* make this absurdly large so we don't have to loop */ 127 static unsigned char out[1024*1024]; 128 + struct z_stream_s z; 129 int ret; 130 131 memset(&z, 0, sizeof(z)); ··· 278 static unsigned char buf[25 * 1024 * 1024]; 279 static unsigned char out[25 * 1024 * 1024]; 280 int len; 281 + struct z_stream_s z; 282 int ret; 283 284 len = read(0, buf, sizeof(buf));

+2 -2

Makefile

··· 1377 UNIT_TEST_PROGRAMS += t-reftable-block 1378 UNIT_TEST_PROGRAMS += t-reftable-merged 1379 UNIT_TEST_PROGRAMS += t-reftable-pq 1380 - UNIT_TEST_PROGRAMS += t-reftable-reader 1381 UNIT_TEST_PROGRAMS += t-reftable-readwrite 1382 UNIT_TEST_PROGRAMS += t-reftable-record 1383 UNIT_TEST_PROGRAMS += t-reftable-stack 0 1384 UNIT_TEST_PROGS = $(patsubst %,$(UNIT_TEST_BIN)/%$X,$(UNIT_TEST_PROGRAMS)) 1385 UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o 1386 UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/lib-reftable.o ··· 2737 REFTABLE_OBJS += reftable/iter.o 2738 REFTABLE_OBJS += reftable/merged.o 2739 REFTABLE_OBJS += reftable/pq.o 2740 - REFTABLE_OBJS += reftable/reader.o 2741 REFTABLE_OBJS += reftable/record.o 2742 REFTABLE_OBJS += reftable/stack.o 2743 REFTABLE_OBJS += reftable/system.o 0 2744 REFTABLE_OBJS += reftable/tree.o 2745 REFTABLE_OBJS += reftable/writer.o 2746

··· 1377 UNIT_TEST_PROGRAMS += t-reftable-block 1378 UNIT_TEST_PROGRAMS += t-reftable-merged 1379 UNIT_TEST_PROGRAMS += t-reftable-pq 0 1380 UNIT_TEST_PROGRAMS += t-reftable-readwrite 1381 UNIT_TEST_PROGRAMS += t-reftable-record 1382 UNIT_TEST_PROGRAMS += t-reftable-stack 1383 + UNIT_TEST_PROGRAMS += t-reftable-table 1384 UNIT_TEST_PROGS = $(patsubst %,$(UNIT_TEST_BIN)/%$X,$(UNIT_TEST_PROGRAMS)) 1385 UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/test-lib.o 1386 UNIT_TEST_OBJS += $(UNIT_TEST_DIR)/lib-reftable.o ··· 2737 REFTABLE_OBJS += reftable/iter.o 2738 REFTABLE_OBJS += reftable/merged.o 2739 REFTABLE_OBJS += reftable/pq.o 0 2740 REFTABLE_OBJS += reftable/record.o 2741 REFTABLE_OBJS += reftable/stack.o 2742 REFTABLE_OBJS += reftable/system.o 2743 + REFTABLE_OBJS += reftable/table.o 2744 REFTABLE_OBJS += reftable/tree.o 2745 REFTABLE_OBJS += reftable/writer.o 2746

+2 -2

compat/zlib-compat.h

··· 4 #ifdef HAVE_ZLIB_NG 5 # include <zlib-ng.h> 6 7 - # define z_stream zng_stream 8 - #define gz_header_s zng_gz_header_s 9 10 # define crc32(crc, buf, len) zng_crc32(crc, buf, len) 11

··· 4 #ifdef HAVE_ZLIB_NG 5 # include <zlib-ng.h> 6 7 + # define z_stream_s zng_stream_s 8 + # define gz_header_s zng_gz_header_s 9 10 # define crc32(crc, buf, len) zng_crc32(crc, buf, len) 11

+1 -1

git-zlib.h

··· 4 #include "compat/zlib-compat.h" 5 6 typedef struct git_zstream { 7 - z_stream z; 8 unsigned long avail_in; 9 unsigned long avail_out; 10 unsigned long total_in;

··· 4 #include "compat/zlib-compat.h" 5 6 typedef struct git_zstream { 7 + struct z_stream_s z; 8 unsigned long avail_in; 9 unsigned long avail_out; 10 unsigned long total_in;

+1 -1

meson.build

··· 446 'reftable/iter.c', 447 'reftable/merged.c', 448 'reftable/pq.c', 449 - 'reftable/reader.c', 450 'reftable/record.c', 451 'reftable/stack.c', 452 'reftable/system.c', 0 453 'reftable/tree.c', 454 'reftable/writer.c', 455 'remote.c',

··· 446 'reftable/iter.c', 447 'reftable/merged.c', 448 'reftable/pq.c', 0 449 'reftable/record.c', 450 'reftable/stack.c', 451 'reftable/system.c', 452 + 'reftable/table.c', 453 'reftable/tree.c', 454 'reftable/writer.c', 455 'remote.c',

+6 -6

reftable/basics.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #define REFTABLE_ALLOW_BANNED_ALLOCATORS 10 #include "basics.h"

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #define REFTABLE_ALLOW_BANNED_ALLOCATORS 10 #include "basics.h"

+6 -13

reftable/basics.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef BASICS_H 10 #define BASICS_H ··· 17 #include "reftable-basics.h" 18 19 #define REFTABLE_UNUSED __attribute__((__unused__)) 20 - 21 - struct reftable_buf { 22 - size_t alloc; 23 - size_t len; 24 - char *buf; 25 - }; 26 - #define REFTABLE_BUF_INIT { 0 } 27 28 /* 29 * Initialize the buffer such that it is ready for use. This is equivalent to

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef BASICS_H 10 #define BASICS_H ··· 17 #include "reftable-basics.h" 18 19 #define REFTABLE_UNUSED __attribute__((__unused__)) 0 0 0 0 0 0 0 20 21 /* 22 * Initialize the buffer such that it is ready for use. This is equivalent to

+175 -109

reftable/block.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "block.h" 10 11 #include "blocksource.h" 12 #include "constants.h" 0 13 #include "record.h" 14 #include "reftable-error.h" 15 #include "system.h" ··· 160 * Log records are stored zlib-compressed. Note that the compression 161 * also spans over the restart points we have just written. 162 */ 163 - if (block_writer_type(w) == BLOCK_TYPE_LOG) { 164 int block_header_skip = 4 + w->header_off; 165 uLongf src_len = w->next - block_header_skip, compressed_len; 166 int ret; ··· 210 return w->next; 211 } 212 213 - int block_reader_init(struct block_reader *br, struct reftable_block *block, 214 - uint32_t header_off, uint32_t table_block_size, 215 - uint32_t hash_size) 216 { 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 217 uint32_t full_block_size = table_block_size; 218 - uint8_t typ = block->data[header_off]; 219 - uint32_t sz = reftable_get_be24(block->data + header_off + 1); 220 - int err = 0; 221 - uint16_t restart_count = 0; 222 - uint32_t restart_start = 0; 223 - uint8_t *restart_bytes = NULL; 224 225 - reftable_block_done(&br->block); 0 0 226 227 - if (!reftable_is_block_type(typ)) { 228 - err = REFTABLE_FORMAT_ERROR; 0 229 goto done; 230 } 231 232 - if (typ == BLOCK_TYPE_LOG) { 233 - uint32_t block_header_skip = 4 + header_off; 234 - uLong dst_len = sz - block_header_skip; 235 - uLong src_len = block->len - block_header_skip; 0 0 0 0 0 0 0 236 237 /* Log blocks specify the *uncompressed* size in their header. */ 238 - REFTABLE_ALLOC_GROW_OR_NULL(br->uncompressed_data, sz, 239 - br->uncompressed_cap); 240 - if (!br->uncompressed_data) { 241 err = REFTABLE_OUT_OF_MEMORY_ERROR; 242 goto done; 243 } 244 245 /* Copy over the block header verbatim. It's not compressed. */ 246 - memcpy(br->uncompressed_data, block->data, block_header_skip); 247 248 - if (!br->zstream) { 249 - REFTABLE_CALLOC_ARRAY(br->zstream, 1); 250 - if (!br->zstream) { 251 err = REFTABLE_OUT_OF_MEMORY_ERROR; 252 goto done; 253 } 254 255 - err = inflateInit(br->zstream); 256 } else { 257 - err = inflateReset(br->zstream); 258 } 259 if (err != Z_OK) { 260 err = REFTABLE_ZLIB_ERROR; 261 goto done; 262 } 263 264 - br->zstream->next_in = block->data + block_header_skip; 265 - br->zstream->avail_in = src_len; 266 - br->zstream->next_out = br->uncompressed_data + block_header_skip; 267 - br->zstream->avail_out = dst_len; 268 269 /* 270 * We know both input as well as output size, and we know that ··· 273 * here to instruct zlib to inflate the data in one go, which 274 * is more efficient than using `Z_NO_FLUSH`. 275 */ 276 - err = inflate(br->zstream, Z_FINISH); 277 if (err != Z_STREAM_END) { 278 err = REFTABLE_ZLIB_ERROR; 279 goto done; 280 } 281 err = 0; 282 283 - if (br->zstream->total_out + block_header_skip != sz) { 284 err = REFTABLE_FORMAT_ERROR; 285 goto done; 286 } 287 288 /* We're done with the input data. */ 289 - reftable_block_done(block); 290 - block->data = br->uncompressed_data; 291 - block->len = sz; 292 - full_block_size = src_len + block_header_skip - br->zstream->avail_in; 293 } else if (full_block_size == 0) { 294 - full_block_size = sz; 295 - } else if (sz < full_block_size && sz < block->len && 296 - block->data[sz] != 0) { 297 /* If the block is smaller than the full block size, it is 298 padded (data followed by '\0') or the next block is 299 unaligned. */ 300 - full_block_size = sz; 301 } 302 303 - restart_count = reftable_get_be16(block->data + sz - 2); 304 - restart_start = sz - 2 - 3 * restart_count; 305 - restart_bytes = block->data + restart_start; 306 307 - /* transfer ownership. */ 308 - br->block = *block; 309 - block->data = NULL; 310 - block->len = 0; 0 0 311 312 - br->hash_size = hash_size; 313 - br->block_len = restart_start; 314 - br->full_block_size = full_block_size; 315 - br->header_off = header_off; 316 - br->restart_count = restart_count; 317 - br->restart_bytes = restart_bytes; 318 319 done: 0 0 320 return err; 321 } 322 323 - void block_reader_release(struct block_reader *br) 324 { 325 - inflateEnd(br->zstream); 326 - reftable_free(br->zstream); 327 - reftable_free(br->uncompressed_data); 328 - reftable_block_done(&br->block); 0 329 } 330 331 - uint8_t block_reader_type(const struct block_reader *r) 332 { 333 - return r->block.data[r->header_off]; 334 } 335 336 - int block_reader_first_key(const struct block_reader *br, struct reftable_buf *key) 337 { 338 - int off = br->header_off + 4, n; 339 struct string_view in = { 340 - .buf = br->block.data + off, 341 - .len = br->block_len - off, 342 }; 343 uint8_t extra = 0; 344 ··· 353 return 0; 354 } 355 356 - static uint32_t block_reader_restart_offset(const struct block_reader *br, size_t idx) 357 { 358 - return reftable_get_be24(br->restart_bytes + 3 * idx); 359 } 360 361 - void block_iter_seek_start(struct block_iter *it, const struct block_reader *br) 0 0 0 0 0 0 362 { 363 - it->block = br->block.data; 364 - it->block_len = br->block_len; 365 - it->hash_size = br->hash_size; 366 reftable_buf_reset(&it->last_key); 367 - it->next_off = br->header_off + 4; 368 } 369 370 struct restart_needle_less_args { 371 int error; 372 struct reftable_buf needle; 373 - const struct block_reader *reader; 374 }; 375 376 static int restart_needle_less(size_t idx, void *_args) 377 { 378 struct restart_needle_less_args *args = _args; 379 - uint32_t off = block_reader_restart_offset(args->reader, idx); 380 struct string_view in = { 381 - .buf = args->reader->block.data + off, 382 - .len = args->reader->block_len - off, 383 }; 384 uint64_t prefix_len, suffix_len; 385 uint8_t extra; ··· 412 int block_iter_next(struct block_iter *it, struct reftable_record *rec) 413 { 414 struct string_view in = { 415 - .buf = (unsigned char *) it->block + it->next_off, 416 - .len = it->block_len - it->next_off, 417 }; 418 struct string_view start = in; 419 uint8_t extra = 0; 420 int n = 0; 421 422 - if (it->next_off >= it->block_len) 423 return 1; 424 425 n = reftable_decode_key(&it->last_key, &extra, in); ··· 429 return REFTABLE_FORMAT_ERROR; 430 431 string_view_consume(&in, n); 432 - n = reftable_record_decode(rec, it->last_key, extra, in, it->hash_size, 433 &it->scratch); 434 if (n < 0) 435 return -1; ··· 444 reftable_buf_reset(&it->last_key); 445 it->next_off = 0; 446 it->block = NULL; 447 - it->block_len = 0; 448 - it->hash_size = 0; 449 } 450 451 void block_iter_close(struct block_iter *it) ··· 454 reftable_buf_release(&it->scratch); 455 } 456 457 - int block_iter_seek_key(struct block_iter *it, const struct block_reader *br, 458 - struct reftable_buf *want) 459 { 460 struct restart_needle_less_args args = { 461 .needle = *want, 462 - .reader = br, 463 }; 464 struct reftable_record rec; 465 int err = 0; ··· 477 * restart point. While that works alright, we would end up scanning 478 * too many record. 479 */ 480 - i = binsearch(br->restart_count, &restart_needle_less, &args); 481 if (args.error) { 482 err = REFTABLE_FORMAT_ERROR; 483 goto done; ··· 502 * starting from the preceding restart point. 503 */ 504 if (i > 0) 505 - it->next_off = block_reader_restart_offset(br, i - 1); 506 else 507 - it->next_off = br->header_off + 4; 508 - it->block = br->block.data; 509 - it->block_len = br->block_len; 510 - it->hash_size = br->hash_size; 511 512 - err = reftable_record_init(&rec, block_reader_type(br)); 513 if (err < 0) 514 goto done; 515 516 /* 517 * We're looking for the last entry less than the wanted key so that 518 * the next call to `block_reader_next()` would yield the wanted 519 - * record. We thus don't want to position our reader at the sought 520 * after record, but one before. To do so, we have to go one entry too 521 * far and then back up. 522 */ ··· 561 return err; 562 } 563 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 564 void block_writer_release(struct block_writer *bw) 565 { 566 deflateEnd(bw->zstream); ··· 571 reftable_buf_release(&bw->last_key); 572 /* the block is not owned. */ 573 } 574 - 575 - void reftable_block_done(struct reftable_block *blockp) 576 - { 577 - struct reftable_block_source source = blockp->source; 578 - if (blockp && source.ops) 579 - source.ops->return_block(source.arg, blockp); 580 - blockp->data = NULL; 581 - blockp->len = 0; 582 - blockp->source.ops = NULL; 583 - blockp->source.arg = NULL; 584 - }

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "block.h" 10 11 #include "blocksource.h" 12 #include "constants.h" 13 + #include "iter.h" 14 #include "record.h" 15 #include "reftable-error.h" 16 #include "system.h" ··· 161 * Log records are stored zlib-compressed. Note that the compression 162 * also spans over the restart points we have just written. 163 */ 164 + if (block_writer_type(w) == REFTABLE_BLOCK_TYPE_LOG) { 165 int block_header_skip = 4 + w->header_off; 166 uLongf src_len = w->next - block_header_skip, compressed_len; 167 int ret; ··· 211 return w->next; 212 } 213 214 + static int read_block(struct reftable_block_source *source, 215 + struct reftable_block_data *dest, uint64_t off, 216 + uint32_t sz) 217 { 218 + size_t size = block_source_size(source); 219 + block_source_release_data(dest); 220 + if (off >= size) 221 + return 0; 222 + if (off + sz > size) 223 + sz = size - off; 224 + return block_source_read_data(source, dest, off, sz); 225 + } 226 + 227 + int reftable_block_init(struct reftable_block *block, 228 + struct reftable_block_source *source, 229 + uint32_t offset, uint32_t header_size, 230 + uint32_t table_block_size, uint32_t hash_size) 231 + { 232 + uint32_t guess_block_size = table_block_size ? 233 + table_block_size : DEFAULT_BLOCK_SIZE; 234 uint32_t full_block_size = table_block_size; 235 + uint16_t restart_count; 236 + uint32_t restart_off; 237 + uint32_t block_size; 238 + uint8_t block_type; 239 + int err; 0 240 241 + err = read_block(source, &block->block_data, offset, guess_block_size); 242 + if (err < 0) 243 + goto done; 244 245 + block_type = block->block_data.data[header_size]; 246 + if (!reftable_is_block_type(block_type)) { 247 + err = REFTABLE_FORMAT_ERROR; 248 goto done; 249 } 250 251 + block_size = reftable_get_be24(block->block_data.data + header_size + 1); 252 + if (block_size > guess_block_size) { 253 + err = read_block(source, &block->block_data, offset, block_size); 254 + if (err < 0) 255 + goto done; 256 + } 257 + 258 + if (block_type == REFTABLE_BLOCK_TYPE_LOG) { 259 + uint32_t block_header_skip = 4 + header_size; 260 + uLong dst_len = block_size - block_header_skip; 261 + uLong src_len = block->block_data.len - block_header_skip; 262 263 /* Log blocks specify the *uncompressed* size in their header. */ 264 + REFTABLE_ALLOC_GROW_OR_NULL(block->uncompressed_data, block_size, 265 + block->uncompressed_cap); 266 + if (!block->uncompressed_data) { 267 err = REFTABLE_OUT_OF_MEMORY_ERROR; 268 goto done; 269 } 270 271 /* Copy over the block header verbatim. It's not compressed. */ 272 + memcpy(block->uncompressed_data, block->block_data.data, block_header_skip); 273 274 + if (!block->zstream) { 275 + REFTABLE_CALLOC_ARRAY(block->zstream, 1); 276 + if (!block->zstream) { 277 err = REFTABLE_OUT_OF_MEMORY_ERROR; 278 goto done; 279 } 280 281 + err = inflateInit(block->zstream); 282 } else { 283 + err = inflateReset(block->zstream); 284 } 285 if (err != Z_OK) { 286 err = REFTABLE_ZLIB_ERROR; 287 goto done; 288 } 289 290 + block->zstream->next_in = block->block_data.data + block_header_skip; 291 + block->zstream->avail_in = src_len; 292 + block->zstream->next_out = block->uncompressed_data + block_header_skip; 293 + block->zstream->avail_out = dst_len; 294 295 /* 296 * We know both input as well as output size, and we know that ··· 299 * here to instruct zlib to inflate the data in one go, which 300 * is more efficient than using `Z_NO_FLUSH`. 301 */ 302 + err = inflate(block->zstream, Z_FINISH); 303 if (err != Z_STREAM_END) { 304 err = REFTABLE_ZLIB_ERROR; 305 goto done; 306 } 307 err = 0; 308 309 + if (block->zstream->total_out + block_header_skip != block_size) { 310 err = REFTABLE_FORMAT_ERROR; 311 goto done; 312 } 313 314 /* We're done with the input data. */ 315 + block_source_release_data(&block->block_data); 316 + block->block_data.data = block->uncompressed_data; 317 + block->block_data.len = block_size; 318 + full_block_size = src_len + block_header_skip - block->zstream->avail_in; 319 } else if (full_block_size == 0) { 320 + full_block_size = block_size; 321 + } else if (block_size < full_block_size && block_size < block->block_data.len && 322 + block->block_data.data[block_size] != 0) { 323 /* If the block is smaller than the full block size, it is 324 padded (data followed by '\0') or the next block is 325 unaligned. */ 326 + full_block_size = block_size; 327 } 328 329 + restart_count = reftable_get_be16(block->block_data.data + block_size - 2); 330 + restart_off = block_size - 2 - 3 * restart_count; 0 331 332 + block->block_type = block_type; 333 + block->hash_size = hash_size; 334 + block->restart_off = restart_off; 335 + block->full_block_size = full_block_size; 336 + block->header_off = header_size; 337 + block->restart_count = restart_count; 338 339 + err = 0; 0 0 0 0 0 340 341 done: 342 + if (err < 0) 343 + reftable_block_release(block); 344 return err; 345 } 346 347 + void reftable_block_release(struct reftable_block *block) 348 { 349 + inflateEnd(block->zstream); 350 + reftable_free(block->zstream); 351 + reftable_free(block->uncompressed_data); 352 + block_source_release_data(&block->block_data); 353 + memset(block, 0, sizeof(*block)); 354 } 355 356 + uint8_t reftable_block_type(const struct reftable_block *b) 357 { 358 + return b->block_data.data[b->header_off]; 359 } 360 361 + int reftable_block_first_key(const struct reftable_block *block, struct reftable_buf *key) 362 { 363 + int off = block->header_off + 4, n; 364 struct string_view in = { 365 + .buf = block->block_data.data + off, 366 + .len = block->restart_off - off, 367 }; 368 uint8_t extra = 0; 369 ··· 378 return 0; 379 } 380 381 + static uint32_t block_restart_offset(const struct reftable_block *b, size_t idx) 382 { 383 + return reftable_get_be24(b->block_data.data + b->restart_off + 3 * idx); 384 } 385 386 + void block_iter_init(struct block_iter *it, const struct reftable_block *block) 387 + { 388 + it->block = block; 389 + block_iter_seek_start(it); 390 + } 391 + 392 + void block_iter_seek_start(struct block_iter *it) 393 { 0 0 0 394 reftable_buf_reset(&it->last_key); 395 + it->next_off = it->block->header_off + 4; 396 } 397 398 struct restart_needle_less_args { 399 int error; 400 struct reftable_buf needle; 401 + const struct reftable_block *block; 402 }; 403 404 static int restart_needle_less(size_t idx, void *_args) 405 { 406 struct restart_needle_less_args *args = _args; 407 + uint32_t off = block_restart_offset(args->block, idx); 408 struct string_view in = { 409 + .buf = args->block->block_data.data + off, 410 + .len = args->block->restart_off - off, 411 }; 412 uint64_t prefix_len, suffix_len; 413 uint8_t extra; ··· 440 int block_iter_next(struct block_iter *it, struct reftable_record *rec) 441 { 442 struct string_view in = { 443 + .buf = (unsigned char *) it->block->block_data.data + it->next_off, 444 + .len = it->block->restart_off - it->next_off, 445 }; 446 struct string_view start = in; 447 uint8_t extra = 0; 448 int n = 0; 449 450 + if (it->next_off >= it->block->restart_off) 451 return 1; 452 453 n = reftable_decode_key(&it->last_key, &extra, in); ··· 457 return REFTABLE_FORMAT_ERROR; 458 459 string_view_consume(&in, n); 460 + n = reftable_record_decode(rec, it->last_key, extra, in, it->block->hash_size, 461 &it->scratch); 462 if (n < 0) 463 return -1; ··· 472 reftable_buf_reset(&it->last_key); 473 it->next_off = 0; 474 it->block = NULL; 0 0 475 } 476 477 void block_iter_close(struct block_iter *it) ··· 480 reftable_buf_release(&it->scratch); 481 } 482 483 + int block_iter_seek_key(struct block_iter *it, struct reftable_buf *want) 0 484 { 485 struct restart_needle_less_args args = { 486 .needle = *want, 487 + .block = it->block, 488 }; 489 struct reftable_record rec; 490 int err = 0; ··· 502 * restart point. While that works alright, we would end up scanning 503 * too many record. 504 */ 505 + i = binsearch(it->block->restart_count, &restart_needle_less, &args); 506 if (args.error) { 507 err = REFTABLE_FORMAT_ERROR; 508 goto done; ··· 527 * starting from the preceding restart point. 528 */ 529 if (i > 0) 530 + it->next_off = block_restart_offset(it->block, i - 1); 531 else 532 + it->next_off = it->block->header_off + 4; 0 0 0 533 534 + err = reftable_record_init(&rec, reftable_block_type(it->block)); 535 if (err < 0) 536 goto done; 537 538 /* 539 * We're looking for the last entry less than the wanted key so that 540 * the next call to `block_reader_next()` would yield the wanted 541 + * record. We thus don't want to position our iterator at the sought 542 * after record, but one before. To do so, we have to go one entry too 543 * far and then back up. 544 */ ··· 583 return err; 584 } 585 586 + static int block_iter_seek_void(void *it, struct reftable_record *want) 587 + { 588 + struct reftable_buf buf = REFTABLE_BUF_INIT; 589 + struct block_iter *bi = it; 590 + int err; 591 + 592 + if (bi->block->block_type != want->type) 593 + return REFTABLE_API_ERROR; 594 + 595 + err = reftable_record_key(want, &buf); 596 + if (err < 0) 597 + goto out; 598 + 599 + err = block_iter_seek_key(it, &buf); 600 + if (err < 0) 601 + goto out; 602 + 603 + err = 0; 604 + 605 + out: 606 + reftable_buf_release(&buf); 607 + return err; 608 + } 609 + 610 + static int block_iter_next_void(void *it, struct reftable_record *rec) 611 + { 612 + return block_iter_next(it, rec); 613 + } 614 + 615 + static void block_iter_close_void(void *it) 616 + { 617 + block_iter_close(it); 618 + } 619 + 620 + static struct reftable_iterator_vtable block_iter_vtable = { 621 + .seek = &block_iter_seek_void, 622 + .next = &block_iter_next_void, 623 + .close = &block_iter_close_void, 624 + }; 625 + 626 + int reftable_block_init_iterator(const struct reftable_block *b, 627 + struct reftable_iterator *it) 628 + { 629 + struct block_iter *bi; 630 + 631 + REFTABLE_CALLOC_ARRAY(bi, 1); 632 + block_iter_init(bi, b); 633 + 634 + assert(!it->ops); 635 + it->iter_arg = bi; 636 + it->ops = &block_iter_vtable; 637 + 638 + return 0; 639 + } 640 + 641 void block_writer_release(struct block_writer *bw) 642 { 643 deflateEnd(bw->zstream); ··· 648 reftable_buf_release(&bw->last_key); 649 /* the block is not owned. */ 650 } 0 0 0 0 0 0 0 0 0 0 0

+26 -59

reftable/block.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef BLOCK_H 10 #define BLOCK_H 11 12 #include "basics.h" 13 #include "record.h" 0 14 #include "reftable-blocksource.h" 15 16 /* ··· 18 * allocation overhead. 19 */ 20 struct block_writer { 21 - z_stream *zstream; 22 unsigned char *compressed; 23 size_t compressed_cap; 24 ··· 62 /* clears out internally allocated block_writer members. */ 63 void block_writer_release(struct block_writer *bw); 64 65 - struct z_stream; 66 - 67 - /* Read a block. */ 68 - struct block_reader { 69 - /* offset of the block header; nonzero for the first block in a 70 - * reftable. */ 71 - uint32_t header_off; 72 - 73 - /* the memory block */ 74 - struct reftable_block block; 75 - uint32_t hash_size; 76 - 77 - /* Uncompressed data for log entries. */ 78 - z_stream *zstream; 79 - unsigned char *uncompressed_data; 80 - size_t uncompressed_cap; 81 - 82 - /* size of the data, excluding restart data. */ 83 - uint32_t block_len; 84 - uint8_t *restart_bytes; 85 - uint16_t restart_count; 86 - 87 - /* size of the data in the file. For log blocks, this is the compressed 88 - * size. */ 89 - uint32_t full_block_size; 90 - }; 91 - 92 - /* initializes a block reader. */ 93 - int block_reader_init(struct block_reader *br, struct reftable_block *bl, 94 - uint32_t header_off, uint32_t table_block_size, 95 - uint32_t hash_size); 96 - 97 - void block_reader_release(struct block_reader *br); 98 - 99 - /* Returns the block type (eg. 'r' for refs) */ 100 - uint8_t block_reader_type(const struct block_reader *r); 101 - 102 - /* Decodes the first key in the block */ 103 - int block_reader_first_key(const struct block_reader *br, struct reftable_buf *key); 104 - 105 - /* Iterate over entries in a block */ 106 struct block_iter { 107 /* offset within the block of the next entry to read. */ 108 uint32_t next_off; 109 - const unsigned char *block; 110 - size_t block_len; 111 - uint32_t hash_size; 112 113 /* key for last entry we read. */ 114 struct reftable_buf last_key; ··· 120 .scratch = REFTABLE_BUF_INIT, \ 121 } 122 123 - /* Position `it` at start of the block */ 124 - void block_iter_seek_start(struct block_iter *it, const struct block_reader *br); 0 0 0 0 0 125 126 - /* Position `it` to the `want` key in the block */ 127 - int block_iter_seek_key(struct block_iter *it, const struct block_reader *br, 128 - struct reftable_buf *want); 0 0 0 0 0 0 129 130 /* return < 0 for error, 0 for OK, > 0 for EOF. */ 131 int block_iter_next(struct block_iter *it, struct reftable_record *rec); ··· 141 142 /* size of file footer, depending on format version */ 143 size_t footer_size(int version); 144 - 145 - /* returns a block to its source. */ 146 - void reftable_block_done(struct reftable_block *ret); 147 148 #endif

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef BLOCK_H 10 #define BLOCK_H 11 12 #include "basics.h" 13 #include "record.h" 14 + #include "reftable-block.h" 15 #include "reftable-blocksource.h" 16 17 /* ··· 19 * allocation overhead. 20 */ 21 struct block_writer { 22 + struct z_stream_s *zstream; 23 unsigned char *compressed; 24 size_t compressed_cap; 25 ··· 63 /* clears out internally allocated block_writer members. */ 64 void block_writer_release(struct block_writer *bw); 65 66 + /* Iterator for records contained in a single block. */ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 67 struct block_iter { 68 /* offset within the block of the next entry to read. */ 69 uint32_t next_off; 70 + const struct reftable_block *block; 0 0 71 72 /* key for last entry we read. */ 73 struct reftable_buf last_key; ··· 79 .scratch = REFTABLE_BUF_INIT, \ 80 } 81 82 + /* 83 + * Initialize the block iterator with the given block. The iterator will be 84 + * positioned at the first record contained in the block. The block must remain 85 + * valid until the end of the iterator's lifetime. It is valid to re-initialize 86 + * iterators multiple times. 87 + */ 88 + void block_iter_init(struct block_iter *it, const struct reftable_block *block); 89 90 + /* Position the initialized iterator at the first record of its block. */ 91 + void block_iter_seek_start(struct block_iter *it); 92 + 93 + /* 94 + * Position the initialized iterator at the desired record key. It is not an 95 + * error in case the record cannot be found. If so, a subsequent call to 96 + * `block_iter_next()` will indicate that the iterator is exhausted. 97 + */ 98 + int block_iter_seek_key(struct block_iter *it, struct reftable_buf *want); 99 100 /* return < 0 for error, 0 for OK, > 0 for EOF. */ 101 int block_iter_next(struct block_iter *it, struct reftable_record *rec); ··· 111 112 /* size of file footer, depending on format version */ 113 size_t footer_size(int version); 0 0 0 114 115 #endif

+51 -16

reftable/blocksource.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "system.h" 10 ··· 13 #include "reftable-blocksource.h" 14 #include "reftable-error.h" 15 16 - static void reftable_buf_return_block(void *b REFTABLE_UNUSED, struct reftable_block *dest) 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 17 { 18 if (dest->len) 19 memset(dest->data, 0xff, dest->len); ··· 24 { 25 } 26 27 - static ssize_t reftable_buf_read_block(void *v, struct reftable_block *dest, 28 - uint64_t off, uint32_t size) 29 { 30 struct reftable_buf *b = v; 31 assert(off + size <= b->len); ··· 44 45 static struct reftable_block_source_vtable reftable_buf_vtable = { 46 .size = &reftable_buf_size, 47 - .read_block = &reftable_buf_read_block, 48 - .return_block = &reftable_buf_return_block, 49 .close = &reftable_buf_close, 50 }; 51 ··· 67 return ((struct file_block_source *)b)->size; 68 } 69 70 - static void file_return_block(void *b REFTABLE_UNUSED, struct reftable_block *dest REFTABLE_UNUSED) 71 { 72 } 73 ··· 78 reftable_free(b); 79 } 80 81 - static ssize_t file_read_block(void *v, struct reftable_block *dest, uint64_t off, 82 - uint32_t size) 83 { 84 struct file_block_source *b = v; 85 assert(off + size <= b->size); ··· 90 91 static struct reftable_block_source_vtable file_vtable = { 92 .size = &file_size, 93 - .read_block = &file_read_block, 94 - .return_block = &file_return_block, 95 .close = &file_close, 96 }; 97

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "system.h" 10 ··· 13 #include "reftable-blocksource.h" 14 #include "reftable-error.h" 15 16 + void block_source_release_data(struct reftable_block_data *data) 17 + { 18 + struct reftable_block_source source = data->source; 19 + if (data && source.ops) 20 + source.ops->release_data(source.arg, data); 21 + data->data = NULL; 22 + data->len = 0; 23 + data->source.ops = NULL; 24 + data->source.arg = NULL; 25 + } 26 + 27 + void block_source_close(struct reftable_block_source *source) 28 + { 29 + if (!source->ops) { 30 + return; 31 + } 32 + 33 + source->ops->close(source->arg); 34 + source->ops = NULL; 35 + } 36 + 37 + ssize_t block_source_read_data(struct reftable_block_source *source, 38 + struct reftable_block_data *dest, uint64_t off, 39 + uint32_t size) 40 + { 41 + ssize_t result = source->ops->read_data(source->arg, dest, off, size); 42 + dest->source = *source; 43 + return result; 44 + } 45 + 46 + uint64_t block_source_size(struct reftable_block_source *source) 47 + { 48 + return source->ops->size(source->arg); 49 + } 50 + 51 + static void reftable_buf_release_data(void *b REFTABLE_UNUSED, struct reftable_block_data *dest) 52 { 53 if (dest->len) 54 memset(dest->data, 0xff, dest->len); ··· 59 { 60 } 61 62 + static ssize_t reftable_buf_read_data(void *v, struct reftable_block_data *dest, 63 + uint64_t off, uint32_t size) 64 { 65 struct reftable_buf *b = v; 66 assert(off + size <= b->len); ··· 79 80 static struct reftable_block_source_vtable reftable_buf_vtable = { 81 .size = &reftable_buf_size, 82 + .read_data = &reftable_buf_read_data, 83 + .release_data = &reftable_buf_release_data, 84 .close = &reftable_buf_close, 85 }; 86 ··· 102 return ((struct file_block_source *)b)->size; 103 } 104 105 + static void file_release_data(void *b REFTABLE_UNUSED, struct reftable_block_data *dest REFTABLE_UNUSED) 106 { 107 } 108 ··· 113 reftable_free(b); 114 } 115 116 + static ssize_t file_read_data(void *v, struct reftable_block_data *dest, uint64_t off, 117 + uint32_t size) 118 { 119 struct file_block_source *b = v; 120 assert(off + size <= b->size); ··· 125 126 static struct reftable_block_source_vtable file_vtable = { 127 .size = &file_size, 128 + .read_data = &file_read_data, 129 + .release_data = &file_release_data, 130 .close = &file_close, 131 }; 132

+32 -7

reftable/blocksource.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef BLOCKSOURCE_H 10 #define BLOCKSOURCE_H ··· 12 #include "system.h" 13 14 struct reftable_block_source; 0 15 struct reftable_buf; 16 17 - /* Create an in-memory block source for reading reftables */ 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 18 void block_source_from_buf(struct reftable_block_source *bs, 19 struct reftable_buf *buf); 20

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef BLOCKSOURCE_H 10 #define BLOCKSOURCE_H ··· 12 #include "system.h" 13 14 struct reftable_block_source; 15 + struct reftable_block_data; 16 struct reftable_buf; 17 18 + /* 19 + * Close the block source and the underlying resource. This is a no-op in case 20 + * the block source is zero-initialized. 21 + */ 22 + void block_source_close(struct reftable_block_source *source); 23 + 24 + /* 25 + * Read a block of length `size` from the source at the given `off`. 26 + */ 27 + ssize_t block_source_read_data(struct reftable_block_source *source, 28 + struct reftable_block_data *dest, uint64_t off, 29 + uint32_t size); 30 + 31 + /* 32 + * Return the total length of the underlying resource. 33 + */ 34 + uint64_t block_source_size(struct reftable_block_source *source); 35 + 36 + /* 37 + * Return a block to its original source, releasing any resources associated 38 + * with it. 39 + */ 40 + void block_source_release_data(struct reftable_block_data *data); 41 + 42 + /* Create an in-memory block source for reading reftables. */ 43 void block_source_from_buf(struct reftable_block_source *bs, 44 struct reftable_buf *buf); 45

+7 -11

reftable/constants.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef CONSTANTS_H 10 #define CONSTANTS_H 11 12 - #define BLOCK_TYPE_LOG 'g' 13 - #define BLOCK_TYPE_INDEX 'i' 14 - #define BLOCK_TYPE_REF 'r' 15 - #define BLOCK_TYPE_OBJ 'o' 16 - #define BLOCK_TYPE_ANY 0 17 18 #define MAX_RESTARTS ((1 << 16) - 1) 19 #define DEFAULT_BLOCK_SIZE 4096

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef CONSTANTS_H 10 #define CONSTANTS_H 11 12 + #include "reftable-constants.h" 0 0 0 0 13 14 #define MAX_RESTARTS ((1 << 16) - 1) 15 #define DEFAULT_BLOCK_SIZE 4096

+6 -6

reftable/error.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "system.h" 10 #include "reftable-error.h"

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "system.h" 10 #include "reftable-error.h"

+18 -18

reftable/iter.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "iter.h" 10 11 #include "system.h" 12 13 #include "block.h" 0 14 #include "constants.h" 15 - #include "reader.h" 16 #include "reftable-error.h" 0 17 18 int iterator_seek(struct reftable_iterator *it, struct reftable_record *want) 19 { ··· 113 { 114 struct indexed_table_ref_iter *it = p; 115 block_iter_close(&it->cur); 116 - reftable_block_done(&it->block_reader.block); 117 reftable_free(it->offsets); 118 reftable_buf_release(&it->oid); 119 } ··· 127 return 1; 128 } 129 130 - reftable_block_done(&it->block_reader.block); 131 132 off = it->offsets[it->offset_idx++]; 133 - err = reader_init_block_reader(it->r, &it->block_reader, off, 134 - BLOCK_TYPE_REF); 135 if (err < 0) { 136 return err; 137 } ··· 139 /* indexed block does not exist. */ 140 return REFTABLE_FORMAT_ERROR; 141 } 142 - block_iter_seek_start(&it->cur, &it->block_reader); 143 return 0; 144 } 145 ··· 181 } 182 183 int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, 184 - struct reftable_reader *r, uint8_t *oid, 185 int oid_len, uint64_t *offsets, int offset_len) 186 { 187 struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT; ··· 195 } 196 197 *itr = empty; 198 - itr->r = r; 199 200 err = reftable_buf_add(&itr->oid, oid, oid_len); 201 if (err < 0) ··· 246 const char *name) 247 { 248 struct reftable_record want = { 249 - .type = BLOCK_TYPE_REF, 250 .u.ref = { 251 .refname = (char *)name, 252 }, ··· 258 struct reftable_ref_record *ref) 259 { 260 struct reftable_record rec = { 261 - .type = BLOCK_TYPE_REF, 262 .u = { 263 .ref = *ref 264 }, ··· 272 const char *name, uint64_t update_index) 273 { 274 struct reftable_record want = { 275 - .type = BLOCK_TYPE_LOG, 276 .u.log = { 277 .refname = (char *)name, 278 .update_index = update_index, ··· 291 struct reftable_log_record *log) 292 { 293 struct reftable_record rec = { 294 - .type = BLOCK_TYPE_LOG, 295 .u = { 296 .log = *log, 297 },

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "iter.h" 10 11 #include "system.h" 12 13 #include "block.h" 14 + #include "blocksource.h" 15 #include "constants.h" 0 16 #include "reftable-error.h" 17 + #include "table.h" 18 19 int iterator_seek(struct reftable_iterator *it, struct reftable_record *want) 20 { ··· 114 { 115 struct indexed_table_ref_iter *it = p; 116 block_iter_close(&it->cur); 117 + block_source_release_data(&it->block.block_data); 118 reftable_free(it->offsets); 119 reftable_buf_release(&it->oid); 120 } ··· 128 return 1; 129 } 130 131 + block_source_release_data(&it->block.block_data); 132 133 off = it->offsets[it->offset_idx++]; 134 + err = table_init_block(it->table, &it->block, off, REFTABLE_BLOCK_TYPE_REF); 0 135 if (err < 0) { 136 return err; 137 } ··· 139 /* indexed block does not exist. */ 140 return REFTABLE_FORMAT_ERROR; 141 } 142 + block_iter_init(&it->cur, &it->block); 143 return 0; 144 } 145 ··· 181 } 182 183 int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, 184 + struct reftable_table *t, uint8_t *oid, 185 int oid_len, uint64_t *offsets, int offset_len) 186 { 187 struct indexed_table_ref_iter empty = INDEXED_TABLE_REF_ITER_INIT; ··· 195 } 196 197 *itr = empty; 198 + itr->table = t; 199 200 err = reftable_buf_add(&itr->oid, oid, oid_len); 201 if (err < 0) ··· 246 const char *name) 247 { 248 struct reftable_record want = { 249 + .type = REFTABLE_BLOCK_TYPE_REF, 250 .u.ref = { 251 .refname = (char *)name, 252 }, ··· 258 struct reftable_ref_record *ref) 259 { 260 struct reftable_record rec = { 261 + .type = REFTABLE_BLOCK_TYPE_REF, 262 .u = { 263 .ref = *ref 264 }, ··· 272 const char *name, uint64_t update_index) 273 { 274 struct reftable_record want = { 275 + .type = REFTABLE_BLOCK_TYPE_LOG, 276 .u.log = { 277 .refname = (char *)name, 278 .update_index = update_index, ··· 291 struct reftable_log_record *log) 292 { 293 struct reftable_record rec = { 294 + .type = REFTABLE_BLOCK_TYPE_LOG, 295 .u = { 296 .log = *log, 297 },

+9 -9

reftable/iter.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef ITER_H 10 #define ITER_H ··· 59 * but using the object index. 60 */ 61 struct indexed_table_ref_iter { 62 - struct reftable_reader *r; 63 struct reftable_buf oid; 64 65 /* mutable */ ··· 68 /* Points to the next offset to read. */ 69 int offset_idx; 70 int offset_len; 71 - struct block_reader block_reader; 72 struct block_iter cur; 73 int is_finished; 74 }; ··· 83 84 /* Takes ownership of `offsets` */ 85 int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, 86 - struct reftable_reader *r, uint8_t *oid, 87 int oid_len, uint64_t *offsets, int offset_len); 88 89 #endif

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef ITER_H 10 #define ITER_H ··· 59 * but using the object index. 60 */ 61 struct indexed_table_ref_iter { 62 + struct reftable_table *table; 63 struct reftable_buf oid; 64 65 /* mutable */ ··· 68 /* Points to the next offset to read. */ 69 int offset_idx; 70 int offset_len; 71 + struct reftable_block block; 72 struct block_iter cur; 73 int is_finished; 74 }; ··· 83 84 /* Takes ownership of `offsets` */ 85 int indexed_table_ref_iter_new(struct indexed_table_ref_iter **dest, 86 + struct reftable_table *t, uint8_t *oid, 87 int oid_len, uint64_t *offsets, int offset_len); 88 89 #endif

+21 -21

reftable/merged.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "merged.h" 10 11 #include "constants.h" 12 #include "iter.h" 13 #include "pq.h" 14 - #include "reader.h" 15 #include "record.h" 16 #include "reftable-merged.h" 17 #include "reftable-error.h" 18 #include "system.h" 0 19 20 struct merged_subiter { 21 struct reftable_iterator iter; ··· 192 } 193 194 int reftable_merged_table_new(struct reftable_merged_table **dest, 195 - struct reftable_reader **readers, size_t n, 196 enum reftable_hash hash_id) 197 { 198 struct reftable_merged_table *m = NULL; ··· 200 uint64_t first_min = 0; 201 202 for (size_t i = 0; i < n; i++) { 203 - uint64_t min = reftable_reader_min_update_index(readers[i]); 204 - uint64_t max = reftable_reader_max_update_index(readers[i]); 205 206 - if (reftable_reader_hash_id(readers[i]) != hash_id) { 207 return REFTABLE_FORMAT_ERROR; 208 } 209 if (i == 0 || min < first_min) { ··· 218 if (!m) 219 return REFTABLE_OUT_OF_MEMORY_ERROR; 220 221 - m->readers = readers; 222 - m->readers_len = n; 223 m->min = first_min; 224 m->max = last_max; 225 m->hash_id = hash_id; ··· 254 struct merged_iter *mi = NULL; 255 int ret; 256 257 - if (mt->readers_len) { 258 - REFTABLE_CALLOC_ARRAY(subiters, mt->readers_len); 259 if (!subiters) { 260 ret = REFTABLE_OUT_OF_MEMORY_ERROR; 261 goto out; 262 } 263 } 264 265 - for (size_t i = 0; i < mt->readers_len; i++) { 266 ret = reftable_record_init(&subiters[i].rec, typ); 267 if (ret < 0) 268 goto out; 269 270 - ret = reader_init_iter(mt->readers[i], &subiters[i].iter, typ); 271 if (ret < 0) 272 goto out; 273 } ··· 280 mi->advance_index = -1; 281 mi->suppress_deletions = mt->suppress_deletions; 282 mi->subiters = subiters; 283 - mi->subiters_len = mt->readers_len; 284 285 iterator_from_merged_iter(it, mi); 286 ret = 0; 287 288 out: 289 if (ret < 0) { 290 - for (size_t i = 0; subiters && i < mt->readers_len; i++) { 291 reftable_iterator_destroy(&subiters[i].iter); 292 reftable_record_release(&subiters[i].rec); 293 } ··· 301 int reftable_merged_table_init_ref_iterator(struct reftable_merged_table *mt, 302 struct reftable_iterator *it) 303 { 304 - return merged_table_init_iter(mt, it, BLOCK_TYPE_REF); 305 } 306 307 int reftable_merged_table_init_log_iterator(struct reftable_merged_table *mt, 308 struct reftable_iterator *it) 309 { 310 - return merged_table_init_iter(mt, it, BLOCK_TYPE_LOG); 311 } 312 313 enum reftable_hash reftable_merged_table_hash_id(struct reftable_merged_table *mt)

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "merged.h" 10 11 #include "constants.h" 12 #include "iter.h" 13 #include "pq.h" 0 14 #include "record.h" 15 #include "reftable-merged.h" 16 #include "reftable-error.h" 17 #include "system.h" 18 + #include "table.h" 19 20 struct merged_subiter { 21 struct reftable_iterator iter; ··· 192 } 193 194 int reftable_merged_table_new(struct reftable_merged_table **dest, 195 + struct reftable_table **tables, size_t n, 196 enum reftable_hash hash_id) 197 { 198 struct reftable_merged_table *m = NULL; ··· 200 uint64_t first_min = 0; 201 202 for (size_t i = 0; i < n; i++) { 203 + uint64_t min = reftable_table_min_update_index(tables[i]); 204 + uint64_t max = reftable_table_max_update_index(tables[i]); 205 206 + if (reftable_table_hash_id(tables[i]) != hash_id) { 207 return REFTABLE_FORMAT_ERROR; 208 } 209 if (i == 0 || min < first_min) { ··· 218 if (!m) 219 return REFTABLE_OUT_OF_MEMORY_ERROR; 220 221 + m->tables = tables; 222 + m->tables_len = n; 223 m->min = first_min; 224 m->max = last_max; 225 m->hash_id = hash_id; ··· 254 struct merged_iter *mi = NULL; 255 int ret; 256 257 + if (mt->tables_len) { 258 + REFTABLE_CALLOC_ARRAY(subiters, mt->tables_len); 259 if (!subiters) { 260 ret = REFTABLE_OUT_OF_MEMORY_ERROR; 261 goto out; 262 } 263 } 264 265 + for (size_t i = 0; i < mt->tables_len; i++) { 266 ret = reftable_record_init(&subiters[i].rec, typ); 267 if (ret < 0) 268 goto out; 269 270 + ret = table_init_iter(mt->tables[i], &subiters[i].iter, typ); 271 if (ret < 0) 272 goto out; 273 } ··· 280 mi->advance_index = -1; 281 mi->suppress_deletions = mt->suppress_deletions; 282 mi->subiters = subiters; 283 + mi->subiters_len = mt->tables_len; 284 285 iterator_from_merged_iter(it, mi); 286 ret = 0; 287 288 out: 289 if (ret < 0) { 290 + for (size_t i = 0; subiters && i < mt->tables_len; i++) { 291 reftable_iterator_destroy(&subiters[i].iter); 292 reftable_record_release(&subiters[i].rec); 293 } ··· 301 int reftable_merged_table_init_ref_iterator(struct reftable_merged_table *mt, 302 struct reftable_iterator *it) 303 { 304 + return merged_table_init_iter(mt, it, REFTABLE_BLOCK_TYPE_REF); 305 } 306 307 int reftable_merged_table_init_log_iterator(struct reftable_merged_table *mt, 308 struct reftable_iterator *it) 309 { 310 + return merged_table_init_iter(mt, it, REFTABLE_BLOCK_TYPE_LOG); 311 } 312 313 enum reftable_hash reftable_merged_table_hash_id(struct reftable_merged_table *mt)

+8 -8

reftable/merged.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef MERGED_H 10 #define MERGED_H ··· 13 #include "reftable-basics.h" 14 15 struct reftable_merged_table { 16 - struct reftable_reader **readers; 17 - size_t readers_len; 18 enum reftable_hash hash_id; 19 20 /* If unset, produce deletions. This is useful for compaction. For the

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef MERGED_H 10 #define MERGED_H ··· 13 #include "reftable-basics.h" 14 15 struct reftable_merged_table { 16 + struct reftable_table **tables; 17 + size_t tables_len; 18 enum reftable_hash hash_id; 19 20 /* If unset, produce deletions. This is useful for compaction. For the

+6 -6

reftable/pq.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "pq.h" 10

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "pq.h" 10

+6 -6

reftable/pq.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef PQ_H 10 #define PQ_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef PQ_H 10 #define PQ_H

-879

reftable/reader.c

··· 1 - /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 - 9 - #include "reader.h" 10 - 11 - #include "system.h" 12 - #include "block.h" 13 - #include "constants.h" 14 - #include "iter.h" 15 - #include "record.h" 16 - #include "reftable-error.h" 17 - 18 - uint64_t block_source_size(struct reftable_block_source *source) 19 - { 20 - return source->ops->size(source->arg); 21 - } 22 - 23 - ssize_t block_source_read_block(struct reftable_block_source *source, 24 - struct reftable_block *dest, uint64_t off, 25 - uint32_t size) 26 - { 27 - ssize_t result = source->ops->read_block(source->arg, dest, off, size); 28 - dest->source = *source; 29 - return result; 30 - } 31 - 32 - void block_source_close(struct reftable_block_source *source) 33 - { 34 - if (!source->ops) { 35 - return; 36 - } 37 - 38 - source->ops->close(source->arg); 39 - source->ops = NULL; 40 - } 41 - 42 - static struct reftable_reader_offsets * 43 - reader_offsets_for(struct reftable_reader *r, uint8_t typ) 44 - { 45 - switch (typ) { 46 - case BLOCK_TYPE_REF: 47 - return &r->ref_offsets; 48 - case BLOCK_TYPE_LOG: 49 - return &r->log_offsets; 50 - case BLOCK_TYPE_OBJ: 51 - return &r->obj_offsets; 52 - } 53 - abort(); 54 - } 55 - 56 - static int reader_get_block(struct reftable_reader *r, 57 - struct reftable_block *dest, uint64_t off, 58 - uint32_t sz) 59 - { 60 - ssize_t bytes_read; 61 - if (off >= r->size) 62 - return 0; 63 - if (off + sz > r->size) 64 - sz = r->size - off; 65 - 66 - bytes_read = block_source_read_block(&r->source, dest, off, sz); 67 - if (bytes_read < 0) 68 - return (int)bytes_read; 69 - 70 - return 0; 71 - } 72 - 73 - enum reftable_hash reftable_reader_hash_id(struct reftable_reader *r) 74 - { 75 - return r->hash_id; 76 - } 77 - 78 - const char *reader_name(struct reftable_reader *r) 79 - { 80 - return r->name; 81 - } 82 - 83 - static int parse_footer(struct reftable_reader *r, uint8_t *footer, 84 - uint8_t *header) 85 - { 86 - uint8_t *f = footer; 87 - uint8_t first_block_typ; 88 - int err = 0; 89 - uint32_t computed_crc; 90 - uint32_t file_crc; 91 - 92 - if (memcmp(f, "REFT", 4)) { 93 - err = REFTABLE_FORMAT_ERROR; 94 - goto done; 95 - } 96 - f += 4; 97 - 98 - if (memcmp(footer, header, header_size(r->version))) { 99 - err = REFTABLE_FORMAT_ERROR; 100 - goto done; 101 - } 102 - 103 - f++; 104 - r->block_size = reftable_get_be24(f); 105 - 106 - f += 3; 107 - r->min_update_index = reftable_get_be64(f); 108 - f += 8; 109 - r->max_update_index = reftable_get_be64(f); 110 - f += 8; 111 - 112 - if (r->version == 1) { 113 - r->hash_id = REFTABLE_HASH_SHA1; 114 - } else { 115 - switch (reftable_get_be32(f)) { 116 - case REFTABLE_FORMAT_ID_SHA1: 117 - r->hash_id = REFTABLE_HASH_SHA1; 118 - break; 119 - case REFTABLE_FORMAT_ID_SHA256: 120 - r->hash_id = REFTABLE_HASH_SHA256; 121 - break; 122 - default: 123 - err = REFTABLE_FORMAT_ERROR; 124 - goto done; 125 - } 126 - 127 - f += 4; 128 - } 129 - 130 - r->ref_offsets.index_offset = reftable_get_be64(f); 131 - f += 8; 132 - 133 - r->obj_offsets.offset = reftable_get_be64(f); 134 - f += 8; 135 - 136 - r->object_id_len = r->obj_offsets.offset & ((1 << 5) - 1); 137 - r->obj_offsets.offset >>= 5; 138 - 139 - r->obj_offsets.index_offset = reftable_get_be64(f); 140 - f += 8; 141 - r->log_offsets.offset = reftable_get_be64(f); 142 - f += 8; 143 - r->log_offsets.index_offset = reftable_get_be64(f); 144 - f += 8; 145 - 146 - computed_crc = crc32(0, footer, f - footer); 147 - file_crc = reftable_get_be32(f); 148 - f += 4; 149 - if (computed_crc != file_crc) { 150 - err = REFTABLE_FORMAT_ERROR; 151 - goto done; 152 - } 153 - 154 - first_block_typ = header[header_size(r->version)]; 155 - r->ref_offsets.is_present = (first_block_typ == BLOCK_TYPE_REF); 156 - r->ref_offsets.offset = 0; 157 - r->log_offsets.is_present = (first_block_typ == BLOCK_TYPE_LOG || 158 - r->log_offsets.offset > 0); 159 - r->obj_offsets.is_present = r->obj_offsets.offset > 0; 160 - if (r->obj_offsets.is_present && !r->object_id_len) { 161 - err = REFTABLE_FORMAT_ERROR; 162 - goto done; 163 - } 164 - 165 - err = 0; 166 - done: 167 - return err; 168 - } 169 - 170 - struct table_iter { 171 - struct reftable_reader *r; 172 - uint8_t typ; 173 - uint64_t block_off; 174 - struct block_reader br; 175 - struct block_iter bi; 176 - int is_finished; 177 - }; 178 - 179 - static int table_iter_init(struct table_iter *ti, struct reftable_reader *r) 180 - { 181 - struct block_iter bi = BLOCK_ITER_INIT; 182 - memset(ti, 0, sizeof(*ti)); 183 - reftable_reader_incref(r); 184 - ti->r = r; 185 - ti->bi = bi; 186 - return 0; 187 - } 188 - 189 - static int table_iter_next_in_block(struct table_iter *ti, 190 - struct reftable_record *rec) 191 - { 192 - int res = block_iter_next(&ti->bi, rec); 193 - if (res == 0 && reftable_record_type(rec) == BLOCK_TYPE_REF) { 194 - rec->u.ref.update_index += ti->r->min_update_index; 195 - } 196 - 197 - return res; 198 - } 199 - 200 - static void table_iter_block_done(struct table_iter *ti) 201 - { 202 - block_reader_release(&ti->br); 203 - block_iter_reset(&ti->bi); 204 - } 205 - 206 - static int32_t extract_block_size(uint8_t *data, uint8_t *typ, uint64_t off, 207 - int version) 208 - { 209 - int32_t result = 0; 210 - 211 - if (off == 0) { 212 - data += header_size(version); 213 - } 214 - 215 - *typ = data[0]; 216 - if (reftable_is_block_type(*typ)) { 217 - result = reftable_get_be24(data + 1); 218 - } 219 - return result; 220 - } 221 - 222 - int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, 223 - uint64_t next_off, uint8_t want_typ) 224 - { 225 - int32_t guess_block_size = r->block_size ? r->block_size : 226 - DEFAULT_BLOCK_SIZE; 227 - struct reftable_block block = { NULL }; 228 - uint8_t block_typ = 0; 229 - int err = 0; 230 - uint32_t header_off = next_off ? 0 : header_size(r->version); 231 - int32_t block_size = 0; 232 - 233 - if (next_off >= r->size) 234 - return 1; 235 - 236 - err = reader_get_block(r, &block, next_off, guess_block_size); 237 - if (err < 0) 238 - goto done; 239 - 240 - block_size = extract_block_size(block.data, &block_typ, next_off, 241 - r->version); 242 - if (block_size < 0) { 243 - err = block_size; 244 - goto done; 245 - } 246 - if (want_typ != BLOCK_TYPE_ANY && block_typ != want_typ) { 247 - err = 1; 248 - goto done; 249 - } 250 - 251 - if (block_size > guess_block_size) { 252 - reftable_block_done(&block); 253 - err = reader_get_block(r, &block, next_off, block_size); 254 - if (err < 0) { 255 - goto done; 256 - } 257 - } 258 - 259 - err = block_reader_init(br, &block, header_off, r->block_size, 260 - hash_size(r->hash_id)); 261 - done: 262 - reftable_block_done(&block); 263 - 264 - return err; 265 - } 266 - 267 - static void table_iter_close(struct table_iter *ti) 268 - { 269 - table_iter_block_done(ti); 270 - block_iter_close(&ti->bi); 271 - reftable_reader_decref(ti->r); 272 - } 273 - 274 - static int table_iter_next_block(struct table_iter *ti) 275 - { 276 - uint64_t next_block_off = ti->block_off + ti->br.full_block_size; 277 - int err; 278 - 279 - err = reader_init_block_reader(ti->r, &ti->br, next_block_off, ti->typ); 280 - if (err > 0) 281 - ti->is_finished = 1; 282 - if (err) 283 - return err; 284 - 285 - ti->block_off = next_block_off; 286 - ti->is_finished = 0; 287 - block_iter_seek_start(&ti->bi, &ti->br); 288 - 289 - return 0; 290 - } 291 - 292 - static int table_iter_next(struct table_iter *ti, struct reftable_record *rec) 293 - { 294 - if (reftable_record_type(rec) != ti->typ) 295 - return REFTABLE_API_ERROR; 296 - 297 - while (1) { 298 - int err; 299 - 300 - if (ti->is_finished) 301 - return 1; 302 - 303 - /* 304 - * Check whether the current block still has more records. If 305 - * so, return it. If the iterator returns positive then the 306 - * current block has been exhausted. 307 - */ 308 - err = table_iter_next_in_block(ti, rec); 309 - if (err <= 0) 310 - return err; 311 - 312 - /* 313 - * Otherwise, we need to continue to the next block in the 314 - * table and retry. If there are no more blocks then the 315 - * iterator is drained. 316 - */ 317 - err = table_iter_next_block(ti); 318 - if (err) { 319 - ti->is_finished = 1; 320 - return err; 321 - } 322 - } 323 - } 324 - 325 - static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ) 326 - { 327 - int err; 328 - 329 - err = reader_init_block_reader(ti->r, &ti->br, off, typ); 330 - if (err != 0) 331 - return err; 332 - 333 - ti->typ = block_reader_type(&ti->br); 334 - ti->block_off = off; 335 - block_iter_seek_start(&ti->bi, &ti->br); 336 - ti->is_finished = 0; 337 - return 0; 338 - } 339 - 340 - static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index) 341 - { 342 - struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ); 343 - uint64_t off = offs->offset; 344 - if (index) { 345 - off = offs->index_offset; 346 - if (off == 0) { 347 - return 1; 348 - } 349 - typ = BLOCK_TYPE_INDEX; 350 - } 351 - 352 - return table_iter_seek_to(ti, off, typ); 353 - } 354 - 355 - static int table_iter_seek_linear(struct table_iter *ti, 356 - struct reftable_record *want) 357 - { 358 - struct reftable_buf want_key = REFTABLE_BUF_INIT; 359 - struct reftable_buf got_key = REFTABLE_BUF_INIT; 360 - struct reftable_record rec; 361 - int err; 362 - 363 - err = reftable_record_init(&rec, reftable_record_type(want)); 364 - if (err < 0) 365 - goto done; 366 - 367 - err = reftable_record_key(want, &want_key); 368 - if (err < 0) 369 - goto done; 370 - 371 - /* 372 - * First we need to locate the block that must contain our record. To 373 - * do so we scan through blocks linearly until we find the first block 374 - * whose first key is bigger than our wanted key. Once we have found 375 - * that block we know that the key must be contained in the preceding 376 - * block. 377 - * 378 - * This algorithm is somewhat unfortunate because it means that we 379 - * always have to seek one block too far and then back up. But as we 380 - * can only decode the _first_ key of a block but not its _last_ key we 381 - * have no other way to do this. 382 - */ 383 - while (1) { 384 - struct table_iter next = *ti; 385 - 386 - /* 387 - * We must be careful to not modify underlying data of `ti` 388 - * because we may find that `next` does not contain our desired 389 - * block, but that `ti` does. In that case, we would discard 390 - * `next` and continue with `ti`. 391 - * 392 - * This also means that we cannot reuse allocated memory for 393 - * `next` here. While it would be great if we could, it should 394 - * in practice not be too bad given that we should only ever 395 - * end up doing linear seeks with at most three blocks. As soon 396 - * as we have more than three blocks we would have an index, so 397 - * we would not do a linear search there anymore. 398 - */ 399 - memset(&next.br.block, 0, sizeof(next.br.block)); 400 - next.br.zstream = NULL; 401 - next.br.uncompressed_data = NULL; 402 - next.br.uncompressed_cap = 0; 403 - 404 - err = table_iter_next_block(&next); 405 - if (err < 0) 406 - goto done; 407 - if (err > 0) 408 - break; 409 - 410 - err = block_reader_first_key(&next.br, &got_key); 411 - if (err < 0) 412 - goto done; 413 - 414 - if (reftable_buf_cmp(&got_key, &want_key) > 0) { 415 - table_iter_block_done(&next); 416 - break; 417 - } 418 - 419 - table_iter_block_done(ti); 420 - *ti = next; 421 - } 422 - 423 - /* 424 - * We have located the block that must contain our record, so we seek 425 - * the wanted key inside of it. If the block does not contain our key 426 - * we know that the corresponding record does not exist. 427 - */ 428 - err = block_iter_seek_key(&ti->bi, &ti->br, &want_key); 429 - if (err < 0) 430 - goto done; 431 - err = 0; 432 - 433 - done: 434 - reftable_record_release(&rec); 435 - reftable_buf_release(&want_key); 436 - reftable_buf_release(&got_key); 437 - return err; 438 - } 439 - 440 - static int table_iter_seek_indexed(struct table_iter *ti, 441 - struct reftable_record *rec) 442 - { 443 - struct reftable_record want_index = { 444 - .type = BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT } 445 - }; 446 - struct reftable_record index_result = { 447 - .type = BLOCK_TYPE_INDEX, 448 - .u.idx = { .last_key = REFTABLE_BUF_INIT }, 449 - }; 450 - int err; 451 - 452 - err = reftable_record_key(rec, &want_index.u.idx.last_key); 453 - if (err < 0) 454 - goto done; 455 - 456 - /* 457 - * The index may consist of multiple levels, where each level may have 458 - * multiple index blocks. We start by doing a linear search in the 459 - * highest layer that identifies the relevant index block as well as 460 - * the record inside that block that corresponds to our wanted key. 461 - */ 462 - err = table_iter_seek_linear(ti, &want_index); 463 - if (err < 0) 464 - goto done; 465 - 466 - /* 467 - * Traverse down the levels until we find a non-index entry. 468 - */ 469 - while (1) { 470 - /* 471 - * In case we seek a record that does not exist the index iter 472 - * will tell us that the iterator is over. This works because 473 - * the last index entry of the current level will contain the 474 - * last key it knows about. So in case our seeked key is larger 475 - * than the last indexed key we know that it won't exist. 476 - * 477 - * There is one subtlety in the layout of the index section 478 - * that makes this work as expected: the highest-level index is 479 - * at end of the section and will point backwards and thus we 480 - * start reading from the end of the index section, not the 481 - * beginning. 482 - * 483 - * If that wasn't the case and the order was reversed then the 484 - * linear seek would seek into the lower levels and traverse 485 - * all levels of the index only to find out that the key does 486 - * not exist. 487 - */ 488 - err = table_iter_next(ti, &index_result); 489 - if (err != 0) 490 - goto done; 491 - 492 - err = table_iter_seek_to(ti, index_result.u.idx.offset, 0); 493 - if (err != 0) 494 - goto done; 495 - 496 - err = block_iter_seek_key(&ti->bi, &ti->br, &want_index.u.idx.last_key); 497 - if (err < 0) 498 - goto done; 499 - 500 - if (ti->typ == reftable_record_type(rec)) { 501 - err = 0; 502 - break; 503 - } 504 - 505 - if (ti->typ != BLOCK_TYPE_INDEX) { 506 - err = REFTABLE_FORMAT_ERROR; 507 - goto done; 508 - } 509 - } 510 - 511 - done: 512 - reftable_record_release(&want_index); 513 - reftable_record_release(&index_result); 514 - return err; 515 - } 516 - 517 - static int table_iter_seek(struct table_iter *ti, 518 - struct reftable_record *want) 519 - { 520 - uint8_t typ = reftable_record_type(want); 521 - struct reftable_reader_offsets *offs = reader_offsets_for(ti->r, typ); 522 - int err; 523 - 524 - err = table_iter_seek_start(ti, reftable_record_type(want), 525 - !!offs->index_offset); 526 - if (err < 0) 527 - goto out; 528 - 529 - if (offs->index_offset) 530 - err = table_iter_seek_indexed(ti, want); 531 - else 532 - err = table_iter_seek_linear(ti, want); 533 - if (err) 534 - goto out; 535 - 536 - out: 537 - return err; 538 - } 539 - 540 - static int table_iter_seek_void(void *ti, struct reftable_record *want) 541 - { 542 - return table_iter_seek(ti, want); 543 - } 544 - 545 - static int table_iter_next_void(void *ti, struct reftable_record *rec) 546 - { 547 - return table_iter_next(ti, rec); 548 - } 549 - 550 - static void table_iter_close_void(void *ti) 551 - { 552 - table_iter_close(ti); 553 - } 554 - 555 - static struct reftable_iterator_vtable table_iter_vtable = { 556 - .seek = &table_iter_seek_void, 557 - .next = &table_iter_next_void, 558 - .close = &table_iter_close_void, 559 - }; 560 - 561 - static void iterator_from_table_iter(struct reftable_iterator *it, 562 - struct table_iter *ti) 563 - { 564 - assert(!it->ops); 565 - it->iter_arg = ti; 566 - it->ops = &table_iter_vtable; 567 - } 568 - 569 - int reader_init_iter(struct reftable_reader *r, 570 - struct reftable_iterator *it, 571 - uint8_t typ) 572 - { 573 - struct reftable_reader_offsets *offs = reader_offsets_for(r, typ); 574 - 575 - if (offs->is_present) { 576 - struct table_iter *ti; 577 - REFTABLE_ALLOC_ARRAY(ti, 1); 578 - if (!ti) 579 - return REFTABLE_OUT_OF_MEMORY_ERROR; 580 - 581 - table_iter_init(ti, r); 582 - iterator_from_table_iter(it, ti); 583 - } else { 584 - iterator_set_empty(it); 585 - } 586 - 587 - return 0; 588 - } 589 - 590 - int reftable_reader_init_ref_iterator(struct reftable_reader *r, 591 - struct reftable_iterator *it) 592 - { 593 - return reader_init_iter(r, it, BLOCK_TYPE_REF); 594 - } 595 - 596 - int reftable_reader_init_log_iterator(struct reftable_reader *r, 597 - struct reftable_iterator *it) 598 - { 599 - return reader_init_iter(r, it, BLOCK_TYPE_LOG); 600 - } 601 - 602 - int reftable_reader_new(struct reftable_reader **out, 603 - struct reftable_block_source *source, char const *name) 604 - { 605 - struct reftable_block footer = { 0 }; 606 - struct reftable_block header = { 0 }; 607 - struct reftable_reader *r; 608 - uint64_t file_size = block_source_size(source); 609 - uint32_t read_size; 610 - ssize_t bytes_read; 611 - int err; 612 - 613 - REFTABLE_CALLOC_ARRAY(r, 1); 614 - if (!r) { 615 - err = REFTABLE_OUT_OF_MEMORY_ERROR; 616 - goto done; 617 - } 618 - 619 - /* 620 - * We need one extra byte to read the type of first block. We also 621 - * pretend to always be reading v2 of the format because it is larger. 622 - */ 623 - read_size = header_size(2) + 1; 624 - if (read_size > file_size) { 625 - err = REFTABLE_FORMAT_ERROR; 626 - goto done; 627 - } 628 - 629 - bytes_read = block_source_read_block(source, &header, 0, read_size); 630 - if (bytes_read < 0 || (size_t)bytes_read != read_size) { 631 - err = REFTABLE_IO_ERROR; 632 - goto done; 633 - } 634 - 635 - if (memcmp(header.data, "REFT", 4)) { 636 - err = REFTABLE_FORMAT_ERROR; 637 - goto done; 638 - } 639 - r->version = header.data[4]; 640 - if (r->version != 1 && r->version != 2) { 641 - err = REFTABLE_FORMAT_ERROR; 642 - goto done; 643 - } 644 - 645 - r->size = file_size - footer_size(r->version); 646 - r->source = *source; 647 - r->name = reftable_strdup(name); 648 - if (!r->name) { 649 - err = REFTABLE_OUT_OF_MEMORY_ERROR; 650 - goto done; 651 - } 652 - r->hash_id = 0; 653 - r->refcount = 1; 654 - 655 - bytes_read = block_source_read_block(source, &footer, r->size, 656 - footer_size(r->version)); 657 - if (bytes_read < 0 || (size_t)bytes_read != footer_size(r->version)) { 658 - err = REFTABLE_IO_ERROR; 659 - goto done; 660 - } 661 - 662 - err = parse_footer(r, footer.data, header.data); 663 - if (err) 664 - goto done; 665 - 666 - *out = r; 667 - 668 - done: 669 - reftable_block_done(&footer); 670 - reftable_block_done(&header); 671 - if (err) { 672 - if (r) 673 - reftable_free(r->name); 674 - reftable_free(r); 675 - block_source_close(source); 676 - } 677 - return err; 678 - } 679 - 680 - void reftable_reader_incref(struct reftable_reader *r) 681 - { 682 - r->refcount++; 683 - } 684 - 685 - void reftable_reader_decref(struct reftable_reader *r) 686 - { 687 - if (!r) 688 - return; 689 - if (--r->refcount) 690 - return; 691 - block_source_close(&r->source); 692 - REFTABLE_FREE_AND_NULL(r->name); 693 - reftable_free(r); 694 - } 695 - 696 - static int reftable_reader_refs_for_indexed(struct reftable_reader *r, 697 - struct reftable_iterator *it, 698 - uint8_t *oid) 699 - { 700 - struct reftable_record want = { 701 - .type = BLOCK_TYPE_OBJ, 702 - .u.obj = { 703 - .hash_prefix = oid, 704 - .hash_prefix_len = r->object_id_len, 705 - }, 706 - }; 707 - struct reftable_iterator oit = { NULL }; 708 - struct reftable_record got = { 709 - .type = BLOCK_TYPE_OBJ, 710 - .u.obj = { 0 }, 711 - }; 712 - int err = 0; 713 - struct indexed_table_ref_iter *itr = NULL; 714 - 715 - /* Look through the reverse index. */ 716 - err = reader_init_iter(r, &oit, BLOCK_TYPE_OBJ); 717 - if (err < 0) 718 - goto done; 719 - 720 - err = iterator_seek(&oit, &want); 721 - if (err != 0) 722 - goto done; 723 - 724 - /* read out the reftable_obj_record */ 725 - err = iterator_next(&oit, &got); 726 - if (err < 0) 727 - goto done; 728 - 729 - if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix, 730 - r->object_id_len)) { 731 - /* didn't find it; return empty iterator */ 732 - iterator_set_empty(it); 733 - err = 0; 734 - goto done; 735 - } 736 - 737 - err = indexed_table_ref_iter_new(&itr, r, oid, hash_size(r->hash_id), 738 - got.u.obj.offsets, 739 - got.u.obj.offset_len); 740 - if (err < 0) 741 - goto done; 742 - got.u.obj.offsets = NULL; 743 - iterator_from_indexed_table_ref_iter(it, itr); 744 - 745 - done: 746 - reftable_iterator_destroy(&oit); 747 - reftable_record_release(&got); 748 - return err; 749 - } 750 - 751 - static int reftable_reader_refs_for_unindexed(struct reftable_reader *r, 752 - struct reftable_iterator *it, 753 - uint8_t *oid) 754 - { 755 - struct table_iter *ti; 756 - struct filtering_ref_iterator *filter = NULL; 757 - struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT; 758 - uint32_t oid_len = hash_size(r->hash_id); 759 - int err; 760 - 761 - REFTABLE_ALLOC_ARRAY(ti, 1); 762 - if (!ti) { 763 - err = REFTABLE_OUT_OF_MEMORY_ERROR; 764 - goto out; 765 - } 766 - 767 - table_iter_init(ti, r); 768 - err = table_iter_seek_start(ti, BLOCK_TYPE_REF, 0); 769 - if (err < 0) 770 - goto out; 771 - 772 - filter = reftable_malloc(sizeof(*filter)); 773 - if (!filter) { 774 - err = REFTABLE_OUT_OF_MEMORY_ERROR; 775 - goto out; 776 - } 777 - *filter = empty; 778 - 779 - err = reftable_buf_add(&filter->oid, oid, oid_len); 780 - if (err < 0) 781 - goto out; 782 - 783 - iterator_from_table_iter(&filter->it, ti); 784 - 785 - iterator_from_filtering_ref_iterator(it, filter); 786 - 787 - err = 0; 788 - 789 - out: 790 - if (err < 0) { 791 - if (ti) 792 - table_iter_close(ti); 793 - reftable_free(ti); 794 - } 795 - return err; 796 - } 797 - 798 - int reftable_reader_refs_for(struct reftable_reader *r, 799 - struct reftable_iterator *it, uint8_t *oid) 800 - { 801 - if (r->obj_offsets.is_present) 802 - return reftable_reader_refs_for_indexed(r, it, oid); 803 - return reftable_reader_refs_for_unindexed(r, it, oid); 804 - } 805 - 806 - uint64_t reftable_reader_max_update_index(struct reftable_reader *r) 807 - { 808 - return r->max_update_index; 809 - } 810 - 811 - uint64_t reftable_reader_min_update_index(struct reftable_reader *r) 812 - { 813 - return r->min_update_index; 814 - } 815 - 816 - int reftable_reader_print_blocks(const char *tablename) 817 - { 818 - struct { 819 - const char *name; 820 - int type; 821 - } sections[] = { 822 - { 823 - .name = "ref", 824 - .type = BLOCK_TYPE_REF, 825 - }, 826 - { 827 - .name = "obj", 828 - .type = BLOCK_TYPE_OBJ, 829 - }, 830 - { 831 - .name = "log", 832 - .type = BLOCK_TYPE_LOG, 833 - }, 834 - }; 835 - struct reftable_block_source src = { 0 }; 836 - struct reftable_reader *r = NULL; 837 - struct table_iter ti = { 0 }; 838 - size_t i; 839 - int err; 840 - 841 - err = reftable_block_source_from_file(&src, tablename); 842 - if (err < 0) 843 - goto done; 844 - 845 - err = reftable_reader_new(&r, &src, tablename); 846 - if (err < 0) 847 - goto done; 848 - 849 - table_iter_init(&ti, r); 850 - 851 - printf("header:\n"); 852 - printf(" block_size: %d\n", r->block_size); 853 - 854 - for (i = 0; i < sizeof(sections) / sizeof(*sections); i++) { 855 - err = table_iter_seek_start(&ti, sections[i].type, 0); 856 - if (err < 0) 857 - goto done; 858 - if (err > 0) 859 - continue; 860 - 861 - printf("%s:\n", sections[i].name); 862 - 863 - while (1) { 864 - printf(" - length: %u\n", ti.br.block_len); 865 - printf(" restarts: %u\n", ti.br.restart_count); 866 - 867 - err = table_iter_next_block(&ti); 868 - if (err < 0) 869 - goto done; 870 - if (err > 0) 871 - break; 872 - } 873 - } 874 - 875 - done: 876 - reftable_reader_decref(r); 877 - table_iter_close(&ti); 878 - return err; 879 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

-67

reftable/reader.h

··· 1 - /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 - 9 - #ifndef READER_H 10 - #define READER_H 11 - 12 - #include "block.h" 13 - #include "record.h" 14 - #include "reftable-iterator.h" 15 - #include "reftable-reader.h" 16 - 17 - uint64_t block_source_size(struct reftable_block_source *source); 18 - 19 - ssize_t block_source_read_block(struct reftable_block_source *source, 20 - struct reftable_block *dest, uint64_t off, 21 - uint32_t size); 22 - void block_source_close(struct reftable_block_source *source); 23 - 24 - /* metadata for a block type */ 25 - struct reftable_reader_offsets { 26 - int is_present; 27 - uint64_t offset; 28 - uint64_t index_offset; 29 - }; 30 - 31 - /* The state for reading a reftable file. */ 32 - struct reftable_reader { 33 - /* for convenience, associate a name with the instance. */ 34 - char *name; 35 - struct reftable_block_source source; 36 - 37 - /* Size of the file, excluding the footer. */ 38 - uint64_t size; 39 - 40 - /* The hash function used for ref records. */ 41 - enum reftable_hash hash_id; 42 - 43 - uint32_t block_size; 44 - uint64_t min_update_index; 45 - uint64_t max_update_index; 46 - /* Length of the OID keys in the 'o' section */ 47 - int object_id_len; 48 - int version; 49 - 50 - struct reftable_reader_offsets ref_offsets; 51 - struct reftable_reader_offsets obj_offsets; 52 - struct reftable_reader_offsets log_offsets; 53 - 54 - uint64_t refcount; 55 - }; 56 - 57 - const char *reader_name(struct reftable_reader *r); 58 - 59 - int reader_init_iter(struct reftable_reader *r, 60 - struct reftable_iterator *it, 61 - uint8_t typ); 62 - 63 - /* initialize a block reader to read from `r` */ 64 - int reader_init_block_reader(struct reftable_reader *r, struct block_reader *br, 65 - uint64_t next_off, uint8_t want_typ); 66 - 67 - #endif

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

+26 -26

reftable/record.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 /* record.c - methods for different types of records. */ 10 ··· 69 int reftable_is_block_type(uint8_t typ) 70 { 71 switch (typ) { 72 - case BLOCK_TYPE_REF: 73 - case BLOCK_TYPE_LOG: 74 - case BLOCK_TYPE_OBJ: 75 - case BLOCK_TYPE_INDEX: 76 return 1; 77 } 78 return 0; ··· 459 460 static struct reftable_record_vtable reftable_ref_record_vtable = { 461 .key = &reftable_ref_record_key, 462 - .type = BLOCK_TYPE_REF, 463 .copy_from = &reftable_ref_record_copy_from, 464 .val_type = &reftable_ref_record_val_type, 465 .encode = &reftable_ref_record_encode, ··· 659 660 static struct reftable_record_vtable reftable_obj_record_vtable = { 661 .key = &reftable_obj_record_key, 662 - .type = BLOCK_TYPE_OBJ, 663 .copy_from = &reftable_obj_record_copy_from, 664 .val_type = &reftable_obj_record_val_type, 665 .encode = &reftable_obj_record_encode, ··· 1030 1031 static struct reftable_record_vtable reftable_log_record_vtable = { 1032 .key = &reftable_log_record_key, 1033 - .type = BLOCK_TYPE_LOG, 1034 .copy_from = &reftable_log_record_copy_from, 1035 .val_type = &reftable_log_record_val_type, 1036 .encode = &reftable_log_record_encode, ··· 1132 1133 static struct reftable_record_vtable reftable_index_record_vtable = { 1134 .key = &reftable_index_record_key, 1135 - .type = BLOCK_TYPE_INDEX, 1136 .copy_from = &reftable_index_record_copy_from, 1137 .val_type = &reftable_index_record_val_type, 1138 .encode = &reftable_index_record_encode, ··· 1275 static void *reftable_record_data(struct reftable_record *rec) 1276 { 1277 switch (rec->type) { 1278 - case BLOCK_TYPE_REF: 1279 return &rec->u.ref; 1280 - case BLOCK_TYPE_LOG: 1281 return &rec->u.log; 1282 - case BLOCK_TYPE_INDEX: 1283 return &rec->u.idx; 1284 - case BLOCK_TYPE_OBJ: 1285 return &rec->u.obj; 1286 } 1287 abort(); ··· 1291 reftable_record_vtable(struct reftable_record *rec) 1292 { 1293 switch (rec->type) { 1294 - case BLOCK_TYPE_REF: 1295 return &reftable_ref_record_vtable; 1296 - case BLOCK_TYPE_LOG: 1297 return &reftable_log_record_vtable; 1298 - case BLOCK_TYPE_INDEX: 1299 return &reftable_index_record_vtable; 1300 - case BLOCK_TYPE_OBJ: 1301 return &reftable_obj_record_vtable; 1302 } 1303 abort(); ··· 1309 rec->type = typ; 1310 1311 switch (typ) { 1312 - case BLOCK_TYPE_REF: 1313 - case BLOCK_TYPE_LOG: 1314 - case BLOCK_TYPE_OBJ: 1315 return 0; 1316 - case BLOCK_TYPE_INDEX: 1317 reftable_buf_init(&rec->u.idx.last_key); 1318 return 0; 1319 default:

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 /* record.c - methods for different types of records. */ 10 ··· 69 int reftable_is_block_type(uint8_t typ) 70 { 71 switch (typ) { 72 + case REFTABLE_BLOCK_TYPE_REF: 73 + case REFTABLE_BLOCK_TYPE_LOG: 74 + case REFTABLE_BLOCK_TYPE_OBJ: 75 + case REFTABLE_BLOCK_TYPE_INDEX: 76 return 1; 77 } 78 return 0; ··· 459 460 static struct reftable_record_vtable reftable_ref_record_vtable = { 461 .key = &reftable_ref_record_key, 462 + .type = REFTABLE_BLOCK_TYPE_REF, 463 .copy_from = &reftable_ref_record_copy_from, 464 .val_type = &reftable_ref_record_val_type, 465 .encode = &reftable_ref_record_encode, ··· 659 660 static struct reftable_record_vtable reftable_obj_record_vtable = { 661 .key = &reftable_obj_record_key, 662 + .type = REFTABLE_BLOCK_TYPE_OBJ, 663 .copy_from = &reftable_obj_record_copy_from, 664 .val_type = &reftable_obj_record_val_type, 665 .encode = &reftable_obj_record_encode, ··· 1030 1031 static struct reftable_record_vtable reftable_log_record_vtable = { 1032 .key = &reftable_log_record_key, 1033 + .type = REFTABLE_BLOCK_TYPE_LOG, 1034 .copy_from = &reftable_log_record_copy_from, 1035 .val_type = &reftable_log_record_val_type, 1036 .encode = &reftable_log_record_encode, ··· 1132 1133 static struct reftable_record_vtable reftable_index_record_vtable = { 1134 .key = &reftable_index_record_key, 1135 + .type = REFTABLE_BLOCK_TYPE_INDEX, 1136 .copy_from = &reftable_index_record_copy_from, 1137 .val_type = &reftable_index_record_val_type, 1138 .encode = &reftable_index_record_encode, ··· 1275 static void *reftable_record_data(struct reftable_record *rec) 1276 { 1277 switch (rec->type) { 1278 + case REFTABLE_BLOCK_TYPE_REF: 1279 return &rec->u.ref; 1280 + case REFTABLE_BLOCK_TYPE_LOG: 1281 return &rec->u.log; 1282 + case REFTABLE_BLOCK_TYPE_INDEX: 1283 return &rec->u.idx; 1284 + case REFTABLE_BLOCK_TYPE_OBJ: 1285 return &rec->u.obj; 1286 } 1287 abort(); ··· 1291 reftable_record_vtable(struct reftable_record *rec) 1292 { 1293 switch (rec->type) { 1294 + case REFTABLE_BLOCK_TYPE_REF: 1295 return &reftable_ref_record_vtable; 1296 + case REFTABLE_BLOCK_TYPE_LOG: 1297 return &reftable_log_record_vtable; 1298 + case REFTABLE_BLOCK_TYPE_INDEX: 1299 return &reftable_index_record_vtable; 1300 + case REFTABLE_BLOCK_TYPE_OBJ: 1301 return &reftable_obj_record_vtable; 1302 } 1303 abort(); ··· 1309 rec->type = typ; 1310 1311 switch (typ) { 1312 + case REFTABLE_BLOCK_TYPE_REF: 1313 + case REFTABLE_BLOCK_TYPE_LOG: 1314 + case REFTABLE_BLOCK_TYPE_OBJ: 1315 return 0; 1316 + case REFTABLE_BLOCK_TYPE_INDEX: 1317 reftable_buf_init(&rec->u.idx.last_key); 1318 return 0; 1319 default:

+6 -6

reftable/record.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef RECORD_H 10 #define RECORD_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef RECORD_H 10 #define RECORD_H

+9 -1

reftable/reftable-basics.h

··· 4 * Use of this source code is governed by a BSD-style 5 * license that can be found in the LICENSE file or at 6 * https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef REFTABLE_BASICS_H 10 #define REFTABLE_BASICS_H 11 12 #include <stddef.h> 0 0 0 0 0 0 0 0 13 14 /* 15 * Hash functions understood by the reftable library. Note that the values are

··· 4 * Use of this source code is governed by a BSD-style 5 * license that can be found in the LICENSE file or at 6 * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef REFTABLE_BASICS_H 10 #define REFTABLE_BASICS_H 11 12 #include <stddef.h> 13 + 14 + /* A buffer that contains arbitrary byte slices. */ 15 + struct reftable_buf { 16 + size_t alloc; 17 + size_t len; 18 + char *buf; 19 + }; 20 + #define REFTABLE_BUF_INIT { 0 } 21 22 /* 23 * Hash functions understood by the reftable library. Note that the values are

+74

reftable/reftable-block.h

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 + 9 + #ifndef REFTABLE_BLOCK_H 10 + #define REFTABLE_BLOCK_H 11 + 12 + #include <stdint.h> 13 + 14 + #include "reftable-basics.h" 15 + #include "reftable-blocksource.h" 16 + #include "reftable-iterator.h" 17 + 18 + struct z_stream_s; 19 + 20 + /* 21 + * A block part of a reftable. Contains records as well as some metadata 22 + * describing them. 23 + */ 24 + struct reftable_block { 25 + /* 26 + * Offset of the block header; nonzero for the first block in a 27 + * reftable. 28 + */ 29 + uint32_t header_off; 30 + 31 + /* The memory block. */ 32 + struct reftable_block_data block_data; 33 + uint32_t hash_size; 34 + 35 + /* Uncompressed data for log entries. */ 36 + struct z_stream_s *zstream; 37 + unsigned char *uncompressed_data; 38 + size_t uncompressed_cap; 39 + 40 + /* 41 + * Restart point data. Restart points are located after the block's 42 + * record data. 43 + */ 44 + uint16_t restart_count; 45 + uint32_t restart_off; 46 + 47 + /* 48 + * Size of the data in the file. For log blocks, this is the compressed 49 + * size. 50 + */ 51 + uint32_t full_block_size; 52 + uint8_t block_type; 53 + }; 54 + 55 + /* Initialize a reftable block from the given block source. */ 56 + int reftable_block_init(struct reftable_block *b, 57 + struct reftable_block_source *source, 58 + uint32_t offset, uint32_t header_size, 59 + uint32_t table_block_size, uint32_t hash_size); 60 + 61 + /* Release resources allocated by the block. */ 62 + void reftable_block_release(struct reftable_block *b); 63 + 64 + /* Initialize a generic record iterator from the given block. */ 65 + int reftable_block_init_iterator(const struct reftable_block *b, 66 + struct reftable_iterator *it); 67 + 68 + /* Returns the block type (eg. 'r' for refs). */ 69 + uint8_t reftable_block_type(const struct reftable_block *b); 70 + 71 + /* Decodes the first key in the block. */ 72 + int reftable_block_first_key(const struct reftable_block *b, struct reftable_buf *key); 73 + 74 + #endif /* REFTABLE_BLOCK_H */

+15 -14

reftable/reftable-blocksource.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef REFTABLE_BLOCKSOURCE_H 10 #define REFTABLE_BLOCKSOURCE_H 11 12 #include <stdint.h> 13 14 - /* block_source is a generic wrapper for a seekable readable file. 0 15 */ 16 struct reftable_block_source { 17 struct reftable_block_source_vtable *ops; ··· 20 21 /* a contiguous segment of bytes. It keeps track of its generating block_source 22 * so it can return itself into the pool. */ 23 - struct reftable_block { 24 uint8_t *data; 25 size_t len; 26 struct reftable_block_source source; ··· 28 29 /* block_source_vtable are the operations that make up block_source */ 30 struct reftable_block_source_vtable { 31 - /* returns the size of a block source */ 32 uint64_t (*size)(void *source); 33 34 /* 35 * Reads a segment from the block source. It is an error to read beyond 36 * the end of the block. 37 */ 38 - ssize_t (*read_block)(void *source, struct reftable_block *dest, 39 - uint64_t off, uint32_t size); 40 41 - /* mark the block as read; may return the data back to malloc */ 42 - void (*return_block)(void *source, struct reftable_block *blockp); 43 44 - /* release all resources associated with the block source */ 45 void (*close)(void *source); 46 }; 47

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef REFTABLE_BLOCKSOURCE_H 10 #define REFTABLE_BLOCKSOURCE_H 11 12 #include <stdint.h> 13 14 + /* 15 + * Generic wrapper for a seekable readable file. 16 */ 17 struct reftable_block_source { 18 struct reftable_block_source_vtable *ops; ··· 21 22 /* a contiguous segment of bytes. It keeps track of its generating block_source 23 * so it can return itself into the pool. */ 24 + struct reftable_block_data { 25 uint8_t *data; 26 size_t len; 27 struct reftable_block_source source; ··· 29 30 /* block_source_vtable are the operations that make up block_source */ 31 struct reftable_block_source_vtable { 32 + /* Returns the size of a block source. */ 33 uint64_t (*size)(void *source); 34 35 /* 36 * Reads a segment from the block source. It is an error to read beyond 37 * the end of the block. 38 */ 39 + ssize_t (*read_data)(void *source, struct reftable_block_data *dest, 40 + uint64_t off, uint32_t size); 41 42 + /* Mark the block as read; may release the data. */ 43 + void (*release_data)(void *source, struct reftable_block_data *data); 44 45 + /* Release all resources associated with the block source. */ 46 void (*close)(void *source); 47 }; 48

+18

reftable/reftable-constants.h

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 + 9 + #ifndef REFTABLE_CONSTANTS_H 10 + #define REFTABLE_CONSTANTS_H 11 + 12 + #define REFTABLE_BLOCK_TYPE_LOG 'g' 13 + #define REFTABLE_BLOCK_TYPE_INDEX 'i' 14 + #define REFTABLE_BLOCK_TYPE_REF 'r' 15 + #define REFTABLE_BLOCK_TYPE_OBJ 'o' 16 + #define REFTABLE_BLOCK_TYPE_ANY 0 17 + 18 + #endif /* REFTABLE_CONSTANTS_H */

+6 -6

reftable/reftable-error.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef REFTABLE_ERROR_H 10 #define REFTABLE_ERROR_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef REFTABLE_ERROR_H 10 #define REFTABLE_ERROR_H

+6 -6

reftable/reftable-iterator.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef REFTABLE_ITERATOR_H 10 #define REFTABLE_ITERATOR_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef REFTABLE_ITERATOR_H 10 #define REFTABLE_ITERATOR_H

+9 -9

reftable/reftable-merged.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef REFTABLE_MERGED_H 10 #define REFTABLE_MERGED_H ··· 26 /* A merged table is implements seeking/iterating over a stack of tables. */ 27 struct reftable_merged_table; 28 29 - struct reftable_reader; 30 31 /* 32 - * reftable_merged_table_new creates a new merged table. The readers must be 33 * kept alive as long as the merged table is still in use. 34 */ 35 int reftable_merged_table_new(struct reftable_merged_table **dest, 36 - struct reftable_reader **readers, size_t n, 37 enum reftable_hash hash_id); 38 39 /* Initialize a merged table iterator for reading refs. */

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef REFTABLE_MERGED_H 10 #define REFTABLE_MERGED_H ··· 26 /* A merged table is implements seeking/iterating over a stack of tables. */ 27 struct reftable_merged_table; 28 29 + struct reftable_table; 30 31 /* 32 + * reftable_merged_table_new creates a new merged table. The tables must be 33 * kept alive as long as the merged table is still in use. 34 */ 35 int reftable_merged_table_new(struct reftable_merged_table **dest, 36 + struct reftable_table **tables, size_t n, 37 enum reftable_hash hash_id); 38 39 /* Initialize a merged table iterator for reading refs. */

-72

reftable/reftable-reader.h

··· 1 - /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 - 9 - #ifndef REFTABLE_READER_H 10 - #define REFTABLE_READER_H 11 - 12 - #include "reftable-iterator.h" 13 - #include "reftable-blocksource.h" 14 - 15 - /* 16 - * Reading single tables 17 - * 18 - * The follow routines are for reading single files. For an 19 - * application-level interface, skip ahead to struct 20 - * reftable_merged_table and struct reftable_stack. 21 - */ 22 - 23 - /* The reader struct is a handle to an open reftable file. */ 24 - struct reftable_reader; 25 - 26 - /* reftable_reader_new opens a reftable for reading. If successful, 27 - * returns 0 code and sets pp. The name is used for creating a 28 - * stack. Typically, it is the basename of the file. The block source 29 - * `src` is owned by the reader, and is closed on calling 30 - * reftable_reader_destroy(). On error, the block source `src` is 31 - * closed as well. 32 - */ 33 - int reftable_reader_new(struct reftable_reader **pp, 34 - struct reftable_block_source *src, const char *name); 35 - 36 - /* 37 - * Manage the reference count of the reftable reader. A newly initialized 38 - * reader starts with a refcount of 1 and will be deleted once the refcount has 39 - * reached 0. 40 - * 41 - * This is required because readers may have longer lifetimes than the stack 42 - * they belong to. The stack may for example be reloaded while the old tables 43 - * are still being accessed by an iterator. 44 - */ 45 - void reftable_reader_incref(struct reftable_reader *reader); 46 - void reftable_reader_decref(struct reftable_reader *reader); 47 - 48 - /* Initialize a reftable iterator for reading refs. */ 49 - int reftable_reader_init_ref_iterator(struct reftable_reader *r, 50 - struct reftable_iterator *it); 51 - 52 - /* Initialize a reftable iterator for reading logs. */ 53 - int reftable_reader_init_log_iterator(struct reftable_reader *r, 54 - struct reftable_iterator *it); 55 - 56 - /* returns the hash ID used in this table. */ 57 - enum reftable_hash reftable_reader_hash_id(struct reftable_reader *r); 58 - 59 - /* return an iterator for the refs pointing to `oid`. */ 60 - int reftable_reader_refs_for(struct reftable_reader *r, 61 - struct reftable_iterator *it, uint8_t *oid); 62 - 63 - /* return the max_update_index for a table */ 64 - uint64_t reftable_reader_max_update_index(struct reftable_reader *r); 65 - 66 - /* return the min_update_index for a table */ 67 - uint64_t reftable_reader_min_update_index(struct reftable_reader *r); 68 - 69 - /* print blocks onto stdout for debugging. */ 70 - int reftable_reader_print_blocks(const char *tablename); 71 - 72 - #endif

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

+6 -6

reftable/reftable-record.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef REFTABLE_RECORD_H 10 #define REFTABLE_RECORD_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef REFTABLE_RECORD_H 10 #define REFTABLE_RECORD_H

+6 -6

reftable/reftable-stack.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef REFTABLE_STACK_H 10 #define REFTABLE_STACK_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef REFTABLE_STACK_H 10 #define REFTABLE_STACK_H

+115

reftable/reftable-table.h

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 + 9 + #ifndef REFTABLE_TABLE_H 10 + #define REFTABLE_TABLE_H 11 + 12 + #include "reftable-iterator.h" 13 + #include "reftable-block.h" 14 + #include "reftable-blocksource.h" 15 + 16 + /* 17 + * Reading single tables 18 + * 19 + * The follow routines are for reading single files. For an 20 + * application-level interface, skip ahead to struct 21 + * reftable_merged_table and struct reftable_stack. 22 + */ 23 + 24 + /* Metadata for a block type. */ 25 + struct reftable_table_offsets { 26 + int is_present; 27 + uint64_t offset; 28 + uint64_t index_offset; 29 + }; 30 + 31 + /* The table struct is a handle to an open reftable file. */ 32 + struct reftable_table { 33 + /* for convenience, associate a name with the instance. */ 34 + char *name; 35 + struct reftable_block_source source; 36 + 37 + /* Size of the file, excluding the footer. */ 38 + uint64_t size; 39 + 40 + /* The hash function used for ref records. */ 41 + enum reftable_hash hash_id; 42 + 43 + uint32_t block_size; 44 + uint64_t min_update_index; 45 + uint64_t max_update_index; 46 + /* Length of the OID keys in the 'o' section */ 47 + int object_id_len; 48 + int version; 49 + 50 + struct reftable_table_offsets ref_offsets; 51 + struct reftable_table_offsets obj_offsets; 52 + struct reftable_table_offsets log_offsets; 53 + 54 + uint64_t refcount; 55 + }; 56 + 57 + /* reftable_table_new opens a reftable for reading. If successful, 58 + * returns 0 code and sets pp. The name is used for creating a 59 + * stack. Typically, it is the basename of the file. The block source 60 + * `src` is owned by the table, and is closed on calling 61 + * reftable_table_destroy(). On error, the block source `src` is 62 + * closed as well. 63 + */ 64 + int reftable_table_new(struct reftable_table **out, 65 + struct reftable_block_source *src, const char *name); 66 + 67 + /* 68 + * Manage the reference count of the reftable table. A newly initialized 69 + * table starts with a refcount of 1 and will be deleted once the refcount has 70 + * reached 0. 71 + * 72 + * This is required because tables may have longer lifetimes than the stack 73 + * they belong to. The stack may for example be reloaded while the old tables 74 + * are still being accessed by an iterator. 75 + */ 76 + void reftable_table_incref(struct reftable_table *table); 77 + void reftable_table_decref(struct reftable_table *table); 78 + 79 + /* Initialize a reftable iterator for reading refs. */ 80 + int reftable_table_init_ref_iterator(struct reftable_table *t, 81 + struct reftable_iterator *it); 82 + 83 + /* Initialize a reftable iterator for reading logs. */ 84 + int reftable_table_init_log_iterator(struct reftable_table *t, 85 + struct reftable_iterator *it); 86 + 87 + /* returns the hash ID used in this table. */ 88 + enum reftable_hash reftable_table_hash_id(struct reftable_table *t); 89 + 90 + /* return an iterator for the refs pointing to `oid`. */ 91 + int reftable_table_refs_for(struct reftable_table *t, 92 + struct reftable_iterator *it, uint8_t *oid); 93 + 94 + /* return the max_update_index for a table */ 95 + uint64_t reftable_table_max_update_index(struct reftable_table *t); 96 + 97 + /* return the min_update_index for a table */ 98 + uint64_t reftable_table_min_update_index(struct reftable_table *t); 99 + 100 + /* 101 + * An iterator that iterates through the blocks contained in a given table. 102 + */ 103 + struct reftable_table_iterator { 104 + void *iter_arg; 105 + }; 106 + 107 + int reftable_table_iterator_init(struct reftable_table_iterator *it, 108 + struct reftable_table *t); 109 + 110 + void reftable_table_iterator_release(struct reftable_table_iterator *it); 111 + 112 + int reftable_table_iterator_next(struct reftable_table_iterator *it, 113 + const struct reftable_block **out); 114 + 115 + #endif

+6 -6

reftable/reftable-writer.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef REFTABLE_WRITER_H 10 #define REFTABLE_WRITER_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef REFTABLE_WRITER_H 10 #define REFTABLE_WRITER_H

+94 -94

reftable/stack.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "stack.h" 10 11 #include "system.h" 12 #include "constants.h" 13 #include "merged.h" 14 - #include "reader.h" 15 #include "reftable-error.h" 16 #include "reftable-record.h" 17 #include "reftable-merged.h" 0 18 #include "writer.h" 19 20 static int stack_try_add(struct reftable_stack *st, ··· 203 struct reftable_iterator *it) 204 { 205 return merged_table_init_iter(reftable_stack_merged_table(st), 206 - it, BLOCK_TYPE_REF); 207 } 208 209 int reftable_stack_init_log_iterator(struct reftable_stack *st, 210 struct reftable_iterator *it) 211 { 212 return merged_table_init_iter(reftable_stack_merged_table(st), 213 - it, BLOCK_TYPE_LOG); 214 } 215 216 struct reftable_merged_table * ··· 248 REFTABLE_FREE_AND_NULL(names); 249 } 250 251 - if (st->readers) { 252 struct reftable_buf filename = REFTABLE_BUF_INIT; 253 254 - for (size_t i = 0; i < st->readers_len; i++) { 255 - const char *name = reader_name(st->readers[i]); 256 int try_unlinking = 1; 257 258 reftable_buf_reset(&filename); ··· 260 if (stack_filename(&filename, st, name) < 0) 261 try_unlinking = 0; 262 } 263 - reftable_reader_decref(st->readers[i]); 264 265 if (try_unlinking && filename.len) { 266 /* On Windows, can only unlink after closing. */ ··· 269 } 270 271 reftable_buf_release(&filename); 272 - st->readers_len = 0; 273 - REFTABLE_FREE_AND_NULL(st->readers); 274 } 275 276 if (st->list_fd >= 0) { ··· 284 free_names(names); 285 } 286 287 - static struct reftable_reader **stack_copy_readers(struct reftable_stack *st, 288 - size_t cur_len) 289 { 290 - struct reftable_reader **cur = reftable_calloc(cur_len, sizeof(*cur)); 291 if (!cur) 292 return NULL; 293 for (size_t i = 0; i < cur_len; i++) 294 - cur[i] = st->readers[i]; 295 return cur; 296 } 297 ··· 299 const char **names, 300 int reuse_open) 301 { 302 - size_t cur_len = !st->merged ? 0 : st->merged->readers_len; 303 - struct reftable_reader **cur = NULL; 304 - struct reftable_reader **reused = NULL; 305 - struct reftable_reader **new_readers = NULL; 306 size_t reused_len = 0, reused_alloc = 0, names_len; 307 - size_t new_readers_len = 0; 308 struct reftable_merged_table *new_merged = NULL; 309 struct reftable_buf table_path = REFTABLE_BUF_INIT; 310 int err = 0; 311 size_t i; 312 313 if (cur_len) { 314 - cur = stack_copy_readers(st, cur_len); 315 if (!cur) { 316 err = REFTABLE_OUT_OF_MEMORY_ERROR; 317 goto done; ··· 321 names_len = names_length(names); 322 323 if (names_len) { 324 - new_readers = reftable_calloc(names_len, sizeof(*new_readers)); 325 - if (!new_readers) { 326 err = REFTABLE_OUT_OF_MEMORY_ERROR; 327 goto done; 328 } 329 } 330 331 while (*names) { 332 - struct reftable_reader *rd = NULL; 333 const char *name = *names++; 334 335 /* this is linear; we assume compaction keeps the number of 336 tables under control so this is not quadratic. */ 337 for (i = 0; reuse_open && i < cur_len; i++) { 338 if (cur[i] && 0 == strcmp(cur[i]->name, name)) { 339 - rd = cur[i]; 340 cur[i] = NULL; 341 342 /* 343 * When reloading the stack fails, we end up 344 - * releasing all new readers. This also 345 - * includes the reused readers, even though 346 * they are still in used by the old stack. We 347 * thus need to keep them alive here, which we 348 * do by bumping their refcount. ··· 354 err = REFTABLE_OUT_OF_MEMORY_ERROR; 355 goto done; 356 } 357 - reused[reused_len++] = rd; 358 - reftable_reader_incref(rd); 359 break; 360 } 361 } 362 363 - if (!rd) { 364 struct reftable_block_source src = { NULL }; 365 366 err = stack_filename(&table_path, st, name); ··· 372 if (err < 0) 373 goto done; 374 375 - err = reftable_reader_new(&rd, &src, name); 376 if (err < 0) 377 goto done; 378 } 379 380 - new_readers[new_readers_len] = rd; 381 - new_readers_len++; 382 } 383 384 /* success! */ 385 - err = reftable_merged_table_new(&new_merged, new_readers, 386 - new_readers_len, st->opts.hash_id); 387 if (err < 0) 388 goto done; 389 390 /* 391 - * Close the old, non-reused readers and proactively try to unlink 392 * them. This is done for systems like Windows, where the underlying 393 - * file of such an open reader wouldn't have been possible to be 394 * unlinked by the compacting process. 395 */ 396 for (i = 0; i < cur_len; i++) { 397 if (cur[i]) { 398 - const char *name = reader_name(cur[i]); 399 400 err = stack_filename(&table_path, st, name); 401 if (err < 0) 402 goto done; 403 404 - reftable_reader_decref(cur[i]); 405 unlink(table_path.buf); 406 } 407 } ··· 412 new_merged->suppress_deletions = 1; 413 st->merged = new_merged; 414 415 - if (st->readers) 416 - reftable_free(st->readers); 417 - st->readers = new_readers; 418 - st->readers_len = new_readers_len; 419 - new_readers = NULL; 420 - new_readers_len = 0; 421 422 /* 423 - * Decrement the refcount of reused readers again. This only needs to 424 * happen on the successful case, because on the unsuccessful one we 425 - * decrement their refcount via `new_readers`. 426 */ 427 for (i = 0; i < reused_len; i++) 428 - reftable_reader_decref(reused[i]); 429 430 done: 431 - for (i = 0; i < new_readers_len; i++) 432 - reftable_reader_decref(new_readers[i]); 433 - reftable_free(new_readers); 434 reftable_free(reused); 435 reftable_free(cur); 436 reftable_buf_release(&table_path); ··· 615 /* 616 * It's fine for "tables.list" to not exist. In that 617 * case, we have to refresh when the loaded stack has 618 - * any readers. 619 */ 620 if (errno == ENOENT) 621 - return !!st->readers_len; 622 return REFTABLE_IO_ERROR; 623 } 624 ··· 637 if (err < 0) 638 return err; 639 640 - for (size_t i = 0; i < st->readers_len; i++) { 641 if (!names[i]) { 642 err = 1; 643 goto done; 644 } 645 646 - if (strcmp(st->readers[i]->name, names[i])) { 647 err = 1; 648 goto done; 649 } 650 } 651 652 - if (names[st->merged->readers_len]) { 653 err = 1; 654 goto done; 655 } ··· 792 if (add->new_tables_len == 0) 793 goto done; 794 795 - for (i = 0; i < add->stack->merged->readers_len; i++) { 796 - if ((err = reftable_buf_addstr(&table_list, add->stack->readers[i]->name)) < 0 || 797 (err = reftable_buf_addstr(&table_list, "\n")) < 0) 798 goto done; 799 } ··· 1000 1001 uint64_t reftable_stack_next_update_index(struct reftable_stack *st) 1002 { 1003 - int sz = st->merged->readers_len; 1004 if (sz > 0) 1005 - return reftable_reader_max_update_index(st->readers[sz - 1]) + 1006 1; 1007 return 1; 1008 } ··· 1021 struct reftable_tmpfile tab_file = REFTABLE_TMPFILE_INIT; 1022 int err = 0; 1023 1024 - err = format_name(&next_name, reftable_reader_min_update_index(st->readers[first]), 1025 - reftable_reader_max_update_index(st->readers[last])); 1026 if (err < 0) 1027 goto done; 1028 ··· 1087 int err = 0; 1088 1089 for (size_t i = first; i <= last; i++) 1090 - st->stats.bytes += st->readers[i]->size; 1091 - err = reftable_writer_set_limits(wr, st->readers[first]->min_update_index, 1092 - st->readers[last]->max_update_index); 1093 if (err < 0) 1094 goto done; 1095 1096 - err = reftable_merged_table_new(&mt, st->readers + first, subtabs_len, 1097 st->opts.hash_id); 1098 if (err < 0) 1099 goto done; 1100 1101 - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); 1102 if (err < 0) 1103 goto done; 1104 ··· 1126 } 1127 reftable_iterator_destroy(&it); 1128 1129 - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); 1130 if (err < 0) 1131 goto done; 1132 ··· 1250 table_locks[i] = REFTABLE_FLOCK_INIT; 1251 1252 for (i = last + 1; i > first; i--) { 1253 - err = stack_filename(&table_name, st, reader_name(st->readers[i - 1])); 1254 if (err < 0) 1255 goto done; 1256 ··· 1376 * compacted in the updated "tables.list" file. 1377 */ 1378 for (size_t i = 0; names[i]; i++) { 1379 - if (strcmp(names[i], st->readers[first]->name)) 1380 continue; 1381 1382 /* ··· 1386 * have compacted them. 1387 */ 1388 for (size_t j = 1; j < last - first + 1; j++) { 1389 - const char *old = first + j < st->merged->readers_len ? 1390 - st->readers[first + j]->name : NULL; 1391 const char *new = names[i + j]; 1392 1393 /* ··· 1427 * `fd_read_lines()` uses a `NULL` sentinel to indicate that 1428 * the array is at its end. As we use `free_names()` to free 1429 * the array, we need to include this sentinel value here and 1430 - * thus have to allocate `readers_len + 1` many entries. 1431 */ 1432 - REFTABLE_CALLOC_ARRAY(names, st->merged->readers_len + 1); 1433 if (!names) { 1434 err = REFTABLE_OUT_OF_MEMORY_ERROR; 1435 goto done; 1436 } 1437 1438 - for (size_t i = 0; i < st->merged->readers_len; i++) { 1439 - names[i] = reftable_strdup(st->readers[i]->name); 1440 if (!names[i]) { 1441 err = REFTABLE_OUT_OF_MEMORY_ERROR; 1442 goto done; ··· 1451 * it into place now. 1452 */ 1453 if (!is_empty_table) { 1454 - err = format_name(&new_table_name, st->readers[first]->min_update_index, 1455 - st->readers[last]->max_update_index); 1456 if (err < 0) 1457 goto done; 1458 ··· 1559 int reftable_stack_compact_all(struct reftable_stack *st, 1560 struct reftable_log_expiry_config *config) 1561 { 1562 - size_t last = st->merged->readers_len ? st->merged->readers_len - 1 : 0; 1563 return stack_compact_range(st, 0, last, config, 0); 1564 } 1565 ··· 1650 int overhead = header_size(version) - 1; 1651 uint64_t *sizes; 1652 1653 - REFTABLE_CALLOC_ARRAY(sizes, st->merged->readers_len); 1654 if (!sizes) 1655 return NULL; 1656 1657 - for (size_t i = 0; i < st->merged->readers_len; i++) 1658 - sizes[i] = st->readers[i]->size - overhead; 1659 1660 return sizes; 1661 } ··· 1665 struct segment seg; 1666 uint64_t *sizes; 1667 1668 - if (st->merged->readers_len < 2) 1669 return 0; 1670 1671 sizes = stack_table_sizes_for_compaction(st); 1672 if (!sizes) 1673 return REFTABLE_OUT_OF_MEMORY_ERROR; 1674 1675 - seg = suggest_compaction_segment(sizes, st->merged->readers_len, 1676 st->opts.auto_compaction_factor); 1677 reftable_free(sizes); 1678 ··· 1763 int err = 0; 1764 uint64_t update_idx = 0; 1765 struct reftable_block_source src = { NULL }; 1766 - struct reftable_reader *rd = NULL; 1767 struct reftable_buf table_path = REFTABLE_BUF_INIT; 1768 1769 err = stack_filename(&table_path, st, name); ··· 1774 if (err < 0) 1775 goto done; 1776 1777 - err = reftable_reader_new(&rd, &src, name); 1778 if (err < 0) 1779 goto done; 1780 1781 - update_idx = reftable_reader_max_update_index(rd); 1782 - reftable_reader_decref(rd); 1783 1784 if (update_idx <= max) { 1785 unlink(table_path.buf); ··· 1803 if (!is_table_name(d->d_name)) 1804 continue; 1805 1806 - for (size_t i = 0; !found && i < st->readers_len; i++) 1807 - found = !strcmp(reader_name(st->readers[i]), d->d_name); 1808 if (found) 1809 continue; 1810

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "stack.h" 10 11 #include "system.h" 12 #include "constants.h" 13 #include "merged.h" 0 14 #include "reftable-error.h" 15 #include "reftable-record.h" 16 #include "reftable-merged.h" 17 + #include "table.h" 18 #include "writer.h" 19 20 static int stack_try_add(struct reftable_stack *st, ··· 203 struct reftable_iterator *it) 204 { 205 return merged_table_init_iter(reftable_stack_merged_table(st), 206 + it, REFTABLE_BLOCK_TYPE_REF); 207 } 208 209 int reftable_stack_init_log_iterator(struct reftable_stack *st, 210 struct reftable_iterator *it) 211 { 212 return merged_table_init_iter(reftable_stack_merged_table(st), 213 + it, REFTABLE_BLOCK_TYPE_LOG); 214 } 215 216 struct reftable_merged_table * ··· 248 REFTABLE_FREE_AND_NULL(names); 249 } 250 251 + if (st->tables) { 252 struct reftable_buf filename = REFTABLE_BUF_INIT; 253 254 + for (size_t i = 0; i < st->tables_len; i++) { 255 + const char *name = reftable_table_name(st->tables[i]); 256 int try_unlinking = 1; 257 258 reftable_buf_reset(&filename); ··· 260 if (stack_filename(&filename, st, name) < 0) 261 try_unlinking = 0; 262 } 263 + reftable_table_decref(st->tables[i]); 264 265 if (try_unlinking && filename.len) { 266 /* On Windows, can only unlink after closing. */ ··· 269 } 270 271 reftable_buf_release(&filename); 272 + st->tables_len = 0; 273 + REFTABLE_FREE_AND_NULL(st->tables); 274 } 275 276 if (st->list_fd >= 0) { ··· 284 free_names(names); 285 } 286 287 + static struct reftable_table **stack_copy_tables(struct reftable_stack *st, 288 + size_t cur_len) 289 { 290 + struct reftable_table **cur = reftable_calloc(cur_len, sizeof(*cur)); 291 if (!cur) 292 return NULL; 293 for (size_t i = 0; i < cur_len; i++) 294 + cur[i] = st->tables[i]; 295 return cur; 296 } 297 ··· 299 const char **names, 300 int reuse_open) 301 { 302 + size_t cur_len = !st->merged ? 0 : st->merged->tables_len; 303 + struct reftable_table **cur = NULL; 304 + struct reftable_table **reused = NULL; 305 + struct reftable_table **new_tables = NULL; 306 size_t reused_len = 0, reused_alloc = 0, names_len; 307 + size_t new_tables_len = 0; 308 struct reftable_merged_table *new_merged = NULL; 309 struct reftable_buf table_path = REFTABLE_BUF_INIT; 310 int err = 0; 311 size_t i; 312 313 if (cur_len) { 314 + cur = stack_copy_tables(st, cur_len); 315 if (!cur) { 316 err = REFTABLE_OUT_OF_MEMORY_ERROR; 317 goto done; ··· 321 names_len = names_length(names); 322 323 if (names_len) { 324 + new_tables = reftable_calloc(names_len, sizeof(*new_tables)); 325 + if (!new_tables) { 326 err = REFTABLE_OUT_OF_MEMORY_ERROR; 327 goto done; 328 } 329 } 330 331 while (*names) { 332 + struct reftable_table *table = NULL; 333 const char *name = *names++; 334 335 /* this is linear; we assume compaction keeps the number of 336 tables under control so this is not quadratic. */ 337 for (i = 0; reuse_open && i < cur_len; i++) { 338 if (cur[i] && 0 == strcmp(cur[i]->name, name)) { 339 + table = cur[i]; 340 cur[i] = NULL; 341 342 /* 343 * When reloading the stack fails, we end up 344 + * releasing all new tables. This also 345 + * includes the reused tables, even though 346 * they are still in used by the old stack. We 347 * thus need to keep them alive here, which we 348 * do by bumping their refcount. ··· 354 err = REFTABLE_OUT_OF_MEMORY_ERROR; 355 goto done; 356 } 357 + reused[reused_len++] = table; 358 + reftable_table_incref(table); 359 break; 360 } 361 } 362 363 + if (!table) { 364 struct reftable_block_source src = { NULL }; 365 366 err = stack_filename(&table_path, st, name); ··· 372 if (err < 0) 373 goto done; 374 375 + err = reftable_table_new(&table, &src, name); 376 if (err < 0) 377 goto done; 378 } 379 380 + new_tables[new_tables_len] = table; 381 + new_tables_len++; 382 } 383 384 /* success! */ 385 + err = reftable_merged_table_new(&new_merged, new_tables, 386 + new_tables_len, st->opts.hash_id); 387 if (err < 0) 388 goto done; 389 390 /* 391 + * Close the old, non-reused tables and proactively try to unlink 392 * them. This is done for systems like Windows, where the underlying 393 + * file of such an open table wouldn't have been possible to be 394 * unlinked by the compacting process. 395 */ 396 for (i = 0; i < cur_len; i++) { 397 if (cur[i]) { 398 + const char *name = reftable_table_name(cur[i]); 399 400 err = stack_filename(&table_path, st, name); 401 if (err < 0) 402 goto done; 403 404 + reftable_table_decref(cur[i]); 405 unlink(table_path.buf); 406 } 407 } ··· 412 new_merged->suppress_deletions = 1; 413 st->merged = new_merged; 414 415 + if (st->tables) 416 + reftable_free(st->tables); 417 + st->tables = new_tables; 418 + st->tables_len = new_tables_len; 419 + new_tables = NULL; 420 + new_tables_len = 0; 421 422 /* 423 + * Decrement the refcount of reused tables again. This only needs to 424 * happen on the successful case, because on the unsuccessful one we 425 + * decrement their refcount via `new_tables`. 426 */ 427 for (i = 0; i < reused_len; i++) 428 + reftable_table_decref(reused[i]); 429 430 done: 431 + for (i = 0; i < new_tables_len; i++) 432 + reftable_table_decref(new_tables[i]); 433 + reftable_free(new_tables); 434 reftable_free(reused); 435 reftable_free(cur); 436 reftable_buf_release(&table_path); ··· 615 /* 616 * It's fine for "tables.list" to not exist. In that 617 * case, we have to refresh when the loaded stack has 618 + * any tables. 619 */ 620 if (errno == ENOENT) 621 + return !!st->tables_len; 622 return REFTABLE_IO_ERROR; 623 } 624 ··· 637 if (err < 0) 638 return err; 639 640 + for (size_t i = 0; i < st->tables_len; i++) { 641 if (!names[i]) { 642 err = 1; 643 goto done; 644 } 645 646 + if (strcmp(st->tables[i]->name, names[i])) { 647 err = 1; 648 goto done; 649 } 650 } 651 652 + if (names[st->merged->tables_len]) { 653 err = 1; 654 goto done; 655 } ··· 792 if (add->new_tables_len == 0) 793 goto done; 794 795 + for (i = 0; i < add->stack->merged->tables_len; i++) { 796 + if ((err = reftable_buf_addstr(&table_list, add->stack->tables[i]->name)) < 0 || 797 (err = reftable_buf_addstr(&table_list, "\n")) < 0) 798 goto done; 799 } ··· 1000 1001 uint64_t reftable_stack_next_update_index(struct reftable_stack *st) 1002 { 1003 + int sz = st->merged->tables_len; 1004 if (sz > 0) 1005 + return reftable_table_max_update_index(st->tables[sz - 1]) + 1006 1; 1007 return 1; 1008 } ··· 1021 struct reftable_tmpfile tab_file = REFTABLE_TMPFILE_INIT; 1022 int err = 0; 1023 1024 + err = format_name(&next_name, reftable_table_min_update_index(st->tables[first]), 1025 + reftable_table_max_update_index(st->tables[last])); 1026 if (err < 0) 1027 goto done; 1028 ··· 1087 int err = 0; 1088 1089 for (size_t i = first; i <= last; i++) 1090 + st->stats.bytes += st->tables[i]->size; 1091 + err = reftable_writer_set_limits(wr, st->tables[first]->min_update_index, 1092 + st->tables[last]->max_update_index); 1093 if (err < 0) 1094 goto done; 1095 1096 + err = reftable_merged_table_new(&mt, st->tables + first, subtabs_len, 1097 st->opts.hash_id); 1098 if (err < 0) 1099 goto done; 1100 1101 + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); 1102 if (err < 0) 1103 goto done; 1104 ··· 1126 } 1127 reftable_iterator_destroy(&it); 1128 1129 + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG); 1130 if (err < 0) 1131 goto done; 1132 ··· 1250 table_locks[i] = REFTABLE_FLOCK_INIT; 1251 1252 for (i = last + 1; i > first; i--) { 1253 + err = stack_filename(&table_name, st, reftable_table_name(st->tables[i - 1])); 1254 if (err < 0) 1255 goto done; 1256 ··· 1376 * compacted in the updated "tables.list" file. 1377 */ 1378 for (size_t i = 0; names[i]; i++) { 1379 + if (strcmp(names[i], st->tables[first]->name)) 1380 continue; 1381 1382 /* ··· 1386 * have compacted them. 1387 */ 1388 for (size_t j = 1; j < last - first + 1; j++) { 1389 + const char *old = first + j < st->merged->tables_len ? 1390 + st->tables[first + j]->name : NULL; 1391 const char *new = names[i + j]; 1392 1393 /* ··· 1427 * `fd_read_lines()` uses a `NULL` sentinel to indicate that 1428 * the array is at its end. As we use `free_names()` to free 1429 * the array, we need to include this sentinel value here and 1430 + * thus have to allocate `tables_len + 1` many entries. 1431 */ 1432 + REFTABLE_CALLOC_ARRAY(names, st->merged->tables_len + 1); 1433 if (!names) { 1434 err = REFTABLE_OUT_OF_MEMORY_ERROR; 1435 goto done; 1436 } 1437 1438 + for (size_t i = 0; i < st->merged->tables_len; i++) { 1439 + names[i] = reftable_strdup(st->tables[i]->name); 1440 if (!names[i]) { 1441 err = REFTABLE_OUT_OF_MEMORY_ERROR; 1442 goto done; ··· 1451 * it into place now. 1452 */ 1453 if (!is_empty_table) { 1454 + err = format_name(&new_table_name, st->tables[first]->min_update_index, 1455 + st->tables[last]->max_update_index); 1456 if (err < 0) 1457 goto done; 1458 ··· 1559 int reftable_stack_compact_all(struct reftable_stack *st, 1560 struct reftable_log_expiry_config *config) 1561 { 1562 + size_t last = st->merged->tables_len ? st->merged->tables_len - 1 : 0; 1563 return stack_compact_range(st, 0, last, config, 0); 1564 } 1565 ··· 1650 int overhead = header_size(version) - 1; 1651 uint64_t *sizes; 1652 1653 + REFTABLE_CALLOC_ARRAY(sizes, st->merged->tables_len); 1654 if (!sizes) 1655 return NULL; 1656 1657 + for (size_t i = 0; i < st->merged->tables_len; i++) 1658 + sizes[i] = st->tables[i]->size - overhead; 1659 1660 return sizes; 1661 } ··· 1665 struct segment seg; 1666 uint64_t *sizes; 1667 1668 + if (st->merged->tables_len < 2) 1669 return 0; 1670 1671 sizes = stack_table_sizes_for_compaction(st); 1672 if (!sizes) 1673 return REFTABLE_OUT_OF_MEMORY_ERROR; 1674 1675 + seg = suggest_compaction_segment(sizes, st->merged->tables_len, 1676 st->opts.auto_compaction_factor); 1677 reftable_free(sizes); 1678 ··· 1763 int err = 0; 1764 uint64_t update_idx = 0; 1765 struct reftable_block_source src = { NULL }; 1766 + struct reftable_table *table = NULL; 1767 struct reftable_buf table_path = REFTABLE_BUF_INIT; 1768 1769 err = stack_filename(&table_path, st, name); ··· 1774 if (err < 0) 1775 goto done; 1776 1777 + err = reftable_table_new(&table, &src, name); 1778 if (err < 0) 1779 goto done; 1780 1781 + update_idx = reftable_table_max_update_index(table); 1782 + reftable_table_decref(table); 1783 1784 if (update_idx <= max) { 1785 unlink(table_path.buf); ··· 1803 if (!is_table_name(d->d_name)) 1804 continue; 1805 1806 + for (size_t i = 0; !found && i < st->tables_len; i++) 1807 + found = !strcmp(reftable_table_name(st->tables[i]), d->d_name); 1808 if (found) 1809 continue; 1810

+8 -8

reftable/stack.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef STACK_H 10 #define STACK_H ··· 22 23 struct reftable_write_options opts; 24 25 - struct reftable_reader **readers; 26 - size_t readers_len; 27 struct reftable_merged_table *merged; 28 struct reftable_compaction_stats stats; 29 };

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef STACK_H 10 #define STACK_H ··· 22 23 struct reftable_write_options opts; 24 25 + struct reftable_table **tables; 26 + size_t tables_len; 27 struct reftable_merged_table *merged; 28 struct reftable_compaction_stats stats; 29 };

+6 -6

reftable/system.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef SYSTEM_H 10 #define SYSTEM_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef SYSTEM_H 10 #define SYSTEM_H

+788

reftable/table.c

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 + 9 + #include "table.h" 10 + 11 + #include "system.h" 12 + #include "block.h" 13 + #include "blocksource.h" 14 + #include "constants.h" 15 + #include "iter.h" 16 + #include "record.h" 17 + #include "reftable-error.h" 18 + 19 + static struct reftable_table_offsets * 20 + table_offsets_for(struct reftable_table *t, uint8_t typ) 21 + { 22 + switch (typ) { 23 + case REFTABLE_BLOCK_TYPE_REF: 24 + return &t->ref_offsets; 25 + case REFTABLE_BLOCK_TYPE_LOG: 26 + return &t->log_offsets; 27 + case REFTABLE_BLOCK_TYPE_OBJ: 28 + return &t->obj_offsets; 29 + } 30 + abort(); 31 + } 32 + 33 + enum reftable_hash reftable_table_hash_id(struct reftable_table *t) 34 + { 35 + return t->hash_id; 36 + } 37 + 38 + const char *reftable_table_name(struct reftable_table *t) 39 + { 40 + return t->name; 41 + } 42 + 43 + static int parse_footer(struct reftable_table *t, uint8_t *footer, 44 + uint8_t *header) 45 + { 46 + uint8_t *f = footer; 47 + uint8_t first_block_typ; 48 + int err = 0; 49 + uint32_t computed_crc; 50 + uint32_t file_crc; 51 + 52 + if (memcmp(f, "REFT", 4)) { 53 + err = REFTABLE_FORMAT_ERROR; 54 + goto done; 55 + } 56 + f += 4; 57 + 58 + if (memcmp(footer, header, header_size(t->version))) { 59 + err = REFTABLE_FORMAT_ERROR; 60 + goto done; 61 + } 62 + 63 + f++; 64 + t->block_size = reftable_get_be24(f); 65 + 66 + f += 3; 67 + t->min_update_index = reftable_get_be64(f); 68 + f += 8; 69 + t->max_update_index = reftable_get_be64(f); 70 + f += 8; 71 + 72 + if (t->version == 1) { 73 + t->hash_id = REFTABLE_HASH_SHA1; 74 + } else { 75 + switch (reftable_get_be32(f)) { 76 + case REFTABLE_FORMAT_ID_SHA1: 77 + t->hash_id = REFTABLE_HASH_SHA1; 78 + break; 79 + case REFTABLE_FORMAT_ID_SHA256: 80 + t->hash_id = REFTABLE_HASH_SHA256; 81 + break; 82 + default: 83 + err = REFTABLE_FORMAT_ERROR; 84 + goto done; 85 + } 86 + 87 + f += 4; 88 + } 89 + 90 + t->ref_offsets.index_offset = reftable_get_be64(f); 91 + f += 8; 92 + 93 + t->obj_offsets.offset = reftable_get_be64(f); 94 + f += 8; 95 + 96 + t->object_id_len = t->obj_offsets.offset & ((1 << 5) - 1); 97 + t->obj_offsets.offset >>= 5; 98 + 99 + t->obj_offsets.index_offset = reftable_get_be64(f); 100 + f += 8; 101 + t->log_offsets.offset = reftable_get_be64(f); 102 + f += 8; 103 + t->log_offsets.index_offset = reftable_get_be64(f); 104 + f += 8; 105 + 106 + computed_crc = crc32(0, footer, f - footer); 107 + file_crc = reftable_get_be32(f); 108 + f += 4; 109 + if (computed_crc != file_crc) { 110 + err = REFTABLE_FORMAT_ERROR; 111 + goto done; 112 + } 113 + 114 + first_block_typ = header[header_size(t->version)]; 115 + t->ref_offsets.is_present = (first_block_typ == REFTABLE_BLOCK_TYPE_REF); 116 + t->ref_offsets.offset = 0; 117 + t->log_offsets.is_present = (first_block_typ == REFTABLE_BLOCK_TYPE_LOG || 118 + t->log_offsets.offset > 0); 119 + t->obj_offsets.is_present = t->obj_offsets.offset > 0; 120 + if (t->obj_offsets.is_present && !t->object_id_len) { 121 + err = REFTABLE_FORMAT_ERROR; 122 + goto done; 123 + } 124 + 125 + err = 0; 126 + done: 127 + return err; 128 + } 129 + 130 + struct table_iter { 131 + struct reftable_table *table; 132 + uint8_t typ; 133 + uint64_t block_off; 134 + struct reftable_block block; 135 + struct block_iter bi; 136 + int is_finished; 137 + }; 138 + 139 + static int table_iter_init(struct table_iter *ti, struct reftable_table *t) 140 + { 141 + struct block_iter bi = BLOCK_ITER_INIT; 142 + memset(ti, 0, sizeof(*ti)); 143 + reftable_table_incref(t); 144 + ti->table = t; 145 + ti->bi = bi; 146 + return 0; 147 + } 148 + 149 + static int table_iter_next_in_block(struct table_iter *ti, 150 + struct reftable_record *rec) 151 + { 152 + int res = block_iter_next(&ti->bi, rec); 153 + if (res == 0 && reftable_record_type(rec) == REFTABLE_BLOCK_TYPE_REF) { 154 + rec->u.ref.update_index += ti->table->min_update_index; 155 + } 156 + 157 + return res; 158 + } 159 + 160 + static void table_iter_block_done(struct table_iter *ti) 161 + { 162 + reftable_block_release(&ti->block); 163 + block_iter_reset(&ti->bi); 164 + } 165 + 166 + int table_init_block(struct reftable_table *t, struct reftable_block *block, 167 + uint64_t next_off, uint8_t want_typ) 168 + { 169 + uint32_t header_off = next_off ? 0 : header_size(t->version); 170 + int err; 171 + 172 + if (next_off >= t->size) 173 + return 1; 174 + 175 + err = reftable_block_init(block, &t->source, next_off, header_off, 176 + t->block_size, hash_size(t->hash_id)); 177 + if (err < 0) 178 + goto done; 179 + 180 + if (want_typ != REFTABLE_BLOCK_TYPE_ANY && block->block_type != want_typ) { 181 + err = 1; 182 + goto done; 183 + } 184 + 185 + done: 186 + if (err) 187 + reftable_block_release(block); 188 + return err; 189 + } 190 + 191 + static void table_iter_close(struct table_iter *ti) 192 + { 193 + table_iter_block_done(ti); 194 + block_iter_close(&ti->bi); 195 + reftable_table_decref(ti->table); 196 + } 197 + 198 + static int table_iter_next_block(struct table_iter *ti) 199 + { 200 + uint64_t next_block_off = ti->block_off + ti->block.full_block_size; 201 + int err; 202 + 203 + err = table_init_block(ti->table, &ti->block, next_block_off, ti->typ); 204 + if (err > 0) 205 + ti->is_finished = 1; 206 + if (err) 207 + return err; 208 + 209 + ti->block_off = next_block_off; 210 + ti->is_finished = 0; 211 + block_iter_init(&ti->bi, &ti->block); 212 + 213 + return 0; 214 + } 215 + 216 + static int table_iter_next(struct table_iter *ti, struct reftable_record *rec) 217 + { 218 + if (reftable_record_type(rec) != ti->typ) 219 + return REFTABLE_API_ERROR; 220 + 221 + while (1) { 222 + int err; 223 + 224 + if (ti->is_finished) 225 + return 1; 226 + 227 + /* 228 + * Check whether the current block still has more records. If 229 + * so, return it. If the iterator returns positive then the 230 + * current block has been exhausted. 231 + */ 232 + err = table_iter_next_in_block(ti, rec); 233 + if (err <= 0) 234 + return err; 235 + 236 + /* 237 + * Otherwise, we need to continue to the next block in the 238 + * table and retry. If there are no more blocks then the 239 + * iterator is drained. 240 + */ 241 + err = table_iter_next_block(ti); 242 + if (err) { 243 + ti->is_finished = 1; 244 + return err; 245 + } 246 + } 247 + } 248 + 249 + static int table_iter_seek_to(struct table_iter *ti, uint64_t off, uint8_t typ) 250 + { 251 + int err; 252 + 253 + err = table_init_block(ti->table, &ti->block, off, typ); 254 + if (err != 0) 255 + return err; 256 + 257 + ti->typ = reftable_block_type(&ti->block); 258 + ti->block_off = off; 259 + block_iter_init(&ti->bi, &ti->block); 260 + ti->is_finished = 0; 261 + return 0; 262 + } 263 + 264 + static int table_iter_seek_start(struct table_iter *ti, uint8_t typ, int index) 265 + { 266 + struct reftable_table_offsets *offs = table_offsets_for(ti->table, typ); 267 + uint64_t off = offs->offset; 268 + if (index) { 269 + off = offs->index_offset; 270 + if (off == 0) { 271 + return 1; 272 + } 273 + typ = REFTABLE_BLOCK_TYPE_INDEX; 274 + } 275 + 276 + return table_iter_seek_to(ti, off, typ); 277 + } 278 + 279 + static int table_iter_seek_linear(struct table_iter *ti, 280 + struct reftable_record *want) 281 + { 282 + struct reftable_buf want_key = REFTABLE_BUF_INIT; 283 + struct reftable_buf got_key = REFTABLE_BUF_INIT; 284 + struct reftable_record rec; 285 + int err; 286 + 287 + err = reftable_record_init(&rec, reftable_record_type(want)); 288 + if (err < 0) 289 + goto done; 290 + 291 + err = reftable_record_key(want, &want_key); 292 + if (err < 0) 293 + goto done; 294 + 295 + /* 296 + * First we need to locate the block that must contain our record. To 297 + * do so we scan through blocks linearly until we find the first block 298 + * whose first key is bigger than our wanted key. Once we have found 299 + * that block we know that the key must be contained in the preceding 300 + * block. 301 + * 302 + * This algorithm is somewhat unfortunate because it means that we 303 + * always have to seek one block too far and then back up. But as we 304 + * can only decode the _first_ key of a block but not its _last_ key we 305 + * have no other way to do this. 306 + */ 307 + while (1) { 308 + struct table_iter next = *ti; 309 + 310 + /* 311 + * We must be careful to not modify underlying data of `ti` 312 + * because we may find that `next` does not contain our desired 313 + * block, but that `ti` does. In that case, we would discard 314 + * `next` and continue with `ti`. 315 + * 316 + * This also means that we cannot reuse allocated memory for 317 + * `next` here. While it would be great if we could, it should 318 + * in practice not be too bad given that we should only ever 319 + * end up doing linear seeks with at most three blocks. As soon 320 + * as we have more than three blocks we would have an index, so 321 + * we would not do a linear search there anymore. 322 + */ 323 + memset(&next.block.block_data, 0, sizeof(next.block.block_data)); 324 + next.block.zstream = NULL; 325 + next.block.uncompressed_data = NULL; 326 + next.block.uncompressed_cap = 0; 327 + 328 + err = table_iter_next_block(&next); 329 + if (err < 0) 330 + goto done; 331 + if (err > 0) 332 + break; 333 + 334 + err = reftable_block_first_key(&next.block, &got_key); 335 + if (err < 0) 336 + goto done; 337 + 338 + if (reftable_buf_cmp(&got_key, &want_key) > 0) { 339 + table_iter_block_done(&next); 340 + break; 341 + } 342 + 343 + table_iter_block_done(ti); 344 + *ti = next; 345 + } 346 + 347 + /* 348 + * We have located the block that must contain our record, so we seek 349 + * the wanted key inside of it. If the block does not contain our key 350 + * we know that the corresponding record does not exist. 351 + */ 352 + block_iter_init(&ti->bi, &ti->block); 353 + err = block_iter_seek_key(&ti->bi, &want_key); 354 + if (err < 0) 355 + goto done; 356 + err = 0; 357 + 358 + done: 359 + reftable_record_release(&rec); 360 + reftable_buf_release(&want_key); 361 + reftable_buf_release(&got_key); 362 + return err; 363 + } 364 + 365 + static int table_iter_seek_indexed(struct table_iter *ti, 366 + struct reftable_record *rec) 367 + { 368 + struct reftable_record want_index = { 369 + .type = REFTABLE_BLOCK_TYPE_INDEX, .u.idx = { .last_key = REFTABLE_BUF_INIT } 370 + }; 371 + struct reftable_record index_result = { 372 + .type = REFTABLE_BLOCK_TYPE_INDEX, 373 + .u.idx = { .last_key = REFTABLE_BUF_INIT }, 374 + }; 375 + int err; 376 + 377 + err = reftable_record_key(rec, &want_index.u.idx.last_key); 378 + if (err < 0) 379 + goto done; 380 + 381 + /* 382 + * The index may consist of multiple levels, where each level may have 383 + * multiple index blocks. We start by doing a linear search in the 384 + * highest layer that identifies the relevant index block as well as 385 + * the record inside that block that corresponds to our wanted key. 386 + */ 387 + err = table_iter_seek_linear(ti, &want_index); 388 + if (err < 0) 389 + goto done; 390 + 391 + /* 392 + * Traverse down the levels until we find a non-index entry. 393 + */ 394 + while (1) { 395 + /* 396 + * In case we seek a record that does not exist the index iter 397 + * will tell us that the iterator is over. This works because 398 + * the last index entry of the current level will contain the 399 + * last key it knows about. So in case our seeked key is larger 400 + * than the last indexed key we know that it won't exist. 401 + * 402 + * There is one subtlety in the layout of the index section 403 + * that makes this work as expected: the highest-level index is 404 + * at end of the section and will point backwards and thus we 405 + * start reading from the end of the index section, not the 406 + * beginning. 407 + * 408 + * If that wasn't the case and the order was reversed then the 409 + * linear seek would seek into the lower levels and traverse 410 + * all levels of the index only to find out that the key does 411 + * not exist. 412 + */ 413 + err = table_iter_next(ti, &index_result); 414 + if (err != 0) 415 + goto done; 416 + 417 + err = table_iter_seek_to(ti, index_result.u.idx.offset, 0); 418 + if (err != 0) 419 + goto done; 420 + 421 + block_iter_init(&ti->bi, &ti->block); 422 + 423 + err = block_iter_seek_key(&ti->bi, &want_index.u.idx.last_key); 424 + if (err < 0) 425 + goto done; 426 + 427 + if (ti->typ == reftable_record_type(rec)) { 428 + err = 0; 429 + break; 430 + } 431 + 432 + if (ti->typ != REFTABLE_BLOCK_TYPE_INDEX) { 433 + err = REFTABLE_FORMAT_ERROR; 434 + goto done; 435 + } 436 + } 437 + 438 + done: 439 + reftable_record_release(&want_index); 440 + reftable_record_release(&index_result); 441 + return err; 442 + } 443 + 444 + static int table_iter_seek(struct table_iter *ti, 445 + struct reftable_record *want) 446 + { 447 + uint8_t typ = reftable_record_type(want); 448 + struct reftable_table_offsets *offs = table_offsets_for(ti->table, typ); 449 + int err; 450 + 451 + err = table_iter_seek_start(ti, reftable_record_type(want), 452 + !!offs->index_offset); 453 + if (err < 0) 454 + goto out; 455 + 456 + if (offs->index_offset) 457 + err = table_iter_seek_indexed(ti, want); 458 + else 459 + err = table_iter_seek_linear(ti, want); 460 + if (err) 461 + goto out; 462 + 463 + out: 464 + return err; 465 + } 466 + 467 + static int table_iter_seek_void(void *ti, struct reftable_record *want) 468 + { 469 + return table_iter_seek(ti, want); 470 + } 471 + 472 + static int table_iter_next_void(void *ti, struct reftable_record *rec) 473 + { 474 + return table_iter_next(ti, rec); 475 + } 476 + 477 + static void table_iter_close_void(void *ti) 478 + { 479 + table_iter_close(ti); 480 + } 481 + 482 + static struct reftable_iterator_vtable table_iter_vtable = { 483 + .seek = &table_iter_seek_void, 484 + .next = &table_iter_next_void, 485 + .close = &table_iter_close_void, 486 + }; 487 + 488 + static void iterator_from_table_iter(struct reftable_iterator *it, 489 + struct table_iter *ti) 490 + { 491 + assert(!it->ops); 492 + it->iter_arg = ti; 493 + it->ops = &table_iter_vtable; 494 + } 495 + 496 + int table_init_iter(struct reftable_table *t, 497 + struct reftable_iterator *it, 498 + uint8_t typ) 499 + { 500 + struct reftable_table_offsets *offs = table_offsets_for(t, typ); 501 + 502 + if (offs->is_present) { 503 + struct table_iter *ti; 504 + REFTABLE_ALLOC_ARRAY(ti, 1); 505 + if (!ti) 506 + return REFTABLE_OUT_OF_MEMORY_ERROR; 507 + 508 + table_iter_init(ti, t); 509 + iterator_from_table_iter(it, ti); 510 + } else { 511 + iterator_set_empty(it); 512 + } 513 + 514 + return 0; 515 + } 516 + 517 + int reftable_table_init_ref_iterator(struct reftable_table *t, 518 + struct reftable_iterator *it) 519 + { 520 + return table_init_iter(t, it, REFTABLE_BLOCK_TYPE_REF); 521 + } 522 + 523 + int reftable_table_init_log_iterator(struct reftable_table *t, 524 + struct reftable_iterator *it) 525 + { 526 + return table_init_iter(t, it, REFTABLE_BLOCK_TYPE_LOG); 527 + } 528 + 529 + int reftable_table_new(struct reftable_table **out, 530 + struct reftable_block_source *source, char const *name) 531 + { 532 + struct reftable_block_data footer = { 0 }; 533 + struct reftable_block_data header = { 0 }; 534 + struct reftable_table *t; 535 + uint64_t file_size = block_source_size(source); 536 + uint32_t read_size; 537 + ssize_t bytes_read; 538 + int err; 539 + 540 + REFTABLE_CALLOC_ARRAY(t, 1); 541 + if (!t) { 542 + err = REFTABLE_OUT_OF_MEMORY_ERROR; 543 + goto done; 544 + } 545 + 546 + /* 547 + * We need one extra byte to read the type of first block. We also 548 + * pretend to always be reading v2 of the format because it is larger. 549 + */ 550 + read_size = header_size(2) + 1; 551 + if (read_size > file_size) { 552 + err = REFTABLE_FORMAT_ERROR; 553 + goto done; 554 + } 555 + 556 + bytes_read = block_source_read_data(source, &header, 0, read_size); 557 + if (bytes_read < 0 || (size_t)bytes_read != read_size) { 558 + err = REFTABLE_IO_ERROR; 559 + goto done; 560 + } 561 + 562 + if (memcmp(header.data, "REFT", 4)) { 563 + err = REFTABLE_FORMAT_ERROR; 564 + goto done; 565 + } 566 + t->version = header.data[4]; 567 + if (t->version != 1 && t->version != 2) { 568 + err = REFTABLE_FORMAT_ERROR; 569 + goto done; 570 + } 571 + 572 + t->size = file_size - footer_size(t->version); 573 + t->source = *source; 574 + t->name = reftable_strdup(name); 575 + if (!t->name) { 576 + err = REFTABLE_OUT_OF_MEMORY_ERROR; 577 + goto done; 578 + } 579 + t->hash_id = 0; 580 + t->refcount = 1; 581 + 582 + bytes_read = block_source_read_data(source, &footer, t->size, 583 + footer_size(t->version)); 584 + if (bytes_read < 0 || (size_t)bytes_read != footer_size(t->version)) { 585 + err = REFTABLE_IO_ERROR; 586 + goto done; 587 + } 588 + 589 + err = parse_footer(t, footer.data, header.data); 590 + if (err) 591 + goto done; 592 + 593 + *out = t; 594 + 595 + done: 596 + block_source_release_data(&footer); 597 + block_source_release_data(&header); 598 + if (err) { 599 + if (t) 600 + reftable_free(t->name); 601 + reftable_free(t); 602 + block_source_close(source); 603 + } 604 + return err; 605 + } 606 + 607 + void reftable_table_incref(struct reftable_table *t) 608 + { 609 + t->refcount++; 610 + } 611 + 612 + void reftable_table_decref(struct reftable_table *t) 613 + { 614 + if (!t) 615 + return; 616 + if (--t->refcount) 617 + return; 618 + block_source_close(&t->source); 619 + REFTABLE_FREE_AND_NULL(t->name); 620 + reftable_free(t); 621 + } 622 + 623 + static int reftable_table_refs_for_indexed(struct reftable_table *t, 624 + struct reftable_iterator *it, 625 + uint8_t *oid) 626 + { 627 + struct reftable_record want = { 628 + .type = REFTABLE_BLOCK_TYPE_OBJ, 629 + .u.obj = { 630 + .hash_prefix = oid, 631 + .hash_prefix_len = t->object_id_len, 632 + }, 633 + }; 634 + struct reftable_iterator oit = { NULL }; 635 + struct reftable_record got = { 636 + .type = REFTABLE_BLOCK_TYPE_OBJ, 637 + .u.obj = { 0 }, 638 + }; 639 + int err = 0; 640 + struct indexed_table_ref_iter *itr = NULL; 641 + 642 + /* Look through the reverse index. */ 643 + err = table_init_iter(t, &oit, REFTABLE_BLOCK_TYPE_OBJ); 644 + if (err < 0) 645 + goto done; 646 + 647 + err = iterator_seek(&oit, &want); 648 + if (err != 0) 649 + goto done; 650 + 651 + /* read out the reftable_obj_record */ 652 + err = iterator_next(&oit, &got); 653 + if (err < 0) 654 + goto done; 655 + 656 + if (err > 0 || memcmp(want.u.obj.hash_prefix, got.u.obj.hash_prefix, 657 + t->object_id_len)) { 658 + /* didn't find it; return empty iterator */ 659 + iterator_set_empty(it); 660 + err = 0; 661 + goto done; 662 + } 663 + 664 + err = indexed_table_ref_iter_new(&itr, t, oid, hash_size(t->hash_id), 665 + got.u.obj.offsets, 666 + got.u.obj.offset_len); 667 + if (err < 0) 668 + goto done; 669 + got.u.obj.offsets = NULL; 670 + iterator_from_indexed_table_ref_iter(it, itr); 671 + 672 + done: 673 + reftable_iterator_destroy(&oit); 674 + reftable_record_release(&got); 675 + return err; 676 + } 677 + 678 + static int reftable_table_refs_for_unindexed(struct reftable_table *t, 679 + struct reftable_iterator *it, 680 + uint8_t *oid) 681 + { 682 + struct table_iter *ti; 683 + struct filtering_ref_iterator *filter = NULL; 684 + struct filtering_ref_iterator empty = FILTERING_REF_ITERATOR_INIT; 685 + uint32_t oid_len = hash_size(t->hash_id); 686 + int err; 687 + 688 + REFTABLE_ALLOC_ARRAY(ti, 1); 689 + if (!ti) { 690 + err = REFTABLE_OUT_OF_MEMORY_ERROR; 691 + goto out; 692 + } 693 + 694 + table_iter_init(ti, t); 695 + err = table_iter_seek_start(ti, REFTABLE_BLOCK_TYPE_REF, 0); 696 + if (err < 0) 697 + goto out; 698 + 699 + filter = reftable_malloc(sizeof(*filter)); 700 + if (!filter) { 701 + err = REFTABLE_OUT_OF_MEMORY_ERROR; 702 + goto out; 703 + } 704 + *filter = empty; 705 + 706 + err = reftable_buf_add(&filter->oid, oid, oid_len); 707 + if (err < 0) 708 + goto out; 709 + 710 + iterator_from_table_iter(&filter->it, ti); 711 + 712 + iterator_from_filtering_ref_iterator(it, filter); 713 + 714 + err = 0; 715 + 716 + out: 717 + if (err < 0) { 718 + if (ti) 719 + table_iter_close(ti); 720 + reftable_free(ti); 721 + } 722 + return err; 723 + } 724 + 725 + int reftable_table_refs_for(struct reftable_table *t, 726 + struct reftable_iterator *it, uint8_t *oid) 727 + { 728 + if (t->obj_offsets.is_present) 729 + return reftable_table_refs_for_indexed(t, it, oid); 730 + return reftable_table_refs_for_unindexed(t, it, oid); 731 + } 732 + 733 + uint64_t reftable_table_max_update_index(struct reftable_table *t) 734 + { 735 + return t->max_update_index; 736 + } 737 + 738 + uint64_t reftable_table_min_update_index(struct reftable_table *t) 739 + { 740 + return t->min_update_index; 741 + } 742 + 743 + int reftable_table_iterator_init(struct reftable_table_iterator *it, 744 + struct reftable_table *t) 745 + { 746 + struct table_iter *ti; 747 + int err; 748 + 749 + REFTABLE_ALLOC_ARRAY(ti, 1); 750 + if (!ti) 751 + return REFTABLE_OUT_OF_MEMORY_ERROR; 752 + 753 + err = table_iter_init(ti, t); 754 + if (err < 0) 755 + goto out; 756 + 757 + it->iter_arg = ti; 758 + err = 0; 759 + 760 + out: 761 + if (err < 0) 762 + reftable_free(ti); 763 + return err; 764 + } 765 + 766 + void reftable_table_iterator_release(struct reftable_table_iterator *it) 767 + { 768 + if (!it->iter_arg) 769 + return; 770 + table_iter_close(it->iter_arg); 771 + reftable_free(it->iter_arg); 772 + it->iter_arg = NULL; 773 + } 774 + 775 + int reftable_table_iterator_next(struct reftable_table_iterator *it, 776 + const struct reftable_block **out) 777 + { 778 + struct table_iter *ti = it->iter_arg; 779 + int err; 780 + 781 + err = table_iter_next_block(ti); 782 + if (err) 783 + return err; 784 + 785 + *out = &ti->block; 786 + 787 + return 0; 788 + }

+29

reftable/table.h

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 + 9 + #ifndef TABLE_H 10 + #define TABLE_H 11 + 12 + #include "block.h" 13 + #include "record.h" 14 + #include "reftable-iterator.h" 15 + #include "reftable-table.h" 16 + 17 + const char *reftable_table_name(struct reftable_table *t); 18 + 19 + int table_init_iter(struct reftable_table *t, 20 + struct reftable_iterator *it, 21 + uint8_t typ); 22 + 23 + /* 24 + * Initialize a block by reading from the given table and offset. 25 + */ 26 + int table_init_block(struct reftable_table *t, struct reftable_block *block, 27 + uint64_t next_off, uint8_t want_typ); 28 + 29 + #endif

+6 -6

reftable/tree.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "system.h" 10 #include "tree.h"

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "system.h" 10 #include "tree.h"

+6 -6

reftable/tree.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef TREE_H 10 #define TREE_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef TREE_H 10 #define TREE_H

+17 -17

reftable/writer.c

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #include "writer.h" 10 ··· 172 wp->write_arg = writer_arg; 173 wp->opts = opts; 174 wp->flush = flush_func; 175 - writer_reinit_block_writer(wp, BLOCK_TYPE_REF); 176 177 *out = wp; 178 ··· 342 struct reftable_ref_record *ref) 343 { 344 struct reftable_record rec = { 345 - .type = BLOCK_TYPE_REF, 346 .u = { 347 .ref = *ref 348 }, ··· 406 struct reftable_log_record *log) 407 { 408 struct reftable_record rec = { 409 - .type = BLOCK_TYPE_LOG, 410 .u = { 411 .log = *log, 412 }, 413 }; 414 if (w->block_writer && 415 - block_writer_type(w->block_writer) == BLOCK_TYPE_REF) { 416 int err = writer_finish_public_section(w); 417 if (err < 0) 418 return err; ··· 532 533 max_level++; 534 index_start = w->next; 535 - err = writer_reinit_block_writer(w, BLOCK_TYPE_INDEX); 536 if (err < 0) 537 return err; 538 ··· 544 w->index_cap = 0; 545 for (i = 0; i < idx_len; i++) { 546 struct reftable_record rec = { 547 - .type = BLOCK_TYPE_INDEX, 548 .u = { 549 .idx = idx[i], 550 }, ··· 609 struct write_record_arg *arg = void_arg; 610 struct obj_index_tree_node *entry = key; 611 struct reftable_record 612 - rec = { .type = BLOCK_TYPE_OBJ, 613 .u.obj = { 614 .hash_prefix = (uint8_t *)entry->hash.buf, 615 .hash_prefix_len = arg->w->stats.object_id_len, ··· 639 if (arg->err < 0) 640 goto done; 641 642 - arg->err = writer_reinit_block_writer(arg->w, BLOCK_TYPE_OBJ); 643 if (arg->err < 0) 644 goto done; 645 ··· 684 infix_walk(w->obj_index_tree, &update_common, &common); 685 w->stats.object_id_len = common.max + 1; 686 687 - err = writer_reinit_block_writer(w, BLOCK_TYPE_OBJ); 688 if (err < 0) 689 return err; 690 ··· 708 err = writer_finish_section(w); 709 if (err < 0) 710 return err; 711 - if (typ == BLOCK_TYPE_REF && !w->opts.skip_index_objects && 712 w->stats.ref_stats.index_blocks > 0) { 713 err = writer_dump_object_index(w); 714 if (err < 0) ··· 813 * By default, all records except for log records are padded to the 814 * block size. 815 */ 816 - if (!w->opts.unpadded && typ != BLOCK_TYPE_LOG) 817 padding = w->opts.block_size - raw_bytes; 818 819 bstats = writer_reftable_block_stats(w, typ);

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #include "writer.h" 10 ··· 172 wp->write_arg = writer_arg; 173 wp->opts = opts; 174 wp->flush = flush_func; 175 + writer_reinit_block_writer(wp, REFTABLE_BLOCK_TYPE_REF); 176 177 *out = wp; 178 ··· 342 struct reftable_ref_record *ref) 343 { 344 struct reftable_record rec = { 345 + .type = REFTABLE_BLOCK_TYPE_REF, 346 .u = { 347 .ref = *ref 348 }, ··· 406 struct reftable_log_record *log) 407 { 408 struct reftable_record rec = { 409 + .type = REFTABLE_BLOCK_TYPE_LOG, 410 .u = { 411 .log = *log, 412 }, 413 }; 414 if (w->block_writer && 415 + block_writer_type(w->block_writer) == REFTABLE_BLOCK_TYPE_REF) { 416 int err = writer_finish_public_section(w); 417 if (err < 0) 418 return err; ··· 532 533 max_level++; 534 index_start = w->next; 535 + err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_INDEX); 536 if (err < 0) 537 return err; 538 ··· 544 w->index_cap = 0; 545 for (i = 0; i < idx_len; i++) { 546 struct reftable_record rec = { 547 + .type = REFTABLE_BLOCK_TYPE_INDEX, 548 .u = { 549 .idx = idx[i], 550 }, ··· 609 struct write_record_arg *arg = void_arg; 610 struct obj_index_tree_node *entry = key; 611 struct reftable_record 612 + rec = { .type = REFTABLE_BLOCK_TYPE_OBJ, 613 .u.obj = { 614 .hash_prefix = (uint8_t *)entry->hash.buf, 615 .hash_prefix_len = arg->w->stats.object_id_len, ··· 639 if (arg->err < 0) 640 goto done; 641 642 + arg->err = writer_reinit_block_writer(arg->w, REFTABLE_BLOCK_TYPE_OBJ); 643 if (arg->err < 0) 644 goto done; 645 ··· 684 infix_walk(w->obj_index_tree, &update_common, &common); 685 w->stats.object_id_len = common.max + 1; 686 687 + err = writer_reinit_block_writer(w, REFTABLE_BLOCK_TYPE_OBJ); 688 if (err < 0) 689 return err; 690 ··· 708 err = writer_finish_section(w); 709 if (err < 0) 710 return err; 711 + if (typ == REFTABLE_BLOCK_TYPE_REF && !w->opts.skip_index_objects && 712 w->stats.ref_stats.index_blocks > 0) { 713 err = writer_dump_object_index(w); 714 if (err < 0) ··· 813 * By default, all records except for log records are padded to the 814 * block size. 815 */ 816 + if (!w->opts.unpadded && typ != REFTABLE_BLOCK_TYPE_LOG) 817 padding = w->opts.block_size - raw_bytes; 818 819 bstats = writer_reftable_block_stats(w, typ);

+6 -6

reftable/writer.h

··· 1 /* 2 - Copyright 2020 Google LLC 3 - 4 - Use of this source code is governed by a BSD-style 5 - license that can be found in the LICENSE file or at 6 - https://developers.google.com/open-source/licenses/bsd 7 - */ 8 9 #ifndef WRITER_H 10 #define WRITER_H

··· 1 /* 2 + * Copyright 2020 Google LLC 3 + * 4 + * Use of this source code is governed by a BSD-style 5 + * license that can be found in the LICENSE file or at 6 + * https://developers.google.com/open-source/licenses/bsd 7 + */ 8 9 #ifndef WRITER_H 10 #define WRITER_H

+74 -7

t/helper/test-reftable.c

··· 2 #include "hash.h" 3 #include "hex.h" 4 #include "reftable/system.h" 0 5 #include "reftable/reftable-error.h" 6 #include "reftable/reftable-merged.h" 7 - #include "reftable/reftable-reader.h" 8 #include "reftable/reftable-stack.h" 0 9 #include "test-tool.h" 10 11 static void print_help(void) ··· 20 "\n"); 21 } 22 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 23 static int dump_table(struct reftable_merged_table *mt) 24 { 25 struct reftable_iterator it = { NULL }; ··· 126 { 127 struct reftable_block_source src = { 0 }; 128 struct reftable_merged_table *mt = NULL; 129 - struct reftable_reader *r = NULL; 130 int err; 131 132 err = reftable_block_source_from_file(&src, tablename); 133 if (err < 0) 134 goto done; 135 136 - err = reftable_reader_new(&r, &src, tablename); 137 if (err < 0) 138 goto done; 139 140 - err = reftable_merged_table_new(&mt, &r, 1, 141 - reftable_reader_hash_id(r)); 142 if (err < 0) 143 goto done; 144 ··· 146 147 done: 148 reftable_merged_table_free(mt); 149 - reftable_reader_decref(r); 150 return err; 151 } 152 ··· 184 arg = argv[1]; 185 186 if (opt_dump_blocks) { 187 - err = reftable_reader_print_blocks(arg); 188 } else if (opt_dump_table) { 189 err = dump_reftable(arg); 190 } else if (opt_dump_stack) {

··· 2 #include "hash.h" 3 #include "hex.h" 4 #include "reftable/system.h" 5 + #include "reftable/reftable-constants.h" 6 #include "reftable/reftable-error.h" 7 #include "reftable/reftable-merged.h" 0 8 #include "reftable/reftable-stack.h" 9 + #include "reftable/reftable-table.h" 10 #include "test-tool.h" 11 12 static void print_help(void) ··· 21 "\n"); 22 } 23 24 + static int dump_blocks(const char *tablename) 25 + { 26 + struct reftable_table_iterator ti = { 0 }; 27 + struct reftable_block_source src = { 0 }; 28 + struct reftable_table *table = NULL; 29 + uint8_t section_type = 0; 30 + int err; 31 + 32 + err = reftable_block_source_from_file(&src, tablename); 33 + if (err < 0) 34 + goto done; 35 + 36 + err = reftable_table_new(&table, &src, tablename); 37 + if (err < 0) 38 + goto done; 39 + 40 + err = reftable_table_iterator_init(&ti, table); 41 + if (err < 0) 42 + goto done; 43 + 44 + printf("header:\n"); 45 + printf(" block_size: %d\n", table->block_size); 46 + 47 + while (1) { 48 + const struct reftable_block *block; 49 + 50 + err = reftable_table_iterator_next(&ti, &block); 51 + if (err < 0) 52 + goto done; 53 + if (err > 0) 54 + break; 55 + 56 + if (block->block_type != section_type) { 57 + const char *section; 58 + switch (block->block_type) { 59 + case REFTABLE_BLOCK_TYPE_LOG: 60 + section = "log"; 61 + break; 62 + case REFTABLE_BLOCK_TYPE_REF: 63 + section = "ref"; 64 + break; 65 + case REFTABLE_BLOCK_TYPE_OBJ: 66 + section = "obj"; 67 + break; 68 + case REFTABLE_BLOCK_TYPE_INDEX: 69 + section = "idx"; 70 + break; 71 + default: 72 + err = -1; 73 + goto done; 74 + } 75 + 76 + section_type = block->block_type; 77 + printf("%s:\n", section); 78 + } 79 + 80 + printf(" - length: %u\n", block->restart_off); 81 + printf(" restarts: %u\n", block->restart_count); 82 + } 83 + 84 + done: 85 + reftable_table_iterator_release(&ti); 86 + reftable_table_decref(table); 87 + return err; 88 + } 89 + 90 static int dump_table(struct reftable_merged_table *mt) 91 { 92 struct reftable_iterator it = { NULL }; ··· 193 { 194 struct reftable_block_source src = { 0 }; 195 struct reftable_merged_table *mt = NULL; 196 + struct reftable_table *table = NULL; 197 int err; 198 199 err = reftable_block_source_from_file(&src, tablename); 200 if (err < 0) 201 goto done; 202 203 + err = reftable_table_new(&table, &src, tablename); 204 if (err < 0) 205 goto done; 206 207 + err = reftable_merged_table_new(&mt, &table, 1, 208 + reftable_table_hash_id(table)); 209 if (err < 0) 210 goto done; 211 ··· 213 214 done: 215 reftable_merged_table_free(mt); 216 + reftable_table_decref(table); 217 return err; 218 } 219 ··· 251 arg = argv[1]; 252 253 if (opt_dump_blocks) { 254 + err = dump_blocks(arg); 255 } else if (opt_dump_table) { 256 err = dump_reftable(arg); 257 } else if (opt_dump_stack) {

+1 -1

t/meson.build

··· 58 'unit-tests/t-reftable-block.c', 59 'unit-tests/t-reftable-merged.c', 60 'unit-tests/t-reftable-pq.c', 61 - 'unit-tests/t-reftable-reader.c', 62 'unit-tests/t-reftable-readwrite.c', 63 'unit-tests/t-reftable-record.c', 64 'unit-tests/t-reftable-stack.c', 0 65 ] 66 67 foreach unit_test_program : unit_test_programs

··· 58 'unit-tests/t-reftable-block.c', 59 'unit-tests/t-reftable-merged.c', 60 'unit-tests/t-reftable-pq.c', 0 61 'unit-tests/t-reftable-readwrite.c', 62 'unit-tests/t-reftable-record.c', 63 'unit-tests/t-reftable-stack.c', 64 + 'unit-tests/t-reftable-table.c', 65 ] 66 67 foreach unit_test_program : unit_test_programs

+9

t/t0613-reftable-write-options.sh

··· 93 restarts: 3 94 - length: 3289 95 restarts: 3 0 0 0 96 EOF 97 test-tool dump-reftable -b .git/reftable/*.ref >actual && 98 test_cmp expect actual ··· 241 restarts: 1 242 - length: 80 243 restarts: 1 0 0 0 244 obj: 245 - length: 11 246 restarts: 1 ··· 277 restarts: 1 278 - length: 80 279 restarts: 1 0 0 0 280 EOF 281 test-tool dump-reftable -b .git/reftable/*.ref >actual && 282 test_cmp expect actual

··· 93 restarts: 3 94 - length: 3289 95 restarts: 3 96 + idx: 97 + - length: 103 98 + restarts: 1 99 EOF 100 test-tool dump-reftable -b .git/reftable/*.ref >actual && 101 test_cmp expect actual ··· 244 restarts: 1 245 - length: 80 246 restarts: 1 247 + idx: 248 + - length: 55 249 + restarts: 2 250 obj: 251 - length: 11 252 restarts: 1 ··· 283 restarts: 1 284 - length: 80 285 restarts: 1 286 + idx: 287 + - length: 55 288 + restarts: 2 289 EOF 290 test-tool dump-reftable -b .git/reftable/*.ref >actual && 291 test_cmp expect actual

+146 -68

t/unit-tests/t-reftable-block.c

··· 19 struct reftable_record recs[30]; 20 const size_t N = ARRAY_SIZE(recs); 21 const size_t block_size = 1024; 22 - struct reftable_block block = { 0 }; 23 struct block_writer bw = { 24 .last_key = REFTABLE_BUF_INIT, 25 }; 26 struct reftable_record rec = { 27 - .type = BLOCK_TYPE_REF, 28 }; 29 size_t i = 0; 30 int ret; 31 - struct block_reader br = { 0 }; 32 struct block_iter it = BLOCK_ITER_INIT; 33 - struct reftable_buf want = REFTABLE_BUF_INIT, buf = REFTABLE_BUF_INIT; 0 34 35 - REFTABLE_CALLOC_ARRAY(block.data, block_size); 36 - check(block.data != NULL); 37 - block.len = block_size; 38 - block_source_from_buf(&block.source ,&buf); 39 - ret = block_writer_init(&bw, BLOCK_TYPE_REF, block.data, block_size, 40 header_off, hash_size(REFTABLE_HASH_SHA1)); 41 check(!ret); 42 ··· 62 63 block_writer_release(&bw); 64 65 - block_reader_init(&br, &block, header_off, block_size, REFTABLE_HASH_SIZE_SHA1); 0 66 67 - block_iter_seek_start(&it, &br); 68 69 for (i = 0; ; i++) { 70 ret = block_iter_next(&it, &rec); ··· 77 } 78 79 for (i = 0; i < N; i++) { 80 - block_iter_reset(&it); 81 reftable_record_key(&recs[i], &want); 82 83 - ret = block_iter_seek_key(&it, &br, &want); 84 check_int(ret, ==, 0); 85 86 ret = block_iter_next(&it, &rec); ··· 89 check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1)); 90 91 want.len--; 92 - ret = block_iter_seek_key(&it, &br, &want); 93 check_int(ret, ==, 0); 94 95 ret = block_iter_next(&it, &rec); ··· 97 check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1)); 98 } 99 100 - block_reader_release(&br); 101 block_iter_close(&it); 102 reftable_record_release(&rec); 103 - reftable_block_done(&br.block); 104 reftable_buf_release(&want); 105 - reftable_buf_release(&buf); 106 for (i = 0; i < N; i++) 107 reftable_record_release(&recs[i]); 108 } ··· 113 struct reftable_record recs[30]; 114 const size_t N = ARRAY_SIZE(recs); 115 const size_t block_size = 2048; 116 - struct reftable_block block = { 0 }; 117 struct block_writer bw = { 118 .last_key = REFTABLE_BUF_INIT, 119 }; 120 struct reftable_record rec = { 121 - .type = BLOCK_TYPE_LOG, 122 }; 123 size_t i = 0; 124 int ret; 125 - struct block_reader br = { 0 }; 126 struct block_iter it = BLOCK_ITER_INIT; 127 - struct reftable_buf want = REFTABLE_BUF_INIT, buf = REFTABLE_BUF_INIT; 0 128 129 - REFTABLE_CALLOC_ARRAY(block.data, block_size); 130 - check(block.data != NULL); 131 - block.len = block_size; 132 - block_source_from_buf(&block.source ,&buf); 133 - ret = block_writer_init(&bw, BLOCK_TYPE_LOG, block.data, block_size, 134 header_off, hash_size(REFTABLE_HASH_SHA1)); 135 check(!ret); 136 ··· 151 152 block_writer_release(&bw); 153 154 - block_reader_init(&br, &block, header_off, block_size, REFTABLE_HASH_SIZE_SHA1); 0 155 156 - block_iter_seek_start(&it, &br); 157 158 for (i = 0; ; i++) { 159 ret = block_iter_next(&it, &rec); ··· 166 } 167 168 for (i = 0; i < N; i++) { 169 - block_iter_reset(&it); 170 reftable_buf_reset(&want); 171 check(!reftable_buf_addstr(&want, recs[i].u.log.refname)); 172 173 - ret = block_iter_seek_key(&it, &br, &want); 174 check_int(ret, ==, 0); 175 176 ret = block_iter_next(&it, &rec); ··· 179 check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1)); 180 181 want.len--; 182 - ret = block_iter_seek_key(&it, &br, &want); 183 check_int(ret, ==, 0); 184 185 ret = block_iter_next(&it, &rec); ··· 187 check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1)); 188 } 189 190 - block_reader_release(&br); 191 block_iter_close(&it); 192 reftable_record_release(&rec); 193 - reftable_block_done(&br.block); 194 reftable_buf_release(&want); 195 - reftable_buf_release(&buf); 196 for (i = 0; i < N; i++) 197 reftable_record_release(&recs[i]); 198 } ··· 203 struct reftable_record recs[30]; 204 const size_t N = ARRAY_SIZE(recs); 205 const size_t block_size = 1024; 206 - struct reftable_block block = { 0 }; 207 struct block_writer bw = { 208 .last_key = REFTABLE_BUF_INIT, 209 }; 210 struct reftable_record rec = { 211 - .type = BLOCK_TYPE_OBJ, 212 }; 213 size_t i = 0; 214 int ret; 215 - struct block_reader br = { 0 }; 216 struct block_iter it = BLOCK_ITER_INIT; 217 - struct reftable_buf want = REFTABLE_BUF_INIT, buf = REFTABLE_BUF_INIT; 0 0 0 0 0 218 219 - REFTABLE_CALLOC_ARRAY(block.data, block_size); 220 - check(block.data != NULL); 221 - block.len = block_size; 222 - block_source_from_buf(&block.source, &buf); 223 - ret = block_writer_init(&bw, BLOCK_TYPE_OBJ, block.data, block_size, 224 header_off, hash_size(REFTABLE_HASH_SHA1)); 225 check(!ret); 226 ··· 243 244 block_writer_release(&bw); 245 246 - block_reader_init(&br, &block, header_off, block_size, REFTABLE_HASH_SIZE_SHA1); 0 247 248 - block_iter_seek_start(&it, &br); 249 250 for (i = 0; ; i++) { 251 ret = block_iter_next(&it, &rec); ··· 258 } 259 260 for (i = 0; i < N; i++) { 261 - block_iter_reset(&it); 262 reftable_record_key(&recs[i], &want); 263 264 - ret = block_iter_seek_key(&it, &br, &want); 265 check_int(ret, ==, 0); 266 267 ret = block_iter_next(&it, &rec); ··· 270 check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1)); 271 } 272 273 - block_reader_release(&br); 274 block_iter_close(&it); 275 reftable_record_release(&rec); 276 - reftable_block_done(&br.block); 277 reftable_buf_release(&want); 278 - reftable_buf_release(&buf); 279 for (i = 0; i < N; i++) 280 reftable_record_release(&recs[i]); 281 } ··· 286 struct reftable_record recs[30]; 287 const size_t N = ARRAY_SIZE(recs); 288 const size_t block_size = 1024; 289 - struct reftable_block block = { 0 }; 290 struct block_writer bw = { 291 .last_key = REFTABLE_BUF_INIT, 292 }; 293 struct reftable_record rec = { 294 - .type = BLOCK_TYPE_INDEX, 295 .u.idx.last_key = REFTABLE_BUF_INIT, 296 }; 297 size_t i = 0; 298 int ret; 299 - struct block_reader br = { 0 }; 300 struct block_iter it = BLOCK_ITER_INIT; 301 - struct reftable_buf want = REFTABLE_BUF_INIT, buf = REFTABLE_BUF_INIT; 0 302 303 - REFTABLE_CALLOC_ARRAY(block.data, block_size); 304 - check(block.data != NULL); 305 - block.len = block_size; 306 - block_source_from_buf(&block.source, &buf); 307 - ret = block_writer_init(&bw, BLOCK_TYPE_INDEX, block.data, block_size, 308 header_off, hash_size(REFTABLE_HASH_SHA1)); 309 check(!ret); 310 ··· 314 snprintf(buf, sizeof(buf), "branch%02"PRIuMAX, (uintmax_t)i); 315 316 reftable_buf_init(&recs[i].u.idx.last_key); 317 - recs[i].type = BLOCK_TYPE_INDEX; 318 check(!reftable_buf_addstr(&recs[i].u.idx.last_key, buf)); 319 recs[i].u.idx.offset = i; 320 ··· 327 328 block_writer_release(&bw); 329 330 - block_reader_init(&br, &block, header_off, block_size, REFTABLE_HASH_SIZE_SHA1); 0 331 332 - block_iter_seek_start(&it, &br); 333 334 for (i = 0; ; i++) { 335 ret = block_iter_next(&it, &rec); ··· 342 } 343 344 for (i = 0; i < N; i++) { 345 - block_iter_reset(&it); 346 reftable_record_key(&recs[i], &want); 347 348 - ret = block_iter_seek_key(&it, &br, &want); 349 check_int(ret, ==, 0); 350 351 ret = block_iter_next(&it, &rec); ··· 354 check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1)); 355 356 want.len--; 357 - ret = block_iter_seek_key(&it, &br, &want); 358 check_int(ret, ==, 0); 359 360 ret = block_iter_next(&it, &rec); ··· 362 check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1)); 363 } 364 365 - block_reader_release(&br); 366 block_iter_close(&it); 367 reftable_record_release(&rec); 368 - reftable_block_done(&br.block); 369 reftable_buf_release(&want); 370 - reftable_buf_release(&buf); 371 for (i = 0; i < N; i++) 372 reftable_record_release(&recs[i]); 373 } 374 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 375 int cmd_main(int argc UNUSED, const char *argv[] UNUSED) 376 { 377 TEST(t_index_block_read_write(), "read-write operations on index blocks work"); 378 TEST(t_log_block_read_write(), "read-write operations on log blocks work"); 379 TEST(t_obj_block_read_write(), "read-write operations on obj blocks work"); 380 TEST(t_ref_block_read_write(), "read-write operations on ref blocks work"); 0 381 382 return test_done(); 383 }

··· 19 struct reftable_record recs[30]; 20 const size_t N = ARRAY_SIZE(recs); 21 const size_t block_size = 1024; 22 + struct reftable_block_source source = { 0 }; 23 struct block_writer bw = { 24 .last_key = REFTABLE_BUF_INIT, 25 }; 26 struct reftable_record rec = { 27 + .type = REFTABLE_BLOCK_TYPE_REF, 28 }; 29 size_t i = 0; 30 int ret; 31 + struct reftable_block block = { 0 }; 32 struct block_iter it = BLOCK_ITER_INIT; 33 + struct reftable_buf want = REFTABLE_BUF_INIT; 34 + struct reftable_buf block_data = REFTABLE_BUF_INIT; 35 36 + REFTABLE_CALLOC_ARRAY(block_data.buf, block_size); 37 + check(block_data.buf != NULL); 38 + block_data.len = block_size; 39 + 40 + ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_REF, (uint8_t *) block_data.buf, block_size, 41 header_off, hash_size(REFTABLE_HASH_SHA1)); 42 check(!ret); 43 ··· 63 64 block_writer_release(&bw); 65 66 + block_source_from_buf(&source ,&block_data); 67 + reftable_block_init(&block, &source, 0, header_off, block_size, REFTABLE_HASH_SIZE_SHA1); 68 69 + block_iter_init(&it, &block); 70 71 for (i = 0; ; i++) { 72 ret = block_iter_next(&it, &rec); ··· 79 } 80 81 for (i = 0; i < N; i++) { 0 82 reftable_record_key(&recs[i], &want); 83 84 + ret = block_iter_seek_key(&it, &want); 85 check_int(ret, ==, 0); 86 87 ret = block_iter_next(&it, &rec); ··· 90 check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1)); 91 92 want.len--; 93 + ret = block_iter_seek_key(&it, &want); 94 check_int(ret, ==, 0); 95 96 ret = block_iter_next(&it, &rec); ··· 98 check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1)); 99 } 100 101 + reftable_block_release(&block); 102 block_iter_close(&it); 103 reftable_record_release(&rec); 0 104 reftable_buf_release(&want); 105 + reftable_buf_release(&block_data); 106 for (i = 0; i < N; i++) 107 reftable_record_release(&recs[i]); 108 } ··· 113 struct reftable_record recs[30]; 114 const size_t N = ARRAY_SIZE(recs); 115 const size_t block_size = 2048; 116 + struct reftable_block_source source = { 0 }; 117 struct block_writer bw = { 118 .last_key = REFTABLE_BUF_INIT, 119 }; 120 struct reftable_record rec = { 121 + .type = REFTABLE_BLOCK_TYPE_LOG, 122 }; 123 size_t i = 0; 124 int ret; 125 + struct reftable_block block = { 0 }; 126 struct block_iter it = BLOCK_ITER_INIT; 127 + struct reftable_buf want = REFTABLE_BUF_INIT; 128 + struct reftable_buf block_data = REFTABLE_BUF_INIT; 129 130 + REFTABLE_CALLOC_ARRAY(block_data.buf, block_size); 131 + check(block_data.buf != NULL); 132 + block_data.len = block_size; 133 + 134 + ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_LOG, (uint8_t *) block_data.buf, block_size, 135 header_off, hash_size(REFTABLE_HASH_SHA1)); 136 check(!ret); 137 ··· 152 153 block_writer_release(&bw); 154 155 + block_source_from_buf(&source, &block_data); 156 + reftable_block_init(&block, &source, 0, header_off, block_size, REFTABLE_HASH_SIZE_SHA1); 157 158 + block_iter_init(&it, &block); 159 160 for (i = 0; ; i++) { 161 ret = block_iter_next(&it, &rec); ··· 168 } 169 170 for (i = 0; i < N; i++) { 0 171 reftable_buf_reset(&want); 172 check(!reftable_buf_addstr(&want, recs[i].u.log.refname)); 173 174 + ret = block_iter_seek_key(&it, &want); 175 check_int(ret, ==, 0); 176 177 ret = block_iter_next(&it, &rec); ··· 180 check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1)); 181 182 want.len--; 183 + ret = block_iter_seek_key(&it, &want); 184 check_int(ret, ==, 0); 185 186 ret = block_iter_next(&it, &rec); ··· 188 check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1)); 189 } 190 191 + reftable_block_release(&block); 192 block_iter_close(&it); 193 reftable_record_release(&rec); 0 194 reftable_buf_release(&want); 195 + reftable_buf_release(&block_data); 196 for (i = 0; i < N; i++) 197 reftable_record_release(&recs[i]); 198 } ··· 203 struct reftable_record recs[30]; 204 const size_t N = ARRAY_SIZE(recs); 205 const size_t block_size = 1024; 206 + struct reftable_block_source source = { 0 }; 207 struct block_writer bw = { 208 .last_key = REFTABLE_BUF_INIT, 209 }; 210 struct reftable_record rec = { 211 + .type = REFTABLE_BLOCK_TYPE_OBJ, 212 }; 213 size_t i = 0; 214 int ret; 215 + struct reftable_block block = { 0 }; 216 struct block_iter it = BLOCK_ITER_INIT; 217 + struct reftable_buf want = REFTABLE_BUF_INIT; 218 + struct reftable_buf block_data = REFTABLE_BUF_INIT; 219 + 220 + REFTABLE_CALLOC_ARRAY(block_data.buf, block_size); 221 + check(block_data.buf != NULL); 222 + block_data.len = block_size; 223 224 + ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_OBJ, (uint8_t *) block_data.buf, block_size, 0 0 0 0 225 header_off, hash_size(REFTABLE_HASH_SHA1)); 226 check(!ret); 227 ··· 244 245 block_writer_release(&bw); 246 247 + block_source_from_buf(&source, &block_data); 248 + reftable_block_init(&block, &source, 0, header_off, block_size, REFTABLE_HASH_SIZE_SHA1); 249 250 + block_iter_init(&it, &block); 251 252 for (i = 0; ; i++) { 253 ret = block_iter_next(&it, &rec); ··· 260 } 261 262 for (i = 0; i < N; i++) { 0 263 reftable_record_key(&recs[i], &want); 264 265 + ret = block_iter_seek_key(&it, &want); 266 check_int(ret, ==, 0); 267 268 ret = block_iter_next(&it, &rec); ··· 271 check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1)); 272 } 273 274 + reftable_block_release(&block); 275 block_iter_close(&it); 276 reftable_record_release(&rec); 0 277 reftable_buf_release(&want); 278 + reftable_buf_release(&block_data); 279 for (i = 0; i < N; i++) 280 reftable_record_release(&recs[i]); 281 } ··· 286 struct reftable_record recs[30]; 287 const size_t N = ARRAY_SIZE(recs); 288 const size_t block_size = 1024; 289 + struct reftable_block_source source = { 0 }; 290 struct block_writer bw = { 291 .last_key = REFTABLE_BUF_INIT, 292 }; 293 struct reftable_record rec = { 294 + .type = REFTABLE_BLOCK_TYPE_INDEX, 295 .u.idx.last_key = REFTABLE_BUF_INIT, 296 }; 297 size_t i = 0; 298 int ret; 299 + struct reftable_block block = { 0 }; 300 struct block_iter it = BLOCK_ITER_INIT; 301 + struct reftable_buf want = REFTABLE_BUF_INIT; 302 + struct reftable_buf block_data = REFTABLE_BUF_INIT; 303 304 + REFTABLE_CALLOC_ARRAY(block_data.buf, block_size); 305 + check(block_data.buf != NULL); 306 + block_data.len = block_size; 307 + 308 + ret = block_writer_init(&bw, REFTABLE_BLOCK_TYPE_INDEX, (uint8_t *) block_data.buf, block_size, 309 header_off, hash_size(REFTABLE_HASH_SHA1)); 310 check(!ret); 311 ··· 315 snprintf(buf, sizeof(buf), "branch%02"PRIuMAX, (uintmax_t)i); 316 317 reftable_buf_init(&recs[i].u.idx.last_key); 318 + recs[i].type = REFTABLE_BLOCK_TYPE_INDEX; 319 check(!reftable_buf_addstr(&recs[i].u.idx.last_key, buf)); 320 recs[i].u.idx.offset = i; 321 ··· 328 329 block_writer_release(&bw); 330 331 + block_source_from_buf(&source, &block_data); 332 + reftable_block_init(&block, &source, 0, header_off, block_size, REFTABLE_HASH_SIZE_SHA1); 333 334 + block_iter_init(&it, &block); 335 336 for (i = 0; ; i++) { 337 ret = block_iter_next(&it, &rec); ··· 344 } 345 346 for (i = 0; i < N; i++) { 0 347 reftable_record_key(&recs[i], &want); 348 349 + ret = block_iter_seek_key(&it, &want); 350 check_int(ret, ==, 0); 351 352 ret = block_iter_next(&it, &rec); ··· 355 check(reftable_record_equal(&recs[i], &rec, REFTABLE_HASH_SIZE_SHA1)); 356 357 want.len--; 358 + ret = block_iter_seek_key(&it, &want); 359 check_int(ret, ==, 0); 360 361 ret = block_iter_next(&it, &rec); ··· 363 check(reftable_record_equal(&recs[10 * (i / 10)], &rec, REFTABLE_HASH_SIZE_SHA1)); 364 } 365 366 + reftable_block_release(&block); 367 block_iter_close(&it); 368 reftable_record_release(&rec); 0 369 reftable_buf_release(&want); 370 + reftable_buf_release(&block_data); 371 for (i = 0; i < N; i++) 372 reftable_record_release(&recs[i]); 373 } 374 375 + static void t_block_iterator(void) 376 + { 377 + struct reftable_block_source source = { 0 }; 378 + struct block_writer writer = { 379 + .last_key = REFTABLE_BUF_INIT, 380 + }; 381 + struct reftable_record expected_refs[20]; 382 + struct reftable_ref_record ref = { 0 }; 383 + struct reftable_iterator it = { 0 }; 384 + struct reftable_block block = { 0 }; 385 + struct reftable_buf data; 386 + int err; 387 + 388 + data.len = 1024; 389 + REFTABLE_CALLOC_ARRAY(data.buf, data.len); 390 + check(data.buf != NULL); 391 + 392 + err = block_writer_init(&writer, REFTABLE_BLOCK_TYPE_REF, (uint8_t *) data.buf, data.len, 393 + 0, hash_size(REFTABLE_HASH_SHA1)); 394 + check(!err); 395 + 396 + for (size_t i = 0; i < ARRAY_SIZE(expected_refs); i++) { 397 + expected_refs[i] = (struct reftable_record) { 398 + .type = REFTABLE_BLOCK_TYPE_REF, 399 + .u.ref = { 400 + .value_type = REFTABLE_REF_VAL1, 401 + .refname = xstrfmt("refs/heads/branch-%02"PRIuMAX, (uintmax_t)i), 402 + }, 403 + }; 404 + memset(expected_refs[i].u.ref.value.val1, i, REFTABLE_HASH_SIZE_SHA1); 405 + 406 + err = block_writer_add(&writer, &expected_refs[i]); 407 + check_int(err, ==, 0); 408 + } 409 + 410 + err = block_writer_finish(&writer); 411 + check_int(err, >, 0); 412 + 413 + block_source_from_buf(&source, &data); 414 + reftable_block_init(&block, &source, 0, 0, data.len, REFTABLE_HASH_SIZE_SHA1); 415 + 416 + err = reftable_block_init_iterator(&block, &it); 417 + check_int(err, ==, 0); 418 + 419 + for (size_t i = 0; ; i++) { 420 + err = reftable_iterator_next_ref(&it, &ref); 421 + if (err > 0) { 422 + check_int(i, ==, ARRAY_SIZE(expected_refs)); 423 + break; 424 + } 425 + check_int(err, ==, 0); 426 + 427 + check(reftable_ref_record_equal(&ref, &expected_refs[i].u.ref, 428 + REFTABLE_HASH_SIZE_SHA1)); 429 + } 430 + 431 + err = reftable_iterator_seek_ref(&it, "refs/heads/does-not-exist"); 432 + check_int(err, ==, 0); 433 + err = reftable_iterator_next_ref(&it, &ref); 434 + check_int(err, ==, 1); 435 + 436 + err = reftable_iterator_seek_ref(&it, "refs/heads/branch-13"); 437 + check_int(err, ==, 0); 438 + err = reftable_iterator_next_ref(&it, &ref); 439 + check_int(err, ==, 0); 440 + check(reftable_ref_record_equal(&ref, &expected_refs[13].u.ref, 441 + REFTABLE_HASH_SIZE_SHA1)); 442 + 443 + for (size_t i = 0; i < ARRAY_SIZE(expected_refs); i++) 444 + reftable_free(expected_refs[i].u.ref.refname); 445 + reftable_ref_record_release(&ref); 446 + reftable_iterator_destroy(&it); 447 + reftable_block_release(&block); 448 + block_writer_release(&writer); 449 + reftable_buf_release(&data); 450 + } 451 + 452 int cmd_main(int argc UNUSED, const char *argv[] UNUSED) 453 { 454 TEST(t_index_block_read_write(), "read-write operations on index blocks work"); 455 TEST(t_log_block_read_write(), "read-write operations on log blocks work"); 456 TEST(t_obj_block_read_write(), "read-write operations on obj blocks work"); 457 TEST(t_ref_block_read_write(), "read-write operations on ref blocks work"); 458 + TEST(t_block_iterator(), "block iterator works"); 459 460 return test_done(); 461 }

+43 -43

t/unit-tests/t-reftable-merged.c

··· 11 #include "reftable/blocksource.h" 12 #include "reftable/constants.h" 13 #include "reftable/merged.h" 14 - #include "reftable/reader.h" 15 #include "reftable/reftable-error.h" 16 #include "reftable/reftable-merged.h" 17 #include "reftable/reftable-writer.h" ··· 19 static struct reftable_merged_table * 20 merged_table_from_records(struct reftable_ref_record **refs, 21 struct reftable_block_source **source, 22 - struct reftable_reader ***readers, const size_t *sizes, 23 struct reftable_buf *buf, const size_t n) 24 { 25 struct reftable_merged_table *mt = NULL; ··· 28 }; 29 int err; 30 31 - REFTABLE_CALLOC_ARRAY(*readers, n); 32 - check(*readers != NULL); 33 REFTABLE_CALLOC_ARRAY(*source, n); 34 check(*source != NULL); 35 ··· 37 t_reftable_write_to_buf(&buf[i], refs[i], sizes[i], NULL, 0, &opts); 38 block_source_from_buf(&(*source)[i], &buf[i]); 39 40 - err = reftable_reader_new(&(*readers)[i], &(*source)[i], 41 - "name"); 42 check(!err); 43 } 44 45 - err = reftable_merged_table_new(&mt, *readers, n, REFTABLE_HASH_SHA1); 46 check(!err); 47 return mt; 48 } 49 50 - static void readers_destroy(struct reftable_reader **readers, const size_t n) 51 { 52 for (size_t i = 0; i < n; i++) 53 - reftable_reader_decref(readers[i]); 54 - reftable_free(readers); 55 } 56 57 static void t_merged_single_record(void) ··· 77 size_t sizes[] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; 78 struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; 79 struct reftable_block_source *bs = NULL; 80 - struct reftable_reader **readers = NULL; 81 struct reftable_merged_table *mt = 82 - merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3); 83 struct reftable_ref_record ref = { 0 }; 84 struct reftable_iterator it = { 0 }; 85 int err; 86 87 - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); 88 check(!err); 89 err = reftable_iterator_seek_ref(&it, "a"); 90 check(!err); ··· 94 check(reftable_ref_record_equal(&r2[0], &ref, REFTABLE_HASH_SIZE_SHA1)); 95 reftable_ref_record_release(&ref); 96 reftable_iterator_destroy(&it); 97 - readers_destroy(readers, 3); 98 reftable_merged_table_free(mt); 99 for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) 100 reftable_buf_release(&bufs[i]); ··· 154 size_t sizes[3] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; 155 struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; 156 struct reftable_block_source *bs = NULL; 157 - struct reftable_reader **readers = NULL; 158 struct reftable_merged_table *mt = 159 - merged_table_from_records(refs, &bs, &readers, sizes, bufs, 3); 160 struct reftable_iterator it = { 0 }; 161 int err; 162 struct reftable_ref_record *out = NULL; ··· 164 size_t cap = 0; 165 size_t i; 166 167 - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); 168 check(!err); 169 err = reftable_iterator_seek_ref(&it, "a"); 170 check(!err); ··· 193 194 for (i = 0; i < 3; i++) 195 reftable_buf_release(&bufs[i]); 196 - readers_destroy(readers, 3); 197 reftable_merged_table_free(mt); 198 reftable_free(bs); 199 } ··· 238 REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, 239 }; 240 struct reftable_block_source *sources = NULL; 241 - struct reftable_reader **readers = NULL; 242 struct reftable_ref_record rec = { 0 }; 243 struct reftable_iterator it = { 0 }; 244 struct reftable_merged_table *mt; 245 246 - mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2); 247 - merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); 248 249 for (size_t i = 0; i < 5; i++) { 250 int err = reftable_iterator_seek_ref(&it, "c"); ··· 266 267 for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) 268 reftable_buf_release(&bufs[i]); 269 - readers_destroy(readers, ARRAY_SIZE(refs)); 270 reftable_ref_record_release(&rec); 271 reftable_iterator_destroy(&it); 272 reftable_merged_table_free(mt); ··· 313 REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, 314 }; 315 struct reftable_block_source *sources = NULL; 316 - struct reftable_reader **readers = NULL; 317 struct reftable_ref_record rec = { 0 }; 318 struct reftable_iterator it = { 0 }; 319 struct reftable_merged_table *mt; 320 int err; 321 322 - mt = merged_table_from_records(refs, &sources, &readers, sizes, bufs, 2); 323 - merged_table_init_iter(mt, &it, BLOCK_TYPE_REF); 324 325 err = reftable_iterator_seek_ref(&it, "b"); 326 check(!err); ··· 338 339 for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) 340 reftable_buf_release(&bufs[i]); 341 - readers_destroy(readers, ARRAY_SIZE(refs)); 342 reftable_ref_record_release(&rec); 343 reftable_iterator_destroy(&it); 344 reftable_merged_table_free(mt); ··· 348 static struct reftable_merged_table * 349 merged_table_from_log_records(struct reftable_log_record **logs, 350 struct reftable_block_source **source, 351 - struct reftable_reader ***readers, const size_t *sizes, 352 struct reftable_buf *buf, const size_t n) 353 { 354 struct reftable_merged_table *mt = NULL; ··· 358 }; 359 int err; 360 361 - REFTABLE_CALLOC_ARRAY(*readers, n); 362 - check(*readers != NULL); 363 REFTABLE_CALLOC_ARRAY(*source, n); 364 check(*source != NULL); 365 ··· 367 t_reftable_write_to_buf(&buf[i], NULL, 0, logs[i], sizes[i], &opts); 368 block_source_from_buf(&(*source)[i], &buf[i]); 369 370 - err = reftable_reader_new(&(*readers)[i], &(*source)[i], 371 - "name"); 372 check(!err); 373 } 374 375 - err = reftable_merged_table_new(&mt, *readers, n, REFTABLE_HASH_SHA1); 376 check(!err); 377 return mt; 378 } ··· 435 size_t sizes[3] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; 436 struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; 437 struct reftable_block_source *bs = NULL; 438 - struct reftable_reader **readers = NULL; 439 struct reftable_merged_table *mt = merged_table_from_log_records( 440 - logs, &bs, &readers, sizes, bufs, 3); 441 struct reftable_iterator it = { 0 }; 442 int err; 443 struct reftable_log_record *out = NULL; ··· 445 size_t cap = 0; 446 size_t i; 447 448 - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); 449 check(!err); 450 err = reftable_iterator_seek_log(&it, "a"); 451 check(!err); ··· 469 check(reftable_log_record_equal(want[i], &out[i], 470 REFTABLE_HASH_SIZE_SHA1)); 471 472 - err = merged_table_init_iter(mt, &it, BLOCK_TYPE_LOG); 473 check(!err); 474 err = reftable_iterator_seek_log_at(&it, "a", 2); 475 check(!err); ··· 485 486 for (i = 0; i < 3; i++) 487 reftable_buf_release(&bufs[i]); 488 - readers_destroy(readers, 3); 489 reftable_merged_table_free(mt); 490 reftable_free(bs); 491 } ··· 502 int err; 503 struct reftable_block_source source = { 0 }; 504 uint32_t hash_id; 505 - struct reftable_reader *rd = NULL; 506 struct reftable_merged_table *merged = NULL; 507 508 reftable_writer_set_limits(w, 1, 1); ··· 516 517 block_source_from_buf(&source, &buf); 518 519 - err = reftable_reader_new(&rd, &source, "filename"); 520 check(!err); 521 522 - hash_id = reftable_reader_hash_id(rd); 523 check_int(hash_id, ==, REFTABLE_HASH_SHA1); 524 525 - err = reftable_merged_table_new(&merged, &rd, 1, REFTABLE_HASH_SHA256); 526 check_int(err, ==, REFTABLE_FORMAT_ERROR); 527 - err = reftable_merged_table_new(&merged, &rd, 1, REFTABLE_HASH_SHA1); 528 check(!err); 529 530 - reftable_reader_decref(rd); 531 reftable_merged_table_free(merged); 532 reftable_buf_release(&buf); 533 }

··· 11 #include "reftable/blocksource.h" 12 #include "reftable/constants.h" 13 #include "reftable/merged.h" 14 + #include "reftable/table.h" 15 #include "reftable/reftable-error.h" 16 #include "reftable/reftable-merged.h" 17 #include "reftable/reftable-writer.h" ··· 19 static struct reftable_merged_table * 20 merged_table_from_records(struct reftable_ref_record **refs, 21 struct reftable_block_source **source, 22 + struct reftable_table ***tables, const size_t *sizes, 23 struct reftable_buf *buf, const size_t n) 24 { 25 struct reftable_merged_table *mt = NULL; ··· 28 }; 29 int err; 30 31 + REFTABLE_CALLOC_ARRAY(*tables, n); 32 + check(*tables != NULL); 33 REFTABLE_CALLOC_ARRAY(*source, n); 34 check(*source != NULL); 35 ··· 37 t_reftable_write_to_buf(&buf[i], refs[i], sizes[i], NULL, 0, &opts); 38 block_source_from_buf(&(*source)[i], &buf[i]); 39 40 + err = reftable_table_new(&(*tables)[i], &(*source)[i], 41 + "name"); 42 check(!err); 43 } 44 45 + err = reftable_merged_table_new(&mt, *tables, n, REFTABLE_HASH_SHA1); 46 check(!err); 47 return mt; 48 } 49 50 + static void tables_destroy(struct reftable_table **tables, const size_t n) 51 { 52 for (size_t i = 0; i < n; i++) 53 + reftable_table_decref(tables[i]); 54 + reftable_free(tables); 55 } 56 57 static void t_merged_single_record(void) ··· 77 size_t sizes[] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; 78 struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; 79 struct reftable_block_source *bs = NULL; 80 + struct reftable_table **tables = NULL; 81 struct reftable_merged_table *mt = 82 + merged_table_from_records(refs, &bs, &tables, sizes, bufs, 3); 83 struct reftable_ref_record ref = { 0 }; 84 struct reftable_iterator it = { 0 }; 85 int err; 86 87 + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); 88 check(!err); 89 err = reftable_iterator_seek_ref(&it, "a"); 90 check(!err); ··· 94 check(reftable_ref_record_equal(&r2[0], &ref, REFTABLE_HASH_SIZE_SHA1)); 95 reftable_ref_record_release(&ref); 96 reftable_iterator_destroy(&it); 97 + tables_destroy(tables, 3); 98 reftable_merged_table_free(mt); 99 for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) 100 reftable_buf_release(&bufs[i]); ··· 154 size_t sizes[3] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; 155 struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; 156 struct reftable_block_source *bs = NULL; 157 + struct reftable_table **tables = NULL; 158 struct reftable_merged_table *mt = 159 + merged_table_from_records(refs, &bs, &tables, sizes, bufs, 3); 160 struct reftable_iterator it = { 0 }; 161 int err; 162 struct reftable_ref_record *out = NULL; ··· 164 size_t cap = 0; 165 size_t i; 166 167 + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); 168 check(!err); 169 err = reftable_iterator_seek_ref(&it, "a"); 170 check(!err); ··· 193 194 for (i = 0; i < 3; i++) 195 reftable_buf_release(&bufs[i]); 196 + tables_destroy(tables, 3); 197 reftable_merged_table_free(mt); 198 reftable_free(bs); 199 } ··· 238 REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, 239 }; 240 struct reftable_block_source *sources = NULL; 241 + struct reftable_table **tables = NULL; 242 struct reftable_ref_record rec = { 0 }; 243 struct reftable_iterator it = { 0 }; 244 struct reftable_merged_table *mt; 245 246 + mt = merged_table_from_records(refs, &sources, &tables, sizes, bufs, 2); 247 + merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); 248 249 for (size_t i = 0; i < 5; i++) { 250 int err = reftable_iterator_seek_ref(&it, "c"); ··· 266 267 for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) 268 reftable_buf_release(&bufs[i]); 269 + tables_destroy(tables, ARRAY_SIZE(refs)); 270 reftable_ref_record_release(&rec); 271 reftable_iterator_destroy(&it); 272 reftable_merged_table_free(mt); ··· 313 REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, 314 }; 315 struct reftable_block_source *sources = NULL; 316 + struct reftable_table **tables = NULL; 317 struct reftable_ref_record rec = { 0 }; 318 struct reftable_iterator it = { 0 }; 319 struct reftable_merged_table *mt; 320 int err; 321 322 + mt = merged_table_from_records(refs, &sources, &tables, sizes, bufs, 2); 323 + merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_REF); 324 325 err = reftable_iterator_seek_ref(&it, "b"); 326 check(!err); ··· 338 339 for (size_t i = 0; i < ARRAY_SIZE(bufs); i++) 340 reftable_buf_release(&bufs[i]); 341 + tables_destroy(tables, ARRAY_SIZE(refs)); 342 reftable_ref_record_release(&rec); 343 reftable_iterator_destroy(&it); 344 reftable_merged_table_free(mt); ··· 348 static struct reftable_merged_table * 349 merged_table_from_log_records(struct reftable_log_record **logs, 350 struct reftable_block_source **source, 351 + struct reftable_table ***tables, const size_t *sizes, 352 struct reftable_buf *buf, const size_t n) 353 { 354 struct reftable_merged_table *mt = NULL; ··· 358 }; 359 int err; 360 361 + REFTABLE_CALLOC_ARRAY(*tables, n); 362 + check(*tables != NULL); 363 REFTABLE_CALLOC_ARRAY(*source, n); 364 check(*source != NULL); 365 ··· 367 t_reftable_write_to_buf(&buf[i], NULL, 0, logs[i], sizes[i], &opts); 368 block_source_from_buf(&(*source)[i], &buf[i]); 369 370 + err = reftable_table_new(&(*tables)[i], &(*source)[i], 371 + "name"); 372 check(!err); 373 } 374 375 + err = reftable_merged_table_new(&mt, *tables, n, REFTABLE_HASH_SHA1); 376 check(!err); 377 return mt; 378 } ··· 435 size_t sizes[3] = { ARRAY_SIZE(r1), ARRAY_SIZE(r2), ARRAY_SIZE(r3) }; 436 struct reftable_buf bufs[3] = { REFTABLE_BUF_INIT, REFTABLE_BUF_INIT, REFTABLE_BUF_INIT }; 437 struct reftable_block_source *bs = NULL; 438 + struct reftable_table **tables = NULL; 439 struct reftable_merged_table *mt = merged_table_from_log_records( 440 + logs, &bs, &tables, sizes, bufs, 3); 441 struct reftable_iterator it = { 0 }; 442 int err; 443 struct reftable_log_record *out = NULL; ··· 445 size_t cap = 0; 446 size_t i; 447 448 + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG); 449 check(!err); 450 err = reftable_iterator_seek_log(&it, "a"); 451 check(!err); ··· 469 check(reftable_log_record_equal(want[i], &out[i], 470 REFTABLE_HASH_SIZE_SHA1)); 471 472 + err = merged_table_init_iter(mt, &it, REFTABLE_BLOCK_TYPE_LOG); 473 check(!err); 474 err = reftable_iterator_seek_log_at(&it, "a", 2); 475 check(!err); ··· 485 486 for (i = 0; i < 3; i++) 487 reftable_buf_release(&bufs[i]); 488 + tables_destroy(tables, 3); 489 reftable_merged_table_free(mt); 490 reftable_free(bs); 491 } ··· 502 int err; 503 struct reftable_block_source source = { 0 }; 504 uint32_t hash_id; 505 + struct reftable_table *table = NULL; 506 struct reftable_merged_table *merged = NULL; 507 508 reftable_writer_set_limits(w, 1, 1); ··· 516 517 block_source_from_buf(&source, &buf); 518 519 + err = reftable_table_new(&table, &source, "filename"); 520 check(!err); 521 522 + hash_id = reftable_table_hash_id(table); 523 check_int(hash_id, ==, REFTABLE_HASH_SHA1); 524 525 + err = reftable_merged_table_new(&merged, &table, 1, REFTABLE_HASH_SHA256); 526 check_int(err, ==, REFTABLE_FORMAT_ERROR); 527 + err = reftable_merged_table_new(&merged, &table, 1, REFTABLE_HASH_SHA1); 528 check(!err); 529 530 + reftable_table_decref(table); 531 reftable_merged_table_free(merged); 532 reftable_buf_release(&buf); 533 }

+5 -5

t/unit-tests/t-reftable-pq.c

··· 34 char *last = NULL; 35 36 for (i = 0; i < N; i++) { 37 - check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF)); 38 recs[i].u.ref.refname = xstrfmt("%02"PRIuMAX, (uintmax_t)i); 39 } 40 ··· 57 merged_iter_pqueue_check(&pq); 58 59 check(pq_entry_equal(&top, &e)); 60 - check(reftable_record_type(e.rec) == BLOCK_TYPE_REF); 61 if (last) 62 check_int(strcmp(last, e.rec->u.ref.refname), <, 0); 63 last = e.rec->u.ref.refname; ··· 76 size_t N = ARRAY_SIZE(recs), i; 77 78 for (i = 0; i < N; i++) { 79 - check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF)); 80 recs[i].u.ref.refname = (char *) "refs/heads/master"; 81 } 82 ··· 100 merged_iter_pqueue_check(&pq); 101 102 check(pq_entry_equal(&top, &e)); 103 - check(reftable_record_type(e.rec) == BLOCK_TYPE_REF); 104 check_int(e.index, ==, i); 105 if (last) 106 check_str(last, e.rec->u.ref.refname); ··· 117 size_t N = ARRAY_SIZE(recs), i; 118 119 for (i = 0; i < N; i++) { 120 - check(!reftable_record_init(&recs[i], BLOCK_TYPE_REF)); 121 recs[i].u.ref.refname = (char *) "refs/heads/master"; 122 } 123

··· 34 char *last = NULL; 35 36 for (i = 0; i < N; i++) { 37 + check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF)); 38 recs[i].u.ref.refname = xstrfmt("%02"PRIuMAX, (uintmax_t)i); 39 } 40 ··· 57 merged_iter_pqueue_check(&pq); 58 59 check(pq_entry_equal(&top, &e)); 60 + check(reftable_record_type(e.rec) == REFTABLE_BLOCK_TYPE_REF); 61 if (last) 62 check_int(strcmp(last, e.rec->u.ref.refname), <, 0); 63 last = e.rec->u.ref.refname; ··· 76 size_t N = ARRAY_SIZE(recs), i; 77 78 for (i = 0; i < N; i++) { 79 + check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF)); 80 recs[i].u.ref.refname = (char *) "refs/heads/master"; 81 } 82 ··· 100 merged_iter_pqueue_check(&pq); 101 102 check(pq_entry_equal(&top, &e)); 103 + check(reftable_record_type(e.rec) == REFTABLE_BLOCK_TYPE_REF); 104 check_int(e.index, ==, i); 105 if (last) 106 check_str(last, e.rec->u.ref.refname); ··· 117 size_t N = ARRAY_SIZE(recs), i; 118 119 for (i = 0; i < N; i++) { 120 + check(!reftable_record_init(&recs[i], REFTABLE_BLOCK_TYPE_REF)); 121 recs[i].u.ref.refname = (char *) "refs/heads/master"; 122 } 123

-96

t/unit-tests/t-reftable-reader.c

··· 1 - #include "test-lib.h" 2 - #include "lib-reftable.h" 3 - #include "reftable/blocksource.h" 4 - #include "reftable/reader.h" 5 - 6 - static int t_reader_seek_once(void) 7 - { 8 - struct reftable_ref_record records[] = { 9 - { 10 - .refname = (char *) "refs/heads/main", 11 - .value_type = REFTABLE_REF_VAL1, 12 - .value.val1 = { 42 }, 13 - }, 14 - }; 15 - struct reftable_block_source source = { 0 }; 16 - struct reftable_ref_record ref = { 0 }; 17 - struct reftable_iterator it = { 0 }; 18 - struct reftable_reader *reader; 19 - struct reftable_buf buf = REFTABLE_BUF_INIT; 20 - int ret; 21 - 22 - t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL); 23 - block_source_from_buf(&source, &buf); 24 - 25 - ret = reftable_reader_new(&reader, &source, "name"); 26 - check(!ret); 27 - 28 - reftable_reader_init_ref_iterator(reader, &it); 29 - ret = reftable_iterator_seek_ref(&it, ""); 30 - check(!ret); 31 - ret = reftable_iterator_next_ref(&it, &ref); 32 - check(!ret); 33 - 34 - ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1); 35 - check_int(ret, ==, 1); 36 - 37 - ret = reftable_iterator_next_ref(&it, &ref); 38 - check_int(ret, ==, 1); 39 - 40 - reftable_ref_record_release(&ref); 41 - reftable_iterator_destroy(&it); 42 - reftable_reader_decref(reader); 43 - reftable_buf_release(&buf); 44 - return 0; 45 - } 46 - 47 - static int t_reader_reseek(void) 48 - { 49 - struct reftable_ref_record records[] = { 50 - { 51 - .refname = (char *) "refs/heads/main", 52 - .value_type = REFTABLE_REF_VAL1, 53 - .value.val1 = { 42 }, 54 - }, 55 - }; 56 - struct reftable_block_source source = { 0 }; 57 - struct reftable_ref_record ref = { 0 }; 58 - struct reftable_iterator it = { 0 }; 59 - struct reftable_reader *reader; 60 - struct reftable_buf buf = REFTABLE_BUF_INIT; 61 - int ret; 62 - 63 - t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL); 64 - block_source_from_buf(&source, &buf); 65 - 66 - ret = reftable_reader_new(&reader, &source, "name"); 67 - check(!ret); 68 - 69 - reftable_reader_init_ref_iterator(reader, &it); 70 - 71 - for (size_t i = 0; i < 5; i++) { 72 - ret = reftable_iterator_seek_ref(&it, ""); 73 - check(!ret); 74 - ret = reftable_iterator_next_ref(&it, &ref); 75 - check(!ret); 76 - 77 - ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1); 78 - check_int(ret, ==, 1); 79 - 80 - ret = reftable_iterator_next_ref(&it, &ref); 81 - check_int(ret, ==, 1); 82 - } 83 - 84 - reftable_ref_record_release(&ref); 85 - reftable_iterator_destroy(&it); 86 - reftable_reader_decref(reader); 87 - reftable_buf_release(&buf); 88 - return 0; 89 - } 90 - 91 - int cmd_main(int argc UNUSED, const char *argv[] UNUSED) 92 - { 93 - TEST(t_reader_seek_once(), "reader can seek once"); 94 - TEST(t_reader_reseek(), "reader can reseek multiple times"); 95 - return test_done(); 96 - }

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

+53 -53

t/unit-tests/t-reftable-readwrite.c

··· 12 #include "lib-reftable.h" 13 #include "reftable/basics.h" 14 #include "reftable/blocksource.h" 15 - #include "reftable/reader.h" 16 #include "reftable/reftable-error.h" 17 #include "reftable/reftable-writer.h" 0 18 #include "strbuf.h" 19 20 static const int update_index = 5; ··· 23 { 24 struct reftable_buf buf = REFTABLE_BUF_INIT; 25 struct reftable_block_source source = { 0 }; 26 - struct reftable_block out = { 0 }; 27 int n; 28 uint8_t in[] = "hello"; 29 check(!reftable_buf_add(&buf, in, sizeof(in))); 30 block_source_from_buf(&source, &buf); 31 check_int(block_source_size(&source), ==, 6); 32 - n = block_source_read_block(&source, &out, 0, sizeof(in)); 33 check_int(n, ==, sizeof(in)); 34 check(!memcmp(in, out.data, n)); 35 - reftable_block_done(&out); 36 37 - n = block_source_read_block(&source, &out, 1, 2); 38 check_int(n, ==, 2); 39 check(!memcmp(out.data, "el", 2)); 40 41 - reftable_block_done(&out); 42 block_source_close(&source); 43 reftable_buf_release(&buf); 44 } ··· 204 struct reftable_ref_record ref = { 0 }; 205 struct reftable_log_record log = { 0 }; 206 struct reftable_iterator it = { 0 }; 207 - struct reftable_reader *reader; 208 struct reftable_block_source source = { 0 }; 209 struct reftable_buf buf = REFTABLE_BUF_INIT; 210 struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); ··· 254 255 block_source_from_buf(&source, &buf); 256 257 - err = reftable_reader_new(&reader, &source, "file.log"); 258 check(!err); 259 260 - err = reftable_reader_init_ref_iterator(reader, &it); 261 check(!err); 262 263 err = reftable_iterator_seek_ref(&it, names[N - 1]); ··· 273 reftable_iterator_destroy(&it); 274 reftable_ref_record_release(&ref); 275 276 - err = reftable_reader_init_log_iterator(reader, &it); 277 check(!err); 278 err = reftable_iterator_seek_log(&it, ""); 279 check(!err); ··· 294 /* cleanup. */ 295 reftable_buf_release(&buf); 296 free_names(names); 297 - reftable_reader_decref(reader); 298 } 299 300 static void t_log_zlib_corruption(void) ··· 303 .block_size = 256, 304 }; 305 struct reftable_iterator it = { 0 }; 306 - struct reftable_reader *reader; 307 struct reftable_block_source source = { 0 }; 308 struct reftable_buf buf = REFTABLE_BUF_INIT; 309 struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); ··· 345 346 block_source_from_buf(&source, &buf); 347 348 - err = reftable_reader_new(&reader, &source, "file.log"); 349 check(!err); 350 351 - err = reftable_reader_init_log_iterator(reader, &it); 352 check(!err); 353 err = reftable_iterator_seek_log(&it, "refname"); 354 check_int(err, ==, REFTABLE_ZLIB_ERROR); ··· 356 reftable_iterator_destroy(&it); 357 358 /* cleanup. */ 359 - reftable_reader_decref(reader); 360 reftable_buf_release(&buf); 361 } 362 ··· 367 int N = 50; 368 struct reftable_iterator it = { 0 }; 369 struct reftable_block_source source = { 0 }; 370 - struct reftable_reader *reader; 371 int err = 0; 372 int j = 0; 373 ··· 375 376 block_source_from_buf(&source, &buf); 377 378 - err = reftable_reader_new(&reader, &source, "file.ref"); 379 check(!err); 380 381 - err = reftable_reader_init_ref_iterator(reader, &it); 382 check(!err); 383 err = reftable_iterator_seek_ref(&it, ""); 384 check(!err); ··· 396 check_int(j, ==, N); 397 398 reftable_iterator_destroy(&it); 399 - reftable_reader_decref(reader); 400 reftable_buf_release(&buf); 401 free_names(names); 402 } ··· 417 char **names; 418 struct reftable_buf buf = REFTABLE_BUF_INIT; 419 int N = 50; 420 - struct reftable_reader *reader; 421 struct reftable_block_source source = { 0 }; 422 int err; 423 struct reftable_log_record log = { 0 }; ··· 427 428 block_source_from_buf(&source, &buf); 429 430 - err = reftable_reader_new(&reader, &source, "file.ref"); 431 check(!err); 432 433 - err = reftable_reader_init_ref_iterator(reader, &it); 434 check(!err); 435 err = reftable_iterator_seek_ref(&it, names[0]); 436 check(!err); ··· 441 reftable_buf_release(&buf); 442 free_names(names); 443 reftable_iterator_destroy(&it); 444 - reftable_reader_decref(reader); 445 reftable_buf_release(&buf); 446 } 447 ··· 450 char **names; 451 struct reftable_buf buf = REFTABLE_BUF_INIT; 452 int N = 50; 453 - struct reftable_reader *reader; 454 struct reftable_block_source source = { 0 }; 455 int err; 456 int i = 0; ··· 463 464 block_source_from_buf(&source, &buf); 465 466 - err = reftable_reader_new(&reader, &source, "file.ref"); 467 check(!err); 468 - check_int(hash_id, ==, reftable_reader_hash_id(reader)); 469 470 if (!index) { 471 - reader->ref_offsets.index_offset = 0; 472 } else { 473 - check_int(reader->ref_offsets.index_offset, >, 0); 474 } 475 476 for (i = 1; i < N; i++) { 477 - err = reftable_reader_init_ref_iterator(reader, &it); 478 check(!err); 479 err = reftable_iterator_seek_ref(&it, names[i]); 480 check(!err); ··· 491 check(!reftable_buf_addstr(&pastLast, names[N - 1])); 492 check(!reftable_buf_addstr(&pastLast, "/")); 493 494 - err = reftable_reader_init_ref_iterator(reader, &it); 495 check(!err); 496 err = reftable_iterator_seek_ref(&it, pastLast.buf); 497 if (err == 0) { ··· 507 508 reftable_buf_release(&buf); 509 free_names(names); 510 - reftable_reader_decref(reader); 511 } 512 513 static void t_table_read_write_seek_linear(void) ··· 535 .block_size = 256, 536 }; 537 struct reftable_ref_record ref = { 0 }; 538 - struct reftable_reader *reader; 539 struct reftable_block_source source = { 0 }; 540 struct reftable_buf buf = REFTABLE_BUF_INIT; 541 struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); ··· 585 586 block_source_from_buf(&source, &buf); 587 588 - err = reftable_reader_new(&reader, &source, "file.ref"); 589 check(!err); 590 if (!indexed) 591 - reader->obj_offsets.is_present = 0; 592 593 - err = reftable_reader_init_ref_iterator(reader, &it); 594 check(!err); 595 err = reftable_iterator_seek_ref(&it, ""); 596 check(!err); 597 reftable_iterator_destroy(&it); 598 599 - err = reftable_reader_refs_for(reader, &it, want_hash); 600 check(!err); 601 602 for (j = 0; ; j++) { ··· 613 reftable_buf_release(&buf); 614 free_names(want_names); 615 reftable_iterator_destroy(&it); 616 - reftable_reader_decref(reader); 617 } 618 619 static void t_table_refs_for_no_index(void) ··· 632 struct reftable_buf buf = REFTABLE_BUF_INIT; 633 struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); 634 struct reftable_block_source source = { 0 }; 635 - struct reftable_reader *rd = NULL; 636 struct reftable_ref_record rec = { 0 }; 637 struct reftable_iterator it = { 0 }; 638 int err; ··· 647 648 block_source_from_buf(&source, &buf); 649 650 - err = reftable_reader_new(&rd, &source, "filename"); 651 check(!err); 652 653 - err = reftable_reader_init_ref_iterator(rd, &it); 654 check(!err); 655 err = reftable_iterator_seek_ref(&it, ""); 656 check(!err); ··· 659 check_int(err, >, 0); 660 661 reftable_iterator_destroy(&it); 662 - reftable_reader_decref(rd); 663 reftable_buf_release(&buf); 664 } 665 ··· 803 struct reftable_iterator it = { 0 }; 804 const struct reftable_stats *stats; 805 struct reftable_writer *writer; 806 - struct reftable_reader *reader; 807 char buf[128]; 808 int err, i; 809 ··· 852 check_int(stats->log_stats.index_offset, >, 0); 853 854 block_source_from_buf(&source, &writer_buf); 855 - err = reftable_reader_new(&reader, &source, "filename"); 856 check(!err); 857 858 /* 859 * Seeking the log uses the log index now. In case there is any 860 * confusion regarding indices we would notice here. 861 */ 862 - err = reftable_reader_init_log_iterator(reader, &it); 863 check(!err); 864 err = reftable_iterator_seek_log(&it, ""); 865 check(!err); 866 867 reftable_iterator_destroy(&it); 868 reftable_writer_free(writer); 869 - reftable_reader_decref(reader); 870 reftable_buf_release(&writer_buf); 871 } 872 ··· 880 struct reftable_iterator it = { 0 }; 881 const struct reftable_stats *stats; 882 struct reftable_writer *writer; 883 - struct reftable_reader *reader; 884 int err; 885 886 writer = t_reftable_strbuf_writer(&writer_buf, &opts); ··· 909 check_int(stats->ref_stats.max_index_level, ==, 2); 910 911 block_source_from_buf(&source, &writer_buf); 912 - err = reftable_reader_new(&reader, &source, "filename"); 913 check(!err); 914 915 /* 916 * Seeking the last ref should work as expected. 917 */ 918 - err = reftable_reader_init_ref_iterator(reader, &it); 919 check(!err); 920 err = reftable_iterator_seek_ref(&it, "refs/heads/199"); 921 check(!err); 922 923 reftable_iterator_destroy(&it); 924 reftable_writer_free(writer); 925 - reftable_reader_decref(reader); 926 reftable_buf_release(&writer_buf); 927 reftable_buf_release(&buf); 928 } ··· 931 { 932 struct reftable_buf buf = REFTABLE_BUF_INIT; 933 struct reftable_block_source source = { 0 }; 934 - struct reftable_reader *reader; 935 int err; 936 937 block_source_from_buf(&source, &buf); 938 - err = reftable_reader_new(&reader, &source, "file.log"); 939 check_int(err, ==, REFTABLE_FORMAT_ERROR); 940 } 941 ··· 944 uint8_t zeros[1024] = { 0 }; 945 struct reftable_buf buf = REFTABLE_BUF_INIT; 946 struct reftable_block_source source = { 0 }; 947 - struct reftable_reader *reader; 948 int err; 949 check(!reftable_buf_add(&buf, zeros, sizeof(zeros))); 950 951 block_source_from_buf(&source, &buf); 952 - err = reftable_reader_new(&reader, &source, "file.log"); 953 check_int(err, ==, REFTABLE_FORMAT_ERROR); 954 955 reftable_buf_release(&buf);

··· 12 #include "lib-reftable.h" 13 #include "reftable/basics.h" 14 #include "reftable/blocksource.h" 0 15 #include "reftable/reftable-error.h" 16 #include "reftable/reftable-writer.h" 17 + #include "reftable/table.h" 18 #include "strbuf.h" 19 20 static const int update_index = 5; ··· 23 { 24 struct reftable_buf buf = REFTABLE_BUF_INIT; 25 struct reftable_block_source source = { 0 }; 26 + struct reftable_block_data out = { 0 }; 27 int n; 28 uint8_t in[] = "hello"; 29 check(!reftable_buf_add(&buf, in, sizeof(in))); 30 block_source_from_buf(&source, &buf); 31 check_int(block_source_size(&source), ==, 6); 32 + n = block_source_read_data(&source, &out, 0, sizeof(in)); 33 check_int(n, ==, sizeof(in)); 34 check(!memcmp(in, out.data, n)); 35 + block_source_release_data(&out); 36 37 + n = block_source_read_data(&source, &out, 1, 2); 38 check_int(n, ==, 2); 39 check(!memcmp(out.data, "el", 2)); 40 41 + block_source_release_data(&out); 42 block_source_close(&source); 43 reftable_buf_release(&buf); 44 } ··· 204 struct reftable_ref_record ref = { 0 }; 205 struct reftable_log_record log = { 0 }; 206 struct reftable_iterator it = { 0 }; 207 + struct reftable_table *table; 208 struct reftable_block_source source = { 0 }; 209 struct reftable_buf buf = REFTABLE_BUF_INIT; 210 struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); ··· 254 255 block_source_from_buf(&source, &buf); 256 257 + err = reftable_table_new(&table, &source, "file.log"); 258 check(!err); 259 260 + err = reftable_table_init_ref_iterator(table, &it); 261 check(!err); 262 263 err = reftable_iterator_seek_ref(&it, names[N - 1]); ··· 273 reftable_iterator_destroy(&it); 274 reftable_ref_record_release(&ref); 275 276 + err = reftable_table_init_log_iterator(table, &it); 277 check(!err); 278 err = reftable_iterator_seek_log(&it, ""); 279 check(!err); ··· 294 /* cleanup. */ 295 reftable_buf_release(&buf); 296 free_names(names); 297 + reftable_table_decref(table); 298 } 299 300 static void t_log_zlib_corruption(void) ··· 303 .block_size = 256, 304 }; 305 struct reftable_iterator it = { 0 }; 306 + struct reftable_table *table; 307 struct reftable_block_source source = { 0 }; 308 struct reftable_buf buf = REFTABLE_BUF_INIT; 309 struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); ··· 345 346 block_source_from_buf(&source, &buf); 347 348 + err = reftable_table_new(&table, &source, "file.log"); 349 check(!err); 350 351 + err = reftable_table_init_log_iterator(table, &it); 352 check(!err); 353 err = reftable_iterator_seek_log(&it, "refname"); 354 check_int(err, ==, REFTABLE_ZLIB_ERROR); ··· 356 reftable_iterator_destroy(&it); 357 358 /* cleanup. */ 359 + reftable_table_decref(table); 360 reftable_buf_release(&buf); 361 } 362 ··· 367 int N = 50; 368 struct reftable_iterator it = { 0 }; 369 struct reftable_block_source source = { 0 }; 370 + struct reftable_table *table; 371 int err = 0; 372 int j = 0; 373 ··· 375 376 block_source_from_buf(&source, &buf); 377 378 + err = reftable_table_new(&table, &source, "file.ref"); 379 check(!err); 380 381 + err = reftable_table_init_ref_iterator(table, &it); 382 check(!err); 383 err = reftable_iterator_seek_ref(&it, ""); 384 check(!err); ··· 396 check_int(j, ==, N); 397 398 reftable_iterator_destroy(&it); 399 + reftable_table_decref(table); 400 reftable_buf_release(&buf); 401 free_names(names); 402 } ··· 417 char **names; 418 struct reftable_buf buf = REFTABLE_BUF_INIT; 419 int N = 50; 420 + struct reftable_table *table; 421 struct reftable_block_source source = { 0 }; 422 int err; 423 struct reftable_log_record log = { 0 }; ··· 427 428 block_source_from_buf(&source, &buf); 429 430 + err = reftable_table_new(&table, &source, "file.ref"); 431 check(!err); 432 433 + err = reftable_table_init_ref_iterator(table, &it); 434 check(!err); 435 err = reftable_iterator_seek_ref(&it, names[0]); 436 check(!err); ··· 441 reftable_buf_release(&buf); 442 free_names(names); 443 reftable_iterator_destroy(&it); 444 + reftable_table_decref(table); 445 reftable_buf_release(&buf); 446 } 447 ··· 450 char **names; 451 struct reftable_buf buf = REFTABLE_BUF_INIT; 452 int N = 50; 453 + struct reftable_table *table; 454 struct reftable_block_source source = { 0 }; 455 int err; 456 int i = 0; ··· 463 464 block_source_from_buf(&source, &buf); 465 466 + err = reftable_table_new(&table, &source, "file.ref"); 467 check(!err); 468 + check_int(hash_id, ==, reftable_table_hash_id(table)); 469 470 if (!index) { 471 + table->ref_offsets.index_offset = 0; 472 } else { 473 + check_int(table->ref_offsets.index_offset, >, 0); 474 } 475 476 for (i = 1; i < N; i++) { 477 + err = reftable_table_init_ref_iterator(table, &it); 478 check(!err); 479 err = reftable_iterator_seek_ref(&it, names[i]); 480 check(!err); ··· 491 check(!reftable_buf_addstr(&pastLast, names[N - 1])); 492 check(!reftable_buf_addstr(&pastLast, "/")); 493 494 + err = reftable_table_init_ref_iterator(table, &it); 495 check(!err); 496 err = reftable_iterator_seek_ref(&it, pastLast.buf); 497 if (err == 0) { ··· 507 508 reftable_buf_release(&buf); 509 free_names(names); 510 + reftable_table_decref(table); 511 } 512 513 static void t_table_read_write_seek_linear(void) ··· 535 .block_size = 256, 536 }; 537 struct reftable_ref_record ref = { 0 }; 538 + struct reftable_table *table; 539 struct reftable_block_source source = { 0 }; 540 struct reftable_buf buf = REFTABLE_BUF_INIT; 541 struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); ··· 585 586 block_source_from_buf(&source, &buf); 587 588 + err = reftable_table_new(&table, &source, "file.ref"); 589 check(!err); 590 if (!indexed) 591 + table->obj_offsets.is_present = 0; 592 593 + err = reftable_table_init_ref_iterator(table, &it); 594 check(!err); 595 err = reftable_iterator_seek_ref(&it, ""); 596 check(!err); 597 reftable_iterator_destroy(&it); 598 599 + err = reftable_table_refs_for(table, &it, want_hash); 600 check(!err); 601 602 for (j = 0; ; j++) { ··· 613 reftable_buf_release(&buf); 614 free_names(want_names); 615 reftable_iterator_destroy(&it); 616 + reftable_table_decref(table); 617 } 618 619 static void t_table_refs_for_no_index(void) ··· 632 struct reftable_buf buf = REFTABLE_BUF_INIT; 633 struct reftable_writer *w = t_reftable_strbuf_writer(&buf, &opts); 634 struct reftable_block_source source = { 0 }; 635 + struct reftable_table *table = NULL; 636 struct reftable_ref_record rec = { 0 }; 637 struct reftable_iterator it = { 0 }; 638 int err; ··· 647 648 block_source_from_buf(&source, &buf); 649 650 + err = reftable_table_new(&table, &source, "filename"); 651 check(!err); 652 653 + err = reftable_table_init_ref_iterator(table, &it); 654 check(!err); 655 err = reftable_iterator_seek_ref(&it, ""); 656 check(!err); ··· 659 check_int(err, >, 0); 660 661 reftable_iterator_destroy(&it); 662 + reftable_table_decref(table); 663 reftable_buf_release(&buf); 664 } 665 ··· 803 struct reftable_iterator it = { 0 }; 804 const struct reftable_stats *stats; 805 struct reftable_writer *writer; 806 + struct reftable_table *table; 807 char buf[128]; 808 int err, i; 809 ··· 852 check_int(stats->log_stats.index_offset, >, 0); 853 854 block_source_from_buf(&source, &writer_buf); 855 + err = reftable_table_new(&table, &source, "filename"); 856 check(!err); 857 858 /* 859 * Seeking the log uses the log index now. In case there is any 860 * confusion regarding indices we would notice here. 861 */ 862 + err = reftable_table_init_log_iterator(table, &it); 863 check(!err); 864 err = reftable_iterator_seek_log(&it, ""); 865 check(!err); 866 867 reftable_iterator_destroy(&it); 868 reftable_writer_free(writer); 869 + reftable_table_decref(table); 870 reftable_buf_release(&writer_buf); 871 } 872 ··· 880 struct reftable_iterator it = { 0 }; 881 const struct reftable_stats *stats; 882 struct reftable_writer *writer; 883 + struct reftable_table *table; 884 int err; 885 886 writer = t_reftable_strbuf_writer(&writer_buf, &opts); ··· 909 check_int(stats->ref_stats.max_index_level, ==, 2); 910 911 block_source_from_buf(&source, &writer_buf); 912 + err = reftable_table_new(&table, &source, "filename"); 913 check(!err); 914 915 /* 916 * Seeking the last ref should work as expected. 917 */ 918 + err = reftable_table_init_ref_iterator(table, &it); 919 check(!err); 920 err = reftable_iterator_seek_ref(&it, "refs/heads/199"); 921 check(!err); 922 923 reftable_iterator_destroy(&it); 924 reftable_writer_free(writer); 925 + reftable_table_decref(table); 926 reftable_buf_release(&writer_buf); 927 reftable_buf_release(&buf); 928 } ··· 931 { 932 struct reftable_buf buf = REFTABLE_BUF_INIT; 933 struct reftable_block_source source = { 0 }; 934 + struct reftable_table *table; 935 int err; 936 937 block_source_from_buf(&source, &buf); 938 + err = reftable_table_new(&table, &source, "file.log"); 939 check_int(err, ==, REFTABLE_FORMAT_ERROR); 940 } 941 ··· 944 uint8_t zeros[1024] = { 0 }; 945 struct reftable_buf buf = REFTABLE_BUF_INIT; 946 struct reftable_block_source source = { 0 }; 947 + struct reftable_table *table; 948 int err; 949 check(!reftable_buf_add(&buf, zeros, sizeof(zeros))); 950 951 block_source_from_buf(&source, &buf); 952 + err = reftable_table_new(&table, &source, "file.log"); 953 check_int(err, ==, REFTABLE_FORMAT_ERROR); 954 955 reftable_buf_release(&buf);

+20 -20

t/unit-tests/t-reftable-record.c

··· 84 { 85 struct reftable_record in[3] = { 86 { 87 - .type = BLOCK_TYPE_REF, 88 .u.ref.refname = (char *) "refs/heads/master", 89 .u.ref.value_type = REFTABLE_REF_VAL1, 90 }, 91 { 92 - .type = BLOCK_TYPE_REF, 93 .u.ref.refname = (char *) "refs/heads/master", 94 .u.ref.value_type = REFTABLE_REF_DELETION, 95 }, 96 { 97 - .type = BLOCK_TYPE_REF, 98 .u.ref.refname = (char *) "HEAD", 99 .u.ref.value_type = REFTABLE_REF_SYMREF, 100 .u.ref.value.symref = (char *) "refs/heads/master", ··· 141 142 for (int i = REFTABLE_REF_DELETION; i < REFTABLE_NR_REF_VALUETYPES; i++) { 143 struct reftable_record in = { 144 - .type = BLOCK_TYPE_REF, 145 .u.ref.value_type = i, 146 }; 147 - struct reftable_record out = { .type = BLOCK_TYPE_REF }; 148 struct reftable_buf key = REFTABLE_BUF_INIT; 149 uint8_t buffer[1024] = { 0 }; 150 struct string_view dest = { ··· 198 { 199 struct reftable_record in[3] = { 200 { 201 - .type = BLOCK_TYPE_LOG, 202 .u.log.refname = (char *) "refs/heads/master", 203 .u.log.update_index = 42, 204 }, 205 { 206 - .type = BLOCK_TYPE_LOG, 207 .u.log.refname = (char *) "refs/heads/master", 208 .u.log.update_index = 22, 209 }, 210 { 211 - .type = BLOCK_TYPE_LOG, 212 .u.log.refname = (char *) "refs/heads/main", 213 .u.log.update_index = 22, 214 }, ··· 297 check(!reftable_log_record_is_deletion(&in[2])); 298 299 for (size_t i = 0; i < ARRAY_SIZE(in); i++) { 300 - struct reftable_record rec = { .type = BLOCK_TYPE_LOG }; 301 struct reftable_buf key = REFTABLE_BUF_INIT; 302 uint8_t buffer[1024] = { 0 }; 303 struct string_view dest = { ··· 306 }; 307 /* populate out, to check for leaks. */ 308 struct reftable_record out = { 309 - .type = BLOCK_TYPE_LOG, 310 .u.log = { 311 .refname = xstrdup("old name"), 312 .value_type = REFTABLE_LOG_UPDATE, ··· 384 uint64_t offsets[] = { 0, 16, 32, 48, 64, 80, 96, 112}; 385 struct reftable_record in[3] = { 386 { 387 - .type = BLOCK_TYPE_OBJ, 388 .u.obj.hash_prefix = id_bytes, 389 .u.obj.hash_prefix_len = 7, 390 .u.obj.offsets = offsets, 391 .u.obj.offset_len = 8, 392 }, 393 { 394 - .type = BLOCK_TYPE_OBJ, 395 .u.obj.hash_prefix = id_bytes, 396 .u.obj.hash_prefix_len = 7, 397 .u.obj.offsets = offsets, 398 .u.obj.offset_len = 5, 399 }, 400 { 401 - .type = BLOCK_TYPE_OBJ, 402 .u.obj.hash_prefix = id_bytes, 403 .u.obj.hash_prefix_len = 5, 404 }, ··· 450 .len = sizeof(buffer), 451 }; 452 struct reftable_record in = { 453 - .type = BLOCK_TYPE_OBJ, 454 .u = { 455 .obj = recs[i], 456 }, 457 }; 458 struct reftable_buf key = REFTABLE_BUF_INIT; 459 - struct reftable_record out = { .type = BLOCK_TYPE_OBJ }; 460 int n, m; 461 uint8_t extra; 462 ··· 482 { 483 struct reftable_record in[3] = { 484 { 485 - .type = BLOCK_TYPE_INDEX, 486 .u.idx.offset = 22, 487 .u.idx.last_key = REFTABLE_BUF_INIT, 488 }, 489 { 490 - .type = BLOCK_TYPE_INDEX, 491 .u.idx.offset = 32, 492 .u.idx.last_key = REFTABLE_BUF_INIT, 493 }, 494 { 495 - .type = BLOCK_TYPE_INDEX, 496 .u.idx.offset = 32, 497 .u.idx.last_key = REFTABLE_BUF_INIT, 498 }, ··· 523 static void t_reftable_index_record_roundtrip(void) 524 { 525 struct reftable_record in = { 526 - .type = BLOCK_TYPE_INDEX, 527 .u.idx = { 528 .offset = 42, 529 .last_key = REFTABLE_BUF_INIT, ··· 537 struct reftable_buf scratch = REFTABLE_BUF_INIT; 538 struct reftable_buf key = REFTABLE_BUF_INIT; 539 struct reftable_record out = { 540 - .type = BLOCK_TYPE_INDEX, 541 .u.idx = { .last_key = REFTABLE_BUF_INIT }, 542 }; 543 int n, m;

··· 84 { 85 struct reftable_record in[3] = { 86 { 87 + .type = REFTABLE_BLOCK_TYPE_REF, 88 .u.ref.refname = (char *) "refs/heads/master", 89 .u.ref.value_type = REFTABLE_REF_VAL1, 90 }, 91 { 92 + .type = REFTABLE_BLOCK_TYPE_REF, 93 .u.ref.refname = (char *) "refs/heads/master", 94 .u.ref.value_type = REFTABLE_REF_DELETION, 95 }, 96 { 97 + .type = REFTABLE_BLOCK_TYPE_REF, 98 .u.ref.refname = (char *) "HEAD", 99 .u.ref.value_type = REFTABLE_REF_SYMREF, 100 .u.ref.value.symref = (char *) "refs/heads/master", ··· 141 142 for (int i = REFTABLE_REF_DELETION; i < REFTABLE_NR_REF_VALUETYPES; i++) { 143 struct reftable_record in = { 144 + .type = REFTABLE_BLOCK_TYPE_REF, 145 .u.ref.value_type = i, 146 }; 147 + struct reftable_record out = { .type = REFTABLE_BLOCK_TYPE_REF }; 148 struct reftable_buf key = REFTABLE_BUF_INIT; 149 uint8_t buffer[1024] = { 0 }; 150 struct string_view dest = { ··· 198 { 199 struct reftable_record in[3] = { 200 { 201 + .type = REFTABLE_BLOCK_TYPE_LOG, 202 .u.log.refname = (char *) "refs/heads/master", 203 .u.log.update_index = 42, 204 }, 205 { 206 + .type = REFTABLE_BLOCK_TYPE_LOG, 207 .u.log.refname = (char *) "refs/heads/master", 208 .u.log.update_index = 22, 209 }, 210 { 211 + .type = REFTABLE_BLOCK_TYPE_LOG, 212 .u.log.refname = (char *) "refs/heads/main", 213 .u.log.update_index = 22, 214 }, ··· 297 check(!reftable_log_record_is_deletion(&in[2])); 298 299 for (size_t i = 0; i < ARRAY_SIZE(in); i++) { 300 + struct reftable_record rec = { .type = REFTABLE_BLOCK_TYPE_LOG }; 301 struct reftable_buf key = REFTABLE_BUF_INIT; 302 uint8_t buffer[1024] = { 0 }; 303 struct string_view dest = { ··· 306 }; 307 /* populate out, to check for leaks. */ 308 struct reftable_record out = { 309 + .type = REFTABLE_BLOCK_TYPE_LOG, 310 .u.log = { 311 .refname = xstrdup("old name"), 312 .value_type = REFTABLE_LOG_UPDATE, ··· 384 uint64_t offsets[] = { 0, 16, 32, 48, 64, 80, 96, 112}; 385 struct reftable_record in[3] = { 386 { 387 + .type = REFTABLE_BLOCK_TYPE_OBJ, 388 .u.obj.hash_prefix = id_bytes, 389 .u.obj.hash_prefix_len = 7, 390 .u.obj.offsets = offsets, 391 .u.obj.offset_len = 8, 392 }, 393 { 394 + .type = REFTABLE_BLOCK_TYPE_OBJ, 395 .u.obj.hash_prefix = id_bytes, 396 .u.obj.hash_prefix_len = 7, 397 .u.obj.offsets = offsets, 398 .u.obj.offset_len = 5, 399 }, 400 { 401 + .type = REFTABLE_BLOCK_TYPE_OBJ, 402 .u.obj.hash_prefix = id_bytes, 403 .u.obj.hash_prefix_len = 5, 404 }, ··· 450 .len = sizeof(buffer), 451 }; 452 struct reftable_record in = { 453 + .type = REFTABLE_BLOCK_TYPE_OBJ, 454 .u = { 455 .obj = recs[i], 456 }, 457 }; 458 struct reftable_buf key = REFTABLE_BUF_INIT; 459 + struct reftable_record out = { .type = REFTABLE_BLOCK_TYPE_OBJ }; 460 int n, m; 461 uint8_t extra; 462 ··· 482 { 483 struct reftable_record in[3] = { 484 { 485 + .type = REFTABLE_BLOCK_TYPE_INDEX, 486 .u.idx.offset = 22, 487 .u.idx.last_key = REFTABLE_BUF_INIT, 488 }, 489 { 490 + .type = REFTABLE_BLOCK_TYPE_INDEX, 491 .u.idx.offset = 32, 492 .u.idx.last_key = REFTABLE_BUF_INIT, 493 }, 494 { 495 + .type = REFTABLE_BLOCK_TYPE_INDEX, 496 .u.idx.offset = 32, 497 .u.idx.last_key = REFTABLE_BUF_INIT, 498 }, ··· 523 static void t_reftable_index_record_roundtrip(void) 524 { 525 struct reftable_record in = { 526 + .type = REFTABLE_BLOCK_TYPE_INDEX, 527 .u.idx = { 528 .offset = 42, 529 .last_key = REFTABLE_BUF_INIT, ··· 537 struct reftable_buf scratch = REFTABLE_BUF_INIT; 538 struct reftable_buf key = REFTABLE_BUF_INIT; 539 struct reftable_record out = { 540 + .type = REFTABLE_BLOCK_TYPE_INDEX, 541 .u.idx = { .last_key = REFTABLE_BUF_INIT }, 542 }; 543 int n, m;

+33 -33

t/unit-tests/t-reftable-stack.c

··· 12 #include "lib-reftable.h" 13 #include "dir.h" 14 #include "reftable/merged.h" 15 - #include "reftable/reader.h" 16 #include "reftable/reftable-error.h" 17 #include "reftable/stack.h" 0 18 #include "strbuf.h" 19 #include "tempfile.h" 20 #include <dirent.h> ··· 176 err = reftable_stack_read_ref(st, ref.refname, &dest); 177 check(!err); 178 check(reftable_ref_record_equal(&ref, &dest, REFTABLE_HASH_SIZE_SHA1)); 179 - check_int(st->readers_len, >, 0); 180 181 #ifndef GIT_WINDOWS_NATIVE 182 check(!reftable_buf_addstr(&scratch, dir)); ··· 189 check(!reftable_buf_addstr(&scratch, dir)); 190 check(!reftable_buf_addstr(&scratch, "/")); 191 /* do not try at home; not an external API for reftable. */ 192 - check(!reftable_buf_addstr(&scratch, st->readers[0]->name)); 193 err = stat(scratch.buf, &stat_result); 194 check(!err); 195 check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); ··· 402 * all tables in the stack. 403 */ 404 if (i != n) 405 - check_int(st->merged->readers_len, ==, i + 1); 406 else 407 - check_int(st->merged->readers_len, ==, 1); 408 } 409 410 reftable_stack_destroy(st); ··· 430 431 err = reftable_stack_add(st, write_test_ref, &ref); 432 check(!err); 433 - check_int(st->merged->readers_len, ==, 1); 434 check_int(st->stats.attempts, ==, 0); 435 check_int(st->stats.failures, ==, 0); 436 ··· 441 */ 442 check(!reftable_buf_addstr(&table_path, dir)); 443 check(!reftable_buf_addstr(&table_path, "/")); 444 - check(!reftable_buf_addstr(&table_path, st->readers[0]->name)); 445 check(!reftable_buf_addstr(&table_path, ".lock")); 446 write_file_buf(table_path.buf, "", 0); 447 448 ref.update_index = 2; 449 err = reftable_stack_add(st, write_test_ref, &ref); 450 check(!err); 451 - check_int(st->merged->readers_len, ==, 2); 452 check_int(st->stats.attempts, ==, 1); 453 check_int(st->stats.failures, ==, 1); 454 ··· 592 check(!reftable_buf_addstr(&path, dir)); 593 check(!reftable_buf_addstr(&path, "/")); 594 /* do not try at home; not an external API for reftable. */ 595 - check(!reftable_buf_addstr(&path, st->readers[0]->name)); 596 err = stat(path.buf, &stat_result); 597 check(!err); 598 check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); ··· 1026 1027 err = reftable_stack_auto_compact(st); 1028 check(!err); 1029 - check(i < 2 || st->merged->readers_len < 2 * fastlogN(i, 2)); 1030 } 1031 1032 check_int(reftable_stack_compaction_stats(st)->entries_written, <, ··· 1061 err = reftable_stack_add(st, &write_test_ref, &ref); 1062 check(!err); 1063 1064 - check(i < 5 || st->merged->readers_len < 5 * fastlogN(i, 5)); 1065 } 1066 1067 reftable_stack_destroy(st); ··· 1082 check(!err); 1083 1084 write_n_ref_tables(st, 5); 1085 - check_int(st->merged->readers_len, ==, 5); 1086 1087 /* 1088 * Given that all tables we have written should be roughly the same ··· 1091 */ 1092 check(!reftable_buf_addstr(&buf, dir)); 1093 check(!reftable_buf_addstr(&buf, "/")); 1094 - check(!reftable_buf_addstr(&buf, st->readers[2]->name)); 1095 check(!reftable_buf_addstr(&buf, ".lock")); 1096 write_file_buf(buf.buf, "", 0); 1097 ··· 1104 err = reftable_stack_auto_compact(st); 1105 check(!err); 1106 check_int(st->stats.failures, ==, 0); 1107 - check_int(st->merged->readers_len, ==, 4); 1108 1109 reftable_stack_destroy(st); 1110 reftable_buf_release(&buf); ··· 1149 * all tables in the stack. 1150 */ 1151 if (i != n) 1152 - check_int(st->merged->readers_len, ==, i + 1); 1153 else 1154 - check_int(st->merged->readers_len, ==, 1); 1155 } 1156 1157 reftable_stack_destroy(st); ··· 1172 check(!err); 1173 1174 write_n_ref_tables(st, 3); 1175 - check_int(st->merged->readers_len, ==, 3); 1176 1177 /* Lock one of the tables that we're about to compact. */ 1178 check(!reftable_buf_addstr(&buf, dir)); 1179 check(!reftable_buf_addstr(&buf, "/")); 1180 - check(!reftable_buf_addstr(&buf, st->readers[1]->name)); 1181 check(!reftable_buf_addstr(&buf, ".lock")); 1182 write_file_buf(buf.buf, "", 0); 1183 ··· 1188 err = reftable_stack_compact_all(st, NULL); 1189 check_int(err, ==, REFTABLE_LOCK_ERROR); 1190 check_int(st->stats.failures, ==, 1); 1191 - check_int(st->merged->readers_len, ==, 3); 1192 1193 reftable_stack_destroy(st); 1194 reftable_buf_release(&buf); ··· 1222 static void unclean_stack_close(struct reftable_stack *st) 1223 { 1224 /* break abstraction boundary to simulate unclean shutdown. */ 1225 - for (size_t i = 0; i < st->readers_len; i++) 1226 - reftable_reader_decref(st->readers[i]); 1227 - st->readers_len = 0; 1228 - REFTABLE_FREE_AND_NULL(st->readers); 1229 } 1230 1231 static void t_reftable_stack_compaction_concurrent_clean(void) ··· 1275 err = reftable_new_stack(&st1, dir, &opts); 1276 check(!err); 1277 write_n_ref_tables(st1, 2); 1278 - check_int(st1->merged->readers_len, ==, 2); 1279 reftable_stack_init_ref_iterator(st1, &it); 1280 err = reftable_iterator_seek_ref(&it, ""); 1281 check(!err); ··· 1283 /* Set up a second stack for the same directory and compact it. */ 1284 err = reftable_new_stack(&st2, dir, &opts); 1285 check(!err); 1286 - check_int(st2->merged->readers_len, ==, 2); 1287 err = reftable_stack_compact_all(st2, NULL); 1288 check(!err); 1289 - check_int(st2->merged->readers_len, ==, 1); 1290 1291 /* 1292 * Verify that we can continue to use the old iterator even after we ··· 1294 */ 1295 err = reftable_stack_reload(st1); 1296 check(!err); 1297 - check_int(st1->merged->readers_len, ==, 1); 1298 err = reftable_iterator_next_ref(&it, &rec); 1299 check(!err); 1300 check_str(rec.refname, "refs/heads/branch-0000"); ··· 1325 err = reftable_new_stack(&st, dir, &opts); 1326 check(!err); 1327 write_n_ref_tables(st, 2); 1328 - check_int(st->merged->readers_len, ==, 2); 1329 reftable_stack_init_ref_iterator(st, &it); 1330 err = reftable_iterator_seek_ref(&it, ""); 1331 check(!err); 1332 1333 /* 1334 * Update the tables.list file with some garbage data, while reusing 1335 - * our old readers. This should trigger a partial reload of the stack, 1336 - * where we try to reuse our old readers. 1337 */ 1338 - check(!reftable_buf_addstr(&content, st->readers[0]->name)); 1339 check(!reftable_buf_addstr(&content, "\n")); 1340 - check(!reftable_buf_addstr(&content, st->readers[1]->name)); 1341 check(!reftable_buf_addstr(&content, "\n")); 1342 check(!reftable_buf_addstr(&content, "garbage\n")); 1343 check(!reftable_buf_addstr(&table_path, st->list_file)); ··· 1348 1349 err = reftable_stack_reload(st); 1350 check_int(err, ==, -4); 1351 - check_int(st->merged->readers_len, ==, 2); 1352 1353 /* 1354 * Even though the reload has failed, we should be able to continue

··· 12 #include "lib-reftable.h" 13 #include "dir.h" 14 #include "reftable/merged.h" 0 15 #include "reftable/reftable-error.h" 16 #include "reftable/stack.h" 17 + #include "reftable/table.h" 18 #include "strbuf.h" 19 #include "tempfile.h" 20 #include <dirent.h> ··· 176 err = reftable_stack_read_ref(st, ref.refname, &dest); 177 check(!err); 178 check(reftable_ref_record_equal(&ref, &dest, REFTABLE_HASH_SIZE_SHA1)); 179 + check_int(st->tables_len, >, 0); 180 181 #ifndef GIT_WINDOWS_NATIVE 182 check(!reftable_buf_addstr(&scratch, dir)); ··· 189 check(!reftable_buf_addstr(&scratch, dir)); 190 check(!reftable_buf_addstr(&scratch, "/")); 191 /* do not try at home; not an external API for reftable. */ 192 + check(!reftable_buf_addstr(&scratch, st->tables[0]->name)); 193 err = stat(scratch.buf, &stat_result); 194 check(!err); 195 check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); ··· 402 * all tables in the stack. 403 */ 404 if (i != n) 405 + check_int(st->merged->tables_len, ==, i + 1); 406 else 407 + check_int(st->merged->tables_len, ==, 1); 408 } 409 410 reftable_stack_destroy(st); ··· 430 431 err = reftable_stack_add(st, write_test_ref, &ref); 432 check(!err); 433 + check_int(st->merged->tables_len, ==, 1); 434 check_int(st->stats.attempts, ==, 0); 435 check_int(st->stats.failures, ==, 0); 436 ··· 441 */ 442 check(!reftable_buf_addstr(&table_path, dir)); 443 check(!reftable_buf_addstr(&table_path, "/")); 444 + check(!reftable_buf_addstr(&table_path, st->tables[0]->name)); 445 check(!reftable_buf_addstr(&table_path, ".lock")); 446 write_file_buf(table_path.buf, "", 0); 447 448 ref.update_index = 2; 449 err = reftable_stack_add(st, write_test_ref, &ref); 450 check(!err); 451 + check_int(st->merged->tables_len, ==, 2); 452 check_int(st->stats.attempts, ==, 1); 453 check_int(st->stats.failures, ==, 1); 454 ··· 592 check(!reftable_buf_addstr(&path, dir)); 593 check(!reftable_buf_addstr(&path, "/")); 594 /* do not try at home; not an external API for reftable. */ 595 + check(!reftable_buf_addstr(&path, st->tables[0]->name)); 596 err = stat(path.buf, &stat_result); 597 check(!err); 598 check_int((stat_result.st_mode & 0777), ==, opts.default_permissions); ··· 1026 1027 err = reftable_stack_auto_compact(st); 1028 check(!err); 1029 + check(i < 2 || st->merged->tables_len < 2 * fastlogN(i, 2)); 1030 } 1031 1032 check_int(reftable_stack_compaction_stats(st)->entries_written, <, ··· 1061 err = reftable_stack_add(st, &write_test_ref, &ref); 1062 check(!err); 1063 1064 + check(i < 5 || st->merged->tables_len < 5 * fastlogN(i, 5)); 1065 } 1066 1067 reftable_stack_destroy(st); ··· 1082 check(!err); 1083 1084 write_n_ref_tables(st, 5); 1085 + check_int(st->merged->tables_len, ==, 5); 1086 1087 /* 1088 * Given that all tables we have written should be roughly the same ··· 1091 */ 1092 check(!reftable_buf_addstr(&buf, dir)); 1093 check(!reftable_buf_addstr(&buf, "/")); 1094 + check(!reftable_buf_addstr(&buf, st->tables[2]->name)); 1095 check(!reftable_buf_addstr(&buf, ".lock")); 1096 write_file_buf(buf.buf, "", 0); 1097 ··· 1104 err = reftable_stack_auto_compact(st); 1105 check(!err); 1106 check_int(st->stats.failures, ==, 0); 1107 + check_int(st->merged->tables_len, ==, 4); 1108 1109 reftable_stack_destroy(st); 1110 reftable_buf_release(&buf); ··· 1149 * all tables in the stack. 1150 */ 1151 if (i != n) 1152 + check_int(st->merged->tables_len, ==, i + 1); 1153 else 1154 + check_int(st->merged->tables_len, ==, 1); 1155 } 1156 1157 reftable_stack_destroy(st); ··· 1172 check(!err); 1173 1174 write_n_ref_tables(st, 3); 1175 + check_int(st->merged->tables_len, ==, 3); 1176 1177 /* Lock one of the tables that we're about to compact. */ 1178 check(!reftable_buf_addstr(&buf, dir)); 1179 check(!reftable_buf_addstr(&buf, "/")); 1180 + check(!reftable_buf_addstr(&buf, st->tables[1]->name)); 1181 check(!reftable_buf_addstr(&buf, ".lock")); 1182 write_file_buf(buf.buf, "", 0); 1183 ··· 1188 err = reftable_stack_compact_all(st, NULL); 1189 check_int(err, ==, REFTABLE_LOCK_ERROR); 1190 check_int(st->stats.failures, ==, 1); 1191 + check_int(st->merged->tables_len, ==, 3); 1192 1193 reftable_stack_destroy(st); 1194 reftable_buf_release(&buf); ··· 1222 static void unclean_stack_close(struct reftable_stack *st) 1223 { 1224 /* break abstraction boundary to simulate unclean shutdown. */ 1225 + for (size_t i = 0; i < st->tables_len; i++) 1226 + reftable_table_decref(st->tables[i]); 1227 + st->tables_len = 0; 1228 + REFTABLE_FREE_AND_NULL(st->tables); 1229 } 1230 1231 static void t_reftable_stack_compaction_concurrent_clean(void) ··· 1275 err = reftable_new_stack(&st1, dir, &opts); 1276 check(!err); 1277 write_n_ref_tables(st1, 2); 1278 + check_int(st1->merged->tables_len, ==, 2); 1279 reftable_stack_init_ref_iterator(st1, &it); 1280 err = reftable_iterator_seek_ref(&it, ""); 1281 check(!err); ··· 1283 /* Set up a second stack for the same directory and compact it. */ 1284 err = reftable_new_stack(&st2, dir, &opts); 1285 check(!err); 1286 + check_int(st2->merged->tables_len, ==, 2); 1287 err = reftable_stack_compact_all(st2, NULL); 1288 check(!err); 1289 + check_int(st2->merged->tables_len, ==, 1); 1290 1291 /* 1292 * Verify that we can continue to use the old iterator even after we ··· 1294 */ 1295 err = reftable_stack_reload(st1); 1296 check(!err); 1297 + check_int(st1->merged->tables_len, ==, 1); 1298 err = reftable_iterator_next_ref(&it, &rec); 1299 check(!err); 1300 check_str(rec.refname, "refs/heads/branch-0000"); ··· 1325 err = reftable_new_stack(&st, dir, &opts); 1326 check(!err); 1327 write_n_ref_tables(st, 2); 1328 + check_int(st->merged->tables_len, ==, 2); 1329 reftable_stack_init_ref_iterator(st, &it); 1330 err = reftable_iterator_seek_ref(&it, ""); 1331 check(!err); 1332 1333 /* 1334 * Update the tables.list file with some garbage data, while reusing 1335 + * our old tables. This should trigger a partial reload of the stack, 1336 + * where we try to reuse our old tables. 1337 */ 1338 + check(!reftable_buf_addstr(&content, st->tables[0]->name)); 1339 check(!reftable_buf_addstr(&content, "\n")); 1340 + check(!reftable_buf_addstr(&content, st->tables[1]->name)); 1341 check(!reftable_buf_addstr(&content, "\n")); 1342 check(!reftable_buf_addstr(&content, "garbage\n")); 1343 check(!reftable_buf_addstr(&table_path, st->list_file)); ··· 1348 1349 err = reftable_stack_reload(st); 1350 check_int(err, ==, -4); 1351 + check_int(st->merged->tables_len, ==, 2); 1352 1353 /* 1354 * Even though the reload has failed, we should be able to continue

+206

t/unit-tests/t-reftable-table.c

··· 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0

··· 1 + #include "test-lib.h" 2 + #include "lib-reftable.h" 3 + #include "reftable/blocksource.h" 4 + #include "reftable/constants.h" 5 + #include "reftable/iter.h" 6 + #include "reftable/table.h" 7 + #include "strbuf.h" 8 + 9 + static int t_table_seek_once(void) 10 + { 11 + struct reftable_ref_record records[] = { 12 + { 13 + .refname = (char *) "refs/heads/main", 14 + .value_type = REFTABLE_REF_VAL1, 15 + .value.val1 = { 42 }, 16 + }, 17 + }; 18 + struct reftable_block_source source = { 0 }; 19 + struct reftable_ref_record ref = { 0 }; 20 + struct reftable_iterator it = { 0 }; 21 + struct reftable_table *table; 22 + struct reftable_buf buf = REFTABLE_BUF_INIT; 23 + int ret; 24 + 25 + t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL); 26 + block_source_from_buf(&source, &buf); 27 + 28 + ret = reftable_table_new(&table, &source, "name"); 29 + check(!ret); 30 + 31 + reftable_table_init_ref_iterator(table, &it); 32 + ret = reftable_iterator_seek_ref(&it, ""); 33 + check(!ret); 34 + ret = reftable_iterator_next_ref(&it, &ref); 35 + check(!ret); 36 + 37 + ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1); 38 + check_int(ret, ==, 1); 39 + 40 + ret = reftable_iterator_next_ref(&it, &ref); 41 + check_int(ret, ==, 1); 42 + 43 + reftable_ref_record_release(&ref); 44 + reftable_iterator_destroy(&it); 45 + reftable_table_decref(table); 46 + reftable_buf_release(&buf); 47 + return 0; 48 + } 49 + 50 + static int t_table_reseek(void) 51 + { 52 + struct reftable_ref_record records[] = { 53 + { 54 + .refname = (char *) "refs/heads/main", 55 + .value_type = REFTABLE_REF_VAL1, 56 + .value.val1 = { 42 }, 57 + }, 58 + }; 59 + struct reftable_block_source source = { 0 }; 60 + struct reftable_ref_record ref = { 0 }; 61 + struct reftable_iterator it = { 0 }; 62 + struct reftable_table *table; 63 + struct reftable_buf buf = REFTABLE_BUF_INIT; 64 + int ret; 65 + 66 + t_reftable_write_to_buf(&buf, records, ARRAY_SIZE(records), NULL, 0, NULL); 67 + block_source_from_buf(&source, &buf); 68 + 69 + ret = reftable_table_new(&table, &source, "name"); 70 + check(!ret); 71 + 72 + reftable_table_init_ref_iterator(table, &it); 73 + 74 + for (size_t i = 0; i < 5; i++) { 75 + ret = reftable_iterator_seek_ref(&it, ""); 76 + check(!ret); 77 + ret = reftable_iterator_next_ref(&it, &ref); 78 + check(!ret); 79 + 80 + ret = reftable_ref_record_equal(&ref, &records[0], REFTABLE_HASH_SIZE_SHA1); 81 + check_int(ret, ==, 1); 82 + 83 + ret = reftable_iterator_next_ref(&it, &ref); 84 + check_int(ret, ==, 1); 85 + } 86 + 87 + reftable_ref_record_release(&ref); 88 + reftable_iterator_destroy(&it); 89 + reftable_table_decref(table); 90 + reftable_buf_release(&buf); 91 + return 0; 92 + } 93 + 94 + static int t_table_block_iterator(void) 95 + { 96 + struct reftable_block_source source = { 0 }; 97 + struct reftable_table_iterator it = { 0 }; 98 + struct reftable_ref_record *records; 99 + const struct reftable_block *block; 100 + struct reftable_table *table; 101 + struct reftable_buf buf = REFTABLE_BUF_INIT; 102 + struct { 103 + uint8_t block_type; 104 + uint16_t header_off; 105 + uint16_t restart_count; 106 + uint16_t record_count; 107 + } expected_blocks[] = { 108 + { 109 + .block_type = REFTABLE_BLOCK_TYPE_REF, 110 + .header_off = 24, 111 + .restart_count = 10, 112 + .record_count = 158, 113 + }, 114 + { 115 + .block_type = REFTABLE_BLOCK_TYPE_REF, 116 + .restart_count = 10, 117 + .record_count = 159, 118 + }, 119 + { 120 + .block_type = REFTABLE_BLOCK_TYPE_REF, 121 + .restart_count = 10, 122 + .record_count = 159, 123 + }, 124 + { 125 + .block_type = REFTABLE_BLOCK_TYPE_REF, 126 + .restart_count = 2, 127 + .record_count = 24, 128 + }, 129 + { 130 + .block_type = REFTABLE_BLOCK_TYPE_INDEX, 131 + .restart_count = 1, 132 + .record_count = 4, 133 + }, 134 + { 135 + .block_type = REFTABLE_BLOCK_TYPE_OBJ, 136 + .restart_count = 1, 137 + .record_count = 1, 138 + }, 139 + }; 140 + const size_t nrecords = 500; 141 + int ret; 142 + 143 + REFTABLE_CALLOC_ARRAY(records, nrecords); 144 + for (size_t i = 0; i < nrecords; i++) { 145 + records[i].value_type = REFTABLE_REF_VAL1; 146 + records[i].refname = xstrfmt("refs/heads/branch-%03"PRIuMAX, 147 + (uintmax_t) i); 148 + } 149 + 150 + t_reftable_write_to_buf(&buf, records, nrecords, NULL, 0, NULL); 151 + block_source_from_buf(&source, &buf); 152 + 153 + ret = reftable_table_new(&table, &source, "name"); 154 + check(!ret); 155 + 156 + ret = reftable_table_iterator_init(&it, table); 157 + check(!ret); 158 + 159 + for (size_t i = 0; i < ARRAY_SIZE(expected_blocks); i++) { 160 + struct reftable_iterator record_it = { 0 }; 161 + struct reftable_record record = { 162 + .type = expected_blocks[i].block_type, 163 + }; 164 + 165 + ret = reftable_table_iterator_next(&it, &block); 166 + check(!ret); 167 + 168 + check_int(block->block_type, ==, expected_blocks[i].block_type); 169 + check_int(block->header_off, ==, expected_blocks[i].header_off); 170 + check_int(block->restart_count, ==, expected_blocks[i].restart_count); 171 + 172 + ret = reftable_block_init_iterator(block, &record_it); 173 + check(!ret); 174 + 175 + for (size_t j = 0; ; j++) { 176 + ret = iterator_next(&record_it, &record); 177 + if (ret > 0) { 178 + check_int(j, ==, expected_blocks[i].record_count); 179 + break; 180 + } 181 + check(!ret); 182 + } 183 + 184 + reftable_iterator_destroy(&record_it); 185 + reftable_record_release(&record); 186 + } 187 + 188 + ret = reftable_table_iterator_next(&it, &block); 189 + check_int(ret, ==, 1); 190 + 191 + for (size_t i = 0; i < nrecords; i++) 192 + reftable_free(records[i].refname); 193 + reftable_table_iterator_release(&it); 194 + reftable_table_decref(table); 195 + reftable_buf_release(&buf); 196 + reftable_free(records); 197 + return 0; 198 + } 199 + 200 + int cmd_main(int argc UNUSED, const char *argv[] UNUSED) 201 + { 202 + TEST(t_table_seek_once(), "table can seek once"); 203 + TEST(t_table_reseek(), "table can reseek multiple times"); 204 + TEST(t_table_block_iterator(), "table can iterate through blocks"); 205 + return test_done(); 206 + }