Merge remote-tracking branch 'remotes/maxreitz/tags/pull-block-2020-03-11' into staging

+4 -2

block/backup-top.c

··· 38 38 BlockCopyState *bcs; 39 39 BdrvChild *target; 40 40 bool active; 41 + int64_t cluster_size; 41 42 } BDRVBackupTopState; 42 43 43 44 static coroutine_fn int backup_top_co_preadv( ··· 57 58 return 0; 58 59 } 59 60 60 - off = QEMU_ALIGN_DOWN(offset, s->bcs->cluster_size); 61 - end = QEMU_ALIGN_UP(offset + bytes, s->bcs->cluster_size); 61 + off = QEMU_ALIGN_DOWN(offset, s->cluster_size); 62 + end = QEMU_ALIGN_UP(offset + bytes, s->cluster_size); 62 63 63 64 return block_copy(s->bcs, off, end - off, NULL); 64 65 } ··· 238 239 goto fail; 239 240 } 240 241 242 + state->cluster_size = cluster_size; 241 243 state->bcs = block_copy_state_new(top->backing, state->target, 242 244 cluster_size, write_flags, &local_err); 243 245 if (local_err) {

+15 -23

block/backup.c

··· 57 57 BackupBlockJob *s = opaque; 58 58 59 59 s->bytes_read += bytes; 60 - job_progress_update(&s->common.job, bytes); 61 - } 62 - 63 - static void backup_progress_reset_callback(void *opaque) 64 - { 65 - BackupBlockJob *s = opaque; 66 - uint64_t estimate = bdrv_get_dirty_count(s->bcs->copy_bitmap); 67 - 68 - job_progress_set_remaining(&s->common.job, estimate); 69 60 } 70 61 71 62 static int coroutine_fn backup_do_cow(BackupBlockJob *job, ··· 111 102 112 103 if (ret < 0 && job->bitmap_mode == BITMAP_SYNC_MODE_ALWAYS) { 113 104 /* If we failed and synced, merge in the bits we didn't copy: */ 114 - bdrv_dirty_bitmap_merge_internal(bm, job->bcs->copy_bitmap, 105 + bdrv_dirty_bitmap_merge_internal(bm, block_copy_dirty_bitmap(job->bcs), 115 106 NULL, true); 116 107 } 117 108 } ··· 154 145 return; 155 146 } 156 147 157 - bdrv_set_dirty_bitmap(backup_job->bcs->copy_bitmap, 0, backup_job->len); 148 + bdrv_set_dirty_bitmap(block_copy_dirty_bitmap(backup_job->bcs), 0, 149 + backup_job->len); 158 150 } 159 151 160 152 static BlockErrorAction backup_error_action(BackupBlockJob *job, ··· 199 191 BdrvDirtyBitmapIter *bdbi; 200 192 int ret = 0; 201 193 202 - bdbi = bdrv_dirty_iter_new(job->bcs->copy_bitmap); 194 + bdbi = bdrv_dirty_iter_new(block_copy_dirty_bitmap(job->bcs)); 203 195 while ((offset = bdrv_dirty_iter_next(bdbi)) != -1) { 204 196 do { 205 197 if (yield_and_check(job)) { ··· 219 211 return ret; 220 212 } 221 213 222 - static void backup_init_copy_bitmap(BackupBlockJob *job) 214 + static void backup_init_bcs_bitmap(BackupBlockJob *job) 223 215 { 224 216 bool ret; 225 217 uint64_t estimate; 218 + BdrvDirtyBitmap *bcs_bitmap = block_copy_dirty_bitmap(job->bcs); 226 219 227 220 if (job->sync_mode == MIRROR_SYNC_MODE_BITMAP) { 228 - ret = bdrv_dirty_bitmap_merge_internal(job->bcs->copy_bitmap, 229 - job->sync_bitmap, 221 + ret = bdrv_dirty_bitmap_merge_internal(bcs_bitmap, job->sync_bitmap, 230 222 NULL, true); 231 223 assert(ret); 232 224 } else { ··· 235 227 * We can't hog the coroutine to initialize this thoroughly. 236 228 * Set a flag and resume work when we are able to yield safely. 237 229 */ 238 - job->bcs->skip_unallocated = true; 230 + block_copy_set_skip_unallocated(job->bcs, true); 239 231 } 240 - bdrv_set_dirty_bitmap(job->bcs->copy_bitmap, 0, job->len); 232 + bdrv_set_dirty_bitmap(bcs_bitmap, 0, job->len); 241 233 } 242 234 243 - estimate = bdrv_get_dirty_count(job->bcs->copy_bitmap); 235 + estimate = bdrv_get_dirty_count(bcs_bitmap); 244 236 job_progress_set_remaining(&job->common.job, estimate); 245 237 } 246 238 ··· 249 241 BackupBlockJob *s = container_of(job, BackupBlockJob, common.job); 250 242 int ret = 0; 251 243 252 - backup_init_copy_bitmap(s); 244 + backup_init_bcs_bitmap(s); 253 245 254 246 if (s->sync_mode == MIRROR_SYNC_MODE_TOP) { 255 247 int64_t offset = 0; ··· 268 260 269 261 offset += count; 270 262 } 271 - s->bcs->skip_unallocated = false; 263 + block_copy_set_skip_unallocated(s->bcs, false); 272 264 } 273 265 274 266 if (s->sync_mode == MIRROR_SYNC_MODE_NONE) { 275 267 /* 276 - * All bits are set in copy_bitmap to allow any cluster to be copied. 268 + * All bits are set in bcs bitmap to allow any cluster to be copied. 277 269 * This does not actually require them to be copied. 278 270 */ 279 271 while (!job_is_cancelled(job)) { ··· 464 456 job->cluster_size = cluster_size; 465 457 job->len = len; 466 458 467 - block_copy_set_callbacks(bcs, backup_progress_bytes_callback, 468 - backup_progress_reset_callback, job); 459 + block_copy_set_progress_callback(bcs, backup_progress_bytes_callback, job); 460 + block_copy_set_progress_meter(bcs, &job->common.job.progress); 469 461 470 462 /* Required permissions are already taken by backup-top target */ 471 463 block_job_add_bdrv(&job->common, "target", target, 0, BLK_PERM_ALL,

+315 -88

block/block-copy.c

··· 24 24 #define BLOCK_COPY_MAX_BUFFER (1 * MiB) 25 25 #define BLOCK_COPY_MAX_MEM (128 * MiB) 26 26 27 - static void coroutine_fn block_copy_wait_inflight_reqs(BlockCopyState *s, 28 - int64_t start, 29 - int64_t end) 27 + typedef struct BlockCopyInFlightReq { 28 + int64_t offset; 29 + int64_t bytes; 30 + QLIST_ENTRY(BlockCopyInFlightReq) list; 31 + CoQueue wait_queue; /* coroutines blocked on this request */ 32 + } BlockCopyInFlightReq; 33 + 34 + typedef struct BlockCopyState { 35 + /* 36 + * BdrvChild objects are not owned or managed by block-copy. They are 37 + * provided by block-copy user and user is responsible for appropriate 38 + * permissions on these children. 39 + */ 40 + BdrvChild *source; 41 + BdrvChild *target; 42 + BdrvDirtyBitmap *copy_bitmap; 43 + int64_t in_flight_bytes; 44 + int64_t cluster_size; 45 + bool use_copy_range; 46 + int64_t copy_size; 47 + uint64_t len; 48 + QLIST_HEAD(, BlockCopyInFlightReq) inflight_reqs; 49 + 50 + BdrvRequestFlags write_flags; 51 + 52 + /* 53 + * skip_unallocated: 54 + * 55 + * Used by sync=top jobs, which first scan the source node for unallocated 56 + * areas and clear them in the copy_bitmap. During this process, the bitmap 57 + * is thus not fully initialized: It may still have bits set for areas that 58 + * are unallocated and should actually not be copied. 59 + * 60 + * This is indicated by skip_unallocated. 61 + * 62 + * In this case, block_copy() will query the source’s allocation status, 63 + * skip unallocated regions, clear them in the copy_bitmap, and invoke 64 + * block_copy_reset_unallocated() every time it does. 65 + */ 66 + bool skip_unallocated; 67 + 68 + ProgressMeter *progress; 69 + /* progress_bytes_callback: called when some copying progress is done. */ 70 + ProgressBytesCallbackFunc progress_bytes_callback; 71 + void *progress_opaque; 72 + 73 + SharedResource *mem; 74 + } BlockCopyState; 75 + 76 + static BlockCopyInFlightReq *find_conflicting_inflight_req(BlockCopyState *s, 77 + int64_t offset, 78 + int64_t bytes) 30 79 { 31 80 BlockCopyInFlightReq *req; 32 - bool waited; 33 81 34 - do { 35 - waited = false; 36 - QLIST_FOREACH(req, &s->inflight_reqs, list) { 37 - if (end > req->start_byte && start < req->end_byte) { 38 - qemu_co_queue_wait(&req->wait_queue, NULL); 39 - waited = true; 40 - break; 41 - } 82 + QLIST_FOREACH(req, &s->inflight_reqs, list) { 83 + if (offset + bytes > req->offset && offset < req->offset + req->bytes) { 84 + return req; 42 85 } 43 - } while (waited); 86 + } 87 + 88 + return NULL; 44 89 } 45 90 91 + /* 92 + * If there are no intersecting requests return false. Otherwise, wait for the 93 + * first found intersecting request to finish and return true. 94 + */ 95 + static bool coroutine_fn block_copy_wait_one(BlockCopyState *s, int64_t offset, 96 + int64_t bytes) 97 + { 98 + BlockCopyInFlightReq *req = find_conflicting_inflight_req(s, offset, bytes); 99 + 100 + if (!req) { 101 + return false; 102 + } 103 + 104 + qemu_co_queue_wait(&req->wait_queue, NULL); 105 + 106 + return true; 107 + } 108 + 109 + /* Called only on full-dirty region */ 46 110 static void block_copy_inflight_req_begin(BlockCopyState *s, 47 111 BlockCopyInFlightReq *req, 48 - int64_t start, int64_t end) 112 + int64_t offset, int64_t bytes) 49 113 { 50 - req->start_byte = start; 51 - req->end_byte = end; 114 + assert(!find_conflicting_inflight_req(s, offset, bytes)); 115 + 116 + bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes); 117 + s->in_flight_bytes += bytes; 118 + 119 + req->offset = offset; 120 + req->bytes = bytes; 52 121 qemu_co_queue_init(&req->wait_queue); 53 122 QLIST_INSERT_HEAD(&s->inflight_reqs, req, list); 54 123 } 55 124 56 - static void coroutine_fn block_copy_inflight_req_end(BlockCopyInFlightReq *req) 125 + /* 126 + * block_copy_inflight_req_shrink 127 + * 128 + * Drop the tail of the request to be handled later. Set dirty bits back and 129 + * wake up all requests waiting for us (may be some of them are not intersecting 130 + * with shrunk request) 131 + */ 132 + static void coroutine_fn block_copy_inflight_req_shrink(BlockCopyState *s, 133 + BlockCopyInFlightReq *req, int64_t new_bytes) 134 + { 135 + if (new_bytes == req->bytes) { 136 + return; 137 + } 138 + 139 + assert(new_bytes > 0 && new_bytes < req->bytes); 140 + 141 + s->in_flight_bytes -= req->bytes - new_bytes; 142 + bdrv_set_dirty_bitmap(s->copy_bitmap, 143 + req->offset + new_bytes, req->bytes - new_bytes); 144 + 145 + req->bytes = new_bytes; 146 + qemu_co_queue_restart_all(&req->wait_queue); 147 + } 148 + 149 + static void coroutine_fn block_copy_inflight_req_end(BlockCopyState *s, 150 + BlockCopyInFlightReq *req, 151 + int ret) 57 152 { 153 + s->in_flight_bytes -= req->bytes; 154 + if (ret < 0) { 155 + bdrv_set_dirty_bitmap(s->copy_bitmap, req->offset, req->bytes); 156 + } 58 157 QLIST_REMOVE(req, list); 59 158 qemu_co_queue_restart_all(&req->wait_queue); 60 159 } ··· 68 167 bdrv_release_dirty_bitmap(s->copy_bitmap); 69 168 shres_destroy(s->mem); 70 169 g_free(s); 170 + } 171 + 172 + static uint32_t block_copy_max_transfer(BdrvChild *source, BdrvChild *target) 173 + { 174 + return MIN_NON_ZERO(INT_MAX, 175 + MIN_NON_ZERO(source->bs->bl.max_transfer, 176 + target->bs->bl.max_transfer)); 71 177 } 72 178 73 179 BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, ··· 76 182 { 77 183 BlockCopyState *s; 78 184 BdrvDirtyBitmap *copy_bitmap; 79 - uint32_t max_transfer = 80 - MIN_NON_ZERO(INT_MAX, 81 - MIN_NON_ZERO(source->bs->bl.max_transfer, 82 - target->bs->bl.max_transfer)); 83 185 84 186 copy_bitmap = bdrv_create_dirty_bitmap(source->bs, cluster_size, NULL, 85 187 errp); ··· 99 201 .mem = shres_create(BLOCK_COPY_MAX_MEM), 100 202 }; 101 203 102 - if (max_transfer < cluster_size) { 204 + if (block_copy_max_transfer(source, target) < cluster_size) { 103 205 /* 104 206 * copy_range does not respect max_transfer. We don't want to bother 105 207 * with requests smaller than block-copy cluster size, so fallback to ··· 114 216 s->copy_size = cluster_size; 115 217 } else { 116 218 /* 117 - * copy_range does not respect max_transfer (it's a TODO), so we factor 118 - * that in here. 219 + * We enable copy-range, but keep small copy_size, until first 220 + * successful copy_range (look at block_copy_do_copy). 119 221 */ 120 222 s->use_copy_range = true; 121 - s->copy_size = MIN(MAX(cluster_size, BLOCK_COPY_MAX_COPY_RANGE), 122 - QEMU_ALIGN_DOWN(max_transfer, cluster_size)); 223 + s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER); 123 224 } 124 225 125 226 QLIST_INIT(&s->inflight_reqs); ··· 127 228 return s; 128 229 } 129 230 130 - void block_copy_set_callbacks( 231 + void block_copy_set_progress_callback( 131 232 BlockCopyState *s, 132 233 ProgressBytesCallbackFunc progress_bytes_callback, 133 - ProgressResetCallbackFunc progress_reset_callback, 134 234 void *progress_opaque) 135 235 { 136 236 s->progress_bytes_callback = progress_bytes_callback; 137 - s->progress_reset_callback = progress_reset_callback; 138 237 s->progress_opaque = progress_opaque; 139 238 } 140 239 240 + void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm) 241 + { 242 + s->progress = pm; 243 + } 244 + 141 245 /* 142 246 * block_copy_do_copy 143 247 * 144 - * Do copy of cluser-aligned chunk. @end is allowed to exceed s->len only to 145 - * cover last cluster when s->len is not aligned to clusters. 248 + * Do copy of cluster-aligned chunk. Requested region is allowed to exceed 249 + * s->len only to cover last cluster when s->len is not aligned to clusters. 146 250 * 147 251 * No sync here: nor bitmap neighter intersecting requests handling, only copy. 148 252 * 149 253 * Returns 0 on success. 150 254 */ 151 255 static int coroutine_fn block_copy_do_copy(BlockCopyState *s, 152 - int64_t start, int64_t end, 153 - bool *error_is_read) 256 + int64_t offset, int64_t bytes, 257 + bool zeroes, bool *error_is_read) 154 258 { 155 259 int ret; 156 - int nbytes = MIN(end, s->len) - start; 260 + int64_t nbytes = MIN(offset + bytes, s->len) - offset; 157 261 void *bounce_buffer = NULL; 158 262 159 - assert(QEMU_IS_ALIGNED(start, s->cluster_size)); 160 - assert(QEMU_IS_ALIGNED(end, s->cluster_size)); 161 - assert(end < s->len || end == QEMU_ALIGN_UP(s->len, s->cluster_size)); 263 + assert(offset >= 0 && bytes > 0 && INT64_MAX - offset >= bytes); 264 + assert(QEMU_IS_ALIGNED(offset, s->cluster_size)); 265 + assert(QEMU_IS_ALIGNED(bytes, s->cluster_size)); 266 + assert(offset < s->len); 267 + assert(offset + bytes <= s->len || 268 + offset + bytes == QEMU_ALIGN_UP(s->len, s->cluster_size)); 269 + assert(nbytes < INT_MAX); 270 + 271 + if (zeroes) { 272 + ret = bdrv_co_pwrite_zeroes(s->target, offset, nbytes, s->write_flags & 273 + ~BDRV_REQ_WRITE_COMPRESSED); 274 + if (ret < 0) { 275 + trace_block_copy_write_zeroes_fail(s, offset, ret); 276 + if (error_is_read) { 277 + *error_is_read = false; 278 + } 279 + } 280 + return ret; 281 + } 162 282 163 283 if (s->use_copy_range) { 164 - ret = bdrv_co_copy_range(s->source, start, s->target, start, nbytes, 284 + ret = bdrv_co_copy_range(s->source, offset, s->target, offset, nbytes, 165 285 0, s->write_flags); 166 286 if (ret < 0) { 167 - trace_block_copy_copy_range_fail(s, start, ret); 287 + trace_block_copy_copy_range_fail(s, offset, ret); 168 288 s->use_copy_range = false; 169 289 s->copy_size = MAX(s->cluster_size, BLOCK_COPY_MAX_BUFFER); 170 290 /* Fallback to read+write with allocated buffer */ 171 291 } else { 292 + if (s->use_copy_range) { 293 + /* 294 + * Successful copy-range. Now increase copy_size. copy_range 295 + * does not respect max_transfer (it's a TODO), so we factor 296 + * that in here. 297 + * 298 + * Note: we double-check s->use_copy_range for the case when 299 + * parallel block-copy request unsets it during previous 300 + * bdrv_co_copy_range call. 301 + */ 302 + s->copy_size = 303 + MIN(MAX(s->cluster_size, BLOCK_COPY_MAX_COPY_RANGE), 304 + QEMU_ALIGN_DOWN(block_copy_max_transfer(s->source, 305 + s->target), 306 + s->cluster_size)); 307 + } 172 308 goto out; 173 309 } 174 310 } ··· 176 312 /* 177 313 * In case of failed copy_range request above, we may proceed with buffered 178 314 * request larger than BLOCK_COPY_MAX_BUFFER. Still, further requests will 179 - * be properly limited, so don't care too much. 315 + * be properly limited, so don't care too much. Moreover the most likely 316 + * case (copy_range is unsupported for the configuration, so the very first 317 + * copy_range request fails) is handled by setting large copy_size only 318 + * after first successful copy_range. 180 319 */ 181 320 182 321 bounce_buffer = qemu_blockalign(s->source->bs, nbytes); 183 322 184 - ret = bdrv_co_pread(s->source, start, nbytes, bounce_buffer, 0); 323 + ret = bdrv_co_pread(s->source, offset, nbytes, bounce_buffer, 0); 185 324 if (ret < 0) { 186 - trace_block_copy_read_fail(s, start, ret); 325 + trace_block_copy_read_fail(s, offset, ret); 187 326 if (error_is_read) { 188 327 *error_is_read = true; 189 328 } 190 329 goto out; 191 330 } 192 331 193 - ret = bdrv_co_pwrite(s->target, start, nbytes, bounce_buffer, 332 + ret = bdrv_co_pwrite(s->target, offset, nbytes, bounce_buffer, 194 333 s->write_flags); 195 334 if (ret < 0) { 196 - trace_block_copy_write_fail(s, start, ret); 335 + trace_block_copy_write_fail(s, offset, ret); 197 336 if (error_is_read) { 198 337 *error_is_read = false; 199 338 } ··· 206 345 return ret; 207 346 } 208 347 348 + static int block_copy_block_status(BlockCopyState *s, int64_t offset, 349 + int64_t bytes, int64_t *pnum) 350 + { 351 + int64_t num; 352 + BlockDriverState *base; 353 + int ret; 354 + 355 + if (s->skip_unallocated && s->source->bs->backing) { 356 + base = s->source->bs->backing->bs; 357 + } else { 358 + base = NULL; 359 + } 360 + 361 + ret = bdrv_block_status_above(s->source->bs, base, offset, bytes, &num, 362 + NULL, NULL); 363 + if (ret < 0 || num < s->cluster_size) { 364 + /* 365 + * On error or if failed to obtain large enough chunk just fallback to 366 + * copy one cluster. 367 + */ 368 + num = s->cluster_size; 369 + ret = BDRV_BLOCK_ALLOCATED | BDRV_BLOCK_DATA; 370 + } else if (offset + num == s->len) { 371 + num = QEMU_ALIGN_UP(num, s->cluster_size); 372 + } else { 373 + num = QEMU_ALIGN_DOWN(num, s->cluster_size); 374 + } 375 + 376 + *pnum = num; 377 + return ret; 378 + } 379 + 209 380 /* 210 381 * Check if the cluster starting at offset is allocated or not. 211 382 * return via pnum the number of contiguous clusters sharing this allocation. ··· 269 440 270 441 if (!ret) { 271 442 bdrv_reset_dirty_bitmap(s->copy_bitmap, offset, bytes); 272 - s->progress_reset_callback(s->progress_opaque); 443 + progress_set_remaining(s->progress, 444 + bdrv_get_dirty_count(s->copy_bitmap) + 445 + s->in_flight_bytes); 273 446 } 274 447 275 448 *count = bytes; 276 449 return ret; 277 450 } 278 451 279 - int coroutine_fn block_copy(BlockCopyState *s, 280 - int64_t start, uint64_t bytes, 281 - bool *error_is_read) 452 + /* 453 + * block_copy_dirty_clusters 454 + * 455 + * Copy dirty clusters in @offset/@bytes range. 456 + * Returns 1 if dirty clusters found and successfully copied, 0 if no dirty 457 + * clusters found and -errno on failure. 458 + */ 459 + static int coroutine_fn block_copy_dirty_clusters(BlockCopyState *s, 460 + int64_t offset, int64_t bytes, 461 + bool *error_is_read) 282 462 { 283 463 int ret = 0; 284 - int64_t end = bytes + start; /* bytes */ 285 - int64_t status_bytes; 286 - BlockCopyInFlightReq req; 464 + bool found_dirty = false; 287 465 288 466 /* 289 467 * block_copy() user is responsible for keeping source and target in same ··· 292 470 assert(bdrv_get_aio_context(s->source->bs) == 293 471 bdrv_get_aio_context(s->target->bs)); 294 472 295 - assert(QEMU_IS_ALIGNED(start, s->cluster_size)); 296 - assert(QEMU_IS_ALIGNED(end, s->cluster_size)); 473 + assert(QEMU_IS_ALIGNED(offset, s->cluster_size)); 474 + assert(QEMU_IS_ALIGNED(bytes, s->cluster_size)); 297 475 298 - block_copy_wait_inflight_reqs(s, start, bytes); 299 - block_copy_inflight_req_begin(s, &req, start, end); 300 - 301 - while (start < end) { 302 - int64_t next_zero, chunk_end; 476 + while (bytes) { 477 + BlockCopyInFlightReq req; 478 + int64_t next_zero, cur_bytes, status_bytes; 303 479 304 - if (!bdrv_dirty_bitmap_get(s->copy_bitmap, start)) { 305 - trace_block_copy_skip(s, start); 306 - start += s->cluster_size; 480 + if (!bdrv_dirty_bitmap_get(s->copy_bitmap, offset)) { 481 + trace_block_copy_skip(s, offset); 482 + offset += s->cluster_size; 483 + bytes -= s->cluster_size; 307 484 continue; /* already copied */ 308 485 } 309 486 310 - chunk_end = MIN(end, start + s->copy_size); 487 + found_dirty = true; 488 + 489 + cur_bytes = MIN(bytes, s->copy_size); 311 490 312 - next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, start, 313 - chunk_end - start); 491 + next_zero = bdrv_dirty_bitmap_next_zero(s->copy_bitmap, offset, 492 + cur_bytes); 314 493 if (next_zero >= 0) { 315 - assert(next_zero > start); /* start is dirty */ 316 - assert(next_zero < chunk_end); /* no need to do MIN() */ 317 - chunk_end = next_zero; 494 + assert(next_zero > offset); /* offset is dirty */ 495 + assert(next_zero < offset + cur_bytes); /* no need to do MIN() */ 496 + cur_bytes = next_zero - offset; 318 497 } 498 + block_copy_inflight_req_begin(s, &req, offset, cur_bytes); 319 499 320 - if (s->skip_unallocated) { 321 - ret = block_copy_reset_unallocated(s, start, &status_bytes); 322 - if (ret == 0) { 323 - trace_block_copy_skip_range(s, start, status_bytes); 324 - start += status_bytes; 325 - continue; 326 - } 327 - /* Clamp to known allocated region */ 328 - chunk_end = MIN(chunk_end, start + status_bytes); 500 + ret = block_copy_block_status(s, offset, cur_bytes, &status_bytes); 501 + assert(ret >= 0); /* never fail */ 502 + cur_bytes = MIN(cur_bytes, status_bytes); 503 + block_copy_inflight_req_shrink(s, &req, cur_bytes); 504 + if (s->skip_unallocated && !(ret & BDRV_BLOCK_ALLOCATED)) { 505 + block_copy_inflight_req_end(s, &req, 0); 506 + progress_set_remaining(s->progress, 507 + bdrv_get_dirty_count(s->copy_bitmap) + 508 + s->in_flight_bytes); 509 + trace_block_copy_skip_range(s, offset, status_bytes); 510 + offset += status_bytes; 511 + bytes -= status_bytes; 512 + continue; 329 513 } 330 514 331 - trace_block_copy_process(s, start); 515 + trace_block_copy_process(s, offset); 332 516 333 - bdrv_reset_dirty_bitmap(s->copy_bitmap, start, chunk_end - start); 334 - 335 - co_get_from_shres(s->mem, chunk_end - start); 336 - ret = block_copy_do_copy(s, start, chunk_end, error_is_read); 337 - co_put_to_shres(s->mem, chunk_end - start); 517 + co_get_from_shres(s->mem, cur_bytes); 518 + ret = block_copy_do_copy(s, offset, cur_bytes, ret & BDRV_BLOCK_ZERO, 519 + error_is_read); 520 + co_put_to_shres(s->mem, cur_bytes); 521 + block_copy_inflight_req_end(s, &req, ret); 338 522 if (ret < 0) { 339 - bdrv_set_dirty_bitmap(s->copy_bitmap, start, chunk_end - start); 340 - break; 523 + return ret; 341 524 } 342 525 343 - s->progress_bytes_callback(chunk_end - start, s->progress_opaque); 344 - start = chunk_end; 345 - ret = 0; 526 + progress_work_done(s->progress, cur_bytes); 527 + s->progress_bytes_callback(cur_bytes, s->progress_opaque); 528 + offset += cur_bytes; 529 + bytes -= cur_bytes; 346 530 } 347 531 348 - block_copy_inflight_req_end(&req); 532 + return found_dirty; 533 + } 534 + 535 + /* 536 + * block_copy 537 + * 538 + * Copy requested region, accordingly to dirty bitmap. 539 + * Collaborate with parallel block_copy requests: if they succeed it will help 540 + * us. If they fail, we will retry not-copied regions. So, if we return error, 541 + * it means that some I/O operation failed in context of _this_ block_copy call, 542 + * not some parallel operation. 543 + */ 544 + int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes, 545 + bool *error_is_read) 546 + { 547 + int ret; 548 + 549 + do { 550 + ret = block_copy_dirty_clusters(s, offset, bytes, error_is_read); 551 + 552 + if (ret == 0) { 553 + ret = block_copy_wait_one(s, offset, bytes); 554 + } 555 + 556 + /* 557 + * We retry in two cases: 558 + * 1. Some progress done 559 + * Something was copied, which means that there were yield points 560 + * and some new dirty bits may have appeared (due to failed parallel 561 + * block-copy requests). 562 + * 2. We have waited for some intersecting block-copy request 563 + * It may have failed and produced new dirty bits. 564 + */ 565 + } while (ret > 0); 349 566 350 567 return ret; 351 568 } 569 + 570 + BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s) 571 + { 572 + return s->copy_bitmap; 573 + } 574 + 575 + void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip) 576 + { 577 + s->skip_unallocated = skip; 578 + }

+62

block/crypto.c

··· 484 484 } 485 485 486 486 487 + static BlockMeasureInfo *block_crypto_measure(QemuOpts *opts, 488 + BlockDriverState *in_bs, 489 + Error **errp) 490 + { 491 + g_autoptr(QCryptoBlockCreateOptions) create_opts = NULL; 492 + Error *local_err = NULL; 493 + BlockMeasureInfo *info; 494 + uint64_t size; 495 + size_t luks_payload_size; 496 + QDict *cryptoopts; 497 + 498 + /* 499 + * Preallocation mode doesn't affect size requirements but we must consume 500 + * the option. 501 + */ 502 + g_free(qemu_opt_get_del(opts, BLOCK_OPT_PREALLOC)); 503 + 504 + size = qemu_opt_get_size_del(opts, BLOCK_OPT_SIZE, 0); 505 + 506 + if (in_bs) { 507 + int64_t ssize = bdrv_getlength(in_bs); 508 + 509 + if (ssize < 0) { 510 + error_setg_errno(&local_err, -ssize, 511 + "Unable to get image virtual_size"); 512 + goto err; 513 + } 514 + 515 + size = ssize; 516 + } 517 + 518 + cryptoopts = qemu_opts_to_qdict_filtered(opts, NULL, 519 + &block_crypto_create_opts_luks, true); 520 + qdict_put_str(cryptoopts, "format", "luks"); 521 + create_opts = block_crypto_create_opts_init(cryptoopts, &local_err); 522 + qobject_unref(cryptoopts); 523 + if (!create_opts) { 524 + goto err; 525 + } 526 + 527 + if (!qcrypto_block_calculate_payload_offset(create_opts, NULL, 528 + &luks_payload_size, 529 + &local_err)) { 530 + goto err; 531 + } 532 + 533 + /* 534 + * Unallocated blocks are still encrypted so allocation status makes no 535 + * difference to the file size. 536 + */ 537 + info = g_new(BlockMeasureInfo, 1); 538 + info->fully_allocated = luks_payload_size + size; 539 + info->required = luks_payload_size + size; 540 + return info; 541 + 542 + err: 543 + error_propagate(errp, local_err); 544 + return NULL; 545 + } 546 + 547 + 487 548 static int block_crypto_probe_luks(const uint8_t *buf, 488 549 int buf_size, 489 550 const char *filename) { ··· 670 731 .bdrv_co_preadv = block_crypto_co_preadv, 671 732 .bdrv_co_pwritev = block_crypto_co_pwritev, 672 733 .bdrv_getlength = block_crypto_getlength, 734 + .bdrv_measure = block_crypto_measure, 673 735 .bdrv_get_info = block_crypto_get_info_luks, 674 736 .bdrv_get_specific_info = block_crypto_get_specific_info_luks, 675 737

+28 -4

block/curl.c

··· 214 214 { 215 215 BDRVCURLState *s = opaque; 216 216 size_t realsize = size * nmemb; 217 - const char *accept_line = "Accept-Ranges: bytes"; 217 + const char *header = (char *)ptr; 218 + const char *end = header + realsize; 219 + const char *accept_ranges = "accept-ranges:"; 220 + const char *bytes = "bytes"; 218 221 219 - if (realsize >= strlen(accept_line) 220 - && strncmp((char *)ptr, accept_line, strlen(accept_line)) == 0) { 221 - s->accept_range = true; 222 + if (realsize >= strlen(accept_ranges) 223 + && g_ascii_strncasecmp(header, accept_ranges, 224 + strlen(accept_ranges)) == 0) { 225 + 226 + char *p = strchr(header, ':') + 1; 227 + 228 + /* Skip whitespace between the header name and value. */ 229 + while (p < end && *p && g_ascii_isspace(*p)) { 230 + p++; 231 + } 232 + 233 + if (end - p >= strlen(bytes) 234 + && strncmp(p, bytes, strlen(bytes)) == 0) { 235 + 236 + /* Check that there is nothing but whitespace after the value. */ 237 + p += strlen(bytes); 238 + while (p < end && *p && g_ascii_isspace(*p)) { 239 + p++; 240 + } 241 + 242 + if (p == end || !*p) { 243 + s->accept_range = true; 244 + } 245 + } 222 246 } 223 247 224 248 return realsize;

+7 -5

block/qcow2-threads.c

··· 128 128 * @src - source buffer, @src_size bytes 129 129 * 130 130 * Returns: 0 on success 131 - * -1 on fail 131 + * -EIO on fail 132 132 */ 133 133 static ssize_t qcow2_decompress(void *dest, size_t dest_size, 134 134 const void *src, size_t src_size) 135 135 { 136 - int ret = 0; 136 + int ret; 137 137 z_stream strm; 138 138 139 139 memset(&strm, 0, sizeof(strm)); ··· 144 144 145 145 ret = inflateInit2(&strm, -12); 146 146 if (ret != Z_OK) { 147 - return -1; 147 + return -EIO; 148 148 } 149 149 150 150 ret = inflate(&strm, Z_FINISH); 151 - if ((ret != Z_STREAM_END && ret != Z_BUF_ERROR) || strm.avail_out != 0) { 151 + if ((ret == Z_STREAM_END || ret == Z_BUF_ERROR) && strm.avail_out == 0) { 152 152 /* 153 153 * We approve Z_BUF_ERROR because we need @dest buffer to be filled, but 154 154 * @src buffer may be processed partly (because in qcow2 we know size of 155 155 * compressed data with precision of one sector) 156 156 */ 157 - ret = -1; 157 + ret = 0; 158 + } else { 159 + ret = -EIO; 158 160 } 159 161 160 162 inflateEnd(&strm);

+20 -55

block/qcow2.c

··· 2610 2610 2611 2611 qcrypto_block_free(s->crypto); 2612 2612 s->crypto = NULL; 2613 + qapi_free_QCryptoBlockOpenOptions(s->crypto_opts); 2613 2614 2614 2615 g_free(s->unknown_header_fields); 2615 2616 cleanup_unknown_header_ext(bs); ··· 4608 4609 return ret; 4609 4610 } 4610 4611 4611 - static ssize_t qcow2_measure_crypto_hdr_init_func(QCryptoBlock *block, 4612 - size_t headerlen, void *opaque, Error **errp) 4613 - { 4614 - size_t *headerlenp = opaque; 4615 - 4616 - /* Stash away the payload size */ 4617 - *headerlenp = headerlen; 4618 - return 0; 4619 - } 4620 - 4621 - static ssize_t qcow2_measure_crypto_hdr_write_func(QCryptoBlock *block, 4622 - size_t offset, const uint8_t *buf, size_t buflen, 4623 - void *opaque, Error **errp) 4624 - { 4625 - /* Discard the bytes, we're not actually writing to an image */ 4626 - return buflen; 4627 - } 4628 - 4629 - /* Determine the number of bytes for the LUKS payload */ 4630 - static bool qcow2_measure_luks_headerlen(QemuOpts *opts, size_t *len, 4631 - Error **errp) 4632 - { 4633 - QDict *opts_qdict; 4634 - QDict *cryptoopts_qdict; 4635 - QCryptoBlockCreateOptions *cryptoopts; 4636 - QCryptoBlock *crypto; 4637 - 4638 - /* Extract "encrypt." options into a qdict */ 4639 - opts_qdict = qemu_opts_to_qdict(opts, NULL); 4640 - qdict_extract_subqdict(opts_qdict, &cryptoopts_qdict, "encrypt."); 4641 - qobject_unref(opts_qdict); 4642 - 4643 - /* Build QCryptoBlockCreateOptions object from qdict */ 4644 - qdict_put_str(cryptoopts_qdict, "format", "luks"); 4645 - cryptoopts = block_crypto_create_opts_init(cryptoopts_qdict, errp); 4646 - qobject_unref(cryptoopts_qdict); 4647 - if (!cryptoopts) { 4648 - return false; 4649 - } 4650 - 4651 - /* Fake LUKS creation in order to determine the payload size */ 4652 - crypto = qcrypto_block_create(cryptoopts, "encrypt.", 4653 - qcow2_measure_crypto_hdr_init_func, 4654 - qcow2_measure_crypto_hdr_write_func, 4655 - len, errp); 4656 - qapi_free_QCryptoBlockCreateOptions(cryptoopts); 4657 - if (!crypto) { 4658 - return false; 4659 - } 4660 - 4661 - qcrypto_block_free(crypto); 4662 - return true; 4663 - } 4664 - 4665 4612 static BlockMeasureInfo *qcow2_measure(QemuOpts *opts, BlockDriverState *in_bs, 4666 4613 Error **errp) 4667 4614 { ··· 4712 4659 g_free(optstr); 4713 4660 4714 4661 if (has_luks) { 4662 + g_autoptr(QCryptoBlockCreateOptions) create_opts = NULL; 4663 + QDict *opts_qdict; 4664 + QDict *cryptoopts; 4715 4665 size_t headerlen; 4716 4666 4717 - if (!qcow2_measure_luks_headerlen(opts, &headerlen, &local_err)) { 4667 + opts_qdict = qemu_opts_to_qdict(opts, NULL); 4668 + qdict_extract_subqdict(opts_qdict, &cryptoopts, "encrypt."); 4669 + qobject_unref(opts_qdict); 4670 + 4671 + qdict_put_str(cryptoopts, "format", "luks"); 4672 + 4673 + create_opts = block_crypto_create_opts_init(cryptoopts, errp); 4674 + qobject_unref(cryptoopts); 4675 + if (!create_opts) { 4676 + goto err; 4677 + } 4678 + 4679 + if (!qcrypto_block_calculate_payload_offset(create_opts, 4680 + "encrypt.", 4681 + &headerlen, 4682 + &local_err)) { 4718 4683 goto err; 4719 4684 } 4720 4685

+1

block/trace-events

··· 48 48 block_copy_copy_range_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d" 49 49 block_copy_read_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d" 50 50 block_copy_write_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d" 51 + block_copy_write_zeroes_fail(void *bcs, int64_t start, int ret) "bcs %p start %"PRId64" ret %d" 51 52 52 53 # ../blockdev.c 53 54 qmp_block_job_cancel(void *job) "job %p"

+8 -8

blockjob.c

··· 299 299 info->device = g_strdup(job->job.id); 300 300 info->busy = atomic_read(&job->job.busy); 301 301 info->paused = job->job.pause_count > 0; 302 - info->offset = job->job.progress_current; 303 - info->len = job->job.progress_total; 302 + info->offset = job->job.progress.current; 303 + info->len = job->job.progress.total; 304 304 info->speed = job->speed; 305 305 info->io_status = job->iostatus; 306 306 info->ready = job_is_ready(&job->job), ··· 330 330 331 331 qapi_event_send_block_job_cancelled(job_type(&job->job), 332 332 job->job.id, 333 - job->job.progress_total, 334 - job->job.progress_current, 333 + job->job.progress.total, 334 + job->job.progress.current, 335 335 job->speed); 336 336 } 337 337 ··· 350 350 351 351 qapi_event_send_block_job_completed(job_type(&job->job), 352 352 job->job.id, 353 - job->job.progress_total, 354 - job->job.progress_current, 353 + job->job.progress.total, 354 + job->job.progress.current, 355 355 job->speed, 356 356 !!msg, 357 357 msg); ··· 379 379 380 380 qapi_event_send_block_job_ready(job_type(&job->job), 381 381 job->job.id, 382 - job->job.progress_total, 383 - job->job.progress_current, 382 + job->job.progress.total, 383 + job->job.progress.current, 384 384 job->speed); 385 385 } 386 386

+36

crypto/block.c

··· 115 115 } 116 116 117 117 118 + static ssize_t qcrypto_block_headerlen_hdr_init_func(QCryptoBlock *block, 119 + size_t headerlen, void *opaque, Error **errp) 120 + { 121 + size_t *headerlenp = opaque; 122 + 123 + /* Stash away the payload size */ 124 + *headerlenp = headerlen; 125 + return 0; 126 + } 127 + 128 + 129 + static ssize_t qcrypto_block_headerlen_hdr_write_func(QCryptoBlock *block, 130 + size_t offset, const uint8_t *buf, size_t buflen, 131 + void *opaque, Error **errp) 132 + { 133 + /* Discard the bytes, we're not actually writing to an image */ 134 + return buflen; 135 + } 136 + 137 + 138 + bool 139 + qcrypto_block_calculate_payload_offset(QCryptoBlockCreateOptions *create_opts, 140 + const char *optprefix, 141 + size_t *len, 142 + Error **errp) 143 + { 144 + /* Fake LUKS creation in order to determine the payload size */ 145 + g_autoptr(QCryptoBlock) crypto = 146 + qcrypto_block_create(create_opts, optprefix, 147 + qcrypto_block_headerlen_hdr_init_func, 148 + qcrypto_block_headerlen_hdr_write_func, 149 + len, errp); 150 + return crypto != NULL; 151 + } 152 + 153 + 118 154 QCryptoBlockInfo *qcrypto_block_get_info(QCryptoBlock *block, 119 155 Error **errp) 120 156 {

+8 -57

include/block/block-copy.h

··· 18 18 #include "block/block.h" 19 19 #include "qemu/co-shared-resource.h" 20 20 21 - typedef struct BlockCopyInFlightReq { 22 - int64_t start_byte; 23 - int64_t end_byte; 24 - QLIST_ENTRY(BlockCopyInFlightReq) list; 25 - CoQueue wait_queue; /* coroutines blocked on this request */ 26 - } BlockCopyInFlightReq; 27 - 28 21 typedef void (*ProgressBytesCallbackFunc)(int64_t bytes, void *opaque); 29 - typedef void (*ProgressResetCallbackFunc)(void *opaque); 30 - typedef struct BlockCopyState { 31 - /* 32 - * BdrvChild objects are not owned or managed by block-copy. They are 33 - * provided by block-copy user and user is responsible for appropriate 34 - * permissions on these children. 35 - */ 36 - BdrvChild *source; 37 - BdrvChild *target; 38 - BdrvDirtyBitmap *copy_bitmap; 39 - int64_t cluster_size; 40 - bool use_copy_range; 41 - int64_t copy_size; 42 - uint64_t len; 43 - QLIST_HEAD(, BlockCopyInFlightReq) inflight_reqs; 44 - 45 - BdrvRequestFlags write_flags; 46 - 47 - /* 48 - * skip_unallocated: 49 - * 50 - * Used by sync=top jobs, which first scan the source node for unallocated 51 - * areas and clear them in the copy_bitmap. During this process, the bitmap 52 - * is thus not fully initialized: It may still have bits set for areas that 53 - * are unallocated and should actually not be copied. 54 - * 55 - * This is indicated by skip_unallocated. 56 - * 57 - * In this case, block_copy() will query the source’s allocation status, 58 - * skip unallocated regions, clear them in the copy_bitmap, and invoke 59 - * block_copy_reset_unallocated() every time it does. 60 - */ 61 - bool skip_unallocated; 62 - 63 - /* progress_bytes_callback: called when some copying progress is done. */ 64 - ProgressBytesCallbackFunc progress_bytes_callback; 65 - 66 - /* 67 - * progress_reset_callback: called when some bytes reset from copy_bitmap 68 - * (see @skip_unallocated above). The callee is assumed to recalculate how 69 - * many bytes remain based on the dirty bit count of copy_bitmap. 70 - */ 71 - ProgressResetCallbackFunc progress_reset_callback; 72 - void *progress_opaque; 73 - 74 - SharedResource *mem; 75 - } BlockCopyState; 22 + typedef struct BlockCopyState BlockCopyState; 76 23 77 24 BlockCopyState *block_copy_state_new(BdrvChild *source, BdrvChild *target, 78 25 int64_t cluster_size, 79 26 BdrvRequestFlags write_flags, 80 27 Error **errp); 81 28 82 - void block_copy_set_callbacks( 29 + void block_copy_set_progress_callback( 83 30 BlockCopyState *s, 84 31 ProgressBytesCallbackFunc progress_bytes_callback, 85 - ProgressResetCallbackFunc progress_reset_callback, 86 32 void *progress_opaque); 33 + 34 + void block_copy_set_progress_meter(BlockCopyState *s, ProgressMeter *pm); 87 35 88 36 void block_copy_state_free(BlockCopyState *s); 89 37 90 38 int64_t block_copy_reset_unallocated(BlockCopyState *s, 91 39 int64_t offset, int64_t *count); 92 40 93 - int coroutine_fn block_copy(BlockCopyState *s, int64_t start, uint64_t bytes, 41 + int coroutine_fn block_copy(BlockCopyState *s, int64_t offset, int64_t bytes, 94 42 bool *error_is_read); 43 + 44 + BdrvDirtyBitmap *block_copy_dirty_bitmap(BlockCopyState *s); 45 + void block_copy_set_skip_unallocated(BlockCopyState *s, bool skip); 95 46 96 47 #endif /* BLOCK_COPY_H */

+22

include/crypto/block.h

··· 146 146 147 147 148 148 /** 149 + * qcrypto_block_calculate_payload_offset: 150 + * @create_opts: the encryption options 151 + * @optprefix: name prefix for options 152 + * @len: output for number of header bytes before payload 153 + * @errp: pointer to a NULL-initialized error object 154 + * 155 + * Calculate the number of header bytes before the payload in an encrypted 156 + * storage volume. The header is an area before the payload that is reserved 157 + * for encryption metadata. 158 + * 159 + * Returns: true on success, false on error 160 + */ 161 + bool 162 + qcrypto_block_calculate_payload_offset(QCryptoBlockCreateOptions *create_opts, 163 + const char *optprefix, 164 + size_t *len, 165 + Error **errp); 166 + 167 + 168 + /** 149 169 * qcrypto_block_get_info: 150 170 * @block: the block encryption object 151 171 * @errp: pointer to a NULL-initialized error object ··· 269 289 void qcrypto_block_free(QCryptoBlock *block); 270 290 271 291 G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoBlock, qcrypto_block_free) 292 + G_DEFINE_AUTOPTR_CLEANUP_FUNC(QCryptoBlockCreateOptions, 293 + qapi_free_QCryptoBlockCreateOptions) 272 294 273 295 #endif /* QCRYPTO_BLOCK_H */

+2 -9

include/qemu/job.h

··· 28 28 29 29 #include "qapi/qapi-types-job.h" 30 30 #include "qemu/queue.h" 31 + #include "qemu/progress_meter.h" 31 32 #include "qemu/coroutine.h" 32 33 #include "block/aio.h" 33 34 ··· 117 118 /** True if this job should automatically dismiss itself */ 118 119 bool auto_dismiss; 119 120 120 - /** 121 - * Current progress. The unit is arbitrary as long as the ratio between 122 - * progress_current and progress_total represents the estimated percentage 123 - * of work already done. 124 - */ 125 - int64_t progress_current; 126 - 127 - /** Estimated progress_current value at the completion of the job */ 128 - int64_t progress_total; 121 + ProgressMeter progress; 129 122 130 123 /** 131 124 * Return code from @run and/or @prepare callback(s).

+58

include/qemu/progress_meter.h

··· 1 + /* 2 + * Helper functionality for some process progress tracking. 3 + * 4 + * Copyright (c) 2011 IBM Corp. 5 + * Copyright (c) 2012, 2018 Red Hat, Inc. 6 + * Copyright (c) 2020 Virtuozzo International GmbH 7 + * 8 + * Permission is hereby granted, free of charge, to any person obtaining a copy 9 + * of this software and associated documentation files (the "Software"), to deal 10 + * in the Software without restriction, including without limitation the rights 11 + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 + * copies of the Software, and to permit persons to whom the Software is 13 + * furnished to do so, subject to the following conditions: 14 + * 15 + * The above copyright notice and this permission notice shall be included in 16 + * all copies or substantial portions of the Software. 17 + * 18 + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24 + * THE SOFTWARE. 25 + */ 26 + 27 + #ifndef QEMU_PROGRESS_METER_H 28 + #define QEMU_PROGRESS_METER_H 29 + 30 + typedef struct ProgressMeter { 31 + /** 32 + * Current progress. The unit is arbitrary as long as the ratio between 33 + * current and total represents the estimated percentage 34 + * of work already done. 35 + */ 36 + uint64_t current; 37 + 38 + /** Estimated current value at the completion of the process */ 39 + uint64_t total; 40 + } ProgressMeter; 41 + 42 + static inline void progress_work_done(ProgressMeter *pm, uint64_t done) 43 + { 44 + pm->current += done; 45 + } 46 + 47 + static inline void progress_set_remaining(ProgressMeter *pm, uint64_t remaining) 48 + { 49 + pm->total = pm->current + remaining; 50 + } 51 + 52 + static inline void progress_increase_remaining(ProgressMeter *pm, 53 + uint64_t delta) 54 + { 55 + pm->total += delta; 56 + } 57 + 58 + #endif /* QEMU_PROGRESS_METER_H */

+2 -2

job-qmp.c

··· 143 143 .id = g_strdup(job->id), 144 144 .type = job_type(job), 145 145 .status = job->status, 146 - .current_progress = job->progress_current, 147 - .total_progress = job->progress_total, 146 + .current_progress = job->progress.current, 147 + .total_progress = job->progress.total, 148 148 .has_error = !!job->err, 149 149 .error = job->err ? \ 150 150 g_strdup(error_get_pretty(job->err)) : NULL,

+3 -3

job.c

··· 369 369 370 370 void job_progress_update(Job *job, uint64_t done) 371 371 { 372 - job->progress_current += done; 372 + progress_work_done(&job->progress, done); 373 373 } 374 374 375 375 void job_progress_set_remaining(Job *job, uint64_t remaining) 376 376 { 377 - job->progress_total = job->progress_current + remaining; 377 + progress_set_remaining(&job->progress, remaining); 378 378 } 379 379 380 380 void job_progress_increase_remaining(Job *job, uint64_t delta) 381 381 { 382 - job->progress_total += delta; 382 + progress_increase_remaining(&job->progress, delta); 383 383 } 384 384 385 385 void job_event_cancelled(Job *job)

+7 -7

qemu-img.c

··· 817 817 check->corruptions_fixed); 818 818 } 819 819 820 + qapi_free_ImageCheck(check); 821 + check = g_new0(ImageCheck, 1); 820 822 ret = collect_image_check(bs, check, filename, fmt, 0); 821 823 822 824 check->leaks_fixed = leaks_fixed; ··· 882 884 do { 883 885 float progress = 0.0f; 884 886 aio_poll(aio_context, true); 885 - if (job->job.progress_total) { 886 - progress = (float)job->job.progress_current / 887 - job->job.progress_total * 100.f; 887 + if (job->job.progress.total) { 888 + progress = (float)job->job.progress.current / 889 + job->job.progress.total * 100.f; 888 890 } 889 891 qemu_progress_print(progress, 0); 890 892 } while (!job_is_ready(&job->job) && !job_is_completed(&job->job)); ··· 4932 4934 filename = argv[optind]; 4933 4935 } 4934 4936 4935 - if (!filename && 4936 - (object_opts || image_opts || fmt || snapshot_name || sn_opts)) { 4937 - error_report("--object, --image-opts, -f, and -l " 4938 - "require a filename argument."); 4937 + if (!filename && (image_opts || fmt || snapshot_name || sn_opts)) { 4938 + error_report("--image-opts, -f, and -l require a filename argument."); 4939 4939 goto out; 4940 4940 } 4941 4941 if (filename && img_size != UINT64_MAX) {

+1 -1

tests/qemu-iotests/178

··· 50 50 $QEMU_IMG measure # missing arguments 51 51 $QEMU_IMG measure --size 2G "$TEST_IMG" # only one allowed 52 52 $QEMU_IMG measure "$TEST_IMG" a # only one filename allowed 53 - $QEMU_IMG measure --object secret,id=sec0,data=MTIzNDU2,format=base64 # missing filename 53 + $QEMU_IMG measure --object secret,id=sec0,data=MTIzNDU2,format=base64 # size or filename needed 54 54 $QEMU_IMG measure --image-opts # missing filename 55 55 $QEMU_IMG measure -f qcow2 # missing filename 56 56 $QEMU_IMG measure -l snap1 # missing filename

+4 -4

tests/qemu-iotests/178.out.qcow2

··· 5 5 qemu-img: Either --size N or one filename must be specified. 6 6 qemu-img: --size N cannot be used together with a filename. 7 7 qemu-img: At most one filename argument is allowed. 8 - qemu-img: --object, --image-opts, -f, and -l require a filename argument. 9 - qemu-img: --object, --image-opts, -f, and -l require a filename argument. 10 - qemu-img: --object, --image-opts, -f, and -l require a filename argument. 11 - qemu-img: --object, --image-opts, -f, and -l require a filename argument. 8 + qemu-img: Either --size N or one filename must be specified. 9 + qemu-img: --image-opts, -f, and -l require a filename argument. 10 + qemu-img: --image-opts, -f, and -l require a filename argument. 11 + qemu-img: --image-opts, -f, and -l require a filename argument. 12 12 qemu-img: Invalid option list: , 13 13 qemu-img: Invalid parameter 'snapshot.foo' 14 14 qemu-img: Failed in parsing snapshot param 'snapshot.foo'

+4 -4

tests/qemu-iotests/178.out.raw

··· 5 5 qemu-img: Either --size N or one filename must be specified. 6 6 qemu-img: --size N cannot be used together with a filename. 7 7 qemu-img: At most one filename argument is allowed. 8 - qemu-img: --object, --image-opts, -f, and -l require a filename argument. 9 - qemu-img: --object, --image-opts, -f, and -l require a filename argument. 10 - qemu-img: --object, --image-opts, -f, and -l require a filename argument. 11 - qemu-img: --object, --image-opts, -f, and -l require a filename argument. 8 + qemu-img: Either --size N or one filename must be specified. 9 + qemu-img: --image-opts, -f, and -l require a filename argument. 10 + qemu-img: --image-opts, -f, and -l require a filename argument. 11 + qemu-img: --image-opts, -f, and -l require a filename argument. 12 12 qemu-img: Invalid option list: , 13 13 qemu-img: Invalid parameter 'snapshot.foo' 14 14 qemu-img: Failed in parsing snapshot param 'snapshot.foo'

+93

tests/qemu-iotests/288

··· 1 + #!/usr/bin/env bash 2 + # 3 + # qemu-img measure tests for LUKS images 4 + # 5 + # Copyright (C) 2020 Red Hat, Inc. 6 + # 7 + # This program is free software; you can redistribute it and/or modify 8 + # it under the terms of the GNU General Public License as published by 9 + # the Free Software Foundation; either version 2 of the License, or 10 + # (at your option) any later version. 11 + # 12 + # This program is distributed in the hope that it will be useful, 13 + # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 + # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 + # GNU General Public License for more details. 16 + # 17 + # You should have received a copy of the GNU General Public License 18 + # along with this program. If not, see <http://www.gnu.org/licenses/>. 19 + # 20 + 21 + # creator 22 + owner=stefanha@redhat.com 23 + 24 + seq=`basename $0` 25 + echo "QA output created by $seq" 26 + 27 + status=1 # failure is the default! 28 + 29 + _cleanup() 30 + { 31 + _cleanup_test_img 32 + rm -f "$TEST_IMG.converted" 33 + } 34 + trap "_cleanup; exit \$status" 0 1 2 3 15 35 + 36 + # get standard environment, filters and checks 37 + . ./common.rc 38 + . ./common.filter 39 + . ./common.pattern 40 + 41 + _supported_fmt luks 42 + _supported_proto file 43 + _supported_os Linux 44 + 45 + SECRET=secret,id=sec0,data=passphrase 46 + 47 + echo "== measure 1G image file ==" 48 + echo 49 + 50 + $QEMU_IMG measure --object "$SECRET" \ 51 + -O "$IMGFMT" \ 52 + -o key-secret=sec0,iter-time=10 \ 53 + --size 1G 54 + 55 + echo 56 + echo "== create 1G image file (size should be no greater than measured) ==" 57 + echo 58 + 59 + _make_test_img 1G 60 + stat -c "image file size in bytes: %s" "$TEST_IMG_FILE" 61 + 62 + echo 63 + echo "== modified 1G image file (size should be no greater than measured) ==" 64 + echo 65 + 66 + $QEMU_IO --object "$SECRET" --image-opts "$TEST_IMG" -c "write -P 0x51 0x10000 0x400" | _filter_qemu_io | _filter_testdir 67 + stat -c "image file size in bytes: %s" "$TEST_IMG_FILE" 68 + 69 + echo 70 + echo "== measure preallocation=falloc 1G image file ==" 71 + echo 72 + 73 + $QEMU_IMG measure --object "$SECRET" \ 74 + -O "$IMGFMT" \ 75 + -o key-secret=sec0,iter-time=10,preallocation=falloc \ 76 + --size 1G 77 + 78 + echo 79 + echo "== measure with input image file ==" 80 + echo 81 + 82 + IMGFMT=raw IMGKEYSECRET= IMGOPTS= _make_test_img 1G | _filter_imgfmt 83 + QEMU_IO_OPTIONS= IMGOPTSSYNTAX= $QEMU_IO -f raw -c "write -P 0x51 0x10000 0x400" "$TEST_IMG_FILE" | _filter_qemu_io | _filter_testdir 84 + $QEMU_IMG measure --object "$SECRET" \ 85 + -O "$IMGFMT" \ 86 + -o key-secret=sec0,iter-time=10 \ 87 + -f raw \ 88 + "$TEST_IMG_FILE" 89 + 90 + # success, all done 91 + echo "*** done" 92 + rm -f $seq.full 93 + status=0

+30

tests/qemu-iotests/288.out

··· 1 + QA output created by 288 2 + == measure 1G image file == 3 + 4 + required size: 1075810304 5 + fully allocated size: 1075810304 6 + 7 + == create 1G image file (size should be no greater than measured) == 8 + 9 + Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 10 + image file size in bytes: 1075810304 11 + 12 + == modified 1G image file (size should be no greater than measured) == 13 + 14 + wrote 1024/1024 bytes at offset 65536 15 + 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 16 + image file size in bytes: 1075810304 17 + 18 + == measure preallocation=falloc 1G image file == 19 + 20 + required size: 1075810304 21 + fully allocated size: 1075810304 22 + 23 + == measure with input image file == 24 + 25 + Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=1073741824 26 + wrote 1024/1024 bytes at offset 65536 27 + 1 KiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec) 28 + required size: 1075810304 29 + fully allocated size: 1075810304 30 + *** done

+17 -5

tests/qemu-iotests/common.rc

··· 56 56 # peek_file_le 'test.img' 512 2 => 65534 57 57 peek_file_le() 58 58 { 59 - # Wrap in echo $() to strip spaces 60 - echo $(od -j"$2" -N"$3" --endian=little -An -vtu"$3" "$1") 59 + local val=0 shift=0 byte 60 + 61 + # coreutils' od --endian is not portable, so manually assemble bytes. 62 + for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do 63 + val=$(( val | (byte << shift) )) 64 + shift=$((shift + 8)) 65 + done 66 + printf %llu $val 61 67 } 62 68 63 69 # peek_file_be 'test.img' 512 2 => 65279 64 70 peek_file_be() 65 71 { 66 - # Wrap in echo $() to strip spaces 67 - echo $(od -j"$2" -N"$3" --endian=big -An -vtu"$3" "$1") 72 + local val=0 byte 73 + 74 + # coreutils' od --endian is not portable, so manually assemble bytes. 75 + for byte in $(od -j"$2" -N"$3" -An -v -tu1 "$1"); do 76 + val=$(( (val << 8) | byte )) 77 + done 78 + printf %llu $val 68 79 } 69 80 70 - # peek_file_raw 'test.img' 512 2 => '\xff\xfe' 81 + # peek_file_raw 'test.img' 512 2 => '\xff\xfe'. Do not use if the raw data 82 + # is likely to contain \0 or trailing \n. 71 83 peek_file_raw() 72 84 { 73 85 dd if="$1" bs=1 skip="$2" count="$3" status=none

+1

tests/qemu-iotests/group

··· 293 293 283 auto quick 294 294 284 rw 295 295 286 rw quick 296 + 288 quick